Changeset 33483

Timestamp:

Mar 12, 2012, 8:37:16 PM (14 years ago)

Author:

eugene

Message:

do not include stars with too few meas in all secfilt; free unneeded memory in relphot; pass error messages up

Location:

branches/eam_branches/ipp-20111122/Ohana/src

Files:

• libdvo/src/HostTable.c (modified) (3 diffs)
• relphot/src/BrightCatalog.c (modified) (9 diffs)
• relphot/src/bcatalog.c (modified) (6 diffs)
• relphot/src/select_images.c (modified) (1 diff)
• uniphot/src/setphot.c (modified) (1 diff)
• uniphot/src/update_dvo_setphot.c (modified) (1 diff)

branches/eam_branches/ipp-20111122/Ohana/src/libdvo/src/HostTable.c

@@ -211 +211 @@
   // XXX can I set the fd_sets once, since I am not actually closing the fd's?
 
+  int globalStatus = TRUE;
+
   int i;
   int Nmax = 0;
@@ -318 +320 @@
           if (host->status) {
             fprintf (stdout, "job failed on %s\n", host->hostname);
+            globalStatus = FALSE;
           }
         } else {
           host->status = -1;
           fprintf (stdout, "job exited abnormally on %s\n", host->hostname);
+          globalStatus = FALSE;
           continue;
         }
@@ -335 +339 @@
 
 escape:
-  return TRUE;
-}
+  return globalStatus;
+}

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/BrightCatalog.c

@@ -102 +102 @@
       measure[i].photcode  = photcode[i];
     }
-    // fprintf (stderr, "loaded data for %lld measures\n", (long long) Nrow);
+    fprintf (stderr, "loaded data for %lld measure\n", (long long) Nrow);
 
     free (dR      );
@@ -154 +154 @@
       average[i].catID          = catID[i];
     }
-    fprintf (stderr, "loaded data for %lld averages\n", (long long) Nrow);
+    fprintf (stderr, "loaded data for %lld average\n", (long long) Nrow);
 
     free (R             );
@@ -200 +200 @@
       secfilt[i].M_80  = M_80[i];
     }
-    fprintf (stderr, "loaded data for %lld averages\n", (long long) Nrow);
+    fprintf (stderr, "loaded data for %lld secfilt\n", (long long) Nrow);
 
     free (M    );
@@ -557 +557 @@
     dvo_catalog_init (&catalogs->catalog[i], TRUE);
     catalogs->catIDs[i] = 0;
-    catalogs->NAVERAGE[i] = 100;
-    catalogs->NMEASURE[i] = 100;
+    catalogs->NAVERAGE[i] = 256;
+    catalogs->NMEASURE[i] = 256;
     catalogs->catalog[i].Naverage = 0;
     catalogs->catalog[i].Nmeasure = 0;
@@ -591 +591 @@
 
   int i;
+
+  int D_NCATALOG = 16;
 
   int Nsecfilt = catalogs->Nsecfilt;
@@ -632 +634 @@
 
     if (catalogs->Ncatalog >= catalogs->NCATALOG) {
-      catalogs->NCATALOG += 16;
+      catalogs->NCATALOG += D_NCATALOG;
+      D_NCATALOG = MAX(2000, 2*D_NCATALOG);
+
       // fprintf (stderr, "realloc catalogs->catalog: old: %llx  ", (long long) catalogs->catalog);
       REALLOCATE (catalogs->catalog, Catalog, catalogs->NCATALOG);
@@ -651 +655 @@
         dvo_catalog_init (&catalogs->catalog[j], TRUE);
         catalogs->catIDs[j] = 0;
-        catalogs->NAVERAGE[j] = 100;
-        catalogs->NMEASURE[j] = 100;
+        catalogs->NAVERAGE[j] = 256;
+        catalogs->NMEASURE[j] = 256;
         catalogs->catalog[j].Naverage = 0;
         catalogs->catalog[j].Nmeasure = 0;
@@ -684 +688 @@
     catalogs->catalog[Nc].Naverage ++;
     if (catalogs->catalog[Nc].Naverage >= catalogs->NAVERAGE[Nc]) {
-      catalogs->NAVERAGE[Nc] += 100;
+      catalogs->NAVERAGE[Nc] += MAX(catalogs->NAVERAGE[Nc], 1024);
       REALLOCATE (catalogs->catalog[Nc].averageT, AverageTiny, catalogs->NAVERAGE[Nc]);
       REALLOCATE (catalogs->catalog[Nc].secfilt,  SecFilt,     catalogs->NAVERAGE[Nc]*Nsecfilt);
@@ -701 +705 @@
     catalogs->catalog[Nc].Nmeasure ++;
     if (catalogs->catalog[Nc].Nmeasure >= catalogs->NMEASURE[Nc]) {
-      catalogs->NMEASURE[Nc] += 100;
+      catalogs->NMEASURE[Nc] += MAX(catalogs->NMEASURE[Nc], 4096);
       REALLOCATE (catalogs->catalog[Nc].measureT, MeasureTiny, catalogs->NMEASURE[Nc]);
     }          

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/bcatalog.c

@@ -4 +4 @@
   
   off_t i, j, offset;
-  int found, Ns;
+  int found, Ns, *Nvalid;
   off_t NAVERAGE, NMEASURE, Naverage, Nmeasure, Nm;
   float mag;
@@ -27 +27 @@
   subcatalog[0].filename = catalog[0].filename;
 
+  // array to count measures with photcodes->equiv to each of the secfilts
+  ALLOCATE (Nvalid, int, Nsecfilt);
+
   // flags used by the photometry analysis (excluding UBERCAL)
   unsigned int PHOTOM_FLAGS = 
@@ -44 +47 @@
     for (j = 0; j < Nsecfilt; j++) {
       subcatalog[0].secfilt[Nsecfilt*Naverage+j] = catalog[0].secfilt[Nsecfilt*i+j];
+      Nvalid[j] = 0; // reset the Nvalid array for this star
     }
 
@@ -86 +90 @@
       if (!code) continue;
       found = FALSE;
+      int Nsec = -1; // Nsec equivalent for the measurement
       for (Ns = 0; !found && (Ns < Nphotcodes); Ns++) {
-        if (code->equiv == photcodes[Ns][0].code) found = TRUE;
+        if (code->equiv != photcodes[Ns][0].code) continue;
+        found = TRUE;
+        Nsec = GetPhotcodeNsec(code->equiv);
       }
       if (!found) {
@@ -134 +141 @@
         if (mag < ImagMin) { Nimag ++; continue; }
         if (mag > ImagMax) { Nimag ++; continue; }
+      }
+
+      // count this measurement as valid for this secfilt entry
+      if (Nsec > -1) {
+        assert (Nsec < Nsecfilt);
+        Nvalid[Nsec] ++;
       }
 
@@ -162 +175 @@
 
     // XXXX test : what checks do I need to make elsewhere to avoid problems here?
-    if (Nm <= STAR_TOOFEW) { /* enough measurements in band? */
+    if (Nm <= STAR_TOOFEW) { /* enough measurements total? */
       Nmeasure -= Nm;
       Nfew ++;
       continue; 
     }
+    int anySecfiltGood = FALSE;
+    for (Ns = 0; Ns < Nsecfilt; Ns++) { /* enough measurements in at least one band? */
+      if (Nvalid[Ns] <= STAR_TOOFEW) continue;
+      anySecfiltGood = TRUE;
+    }
+    if (!anySecfiltGood) {
+      Nmeasure -= Nm;
+      Nfew ++;
+      continue; 
+    }
+
     subcatalog[0].averageT[Naverage].Nmeasure = Nm;
     Naverage ++;

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/select_images.c

@@ -267 +267 @@
   REALLOCATE (line_number, off_t, MAX (nimage, 1));
   free (skycoords);
+  free (RmaxSky);
+  free (index);
 
   *Nimage  = nimage;

branches/eam_branches/ipp-20111122/Ohana/src/uniphot/src/setphot.c

@@ -43 +43 @@
 
   if (!IMAGES_ONLY) {
-    update_dvo_setphot (image, Nimage, &flatcorrTable);
+    status = update_dvo_setphot (image, Nimage, &flatcorrTable);
   }
 
-  // write image table
+  // write image table (even if some remote clients failed)
   if (UPDATE) {
     dvo_image_save (&db, VERBOSE);
   }
 
+  if (!status) exit (1);
   exit (0);
 }

branches/eam_branches/ipp-20111122/Ohana/src/uniphot/src/update_dvo_setphot.c

@@ -147 +147 @@
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    if (!status) return FALSE;
   }