Changeset 33510

Timestamp:

Mar 13, 2012, 3:41:09 PM (14 years ago)

Author:

eugene

Message:

use int for image<->measure+catalog index (user-set in relphot.h with IDX_T); allow retry for failed remote load jobs

Location:

branches/eam_branches/ipp-20111122/Ohana/src/relphot

Files:

• include/relphot.h (modified) (1 diff)
• src/BrightCatalog.c (modified) (1 diff)
• src/ImageOps.c (modified) (5 diffs)
• src/load_catalogs.c (modified) (3 diffs)

branches/eam_branches/ipp-20111122/Ohana/src/relphot/include/relphot.h

@@ -7 +7 @@
 # define GRID_V2
 # define NO_IMAGE -100
+
+// choose off_t or int depending on full-scale relphot analysis resources
+// # define IDX_T off_t
+# define IDX_T int 
 
 typedef enum {

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/BrightCatalog.c

@@ -35 +35 @@
   if (!f) {
     fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename);
-    goto escape;
+    return NULL;
   }
 

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/ImageOps.c

@@ -1 +1 @@
 # include "relphot.h"
 void plot_setMcal (double *list, int Npts, StatType *stats, float clouds);
+
+// the MeasureToImage, ImageToCatalog, and ImageToMeasure arrays are a substantial part of the memory footprint.
+// total data volume is currently:
+// a) Naverage*sizeof(AverageTiny)      [averages]       : 32
+// b) Naverage*sizeof(Secfilt)*Nsecfilt [secfilt values] : 8 * 32
+// b) Nmeasure*sizeof(Measure)          [measurements]   : 72
+// c) Nmeasure*sizeof(IDX_T)*3          [image idx]      : 3 * 16
+// d) Nmeasure*sizeof(IDX_T)*3          [mosaic idx]     : 3 * 16
+// e) Nimage*sizeof(Image)              [image data]     : 360
+
+// for 3pi analysis in 2012, we have Nmeasure ~ 20 x Naverage
+// so, each measurement costs ~14.5B (Ave + Sec) + 72B (Meas) + 96B (idx)!
+// we are using off_t (64bit) to avoid the 32bit limit of an int, but
+// if we really had 2^31 measurements in a single analysis, we would be using > 350GB of ram...
+// until we reach that point, it is sort of silly to use IDX_T = off_t here.
+
+// with IDX_T = int, each measurement costs ~14.5B (Ave + Sec) + 72B (Meas) + 48B (idx)!
+
+// elsewhere (not in relphot_images), we need to use off_t because a single catalog 
 
 // we have an array of images: image[ImageIndex] (ImageIndex : 0 < Nimage)
@@ -20 +39 @@
 
 // relationships between the measure,catalog set and the images:
-static off_t       **MeasureToImage; // image index from measure,catalog   : MeasureToImage[cat][meas] = ImageIndex
-static off_t       **ImageToCatalog; // catalog for given measure on image : ImageCatalog[ImageIndex][i] = cat (i : 0 < NonImage[ImageIndex])
-static off_t       **ImageToMeasure; // measure for given measure on image : ImageMeasure[ImageIndex][i] = cat (i : 0 < NonImage[ImageIndex])
+static IDX_T       **MeasureToImage; // image index from measure,catalog   : MeasureToImage[cat][meas] = ImageIndex
+static IDX_T       **ImageToCatalog; // catalog for given measure on image : ImageCatalog[ImageIndex][i] = cat (i : 0 < NonImage[ImageIndex])
+static IDX_T       **ImageToMeasure; // measure for given measure on image : ImageMeasure[ImageIndex][i] = cat (i : 0 < NonImage[ImageIndex])
 
 // MeasureToImage was 'bin'
@@ -138 +157 @@
 void initImageBins (Catalog *catalog, int Ncatalog, int doImageList) {
 
-  off_t i, j;
-
-  ALLOCATE (MeasureToImage, off_t *, Ncatalog);
+  IDX_T i, j;
+
+  ALLOCATE (MeasureToImage, IDX_T *, Ncatalog);
   for (i = 0; i < Ncatalog; i++) {
-    ALLOCATE (MeasureToImage[i], off_t, MAX (catalog[i].Nmeasure, 1));
+    ALLOCATE (MeasureToImage[i], IDX_T, MAX (catalog[i].Nmeasure, 1));
     for (j = 0; j < catalog[i].Nmeasure; j++) MeasureToImage[i][j] = -1;
   }
@@ -149 +168 @@
     ALLOCATE (N_onImage, off_t, Nimage);
     ALLOCATE (N_ONIMAGE, off_t, Nimage);
-    ALLOCATE (ImageToCatalog, off_t *, Nimage);
-    ALLOCATE (ImageToMeasure, off_t *, Nimage);
+    ALLOCATE (ImageToCatalog, IDX_T *, Nimage);
+    ALLOCATE (ImageToMeasure, IDX_T *, Nimage);
 
     for (i = 0; i < Nimage; i++) {
       N_onImage[i] = 0;
       N_ONIMAGE[i] = 30;
-      ALLOCATE (ImageToCatalog[i], off_t, N_ONIMAGE[i]);
-      ALLOCATE (ImageToMeasure[i], off_t, N_ONIMAGE[i]);
+      ALLOCATE (ImageToCatalog[i], IDX_T, N_ONIMAGE[i]);
+      ALLOCATE (ImageToMeasure[i], IDX_T, N_ONIMAGE[i]);
     }
   }
@@ -289 +308 @@
     if (N_onImage[idx] == N_ONIMAGE[idx]) {
       N_ONIMAGE[idx] += 30;
-      REALLOCATE (ImageToCatalog[idx], off_t, N_ONIMAGE[idx]);
-      REALLOCATE (ImageToMeasure[idx], off_t, N_ONIMAGE[idx]);
+      REALLOCATE (ImageToCatalog[idx], IDX_T, N_ONIMAGE[idx]);
+      REALLOCATE (ImageToMeasure[idx], IDX_T, N_ONIMAGE[idx]);
     }   
   }

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/load_catalogs.c

@@ -77 +77 @@
     Nstar += catalog[i].Naverage;
     Nmeas += catalog[i].Nmeasure;
+    if ((sizeof(IDX_T) == 8) && (catalog[i].Nmeasure > 0xffffffff)) {
+      fprintf (stderr, "ERROR: using small-sized IDX_T on data with more than 2G detections per table will cause errors\n");
+      fprintf (stderr, "  If you need to do this, and you can afford the RAM, rebuild with IDX_T set to off_t (see relphot.h, ImagesOps.c)\n");
+      exit (3);
+    }
   }
   if (Nstar < 2) { 
@@ -173 +178 @@
       if (!pid) {
         if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
-        exit (1);
+        continue;
       }
       table->hosts[i].pid = pid; // save for future reference
@@ -196 +201 @@
   for (i = 0; i < table->Nhosts; i++) {
 
-    // XXX save this name in table->hosts[]?  it is created above as well...
     char catalogFile[512];
     snprintf (catalogFile, 512, "%s/relphot.catalog.subset.dat", table->hosts[i].pathname);
 
-    BrightCatalog *bcatalog = BrightCatalogLoad (catalogFile);
-    assert (bcatalog);
+    BrightCatalog *bcatalog = NULL;
+    while ((bcatalog = BrightCatalogLoad (catalogFile)) == NULL) {
+      // failed to get the data from this host.  This can happen for various reasons.  Give the user a chance to try again...
+      fprintf (stderr, "failed to read data from %s, stopping operations until this can be fixed\n", table->hosts[i].hostname);
+      fprintf (stderr, "you may run the command manually and send this process the CONT signal\n");
+      int pid = getpid();
+      kill (pid, SIGSTOP);
+      fprintf (stderr, "retrying %s\n", catalogFile);
+    }
     
     BrightCatalogSplit (catalogs, bcatalog);