Changeset 33906

Timestamp:

May 23, 2012, 11:03:24 AM (14 years ago)

Author:

eugene

Message:

save Image IDmap in parallel mode so client need not load the full image table

Location:

branches/eam_branches/ipp-20120405/Ohana/src/dvomerge

Files:

• Makefile (modified) (2 diffs)
• include/dvomerge.h (modified) (2 diffs)
• src/IDmapIO.c (added)
• src/dvo_image_merge_dbs.c (modified) (6 diffs)
• src/dvomergeImageIDs.c (modified) (2 diffs)
• src/dvomergeUpdate.c (modified) (1 diff)
• src/dvomergeUpdate_catalogs.c (modified) (5 diffs)
• src/dvomerge_client.c (modified) (3 diffs)

branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/Makefile

@@ -34 +34 @@
 $(SRC)/dvomergeImageIDs.$(ARCH).o \
 $(SRC)/dvo_image_merge_dbs.$(ARCH).o \
+$(SRC)/IDmapIO.$(ARCH).o \
 $(SRC)/SetSignals.$(ARCH).o \
 $(SRC)/ConfigInit.$(ARCH).o \
@@ -56 +57 @@
 $(SRC)/dvomergeImageIDs.$(ARCH).o \
 $(SRC)/dvo_image_merge_dbs.$(ARCH).o \
+$(SRC)/IDmapIO.$(ARCH).o \
 $(SRC)/SetSignals.$(ARCH).o \
 $(SRC)/ConfigInit.$(ARCH).o \

branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/include/dvomerge.h

@@ -42 +42 @@
 
 typedef struct {
-  off_t Nmap;
-  off_t *old;
-  off_t *new;
+  unsigned int Nmap;
+  unsigned int *old;
+  unsigned int *new;
 } IDmapType;
 
@@ -125 +125 @@
 int        dvomergeFromList       PROTO((int argc, char **argv));
 int        dvomergeUpdate_threaded PROTO((int argc, char **argv));
-int        dvomergeUpdate_catalogs PROTO((char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType IDmap, int *secfiltMap));
+int        dvomergeUpdate_catalogs PROTO((char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap));
 
 int        replace_match           PROTO((Average *average_out, Measure *measure_out, Average *average_in, Measure *measure_in));
+
+int        IDmapSave               PROTO((char *filename, IDmapType *IDmap));
+IDmapType *IDmapLoad               PROTO((char *filename));

branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c

@@ -1 +1 @@
 # include "dvomerge.h"
 
+// utility functions
 void sort_IDmap (IDmapType *IDmap);
-
-off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt) {
-
-  // use bisection to find the starting entry by time
-
-  off_t Nlo, Nhi, N;
-
-  // find the last TGT before start
-  Nlo = 0; Nhi = NimagesTgt;
-  while (Nhi - Nlo > 10) {
-    N = 0.5*(Nlo + Nhi);
-    if (TgtTimes[N] < start) {
-      Nlo = MAX(N, 0);
-    } else {
-      Nhi = MIN(N + 1, NimagesTgt);
-    }
-  }
-
-  // check for the matched mosaic starting from Nlo 
-  // we may have to go much beyond Nlo since stop is not sorted
-  // can we use a sorted version of stop to check when we are beyond the valid range??
-  for (N = Nlo; N < NimagesTgt; N++) { 
-    if (TgtTimes[N] < start) continue;
-    if (TgtTimes[N] > stop) return (-1);
-    if (TgtCodes[N] != photcode) continue;
-    return (TgtIndex[N]);
-  }
-  return (-1);
-}
-
-// sort two times vectors and an index by first time vector
-void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N) {
-
-# define SWAPFUNC(A,B){ e_time tmp_t; off_t tmp_i; short tmp_c;         \
-  tmp_t = S[A]; S[A] = S[B]; S[B] = tmp_t; \
-  tmp_i = I[A]; I[A] = I[B]; I[B] = tmp_i; \
-  tmp_c = C[A]; C[A] = C[B]; C[B] = tmp_c; \
-}
-# define COMPARE(A,B)(S[A] < S[B])
-
-  OHANA_SORT (N, COMPARE, SWAPFUNC);
-
-# undef SWAPFUNC
-# undef COMPARE
-
-}
+void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N);
+off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt);
 
 // we have two tables; 'tgt' contains some exposures from 'src' : find them and match a map
@@ -71 +28 @@
   }
 
-  ALLOCATE (IDmap->old, off_t, NimagesSrc);
-  ALLOCATE (IDmap->new, off_t, NimagesSrc);
+  ALLOCATE (IDmap->old, unsigned int, NimagesSrc);
+  ALLOCATE (IDmap->new, unsigned int, NimagesSrc);
   IDmap->Nmap = NimagesSrc;
 
@@ -130 +87 @@
   }
 
-  ALLOCATE (IDmap->old, off_t, Nimages);
-  ALLOCATE (IDmap->new, off_t, Nimages);
+  ALLOCATE (IDmap->old, unsigned int, Nimages);
+  ALLOCATE (IDmap->new, unsigned int, Nimages);
   IDmap->Nmap = Nimages;
 
@@ -165 +122 @@
 }
 
-// XXX isn't the map just ID_new = ID_old + offset ??
+// map this ID to the new table
+// XXX isn't the map just ID_new = ID_old + offset ?? (probably not)
 off_t dvo_map_image_ID (IDmapType *IDmap, off_t oldID) {
 
@@ -199 +157 @@
 }
 
+// given a table of detections, update their image IDs based on the new map
 int dvo_update_image_IDs (IDmapType *IDmap, Catalog *catalog) {
 
@@ -237 +196 @@
 }
 
+off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt) {
+
+  // use bisection to find the starting entry by time
+
+  off_t Nlo, Nhi, N;
+
+  // find the last TGT before start
+  Nlo = 0; Nhi = NimagesTgt;
+  while (Nhi - Nlo > 10) {
+    N = 0.5*(Nlo + Nhi);
+    if (TgtTimes[N] < start) {
+      Nlo = MAX(N, 0);
+    } else {
+      Nhi = MIN(N + 1, NimagesTgt);
+    }
+  }
+
+  // check for the matched mosaic starting from Nlo 
+  // we may have to go much beyond Nlo since stop is not sorted
+  // can we use a sorted version of stop to check when we are beyond the valid range??
+  for (N = Nlo; N < NimagesTgt; N++) { 
+    if (TgtTimes[N] < start) continue;
+    if (TgtTimes[N] > stop) return (-1);
+    if (TgtCodes[N] != photcode) continue;
+    return (TgtIndex[N]);
+  }
+  return (-1);
+}
+
+// sort two times vectors and an index by first time vector
+void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N) {
+
+# define SWAPFUNC(A,B){ e_time tmp_t; off_t tmp_i; short tmp_c;         \
+  tmp_t = S[A]; S[A] = S[B]; S[B] = tmp_t; \
+  tmp_i = I[A]; I[A] = I[B]; I[B] = tmp_i; \
+  tmp_c = C[A]; C[A] = C[B]; C[B] = tmp_c; \
+}
+# define COMPARE(A,B)(S[A] < S[B])
+
+  OHANA_SORT (N, COMPARE, SWAPFUNC);
+
+# undef SWAPFUNC
+# undef COMPARE
+
+}
+

branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/src/dvomergeImageIDs.c

@@ -1 +1 @@
 # include "dvomerge.h"
 
-/*** update the image table ***/
+/*** generate the IDmap for the given databases, and update the output image table ***/
 int dvomergeImagesUpdate (IDmapType *IDmap, char *input, char *output) { 
 
@@ -53 +53 @@
 }
 
-/*** update the image table ***/
+/*** generate the map for the given 2 databases ***/
 int dvomergeImagesGetMap (IDmapType *IDmap, char *input, char *output) { 
 

branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/src/dvomergeUpdate.c

@@ -105 +105 @@
   SetPhotcodeTable(NULL);
 
-  dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, IDmap, secfiltMap);
+  dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, &IDmap, secfiltMap);
 
   // save the output sky table copy

branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c

@@ -8 +8 @@
 // output tables
 
-int dvomergeUpdate_parallel (char *input, char *output, SkyTable *outsky);
-
-int dvomergeUpdate_catalogs (char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType IDmap, int *secfiltMap) {
+int dvomergeUpdate_parallel (char *input, char *output, SkyTable *outsky, IDmapType *IDmap);
+
+int dvomergeUpdate_catalogs (char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap) {
 
   off_t i, j;
@@ -17 +17 @@
 
   if (PARALLEL && !HOST_ID) {
-    int status = dvomergeUpdate_parallel (input, output, outsky);
+    int status = dvomergeUpdate_parallel (input, output, outsky, IDmap);
     return status;
   }
@@ -145 +145 @@
       LoadCatalog (&outcatalog, outlist[0].regions[j], outcatalog.filename, "w", NsecfiltOutput);
 
-      dvo_update_image_IDs (&IDmap, &incatalog);
+      dvo_update_image_IDs (IDmap, &incatalog);
       merge_catalogs_old (outlist[0].regions[j], &outcatalog, &incatalog, RADIUS, secfiltMap);
 
@@ -185 +185 @@
 }
 
-int dvomergeUpdate_parallel (char *input, char *output, SkyTable *outsky) {
-
-  // launch the dvomergeUpdate_client jobs to the parallel hosts
+// launch the dvomergeUpdate_client jobs to the parallel hosts
+int dvomergeUpdate_parallel (char *input, char *output, SkyTable *outsky, IDmapType *IDmap) {
+
+  // ensure that the paths are absolute path names
+  char *absinput  = abspath (input,  DVO_MAX_PATH);
+  char *absoutput = abspath (output, DVO_MAX_PATH);
+
+  // name of image subset file:
+  char IDmapFilename[DVO_MAX_PATH];
+  snprintf (IDmapFilename, DVO_MAX_PATH, "%s/IDmap.fits", absoutput);
+
+  // save IDmap information
+  if (!IDmapSave (IDmapFilename, IDmap)) {
+    fprintf (stderr, "ERROR: failure to save the image ID map\n");
+    exit (1);
+  }
 
   // load the list of hosts
@@ -203 +216 @@
     free (table->hosts[i].pathname);
     table->hosts[i].pathname = tmppath;
-
-    // ensure that the paths are absolute path names
-    char *absinput  = abspath (input,  DVO_MAX_PATH);
-    char *absoutput = abspath (output, DVO_MAX_PATH);
 
     // options / arguments that can affect relastro_client -update-objects:

branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/src/dvomerge_client.c

@@ -10 +10 @@
   SkyList *inlist;
   char filename[256], *input, *output;
-  IDmapType IDmap;
+  IDmapType *IDmap;
   PhotCodeData *inputPhotcodes;
   PhotCodeData *outputPhotcodes;
@@ -51 +51 @@
   }
 
-  dvomergeImagesGetMap (&IDmap, input, output);
+  char *absoutput = abspath (output, DVO_MAX_PATH);
+  char IDmapFilename[DVO_MAX_PATH];
+  snprintf (IDmapFilename, DVO_MAX_PATH, "%s/IDmap.fits", absoutput);
+
+  // save IDmap information
+  IDmap = IDmapLoad (IDmapFilename);
+  if (!IDmap) {
+    fprintf (stderr, "ERROR: failure to save the image ID map\n");
+    exit (1);
+  }
 
   // load the sky table for the existing database
@@ -74 +83 @@
   SetPhotcodeTable(NULL);
   
-   dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, IDmap, secfiltMap);
+  dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, IDmap, secfiltMap);
 
   exit (0);