Changeset 38986 for trunk/Ohana/src/dvomerge

Timestamp:

Oct 27, 2015, 4:49:06 PM (11 years ago)

Author:

eugene

Message:

extensive work on relphot, relastro, uniphot, dvomerge aiming to the construction and calibration of PV3

Location:

trunk/Ohana

Files:

• . (modified) (1 prop)
• src/dvomerge/Makefile (modified) (8 diffs)
• src/dvomerge/include/dvomerge.h (modified) (3 diffs)
• src/dvomerge/include/dvoutils.h (modified) (2 diffs)
• src/dvomerge/include/dvoverify.h (modified) (2 diffs)
• src/dvomerge/src/LoadCatalog.c (modified) (1 diff)
• src/dvomerge/src/ReadDeleteListExternID.c (modified) (2 diffs)
• src/dvomerge/src/args.c (modified) (2 diffs)
• src/dvomerge/src/build_links.c (modified) (1 diff)
• src/dvomerge/src/dvoconvert.c (modified) (1 diff)
• src/dvomerge/src/dvomergeContinue.c (modified) (1 diff)
• src/dvomerge/src/dvomergeContinue_threaded.c (modified) (2 diffs)
• src/dvomerge/src/dvomergeCreate.c (modified) (1 diff)
• src/dvomerge/src/dvomergeFromList.c (modified) (2 diffs)
• src/dvomerge/src/dvomergeUpdate_catalogs.c (modified) (4 diffs)
• src/dvomerge/src/dvomergeUpdate_threaded.c (modified) (2 diffs)
• src/dvomerge/src/dvomerge_client.c (modified) (2 diffs)
• src/dvomerge/src/dvorepair.c (modified) (1 diff)
• src/dvomerge/src/dvorepairCPT.c (modified) (1 diff)
• src/dvomerge/src/dvorepairDeleteImagesByExternID.c (modified) (9 diffs)
• src/dvomerge/src/dvorepairFixImages.c (modified) (4 diffs)
• src/dvomerge/src/dvorepairFixWarpIDs.c (copied) (copied from branches/eam_branches/ipp-20150625/Ohana/src/dvomerge/src/dvorepairFixWarpIDs.c )
• src/dvomerge/src/dvorepairImageVsMeasure.c (modified) (1 diff)
• src/dvomerge/src/dvorepair_by_objID.c (copied) (copied from branches/eam_branches/ipp-20150625/Ohana/src/dvomerge/src/dvorepair_by_objID.c )
• src/dvomerge/src/dvosecfilt_catalogs.c (modified) (1 diff)
• src/dvomerge/src/dvoutils.c (modified) (1 diff)
• src/dvomerge/src/dvoutils_load_image_index.c (modified) (3 diffs)
• src/dvomerge/src/dvoutils_uniq_images.c (modified) (6 diffs)
• src/dvomerge/src/dvoverify.c (modified) (2 diffs)
• src/dvomerge/src/dvoverify_args.c (modified) (3 diffs)
• src/dvomerge/src/dvoverify_catalogs.c (modified) (2 diffs)
• src/dvomerge/src/dvoverify_client.c (modified) (1 diff)
• src/dvomerge/src/dvoverify_utils.c (modified) (5 diffs)
• src/dvomerge/src/help.c (modified) (1 diff)
• src/dvomerge/src/merge_catalogs_old.c (modified) (2 diffs)
• src/dvomerge/src/repair_catalog_by_objID.c (copied) (copied from branches/eam_branches/ipp-20150625/Ohana/src/dvomerge/src/repair_catalog_by_objID.c )
• src/dvomerge/src/replace_match.c (modified) (4 diffs)
• src/dvomerge/src/replace_tycho.c (modified) (2 diffs)

trunk/Ohana/src/dvomerge/Makefile

@@ -40 +40 @@
 $(SRC)/dvo_image_merge_dbs.$(ARCH).o \
 $(SRC)/IDmapIO.$(ARCH).o \
+$(SRC)/ConfigInit.$(ARCH).o \
 $(SRC)/SetSignals.$(ARCH).o \
-$(SRC)/ConfigInit.$(ARCH).o \
 $(SRC)/Shutdown.$(ARCH).o \
 $(SRC)/help.$(ARCH).o \
@@ -51 +51 @@
 $(SRC)/replace_match.$(ARCH).o \
 $(SRC)/replace_tycho.$(ARCH).o \
+$(SRC)/repair_catalog_by_objID.$(ARCH).o \
 $(SRC)/merge_catalogs_new.$(ARCH).o \
 $(SRC)/merge_catalogs_old.$(ARCH).o
@@ -75 +76 @@
 $(SRC)/replace_match.$(ARCH).o \
 $(SRC)/replace_tycho.$(ARCH).o \
+$(SRC)/repair_catalog_by_objID.$(ARCH).o \
 $(SRC)/merge_catalogs_new.$(ARCH).o \
 $(SRC)/merge_catalogs_old.$(ARCH).o
@@ -122 +124 @@
 $(SRC)/dvorepair.$(ARCH).o \
 $(SRC)/dvorepairFixCPT.$(ARCH).o \
+$(SRC)/dvorepairFixWarpIDs.$(ARCH).o \
 $(SRC)/dvorepairImagesVsMeasures.$(ARCH).o \
 $(SRC)/dvorepairDeleteImageList.$(ARCH).o \
@@ -127 +130 @@
 $(SRC)/dvorepairDeleteImagesByExternID.$(ARCH).o \
 $(SRC)/dvorepairFixImages.$(ARCH).o \
+$(SRC)/dvorepair_by_objID.$(ARCH).o \
+$(SRC)/repair_catalog_by_objID.$(ARCH).o \
 $(SRC)/psps_ids.$(ARCH).o \
 $(SRC)/LoadImages.$(ARCH).o \
 $(SRC)/ReadDeleteList.$(ARCH).o \
 $(SRC)/ReadDeleteListExternID.$(ARCH).o \
+$(SRC)/SetSignals.$(ARCH).o \
+$(SRC)/Shutdown.$(ARCH).o \
 $(SRC)/myIndex.$(ARCH).o \
 $(SRC)/match_image.$(ARCH).o \
@@ -140 +147 @@
 
 DVOVERIFY = \
+$(SRC)/SetSignals.$(ARCH).o \
+$(SRC)/Shutdown.$(ARCH).o \
 $(SRC)/dvoverify.$(ARCH).o \
 $(SRC)/dvoverify_args.$(ARCH).o \
@@ -149 +158 @@
 
 DVOVERIFY_CLIENT = \
+$(SRC)/SetSignals.$(ARCH).o \
+$(SRC)/Shutdown.$(ARCH).o \
 $(SRC)/dvoverify_client.$(ARCH).o \
 $(SRC)/dvoverify_args.$(ARCH).o \
@@ -158 +169 @@
 
 DVOUTILS = \
+$(SRC)/SetSignals.$(ARCH).o \
+$(SRC)/Shutdown.$(ARCH).o \
 $(SRC)/dvoutils.$(ARCH).o \
 $(SRC)/dvoutils_args.$(ARCH).o \

trunk/Ohana/src/dvomerge/include/dvomerge.h

@@ -47 +47 @@
 
 int    MATCHED_TABLES;
+int    REPAIR_BY_OBJID;
 
 char *SINGLE_CPT;
@@ -202 +203 @@
 int        dvomergeUpdate_catalogs PROTO((char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap));
 
-int        replace_match           PROTO((Average *average_out, Measure *measure_out, Average *average_in, Measure *measure_in));
+int        replace_match           PROTO((Average *average_out, Measure *measure_out, off_t *next_meas, Average *average_in, Measure *measure_in));
 
 int        IDmapSave               PROTO((char *filename, IDmapType *IDmap));
@@ -241 +242 @@
 
 void replace_tycho_init ();
-int  replace_tycho (Average *averageInp, Measure *measureInp, Average *averageOut, Measure *measureOut);
+int  replace_tycho (Average *averageInp, Measure *measureInp, off_t *next_meas, Average *averageOut, Measure *measureOut);
+int repair_catalog_by_objID (Catalog *catalog);
+
+int dvorepair_by_objID (int argc, char **argv);
+int dvorepairFixWarpIDs (int argc, char **argv);
+

trunk/Ohana/src/dvomerge/include/dvoutils.h

@@ -22 +22 @@
 } DVOUTILS_OP_TYPE;
 
+typedef struct {
+  int   *externID;
+  short *photcode;
+  int Nimages;
+} ImageData;
+
 int VERBOSE;
 int DVOUTILS_OP;
@@ -30 +36 @@
 int dvoutils_args (int *argc, char **argv);
 int dvoutils_uniq_images (char *filename);
-int *dvoutils_load_image_index (char *filename, int *nindex);
+
+ImageData *dvoutils_load_image_index (char *filename);
 
 Image *dvoutils_load_image_table (char *filename, int *nimage);
+
+int        SetSignals             PROTO((void));
+void       SetProtect             PROTO((int mode));
+void       TrapSignal             PROTO((int sig));
+int        Shutdown               PROTO((char *format, ...)) OHANA_FORMAT(printf, 1, 2);
+

trunk/Ohana/src/dvomerge/include/dvoverify.h

@@ -37 +37 @@
 int    LIST_MISSING;
 
+int    IGNORE_SORTED_STATE;
+
 SkyRegion UserPatch;
 
@@ -62 +64 @@
 
 void FreeImageIDs (void);
+
+int        SetSignals             PROTO((void));
+void       SetProtect             PROTO((int mode));
+void       TrapSignal             PROTO((int sig));
+int        Shutdown               PROTO((char *format, ...)) OHANA_FORMAT(printf, 1, 2);
+

trunk/Ohana/src/dvomerge/src/LoadCatalog.c

@@ -9 +9 @@
 
   // always load all of the data (if any exists)
-  catalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT;
+  // XXXX TEMP HACK : skip GALPHOT
+  // XXXX catalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT;
+  catalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR;
   
   catalog[0].catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data

trunk/Ohana/src/dvomerge/src/ReadDeleteListExternID.c

@@ -47 +47 @@
       // confirm we have 9 fields broken by 8 spaces:
       space = c0; // pointer to track the space-separated words
-      indexPoint = c0; // pointer to the ID on this line
+      indexPoint = strchr(c0, ' '); // pointer to the ID on this line
 
-      for (j = 0; j < 8; j++) {
+      for (j = 0; j < 9; j++) {
         space = strchr(space, ' '); 
         if (!space) {
@@ -62 +62 @@
       indexList[Nindex] = index;
 
-      // fprintf (stderr, "index: %d, line: %s\n", index, c0);
+      fprintf (stderr, "index: %d, line: %s\n", index, c0);
 
       Nindex ++;

trunk/Ohana/src/dvomerge/src/args.c

@@ -94 +94 @@
   if ((N = get_argument (*argc, argv, "-replace-tycho"))) {
     REPLACE_TYCHO = TRUE;
+    remove_argument (N, argc, argv);
+  }
+
+  REPAIR_BY_OBJID = FALSE;
+  if ((N = get_argument (*argc, argv, "-repair-by-objid"))) {
+    REPAIR_BY_OBJID = TRUE;
     remove_argument (N, argc, argv);
   }
@@ -266 +272 @@
   }
 
+  REPAIR_BY_OBJID = FALSE;
+  if ((N = get_argument (*argc, argv, "-repair-by-objid"))) {
+    REPAIR_BY_OBJID = TRUE;
+    remove_argument (N, argc, argv);
+  }
+
   NTHREADS = 0;
   if ((N = get_argument (*argc, argv, "-threads"))) {

trunk/Ohana/src/dvomerge/src/build_links.c

@@ -145 +145 @@
     n = average[i].measureOffset;
     average[i].measureOffset = N;
+    int myObjID = average[i].objID;
     for (k = 0; k < average[i].Nmeasure; k++, N++) {
       if (n == -1) abort();
       tmpmeasure[N] = measure[n]; 
-      if (measure[n].averef != i) abort();
+      myAssert (measure[n].averef == i, "error in averef?");
+      myAssert (measure[n].objID == myObjID, "error in objID?");
       tmpmeasure[N].averef = i;
       n = next_meas[n];

trunk/Ohana/src/dvomerge/src/dvoconvert.c

@@ -94 +94 @@
 
     outcatalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
+
+    SetProtect (TRUE);
+    if (!dvo_catalog_save (&outcatalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", outcatalog.filename); exit (1); }
+    if (!dvo_catalog_unlock (&outcatalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", outcatalog.filename); exit (1); }
+    SetProtect (FALSE);
+
     dvo_catalog_save (&outcatalog, VERBOSE);
     dvo_catalog_unlock (&outcatalog);

trunk/Ohana/src/dvomerge/src/dvomergeContinue.c

@@ -133 +133 @@
       // if we receive a signal which would cause us to exit, wait until the full catalog is written
       SetProtect (TRUE);
-      if (!dvo_catalog_save (&outcatalog, VERBOSE)) {
-        fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]);
-        exit (1);
-      }
+      if (!dvo_catalog_save (&outcatalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", outcatalog.filename); exit (1); }
+      if (!dvo_catalog_unlock (&outcatalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", outcatalog.filename); exit (1); }
       SetProtect (FALSE);
 
-      dvo_catalog_unlock (&outcatalog);
       dvo_catalog_free (&outcatalog);
 

trunk/Ohana/src/dvomerge/src/dvomergeContinue_threaded.c

@@ -77 +77 @@
       SetProtect (TRUE);
       if (!dvo_catalog_save (&outcatalog, VERBOSE)) {
-        fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]);
+        fprintf (stderr, "ERROR: failed to save catalog %s\n", outcatalog.filename);
+        threadData->state = TS_FAIL;
+        continue;
+      }
+      if (!dvo_catalog_unlock (&outcatalog)) {
+        fprintf (stderr, "ERROR: failed to unlock catalog %s\n", outcatalog.filename);
         threadData->state = TS_FAIL;
         continue;
@@ -83 +88 @@
       SetProtect (FALSE);
 
-      dvo_catalog_unlock (&outcatalog);
       dvo_catalog_free (&outcatalog);
 

trunk/Ohana/src/dvomerge/src/dvomergeCreate.c

@@ -178 +178 @@
     SkyListFree (inlist);
 
-    dvo_catalog_save (&outcatalog, VERBOSE);
-    dvo_catalog_unlock (&outcatalog);
+    SetProtect (TRUE);
+    if (!dvo_catalog_save (&outcatalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", outcatalog.filename); exit (1); }
+    if (!dvo_catalog_unlock (&outcatalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", outcatalog.filename); exit (1); }
+    SetProtect (FALSE);
+
     dvo_catalog_free (&outcatalog);
   }

trunk/Ohana/src/dvomerge/src/dvomergeFromList.c

@@ -147 +147 @@
     // if we receive a signal which would cause us to exit, wait until the full catalog is written
     SetProtect (TRUE);
-    if (!dvo_catalog_save (&outcatalog, VERBOSE)) {
-      fprintf (stderr, "ERROR: failed to save catalog %s\n", outputfile);
-      exit (1);
-    }
+    if (!dvo_catalog_save (&outcatalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save catalog %s\n", outcatalog.filename); exit (1); }
+    if (!dvo_catalog_unlock (&outcatalog)) { fprintf (stderr, "ERROR: failed to unlock catalog %s\n", outcatalog.filename); exit (1); }
     SetProtect (FALSE);
     
@@ -158 +156 @@
     OutputStatusFree (outstat, 1);
 
-    dvo_catalog_unlock (&outcatalog);
     dvo_catalog_free (&outcatalog);
     

trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c

@@ -177 +177 @@
       LoadCatalog (&outcatalog, outlist[0].regions[j], outcatalog.filename, "w", NsecfiltOutput);
 
-      // if no catalog already exists, use the input catalog to define the format or the specified format
       if (UPDATE_CATFORMAT) {
         outcatalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT);
       } else {
-        outcatalog.catformat = incatalog.catformat;
-      }
+        // IF no catalog already exists, use the input catalog to define the format
+        if (outcatalog.Naverage_disk == 0) {
+          outcatalog.catformat = incatalog.catformat;
+        }
+      }
+
       if (UPDATE_CATCOMPRESS) {
         outcatalog.catcompress = dvo_catalog_catcompress (UPDATE_CATCOMPRESS);
       } else {
-        outcatalog.catcompress = incatalog.catcompress;
+        // IF no catalog already exists, use the input catalog to define the compression
+        if (outcatalog.Naverage_disk == 0) {
+          outcatalog.catcompress = incatalog.catcompress;
+        }
+      }
+
+      if (REPAIR_BY_OBJID) {
+        // For gpc1 / PV3, I broke some catalogs with dvomerge -replace: the last object can
+        // contaminate the first set of new measurements
+        repair_catalog_by_objID (&outcatalog);
       }
 
@@ -195 +207 @@
       }
 
-      if (!dvo_catalog_backup (&outcatalog, TRUE)) {
+      if (!dvo_catalog_backup (&outcatalog, "~", TRUE)) {
         fprintf (stderr, "ERROR: failed to make backup for catalog %s\n", outlist[0].filename[j]);
         exit (1);
@@ -202 +214 @@
       // if we receive a signal which would cause us to exit, wait until the full catalog is written
       SetProtect (TRUE);
-      if (!dvo_catalog_save (&outcatalog, VERBOSE)) {
-        fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]);
-        exit (1);
-      }
+      if (!dvo_catalog_save (&outcatalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save catalog %s\n", outcatalog.filename); exit (1); }
+      if (!dvo_catalog_unlock (&outcatalog)) { fprintf (stderr, "ERROR: failed to unlock catalog %s\n", outcatalog.filename); exit (1); }
       SetProtect (FALSE);
 
-      if (!dvo_catalog_unlock (&outcatalog)) {
-        fprintf (stderr, "ERROR: failed to unlock catalog %s\n", outlist[0].filename[j]);
-        exit (1);
-      }
-
-      if (!dvo_catalog_unlink_backup (&outcatalog, TRUE)) {
+      if (!dvo_catalog_unlink_backup (&outcatalog, "~", TRUE)) {
         fprintf (stderr, "WARNING: failed to remove backup for catalog %s\n", outlist[0].filename[j]);
       }
@@ -282 +287 @@
 
     // options / arguments that can affect relastro_client -update-objects:
-    char command[DVO_MAX_PATH];
-    snprintf (command, DVO_MAX_PATH, "dvomerge_client %s into %s -hostID %d -hostdir %s -region %f %f %f %f -D ADDSTAR_RADIUS %f", 
-              absinput, absoutput, group->hosts[i][0].hostID, group->hosts[i][0].pathname, 
-              UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, RADIUS
-      );
-
-    char tmpline[DVO_MAX_PATH];
-    if (VERBOSE)             { snprintf (tmpline, DVO_MAX_PATH, "%s -v",                command); strcpy (command, tmpline); }
-    if (VERIFY)              { snprintf (tmpline, DVO_MAX_PATH, "%s -verify",           command); strcpy (command, tmpline); }
-    if (VERIFY_CATALOG_ONLY) { snprintf (tmpline, DVO_MAX_PATH, "%s -verify-catalogs",  command); strcpy (command, tmpline); }
-    if (REPLACE_BY_PHOTCODE) { snprintf (tmpline, DVO_MAX_PATH, "%s -replace",          command); strcpy (command, tmpline); }
-    if (REPLACE_TYCHO)       { snprintf (tmpline, DVO_MAX_PATH, "%s -replace-tycho",    command); strcpy (command, tmpline); }
-    if (PARALLEL_INPUT)      { snprintf (tmpline, DVO_MAX_PATH, "%s -parallel-input",   command); strcpy (command, tmpline); }
-    if (FORCE_MERGE)         { snprintf (tmpline, DVO_MAX_PATH, "%s -force-merge",      command); strcpy (command, tmpline); }
-    if (MATCHED_TABLES)      { snprintf (tmpline, DVO_MAX_PATH, "%s -matched-tables",   command); strcpy (command, tmpline); }
-    if (UPDATE_CATFORMAT)    { snprintf (tmpline, DVO_MAX_PATH, "%s -update-catformat %s", command, UPDATE_CATFORMAT); strcpy (command, tmpline); }
+    char *command = NULL;
+    strextend (&command, "dvomerge_client %s into %s -hostID %d -hostdir %s -region %f %f %f %f -D ADDSTAR_RADIUS %f", 
+               absinput, absoutput, group->hosts[i][0].hostID, group->hosts[i][0].pathname, 
+               UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, RADIUS);
+
+    if (VERBOSE)             { strextend (&command, "-v"); }
+    if (VERIFY)              { strextend (&command, "-verify"); }
+    if (VERIFY_CATALOG_ONLY) { strextend (&command, "-verify-catalogs"); }
+    if (REPLACE_BY_PHOTCODE) { strextend (&command, "-replace"); }
+    if (REPLACE_TYCHO)       { strextend (&command, "-replace-tycho"); }
+    if (PARALLEL_INPUT)      { strextend (&command, "-parallel-input"); }
+    if (FORCE_MERGE)         { strextend (&command, "-force-merge"); }
+    if (MATCHED_TABLES)      { strextend (&command, "-matched-tables"); }
+    if (UPDATE_CATFORMAT)    { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); }
+    if (UPDATE_CATCOMPRESS)  { strextend (&command, "-update-catcompress %s", UPDATE_CATCOMPRESS); }
+    if (REPAIR_BY_OBJID)     { strextend (&command, "-repair-by-objid"); }
 
     // add some config variables:
-    snprintf (tmpline, DVO_MAX_PATH, "%s -D CATMODE %s",    command, CATMODE);   strcpy (command, tmpline);
-    snprintf (tmpline, DVO_MAX_PATH, "%s -D CATFORMAT %s",  command, CATFORMAT); strcpy (command, tmpline);
-    snprintf (tmpline, DVO_MAX_PATH, "%s -D SKY_DEPTH %d",  command, SKY_DEPTH); strcpy (command, tmpline);
+    strextend (&command, "-D CATMODE %s", CATMODE);
+    strextend (&command, "-D CATFORMAT %s", CATFORMAT); 
+    strextend (&command, "-D SKY_DEPTH %d", SKY_DEPTH);
 
     fprintf (stderr, "command: %s\n", command);

trunk/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c

@@ -75 +75 @@
       SetProtect (TRUE);
       if (!dvo_catalog_save (&outcatalog, VERBOSE)) {
-        fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]);
+        fprintf (stderr, "ERROR: failed to save catalog %s\n", outcatalog.filename);
+        threadData->state = TS_FAIL;
+        continue;
+      }
+      if (!dvo_catalog_unlock (&outcatalog)) {
+        fprintf (stderr, "ERROR: failed to save catalog %s\n", outcatalog.filename);
         threadData->state = TS_FAIL;
         continue;
@@ -81 +86 @@
       SetProtect (FALSE);
 
-      dvo_catalog_unlock (&outcatalog);
       dvo_catalog_free (&outcatalog);
 

trunk/Ohana/src/dvomerge/src/dvomerge_client.c

@@ -3 +3 @@
 int main (int argc, char **argv) {
 
+  SetSignals ();
   dvomerge_client_help (argc, argv);
   ConfigInit (&argc, argv);
@@ -83 +84 @@
   // loop over the populatable output tables; check for data in input in the corresponding regions
 
-  SetPhotcodeTable(NULL);
-  
+  if (REPLACE_TYCHO) replace_tycho_init();
+
   dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, IDmap, secfiltMap);
 

trunk/Ohana/src/dvomerge/src/dvorepair.c

@@ -7 +7 @@
   // exit (2);
 
+  SetSignals ();
   dvorepair_help(argc, argv);
   
+  if (!strcmp(argv[1], "-fix-warp-ids")) dvorepairFixWarpIDs(argc, argv);
+
   if (!strcmp(argv[1], "-fix-cpt")) dvorepairFixCPT(argc, argv);
+  if (!strcmp(argv[1], "-fix-cpt-by-objID")) dvorepair_by_objID(argc, argv);
   // if (!strcmp(argv[1], "-fix-tables")) dvorepairFixTables(argc, argv);
   if (!strcmp(argv[1], "-images-vs-measures")) dvorepairImagesVsMeasures(argc, argv);

trunk/Ohana/src/dvomerge/src/dvorepairCPT.c

@@ -39 +39 @@
     exit (2);
   }
+
+  SetSignals ();
 
   imageFilename  = argv[1];

trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c

@@ -16 +16 @@
 int SaveImageTable (Image *image, off_t Nimage, char *catdir, FITS_DB *oldDB);
 Image *DeleteSelectedImages (Image *image, off_t Nimage, int *deleteImage, myIndexType *imageIDindex, off_t *NnewImage);
+int DeleteLensing (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete);
+int DeleteMeasure (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete);
+int RepairAverage (Catalog *catalog);
 
 int dvorepairDeleteImagesByExternID (int argc, char **argv) {
@@ -21 +24 @@
   FITS_DB db;  // database handle pointing to input image table
   
-  int i, j, N, NmeasureNew, Ndelete, Nvalid;
-  off_t Nimage, Nmeasure;
+  int i, N;
+  off_t Nimage;
   int nPass, raPass;
 
   Image *image;
-  Measure *measure;
-  Measure *measureNew;
-
-  char *cpmFilenameSrc = NULL;
-  char *cptFilenameSrc = NULL;
-  char *cpsFilenameSrc = NULL;
-  char *cpmFilenameTgt = NULL;
-  char *cptFilenameTgt = NULL;
-  char *cpsFilenameTgt = NULL;
-
   char *imageFilename = NULL;
-
-  DVOCatFormat catformat;
+  char filename[DVO_MAX_PATH];
 
   N = get_argument (argc, argv, "-delete-images-by-extern-id");
@@ -59 +51 @@
   char *catdir = argv[1];
   char *delList = argv[2];
+
+  sprintf (filename, "%s/Photcodes.dat", catdir);
+  if (!LoadPhotcodes (filename, NULL, FALSE)) {
+    fprintf (stderr, "error reading photcodes from %s\n", catdir);
+    exit (1);
+  }     
 
   // load the image data
@@ -196 +194 @@
   }
   
-  ALLOCATE(cpmFilenameSrc, char, strlen(catdir) + 64);
-  ALLOCATE(cptFilenameSrc, char, strlen(catdir) + 64);
-  ALLOCATE(cpsFilenameSrc, char, strlen(catdir) + 64);
-  ALLOCATE(cpmFilenameTgt, char, strlen(catdir) + 64);
-  ALLOCATE(cptFilenameTgt, char, strlen(catdir) + 64);
-  ALLOCATE(cpsFilenameTgt, char, strlen(catdir) + 64);
-
-  int NdeleteTotal = 0;
+  int NmeasureDelTotal = 0;
+  int NlensingDelTotal = 0;
 
   for (raPass = 0; raPass < nPass; raPass++) {
@@ -231 +223 @@
   
     // loop over the populated input regions
-    int Ncheck = 0;
     for (i = 0; i < inlist[0].Nregions; i++) {
       if (!inlist[0].regions[i][0].table) continue;
 
-      sprintf (cpmFilenameSrc, "%s/%s.cpm", catdir, inlist[0].regions[i][0].name);
-      sprintf (cptFilenameSrc, "%s/%s.cpt", catdir, inlist[0].regions[i][0].name);
-      sprintf (cpsFilenameSrc, "%s/%s.cps", catdir, inlist[0].regions[i][0].name);
-      sprintf (cpmFilenameTgt, "%s/%s.cpm.broken", catdir, inlist[0].regions[i][0].name);
-      sprintf (cptFilenameTgt, "%s/%s.cpt.broken", catdir, inlist[0].regions[i][0].name);
-      sprintf (cpsFilenameTgt, "%s/%s.cps.broken", catdir, inlist[0].regions[i][0].name);
-
-      Header cpmHeaderPHU;
-      Header cpmHeaderTBL;
-      FTable cpmFtable;
-
-      /*** read and examine the CPM file ***/
-      {
-        if (VERBOSE) fprintf (stderr, "check %s\n", cpmFilenameSrc);
-
-        cpmFtable.header = &cpmHeaderTBL;
-
-        // open cpm file
-        FILE *cpmFile = fopen(cpmFilenameSrc, "r");
-        if (!cpmFile) continue;
-        // myAssert(cpmFile, "failed to open cpm file");
-    
-        // load the cpm header
-        if (!gfits_fread_header (cpmFile, &cpmHeaderPHU)) {
-          myAbort("failure to cpm header");
-        }
-
-        // move to TBL header
-        int Nbytes = cpmHeaderPHU.datasize + gfits_data_size (&cpmHeaderPHU);
-        fseeko (cpmFile, Nbytes, SEEK_SET);
-
-        // read cpm TBL header
-        if (!gfits_fread_header (cpmFile, &cpmHeaderTBL)) { 
-          myAbort("can't read header for cpm table");
-        }
-
-        // read Measure table data : format is irrelevant here */
-        if (!gfits_fread_ftable_data (cpmFile, &cpmFtable, TRUE)) { 
-          myAbort("can't read data for cpm table");
-        }
-
-        int NbytesPerRow;
-        gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow);
-
-        myAbort ("fix compression");
-
-        // this function can only handle PS1_V5 and later formats
-        // NOTE: FtableToMeasure (and equivalent) free the data associated with cpmFtable.
-        // We are left with the data in measure.
-        measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE);
-        myAssert(measure, "failed to convert ftable to measure data");
-    
-        Nvalid = (int)(cpmFtable.validsize / NbytesPerRow);
-        Nvalid = MIN(Nmeasure, Nvalid);
-
-        // close the input cpm file
-        fclose(cpmFile);
-
-        // allocate an output array of measures (to replace, if needed)
-        ALLOCATE (measureNew, Measure, Nvalid);
-
-        NmeasureNew = 0;
-        Ndelete = 0;
-
-        // examine all measurements: find ones that need to be deleted
-        for (j = 0; j < Nvalid; j++) {
-          int imageID = measure[j].imageID;
-          myAssert(imageID, "measure is missing an image ID");
-          // XXX this case is valid if we have REF detections (no associated image)
-
-          int N = myIndexGetEntry(imageIDindex, imageID);
-          if (N < 0) {
-            // this detection comes from a non-existant image, delete
-            Ndelete ++;
-            continue;
-          }
-
-          if (deleteImage[N]) {
-            Ndelete ++;
-            continue;
-          }
-          measureNew[NmeasureNew] = measure[j];
-          NmeasureNew ++;
-        }
+      snprintf (filename, DVO_MAX_PATH, "%s/%s.cpt", catdir, inlist[0].regions[i][0].name);
+
+      Catalog catalog;
+
+      // set up the basic catalog info
+      dvo_catalog_init (&catalog, TRUE);
+      catalog.filename  = filename; // XXX only allow non-parallel dbs for now
+      catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT | DVO_LOAD_LENSING;
+      catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
+      
+      if (!dvo_catalog_open (&catalog, inlist[0].regions[i], VERBOSE, "w")) {
+        fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename);
+        exit (1);
       }
-
-      NdeleteTotal += Ndelete;
-
-      // if we actually want to delete any measurements, write out a new cpm file
-      if (Ndelete > 0) {
-
-        fprintf (stderr, "deleting %d of %d (keep %d) detections from %s -> %s\n", (int) Ndelete, (int) Nvalid, (int) NmeasureNew, cpmFilenameSrc, cpmFilenameTgt);
+      if (!catalog.Naverage_disk) {
+        if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename);
+        dvo_catalog_unlock (&catalog);
+        dvo_catalog_free (&catalog);
+        continue;
+      }
       
-        // the CPT and CPS tables need to be regenerated.  This must happen first because, in the process, we also update measure->averef
-        // what about other tables?
-        RepairTableCPT_V1(cptFilenameSrc, cptFilenameTgt, cpsFilenameSrc, cpsFilenameTgt, measureNew, NmeasureNew, image, Nimage, imageIDindex, catformat);
-
-        // convert internal to external format 
-        if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat, TRUE)) {
-          myAbort("trouble converting format");
-        }
-
-        // rename the old cpm file:
-        if (rename (cpmFilenameSrc, cpmFilenameTgt)) {
-          perror ("tried to rename file");
-          exit (2);
-        }
-
-        // create and write the output file
-        FILE *cpmFile = fopen(cpmFilenameSrc, "w");
-        myAssert(cpmFile, "failed to open cpt file");
-        
-        // write PHU header
-        if (!gfits_fwrite_header (cpmFile, &cpmHeaderPHU)) {
-          myAbort("can't write primary header");
-        }
-
-        Matrix matrix;
-
-        // write the PHU matrix; this is probably a NOP, do I have to keep it in?
-        gfits_create_matrix (&cpmHeaderPHU, &matrix);
-        if (!gfits_fwrite_matrix  (cpmFile, &matrix)) {
-          myAbort("can't write primary matrix");
-        }
-        gfits_free_matrix (&matrix);
-        
-        // write the table data
-        if (!gfits_fwrite_ftable_range (cpmFile, &cpmFtable, 0, NmeasureNew, 0, NmeasureNew)) {
-          myAbort("can't write table data");
-        }
-        fclose (cpmFile);
-
-        gfits_free_table (&cpmFtable);
-      } else {
-        if (VERBOSE) fprintf (stderr, "nothing to delete in %s\n", cpmFilenameSrc);
+      /*** delete the measure and lensing entries matching the list of imageIDs ***/
+      int NmeasureDel = 0;
+      DeleteMeasure (&catalog, imageIDindex, deleteImage, &NmeasureDel);
+
+      int NlensingDel = 0;
+      DeleteLensing (&catalog, imageIDindex, deleteImage, &NlensingDel);
+
+      fprintf (stderr, "deleting %d measure, %d lensing (keep %d, %d)\n", NmeasureDel, NlensingDel, (int) catalog.Nmeasure, (int) catalog.Nlensing);
+
+      if (!NmeasureDel && !NlensingDel) {
+        if (VERBOSE) fprintf (stderr, "nothing to delete in %s\n", catalog.filename);
+        dvo_catalog_unlock (&catalog);
+        dvo_catalog_free (&catalog);
+        continue;
       }
 
-      gfits_free_header (&cpmHeaderPHU);
-      gfits_free_header (&cpmHeaderTBL);
-      free (measure);
-      free (measureNew);
-
-      Ncheck ++;
-      if (Ncheck % 1000 == 0) {
-        fprintf (stderr, "%s...", inlist[0].regions[i][0].name);
+      NmeasureDelTotal += NmeasureDel;
+      NlensingDelTotal += NlensingDel;
+
+      // the CPT and CPS tables need to be regenerated.  This must happen first because, in the process, we also update measure->averef
+      // what about other tables?
+      RepairAverage (&catalog);
+
+      if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename);
+      
+      if (!dvo_catalog_backup (&catalog, ".undel", TRUE)) {
+        fprintf (stderr, "ERROR: failed to make backup for catalog %s\n", catalog.filename);
+        exit (1);
       }
+
+      SetProtect (TRUE);
+      if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); }
+      if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); }
+      SetProtect (FALSE);
+      dvo_catalog_free (&catalog);
     }
     fprintf (stderr, "\n");
@@ -386 +287 @@
   }
 
-  fprintf (stderr, "Deleted %d detections\n", NdeleteTotal);
+  fprintf (stderr, "Deleted %d measure, %d lensing detections\n", NmeasureDelTotal, NlensingDelTotal);
 
   off_t NnewImage;
@@ -400 +301 @@
   free (deleteIDs);
 
-  free(cpmFilenameSrc);
-  free(cptFilenameSrc);
-  free(cpsFilenameSrc);
-  free(cpmFilenameTgt);
-  free(cptFilenameTgt);
-  free(cpsFilenameTgt);
-
   SkyTableFree(insky);
 
   gfits_db_free (&db);
+
+  ohana_memcheck (TRUE);
 
   exit (0);
@@ -661 +557 @@
   gfits_scan (&cptHeaderPHU, "NSECFILT",     "%d",      1,  &Nsecfilt);
 
-  myAbort ("test for compression");
+  if (0) {
+    char compressMode[256];
+    if (gfits_scan (&cptHeaderTBL, "DVO_CMP", "%s", 1, compressMode)) {
+      if (strcmp (compressMode, "NONE")) {
+        myAbort ("fix compression");
+      }
+    }
+  }
 
   /* convert internal to external format */
@@ -774 +677 @@
 }
 
+int RepairAverage (Catalog *catalog) {
+
+  off_t i, j, Nave;
+
+  // average is sorted so averef is valid
+  // secfilt is also sorted so sequence is valid (but this is kind of moot since it will be re-calculated)
+  // average.measureOffset is NOT valid
+  // average.lensingOffset is NOT valid
+
+  Average *average = catalog->average;
+  Measure *measure = catalog->measure;
+  Lensing *lensing = catalog->lensing;
+  
+  // reset the values of average.Nmeasure, average.Nlensing
+  for (i = 0; i < catalog->Naverage; i++) {
+    average[i].Nmeasure = 0;
+    average[i].Nlensing = 0;
+  }
+
+  // re-calculate average.Nmeasure
+  for (i = 0; i < catalog->Nmeasure; i++) {
+    Nave = measure[i].averef;
+    myAssert (measure[i].objID == average[Nave].objID, "invalid measure:average match");
+    average[Nave].Nmeasure ++;
+  }
+    
+  // re-calculate average.Nlensing
+  for (i = 0; i < catalog->Nlensing; i++) {
+    Nave = lensing[i].averef;
+    myAssert (lensing[i].objID == average[Nave].objID, "invalid lensing:average match");
+    average[Nave].Nlensing ++;
+  }
+
+  // create a copy of the average table, keeping only the entries with Nmeasure & Nlensing > 0
+  ALLOCATE_PTR (averageNew, Average, catalog->Naverage);
+  ALLOCATE_PTR (averefNew,  int,     catalog->Naverage);
+
+  Nave = 0;
+  for (i = 0; i < catalog->Naverage; i++) {
+    averefNew[i] = -1;
+    if (!average[i].Nmeasure && !average[i].Nlensing) continue;
+
+    averageNew[Nave] = average[i];
+    averefNew[i] = Nave;
+    Nave ++;
+  }
+  off_t NaverageNew = Nave;
+
+  // update measure.averef values
+  for (i = 0; i < catalog->Nmeasure; i++) {
+    Nave = averefNew[measure[i].averef];
+    myAssert (Nave >= 0, "oops");
+    measure[i].averef = Nave;
+  }
+    
+  // update lensing.averef values
+  for (i = 0; i < catalog->Nlensing; i++) {
+    Nave = averefNew[lensing[i].averef];
+    myAssert (Nave >= 0, "oops");
+    lensing[i].averef = Nave;
+  }
+  
+  // XXX need to update measureOffset and lensingOffset
+
+  // measure[] should be blocked and sequential: 
+  // measure[i+1].averef >= measure[i].averef
+
+  // update average.measureOffset values
+  Nave = -1;
+  for (i = 0; i < catalog->Nmeasure; i++) {
+    if (measure[i].averef == Nave) continue;
+    Nave = measure[i].averef;
+    averageNew[Nave].measureOffset = i;
+  }
+
+  // update average.lensingOffset values
+  Nave = -1;
+  for (i = 0; i < catalog->Nlensing; i++) {
+    if (lensing[i].averef == Nave) continue;
+    Nave = lensing[i].averef;
+    averageNew[Nave].lensingOffset = i;
+  }
+
+  // check the result (measure -> average)
+  for (i = 0; i < catalog->Nmeasure; i++) {
+    Nave = measure[i].averef;
+    myAssert(averageNew[Nave].objID == measure[i].objID, "objIDs do not match");
+    myAssert(averageNew[Nave].catID == measure[i].catID, "catIDs do not match");
+  }
+  // check the result (average -> measure)
+  for (i = 0; i < NaverageNew; i++) {
+    int m = averageNew[i].measureOffset;
+    for (j = 0; j < averageNew[i].Nmeasure; j++) {
+      myAssert(averageNew[i].objID == measure[j+m].objID, "objIDs do not match");
+      myAssert(averageNew[i].catID == measure[j+m].catID, "catIDs do not match");
+      myAssert(measure[j+m].averef == i, "averef broken");
+    }
+  }
+
+  // check the result (lensing -> average)
+  for (i = 0; i < catalog->Nlensing; i++) {
+    Nave = lensing[i].averef;
+    myAssert(averageNew[Nave].objID == lensing[i].objID, "objIDs do not match");
+    myAssert(averageNew[Nave].catID == lensing[i].catID, "catIDs do not match");
+  }
+  // check the result (average -> lensing)
+  for (i = 0; i < NaverageNew; i++) {
+    int m = averageNew[i].lensingOffset;
+    for (j = 0; j < averageNew[i].Nlensing; j++) {
+      myAssert(averageNew[i].objID == lensing[j+m].objID, "objIDs do not match");
+      myAssert(averageNew[i].catID == lensing[j+m].catID, "catIDs do not match");
+      myAssert(lensing[j+m].averef == i, "averef broken");
+    }
+  }
+
+  free (catalog->secfilt);
+  ALLOCATE (catalog->secfilt, SecFilt, NaverageNew*catalog->Nsecfilt);
+  for (i = 0; i < NaverageNew*catalog->Nsecfilt; i++) {
+    dvo_secfilt_init (&catalog->secfilt[i], SECFILT_RESET_ALL);
+  }
+
+  free (averefNew);
+  free (catalog->average);
+  catalog->average = averageNew;
+  catalog->Naverage = NaverageNew;
+  catalog->Naverage_disk = NaverageNew;
+
+  catalog->Nsecfilt_disk = NaverageNew*catalog->Nsecfilt;
+
+  return (TRUE);
+}
+
+// delete measure entries
+int DeleteMeasure (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) {
+
+  int j;
+
+  Measure *measure = catalog->measure;
+
+  // allocate an output array of measures (to replace, if needed)
+  ALLOCATE_PTR (measureNew, Measure, catalog->Nmeasure);
+
+  int NmeasureNew = 0;
+  int NmeasureDel = 0;
+
+  // examine all measurements: find ones that need to be deleted
+  for (j = 0; j < catalog->Nmeasure; j++) {
+    int imageID = measure[j].imageID;
+    myAssert(imageID, "measure is missing an image ID");
+    // XXX this case is valid if we have REF detections (no associated image)
+
+    int N = myIndexGetEntry(imageIDindex, imageID);
+    if (N < 0) {
+      // this detection comes from a non-existant image; delete
+      NmeasureDel ++;
+      continue;
+    }
+
+    // measure matches a bad image; delete
+    if (deleteImage[N]) {
+      NmeasureDel ++;
+      continue;
+    }
+
+    // keep this measure
+    measureNew[NmeasureNew] = measure[j];
+    NmeasureNew ++;
+  }
+
+  free (catalog->measure);
+  catalog->measure = measureNew;
+  catalog->Nmeasure = NmeasureNew;
+  catalog->Nmeasure_disk = NmeasureNew;
+
+  *nDelete = NmeasureDel;
+
+  return TRUE;
+}
+
+// delete lensing entries
+int DeleteLensing (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) {
+
+  int j;
+
+  Lensing *lensing = catalog->lensing;
+
+  // allocate an output array of measures (to replace, if needed)
+  ALLOCATE_PTR (lensingNew, Lensing, catalog->Nlensing);
+
+  int NlensingNew = 0;
+  int NlensingDel = 0;
+
+  // examine all lensingments: find ones that need to be deleted
+  for (j = 0; j < catalog->Nlensing; j++) {
+    int imageID = lensing[j].imageID;
+    myAssert(imageID, "lensing is missing an image ID");
+    // XXX this case is valid if we have REF detections (no associated image)
+
+    int N = myIndexGetEntry(imageIDindex, imageID);
+    if (N < 0) {
+      // this detection comes from a non-existant image; delete
+      NlensingDel ++;
+      continue;
+    }
+
+    // lensing matches a bad image; delete
+    if (deleteImage[N]) {
+      NlensingDel ++;
+      continue;
+    }
+
+    // keep this lensing
+    lensingNew[NlensingNew] = lensing[j];
+    NlensingNew ++;
+  }
+
+  free (catalog->lensing);
+  catalog->lensing = lensingNew;
+  catalog->Nlensing = NlensingNew;
+  catalog->Nlensing_disk = NlensingNew;
+
+  *nDelete = NlensingDel;
+
+  return TRUE;
+}
+

trunk/Ohana/src/dvomerge/src/dvorepairFixImages.c

@@ -255 +255 @@
 
   /* setup image table format and lock */
-  gfits_db_init (oldDB);
+  gfits_db_init (&db);
   db.mode   = oldDB->mode;
   db.format = oldDB->format;
@@ -266 +266 @@
   /* load or create the image table */
   myAssert (db.dbstate == LCK_EMPTY, "do not overwrite exiting image table");
-  myAssert (db.format == DVO_FORMAT_PS1_V2, "format mismatch");
+  // myAssert (db.format == DVO_FORMAT_PS1_V2, "format mismatch");
 
   dvo_image_create (&db, GetZeroPoint());
+  gfits_copy_header (&oldDB->header, &db.header);
   
   // replace the existing buffer with the image array
@@ -277 +278 @@
   nbytes = gfits_data_size (db.ftable.header);
   db.ftable.datasize = nbytes;
-  db.ftable.buffer = (char *) imageOut;
-  REALLOCATE (db.ftable.buffer, char, MAX (nbytes, 1));
+
+  free (db.ftable.buffer);
+  ALLOCATE (db.ftable.buffer, char, MAX (nbytes, 1));
+  memcpy (db.ftable.buffer, imageOut, nbytes);
   memset (&db.ftable.buffer[Nx*Nout], ' ', nbytes - Nx*Nout);
 
@@ -295 +298 @@
   dvo_image_unlock (&db);
 
+  gfits_db_free (&db); 
+
   return TRUE;
 }

trunk/Ohana/src/dvomerge/src/dvorepairImageVsMeasure.c

@@ -39 +39 @@
   }
 
+  SetSignals ();
   catdir = argv[1];
   Ntol = atoi(argv[2]);

trunk/Ohana/src/dvomerge/src/dvosecfilt_catalogs.c

@@ -82 +82 @@
     catalog.Nsecfilt_disk = Nsecfilt * catalog.Naverage_disk;
 
-    dvo_catalog_save (&catalog, VERBOSE);
-    dvo_catalog_unlock (&catalog);
+    SetProtect (TRUE);
+    if (!dvo_catalog_save (&catalog, VERBOSE)) { fprintf (stderr, "ERROR: failed to save %s\n", catalog.filename); exit (1); }
+    if (!dvo_catalog_unlock (&catalog)) { fprintf (stderr, "ERROR: failed to unlock %s\n", catalog.filename); exit (1); }
+    SetProtect (FALSE);
+
     dvo_catalog_free (&catalog);
   }

trunk/Ohana/src/dvomerge/src/dvoutils.c

@@ -4 +4 @@
 
   // check various options
+  SetSignals ();
   dvoutils_args (&argc, argv);
 

trunk/Ohana/src/dvomerge/src/dvoutils_load_image_index.c

@@ -2 +2 @@
 
 // load the array of EXTERN_ID from the ImageIndex.fits tables
-int *dvoutils_load_image_index (char *filename, int *nindex) {
+ImageData *dvoutils_load_image_index (char *filename) {
 
   int Ncol;
-  off_t Nrow;
+  off_t Nrow, Nphot;
   Header header;
   Header theader;
@@ -46 +46 @@
   fclose (f);
 
-  int *externID = gfits_get_bintable_column_data (&theader, &ftable, "EXTERN_ID", type, &Nrow, &Ncol);
+  ImageData *imdata = NULL;
+  ALLOCATE (imdata, ImageData, 1);
+
+  imdata->externID = gfits_get_bintable_column_data (&theader, &ftable, "EXTERN_ID", type, &Nrow, &Ncol);
   myAssert (!strcmp(type, "int"), "wrong column type");
+
+  imdata->photcode = gfits_get_bintable_column_data (&theader, &ftable, "PHOTCODE", type, &Nphot, &Ncol);
+  myAssert (Nphot == Nrow, "photcode & extern_id mis-match");
+
+  imdata->Nimages = Nrow;
 
   gfits_free_header (&header);
@@ -54 +62 @@
   gfits_free_table  (&ftable);
 
-  *nindex = Nrow;
-
-  return externID;
+  return imdata;
 }
 

trunk/Ohana/src/dvomerge/src/dvoutils_uniq_images.c

@@ -40 +40 @@
     }
 
-    int Nimage;
-    // Image *image = dvoutils_load_image_table (imFile, &Nimage);
-    int *extern_id = dvoutils_load_image_index (imFile, &Nimage);
-    if (!extern_id) continue;
+    ImageData *imdata = dvoutils_load_image_index (imFile);
+    if (!imdata) continue;
 
     DBlist[Ndb] = strcreate (imFile);
@@ -57 +55 @@
 
     int i;
-    for (i = 0; i < Nimage; i++) {
-      if (!extern_id[i]) continue;
-      myMaxID = MAX(extern_id[i], myMaxID);
-      myMinID = MIN(extern_id[i], myMinID);
+    int Nbad = 0;
+    for (i = 0; i < imdata->Nimages; i++) {
+      if (!imdata->photcode[i] && imdata->externID[i]) myAbort ("invalid combo");
+
+      if (!imdata->photcode[i]) continue;
+      if (!imdata->externID[i]) {
+        Nbad ++;
+        continue;
+      }
+      myMaxID = MAX(imdata->externID[i], myMaxID);
+      myMinID = MIN(imdata->externID[i], myMinID);
     }
 
-    fprintf (stderr, "read %s, %d images, %d - %d vs %d - %d\n", imFile, (int) Nimage, myMinID, myMaxID, minID, maxID);
+    fprintf (stderr, "read %s, %d images, %d - %d vs %d - %d\n", imFile, (int) imdata->Nimages, myMinID, myMaxID, minID, maxID);
+
+    if (Nbad) {
+      fprintf (stderr, "WARNING: %d images without extern_id set\n", (int) Nbad);
+    }
 
     // if this is new, we treat it a bit differently
@@ -124 +133 @@
     }
 
-    for (i = 0; i < Nimage; i++) {
-      if (!extern_id[i]) continue;
-      int n = extern_id[i] - minID;
+    for (i = 0; i < imdata->Nimages; i++) {
+      if (!imdata->externID[i]) continue;
+      int n = imdata->externID[i] - minID;
       if (IDlist[n] == 0) {
         DBfound[n] = Ndb;
@@ -133 +142 @@
       IDlist[n] ++;
       if (VERBOSE && (IDlist[n] > 1)) {
-        fprintf (stderr, "duplicate ext_id %d in file %s\n", extern_id[i], imFile);
+        fprintf (stderr, "duplicate ext_id %d in file %s\n", imdata->externID[i], imFile);
       }
     }
@@ -140 +149 @@
       fprintf (stderr, "------------ %s ----------\n", imFile);
 
-      for (i = 0; i < Nimage; i++) {
-        if (!extern_id[i]) continue;
-        fprintf (stderr, "%d => %d\n", extern_id[i], extern_id[i] - minID);
+      for (i = 0; i < imdata->Nimages; i++) {
+        if (!imdata->externID[i]) continue;
+        fprintf (stderr, "%d => %d\n", imdata->externID[i], imdata->externID[i] - minID);
       }
 
@@ -165 +174 @@
     int n = DBfound[i];
     myAssert (n > -1, "impossible!");
-    fprintf (stdout, "%d : %d -- in %s (%d) : %d \n", i + minID, IDlist[i], DBlist[n], DBfound[i], DBindex[i]);
+    fprintf (stdout, "primary %d : %d -- in %s (%d) : %d \n", i + minID, IDlist[i], DBlist[n], DBfound[i], DBindex[i]);
   }
 

trunk/Ohana/src/dvomerge/src/dvoverify.c

@@ -14 +14 @@
 
   // check various options
+  SetSignals ();
   dvoverify_args (&argc, argv);
   CATDIR = argv[1];
@@ -78 +79 @@
   SkyListFree (skylist);
 
-  ohana_memcheck (TRUE);
-  ohana_memdump (TRUE);
+  ohana_memcheck (VERBOSE);
+  ohana_memdump (VERBOSE);
   exit (0);
 }

trunk/Ohana/src/dvomerge/src/dvoverify_args.c

@@ -24 +24 @@
   if ((N = get_argument (*argc, argv, "-sorted"))) {
     CHECKSORTED = TRUE;
+    remove_argument (N, argc, argv);
+  }
+
+  // in some cases, the header sorted state can claim F but is actually T
+  IGNORE_SORTED_STATE = FALSE;
+  if ((N = get_argument (*argc, argv, "-ignore-sorted-state"))) {
+    IGNORE_SORTED_STATE = TRUE;
     remove_argument (N, argc, argv);
   }
@@ -50 +57 @@
     free (filename);
 
-    ohana_memcheck (TRUE);
-    ohana_memdump (TRUE);
+    ohana_memcheck (VERBOSE);
+    ohana_memdump (VERBOSE);
 
     if (!isGood) exit (1);
@@ -204 +211 @@
   }
 
+  // in some cases, the header sorted state can claim F but is actually T
+  IGNORE_SORTED_STATE = FALSE;
+  if ((N = get_argument (*argc, argv, "-ignore-sorted-state"))) {
+    IGNORE_SORTED_STATE = TRUE;
+    remove_argument (N, argc, argv);
+  }
+
   if (!HOST_ID || !HOSTDIR || (*argc != 2)) {
     fprintf (stderr, "USAGE: dvoverify_client (catdir) -results (file) -hostID ID -hostdir DIR [-region Rmin Rmax Dmin Dmax] [-v] [-s]\n\n");

trunk/Ohana/src/dvomerge/src/dvoverify_catalogs.c

@@ -117 +117 @@
 
     // options / arguments that can affect relastro_client -update-objects:
-    char command[DVO_MAX_PATH];
-    snprintf (command, DVO_MAX_PATH, "dvoverify_client %s -results %s -hostID %d -hostdir %s -region %f %f %f %f", 
-              abscatdir, table->hosts[i].results, table->hosts[i].hostID, table->hosts[i].pathname, 
-              UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax
-      );
+    char *command = NULL;
+    strextend (&command, "dvoverify_client %s -results %s -hostID %d -hostdir %s -region %f %f %f %f", 
+               abscatdir, table->hosts[i].results, table->hosts[i].hostID, table->hosts[i].pathname, 
+               UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
 
-    char tmpline[DVO_MAX_PATH];
-    if (VERBOSE)         { snprintf (tmpline, DVO_MAX_PATH, "%s -v", command); strcpy (command, tmpline); }
-    if (CHECKSORTED)     { snprintf (tmpline, DVO_MAX_PATH, "%s -s", command); strcpy (command, tmpline); }
-    if (!CHECK_IMAGE_ID) { snprintf (tmpline, DVO_MAX_PATH, "%s -skip-image-ids", command); strcpy (command, tmpline); }
-    if (LIST_MISSING)    { snprintf (tmpline, DVO_MAX_PATH, "%s -list-missing", command); strcpy (command, tmpline); }
+    if (VERBOSE)             { strextend (&command, "-v"); }
+    if (CHECKSORTED)         { strextend (&command, "-s"); }
+    if (IGNORE_SORTED_STATE) { strextend (&command, "-ignore-sorted-state"); }
+    if (!CHECK_IMAGE_ID)     { strextend (&command, "-skip-image-ids"); }
+    if (LIST_MISSING)        { strextend (&command, "-list-missing"); }
 
     fprintf (stderr, "command: %s\n", command);
@@ -149 +148 @@
       table->hosts[i].pid = pid; // save for future reference
     }
+    free (command);
   }
 

trunk/Ohana/src/dvomerge/src/dvoverify_client.c

@@ -8 +8 @@
 
   // check various options
+  SetSignals ();
   dvoverify_client_args (&argc, argv);
   CATDIR = argv[1];

trunk/Ohana/src/dvomerge/src/dvoverify_utils.c

@@ -188 +188 @@
   dvo_catalog_init (&catalog, TRUE);
   catalog.filename  = filename;
-  catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ;
+  catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT;
   catalog.Nsecfilt  = 0;
   
@@ -214 +214 @@
 
   // if the table is NOT SORTED, do we have a subset of checks we can make?
-  if (!catalog.sorted) {
+  if (!catalog.sorted && !IGNORE_SORTED_STATE) {
     fprintf (stderr, "!");
     dvo_catalog_unlock (&catalog);
@@ -222 +222 @@
     return TRUE;
   }
-
-  int NmeasureTotal = 0;
-  int measureOffsetOK = TRUE;
-  int NlensingTotal = 0;
-  int lensingOffsetOK = TRUE;
-  int NlensobjTotal = 0;
-  int lensobjOffsetOK = TRUE;
-  for (i = 0; i < catalog.Naverage; i++) {
-    NmeasureTotal += catalog.average[i].Nmeasure;
-    if (VERBOSE && !(NmeasureTotal <= catalog.Nmeasure)) {
-      fprintf (stderr, "NmeasureTotal > catalog.Nmeasure: %d %d %d\n", i, catalog.average[i].Nmeasure, (int) catalog.Nmeasure);
-    }
-    measureOffsetOK &= (catalog.average[i].measureOffset < catalog.Nmeasure);
-    if (VERBOSE && !(catalog.average[i].measureOffset < catalog.Nmeasure)) {
-      fprintf (stderr, "measureOffset >= catalog.Nmeasure: %d %d %d\n", i, catalog.average[i].measureOffset, (int) catalog.Nmeasure);
-    }
-    measureOffsetOK &= (catalog.average[i].measureOffset + catalog.average[i].Nmeasure <= catalog.Nmeasure);
-    if (VERBOSE && !(catalog.average[i].measureOffset + catalog.average[i].Nmeasure <= catalog.Nmeasure)) {
-      fprintf (stderr, "measureOffset + Nmeasure > catalog.Nmeasure : %d %d %d\n", i, catalog.average[i].Nmeasure, (int) catalog.Nmeasure);
-    }
-
-    // sum of Nlensing for each object < Nlensing for catalog
-    NlensingTotal += catalog.average[i].Nlensing;
-    if (VERBOSE && !(NlensingTotal <= catalog.Nlensing)) {
-      fprintf (stderr, "NlensingTotal > catalog.Nlensing: %d %d %d\n", i, catalog.average[i].Nlensing, (int) catalog.Nlensing);
-    }
-    // lensingOffset needs to be in range for each object
-    lensingOffsetOK &= (!catalog.Nlensing || (catalog.average[i].lensingOffset < catalog.Nlensing));
-    if (VERBOSE && catalog.Nlensing && !(catalog.average[i].lensingOffset < catalog.Nlensing)) {
-      fprintf (stderr, "lensingOffset >= catalog.Nlensing: %d %d %d\n", i, catalog.average[i].lensingOffset, (int) catalog.Nlensing);
-    }
-    lensingOffsetOK &= (catalog.average[i].lensingOffset + catalog.average[i].Nlensing <= catalog.Nlensing);
-    if (VERBOSE && !(catalog.average[i].lensingOffset + catalog.average[i].Nlensing <= catalog.Nlensing)) {
-      fprintf (stderr, "lensingOffset + Nlensing > catalog.Nlensing : %d %d %d\n", i, catalog.average[i].Nlensing, (int) catalog.Nlensing);
-    }
-
-    NlensobjTotal += catalog.average[i].Nlensobj;
-    if (VERBOSE && !(NlensobjTotal <= catalog.Nlensobj)) {
-      fprintf (stderr, "NlensobjTotal > catalog.Nlensobj: %d %d %d\n", i, catalog.average[i].Nlensobj, (int) catalog.Nlensobj);
-    }
-    lensobjOffsetOK &= (catalog.average[i].lensobjOffset < catalog.Nlensobj);
-    if (VERBOSE && !(catalog.average[i].lensobjOffset < catalog.Nlensobj)) {
-      fprintf (stderr, "lensobjOffset >= catalog.Nlensobj: %d %d %d\n", i, catalog.average[i].lensobjOffset, (int) catalog.Nlensobj);
-    }
-    lensobjOffsetOK &= (catalog.average[i].lensobjOffset + catalog.average[i].Nlensobj <= catalog.Nlensobj);
-    if (VERBOSE && !(catalog.average[i].lensobjOffset + catalog.average[i].Nlensobj <= catalog.Nlensobj)) {
-      fprintf (stderr, "lensobjOffset + Nlensobj > catalog.Nlensobj : %d %d %d\n", i, catalog.average[i].Nlensobj, (int) catalog.Nlensobj);
-    }
-  }
-
-  if (!measureOffsetOK) {
-    fprintf (stderr, "ERROR: catalog %s has an invalid measureOffset\n", catalog.filename);
-    status = FALSE;
-  }
-
-  if (NmeasureTotal != catalog.Nmeasure) {
-    fprintf (stderr, "ERROR: catalog %s has an invalid Nmeasure\n", catalog.filename);
-    status = FALSE;
-  }
-
-  if (!lensingOffsetOK) {
-    fprintf (stderr, "ERROR: catalog %s has an invalid lensingOffset\n", catalog.filename);
-    status = FALSE;
-  }
-
-  if (NlensingTotal != catalog.Nlensing) {
-    fprintf (stderr, "ERROR: catalog %s has an invalid Nlensing\n", catalog.filename);
-    status = FALSE;
-  }
-
-  if (!lensobjOffsetOK) {
-    fprintf (stderr, "ERROR: catalog %s has an invalid lensobjOffset\n", catalog.filename);
-    status = FALSE;
-  }
-
-  if (NlensobjTotal != catalog.Nlensobj) {
-    fprintf (stderr, "ERROR: catalog %s has an invalid Nlensobj\n", catalog.filename);
-    status = FALSE;
-  }
+  
+# define CHECK_TABLE(NAME) {                                            \
+    int Ntotal = 0; int offsetOK = TRUE; int isOK = FALSE;              \
+    for (i = 0; i < catalog.Naverage; i++) {                            \
+      Ntotal += catalog.average[i].N##NAME;                             \
+      if (VERBOSE && !(Ntotal <= catalog.N##NAME)) {                    \
+        fprintf (stderr, "Ntotal (%s) > catalog.N##NAME: %d %d : %d > %d\n", #NAME, i, Ntotal, catalog.average[i].N##NAME, (int) catalog.N##NAME); \
+      }                                                                 \
+      isOK = (catalog.average[i].N##NAME == 0) || (catalog.average[i].NAME##Offset < catalog.N##NAME); \
+      offsetOK &= isOK;                                                 \
+      if (VERBOSE && !isOK) {                                           \
+        fprintf (stderr, "%sOffset >= catalog.N%s: %d : %d >= %d\n", #NAME, #NAME, i, catalog.average[i].NAME##Offset, (int) catalog.N##NAME); \
+      }                                                                 \
+      isOK = (catalog.average[i].N##NAME == 0) || (catalog.average[i].NAME##Offset + catalog.average[i].N##NAME <= catalog.N##NAME); \
+      offsetOK &= isOK;                                                 \
+      if (VERBOSE && !isOK) {                                           \
+        fprintf (stderr, "%sOffset + N%s > catalog.N%s : %d : %d + %d > %d\n", #NAME, #NAME, #NAME, i, catalog.average[i].N##NAME, catalog.average[i].NAME##Offset, (int) catalog.N##NAME); \
+      } }                                                               \
+    if (!offsetOK) {                                                    \
+      fprintf (stderr, "ERROR: catalog %s has an invalid %sOffset\n", catalog.filename, #NAME); \
+      status = FALSE;                                                   \
+    }                                                                   \
+    if (Ntotal != catalog.N##NAME) {                                    \
+      fprintf (stderr, "ERROR: catalog %s has an invalid N%s\n", catalog.filename, #NAME); \
+      status = FALSE;                                                   \
+    }                                                                   \
+  }
+  
+  CHECK_TABLE(measure);
+  CHECK_TABLE(lensing);
+  CHECK_TABLE(lensobj);
+  CHECK_TABLE(starpar);
+  CHECK_TABLE(galphot);
 
   // if we have a problem with Nmeasure and/or measureOffset values, we
@@ -312 +266 @@
   // check measure <-> average links
   {
-    int objIDsOK = TRUE;
-    int catIDsOK = TRUE;
-    int averefOK = TRUE;
+    int NobjIDsBAD = 0;
+    int NcatIDsBAD = 0;
+    int NaverefBAD = 0;
 
     for (i = 0; i < catalog.Naverage; i++) {
       m = catalog.average[i].measureOffset;
       for (j = 0; j < catalog.average[i].Nmeasure; j++) {
-        objIDsOK &= (catalog.average[i].objID == catalog.measure[m+j].objID);
-        catIDsOK &= (catalog.average[i].catID == catalog.measure[m+j].catID);
-        averefOK &= (catalog.measure[m+j].averef == i);
+        if (catalog.average[i].objID != catalog.measure[m+j].objID) {
+          NobjIDsBAD ++;
+
+          // check if objID matches R,D
+          Measure *measure = &catalog.measure[m+j];
+          int iTest = measure->objID;
+          if (catalog.average[iTest].objID == iTest) {
+            double dR = 3600.0*(catalog.average[iTest].R - measure->R) * cos (RAD_DEG*measure->D);
+            double dD = 3600.0*(catalog.average[iTest].D - measure->D);
+            double dRad = hypot (dR, dD);
+            char *date = ohana_sec_to_date (measure->t);
+            fprintf (stderr, "matches ave %d = %d, %s : %6.3f %5d : %f, %f = %f\n", iTest, measure->objID, date, measure->M, measure->photcode, dR, dD, dRad);
+            free (date);
+          } else {
+            fprintf (stderr, "cannot find averef\n");
+          }
+        }
+        if (catalog.average[i].catID != catalog.measure[m+j].catID) {
+          NcatIDsBAD ++;
+        }
+        if (catalog.measure[m+j].averef != i) {
+          NaverefBAD ++;
+        }
       }
     }
     
-    if (!objIDsOK) {
-      fprintf (stderr, "ERROR: catalog %s has invalid obj IDs\n", catalog.filename);
-      status = FALSE;
-    }
-    if (!catIDsOK) {
-      fprintf (stderr, "ERROR: catalog %s has invalid cat IDs\n", catalog.filename);
-      status = FALSE;
-    }
-    if (!averefOK) {
-      fprintf (stderr, "ERROR: catalog %s has invalid averef values\n", catalog.filename);
+    if (NobjIDsBAD) {
+      fprintf (stderr, "ERROR: catalog %s has %d invalid obj IDs\n", catalog.filename, NobjIDsBAD);
+      status = FALSE;
+    }
+    if (NcatIDsBAD) {
+      fprintf (stderr, "ERROR: catalog %s has %d invalid cat IDs\n", catalog.filename, NcatIDsBAD);
+      status = FALSE;
+    }
+    if (NaverefBAD) {
+      fprintf (stderr, "ERROR: catalog %s has %d invalid averef values\n", catalog.filename, NaverefBAD);
       status = FALSE;
     }
@@ -387 +361 @@
     if (!catIDsOK) {
       fprintf (stderr, "ERROR: catalog %s has invalid lensobj cat IDs\n", catalog.filename);
+      status = FALSE;
+    }
+  }
+
+  // check starpar <-> average links
+  {
+    int objIDsOK = TRUE;
+    int catIDsOK = TRUE;
+    int averefOK = TRUE;
+
+    for (i = 0; i < catalog.Naverage; i++) {
+      m = catalog.average[i].starparOffset;
+      for (j = 0; j < catalog.average[i].Nstarpar; j++) {
+        objIDsOK &= (catalog.average[i].objID == catalog.starpar[m+j].objID);
+        catIDsOK &= (catalog.average[i].catID == catalog.starpar[m+j].catID);
+        averefOK &= (catalog.starpar[m+j].averef == i);
+      }
+    }
+    
+    if (!objIDsOK) {
+      fprintf (stderr, "ERROR: catalog %s has invalid starpar obj IDs\n", catalog.filename);
+      status = FALSE;
+    }
+    if (!catIDsOK) {
+      fprintf (stderr, "ERROR: catalog %s has invalid starpar cat IDs\n", catalog.filename);
+      status = FALSE;
+    }
+    if (!averefOK) {
+      fprintf (stderr, "ERROR: catalog %s has invalid starpar averef values\n", catalog.filename);
+      status = FALSE;
+    }
+  }
+
+  // check galphot <-> average links
+  {
+    int objIDsOK = TRUE;
+    int catIDsOK = TRUE;
+    int averefOK = TRUE;
+
+    for (i = 0; i < catalog.Naverage; i++) {
+      m = catalog.average[i].galphotOffset;
+      for (j = 0; j < catalog.average[i].Ngalphot; j++) {
+        objIDsOK &= (catalog.average[i].objID == catalog.galphot[m+j].objID);
+        catIDsOK &= (catalog.average[i].catID == catalog.galphot[m+j].catID);
+        averefOK &= (catalog.galphot[m+j].averef == i);
+      }
+    }
+    
+    if (!objIDsOK) {
+      fprintf (stderr, "ERROR: catalog %s has invalid galphot obj IDs\n", catalog.filename);
+      status = FALSE;
+    }
+    if (!catIDsOK) {
+      fprintf (stderr, "ERROR: catalog %s has invalid galphot cat IDs\n", catalog.filename);
+      status = FALSE;
+    }
+    if (!averefOK) {
+      fprintf (stderr, "ERROR: catalog %s has invalid galphot averef values\n", catalog.filename);
       status = FALSE;
     }

trunk/Ohana/src/dvomerge/src/help.c

@@ -172 +172 @@
 
   fprintf (stderr, "USAGE\n");
+  fprintf (stderr, "  dvorepair -fix-warp-ids (catdir) (idfile) - regenerate images.dat warp_skyfile_ids (EXTERN_ID) values\n");
   fprintf (stderr, "  dvorepair -fix-cpt (images) (rootlist) - regenerate cpt & cps files from the cpm files\n");
   fprintf (stderr, "  dvorepair -images-vs-measures (catdir) (Ntol) - find images with too many missing detections\n");

trunk/Ohana/src/dvomerge/src/merge_catalogs_old.c

@@ -198 +198 @@
     int Nreplace = 0;
     if (REPLACE_TYCHO) {
-      int Mout = output[0].average[n].measureOffset;  
       int Minp =  input[0].average[N].measureOffset;
-      Nreplace = replace_tycho (&output[0].average[n], &output[0].measure[Mout], &input[0].average[N], &input[0].measure[Minp]);
+      Nreplace = replace_tycho (&output[0].average[n], output[0].measure, next_meas, &input[0].average[N], &input[0].measure[Minp]);
       if (Nreplace == 6) {
         input[0].found_t[N] = Nmeas;
@@ -216 +215 @@
         // index to first measure for this object
         // XXX this does not support lensing measurements
-        int Mout = output[0].average[n].measureOffset;  
-        if (replace_match(&output[0].average[n], &output[0].measure[Mout], &input[0].average[N], &input[0].measure[offset])) {
-          input[0].found_t[N] = Mout;
+        if (replace_match (&output[0].average[n], output[0].measure, next_meas, &input[0].average[N], &input[0].measure[offset])) {
+          input[0].found_t[N] = output[0].average[n].measureOffset;  
           continue;
         }

trunk/Ohana/src/dvomerge/src/replace_match.c

@@ -1 +1 @@
 # include "dvomerge.h"
 
+// XXX this function does not work because it it loops over the average->Nmeasure entries
+// as if they were sorted, but objects which have had entries added to them have elements
+// out of sequence.  I probably need to step through next_meas to find the sequence.
+
 // average and measure pointers to the objects of interest and their first / current measurement
-int replace_match (Average *average_out, Measure *measure_out, Average *average_in, Measure *measure_in) {
+int replace_match (Average *average_out, Measure *measure_out, off_t *next_meas, Average *average_in, Measure *measure_in) {
 
   int Nout;
@@ -8 +12 @@
   unsigned int catID;
 
+  off_t m = average_out->measureOffset;
+
   // find the matching photcode in the object's list of measurements
   for (Nout = 0; Nout < average_out[0].Nmeasure; Nout ++) {
-    if (measure_out[Nout].photcode != measure_in[0].photcode) continue;
+    myAssert (m > -1, "oops");
+    if (measure_out[m].photcode != measure_in[0].photcode) {
+      m = next_meas[m];
+      continue;
+    }
     
     // set the new measurements
-    averef = measure_out[Nout].averef;
-    catID  = measure_out[Nout].catID;
-    measure_out[Nout] = measure_in[0];
+    averef = measure_out[m].averef;
+    catID  = measure_out[m].catID;
+    measure_out[m] = measure_in[0];
 
     // old code: find R,D using average_in[0], the get offset relative to average_out[0].  no longer
@@ -24 +34 @@
     // measure_out[Nout].dD = 3600.0*(average_out[0].D - Din);
 
-    measure_out[Nout].dbFlags  = 0;  // XXX why reset these?
-    measure_out[Nout].averef   = averef;
-    measure_out[Nout].objID    = average_out[0].objID;
-    measure_out[Nout].catID    = catID;
+    measure_out[m].dbFlags  = 0;  // XXX why reset these?
+    measure_out[m].averef   = averef;
+    measure_out[m].objID    = average_out[0].objID;
+    measure_out[m].catID    = catID;
 
-    float dRoff = dvoOffsetR(&measure_out[Nout], average_out);
+    float dRoff = dvoOffsetR(&measure_out[m], average_out);
 
     // rationalize dR
     if (dRoff > +180.0*3600.0) {
       // average on high end of boundary, move star up
-      measure_out[Nout].R += 360.0;
+      measure_out[m].R += 360.0;
       dRoff -= 360.0*3600.0;
     }
     if (dRoff < -180.0*3600.0) {
       // average on low end of boundary, move star down
-      measure_out[Nout].R -= 360.0;
+      measure_out[m].R -= 360.0;
       dRoff += 360.0*3600.0;
     }
@@ -49 +59 @@
       if (fabs(dRoff*cosD) > 10*RADIUS) {
         fprintf (stderr, "surprisingly distant detection: %10.6f,%10.6f vs %10.6f,%10.6f\n", 
-                 average_out[0].R, average_out[0].D, measure_out[Nout].R, measure_out[Nout].D);
+                 average_out[0].R, average_out[0].D, measure_out[m].R, measure_out[m].D);
       }
     }

trunk/Ohana/src/dvomerge/src/replace_tycho.c

@@ -10 +10 @@
 }
 
-int replace_tycho (Average *averageOut, Measure *measureOut, Average *averageInp, Measure *measureInp) {
+int replace_tycho (Average *averageOut, Measure *measureOut, off_t *next_meas, Average *averageInp, Measure *measureInp) {
 
   int i;
@@ -17 +17 @@
   int index[6];
 
+  off_t m = averageOut->measureOffset;
+
   // find all tycho measurements
   for (i = 0; (i < averageOut->Nmeasure) && (Nindex < 6); i++) {
 
+    myAssert (m > -1, "oops");
+
     int valid = FALSE;
-    valid = valid || (measureOut[i].photcode == TYCHO_B);
-    valid = valid || (measureOut[i].photcode == TYCHO_V);
-    if (!valid) continue;
-
-    index[Nindex] = i;
-    Nindex ++;
+    valid = valid || (measureOut[m].photcode == TYCHO_B);
+    valid = valid || (measureOut[m].photcode == TYCHO_V);
+    if (valid) { 
+      index[Nindex] = m;
+      Nindex ++;
+    }
+    m = next_meas[m];
   }