Changeset 39605 for trunk

Timestamp:

Jun 21, 2016, 1:14:24 PM (10 years ago)

Author:

eugene

Message:

track deletions and save cpt, cps, cpm files for backup

Location:

trunk/Ohana/src/delstar

Files:

• include/delstar.h (modified) (2 diffs)
• src/delete_duplicate_measures.c (modified) (12 diffs)

trunk/Ohana/src/delstar/include/delstar.h

@@ -43 +43 @@
   int imageID;
 } MeasureEdge;
+
+typedef struct {
+  off_t NdelWarp;
+  off_t NdelChip;
+  off_t NdelStack;
+  off_t NdelOther;
+  off_t NdelAves;
+  off_t NdelMeas;
+} DeleteMeasureResult;
 
 /* global variables set in parameter file */
@@ -163 +172 @@
 int delete_duplicate_measures ();
 int delete_duplicate_measures_parallel (SkyList *sky);
-int delete_duplicate_measures_catalog (Catalog *catalog);
+DeleteMeasureResult delete_duplicate_measures_catalog (Catalog *catalog);
 
 int delete_fix_LAP (ImageSubset *image, off_t Nimage);

trunk/Ohana/src/delstar/src/delete_duplicate_measures.c

@@ -1 +1 @@
 # include "delstar.h"
+int isGPC1chip (int photcode);
+int isGPC1warp (int photcode);
+int isGPC1stack (int photcode);
+int dvo_catalog_subset_backup (Catalog *catalog, char *suffix);
 
 // this function identifies detections to be deleted as being duplicates based on imageID + detID
@@ -52 +56 @@
     catalog.filename  = HOST_ID ? hostfile : skylist[0].filename[i];
     catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
-    catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
+    catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT;
 
     if (VERBOSE) fprintf (stderr, "deleting from %s\n", catalog.filename);
@@ -68 +72 @@
     }
 
-    if (delete_duplicate_measures_catalog (&catalog)) {
-      // skip if nothing was deleted
-      if (UPDATE) {
-        if (!dvo_catalog_backup (&catalog, "~", TRUE)) {
-          fprintf (stderr, "ERROR: failed to make backup for catalog %s\n", catalog.filename);
-          exit (1);
-        }
-        SetProtect (TRUE);
-        dvo_catalog_save_complete (&catalog, VERBOSE2);
-      }
-    }
+    DeleteMeasureResult result = delete_duplicate_measures_catalog (&catalog);
+    
+    // track number of deletions and only update if modifications are made
+    int Nmods = 0;
+    Nmods += result.NdelWarp;
+    Nmods += result.NdelChip;
+    Nmods += result.NdelStack;
+    Nmods += result.NdelOther;
+    Nmods += result.NdelAves;
+    Nmods += result.NdelMeas;
+    if (!Nmods) {
+      fprintf (stderr, "no changes to %s, no output\n", catalog.filename);
+      dvo_catalog_unlock (&catalog);
+      dvo_catalog_free (&catalog);
+      continue;
+    }
+
+    if (!UPDATE) {
+      dvo_catalog_unlock (&catalog);
+      dvo_catalog_free (&catalog);
+      continue;
+    }
+
+    char history[128];
+    struct timeval now;
+    gettimeofday (&now, (void *) NULL);
+    char *moddate = ohana_sec_to_date (now.tv_sec);
+    snprintf (history, 128, "delete duplicate measurements: %s", moddate);
+    gfits_modify_alt (&catalog.header, "HISTORY", "%S", 0, history); // adds a new entry
+    free (moddate);
+
+    // add metadata to define number of corrections
+    char line[128];
+    snprintf (line, 60, "Chip "OFF_T_FMT", Warp "OFF_T_FMT", Stack "OFF_T_FMT, result.NdelChip, result.NdelWarp, result.NdelStack);
+    gfits_modify (&catalog.header, "DELETE_1", "%s", 1, line);
+    snprintf (line, 60, "Other "OFF_T_FMT", Measure "OFF_T_FMT", Average "OFF_T_FMT, result.NdelOther, result.NdelMeas, result.NdelAves);
+    gfits_modify (&catalog.header, "DELETE_2", "%s", 1, line);
+
+    // save backup of original cpm file
+    if (!dvo_catalog_subset_backup (&catalog, ".dl1")) {
+      fprintf (stderr, "ERROR: failed to make backup cpt table for catalog %s\n", catalog.filename);
+      exit (1);
+    }
+    if (!dvo_catalog_subset_backup (catalog.measure_catalog, ".d1l")) {
+      fprintf (stderr, "ERROR: failed to make backup cpm table for catalog %s\n", catalog.filename);
+      exit (1);
+    }
+    if (!dvo_catalog_subset_backup (catalog.secfilt_catalog, ".dl1")) {
+      fprintf (stderr, "ERROR: failed to make backup cps table for catalog %s\n", catalog.filename);
+      exit (1);
+    }
+
+    // XXX something of a hack : I only want to save average, measure, secfilt.  
+    catalog.Nmissing = catalog.Nmissing_off;
+    catalog.Nlensing = catalog.Nlensing_off;
+    catalog.Nlensobj = catalog.Nlensobj_off;
+    catalog.Nstarpar = catalog.Nstarpar_off;
+    catalog.Ngalphot = catalog.Ngalphot_off;
+
+    catalog.Nmissing_off = 0;
+    catalog.Nlensing_off = 0;
+    catalog.Nlensobj_off = 0;
+    catalog.Nstarpar_off = 0;
+    catalog.Ngalphot_off = 0;
+
+    SetProtect (TRUE);
+    dvo_catalog_save_complete (&catalog, VERBOSE2);
     dvo_catalog_unlock (&catalog);
     SetProtect (FALSE);
@@ -89 +149 @@
   FreePhotcodeTable ();
 
+  return TRUE;
+}
+
+int dvo_catalog_subset_backup (Catalog *catalog, char *suffix) {
+
+  int dbstate;
+
+  char tmpfilename[DVO_MAX_PATH];
+  int status = snprintf (tmpfilename, DVO_MAX_PATH, "%s%s", catalog->filename, suffix);
+  if (status >= DVO_MAX_PATH) {
+    fprintf (stderr, "path name too long: %s\n", catalog->filename);
+    return FALSE;
+  }
+      
+  // play it safe: do not overwrite an existing backup file
+  struct stat fileStats;
+  status = stat (tmpfilename, &fileStats);
+  if (!status) {
+    fprintf (stderr, "ERROR: backup file %s already exists, exiting\n", tmpfilename);
+    return FALSE;
+  }
+  
+  // some error accessing the file.  there is only one acceptable error: file not found
+  if (status && (errno != ENOENT)) {
+    perror ("problem with output target");
+    return FALSE;
+  }
+
+  if (fflush (catalog[0].f)) {
+    perror ("fflush: ");
+    fprintf (stderr, "failed to flush file %s\n", catalog[0].filename);
+    return FALSE;
+  }
+
+  // closes f but does not set back to NULL
+  if (!fclearlockfile (catalog[0].filename, catalog[0].f, catalog[0].lockmode, &dbstate)) {
+    fprintf (stderr, "failed to unlock or close file\n");
+    return FALSE;
+  }
+
+  status = rename (catalog->filename, tmpfilename);
+  if (status) {
+    fprintf (stderr, "failed to rename catalog %s\n", catalog->filename);
+    return FALSE;
+  }
+
+  // re-lock file, create stream f 
+  catalog[0].f = fsetlockfile (catalog[0].filename, 3600.0, catalog[0].lockmode, &dbstate);
+  if (catalog[0].f == NULL)   return FALSE;
+  if (dbstate != LCK_EMPTY)   return FALSE;
+
+  if (fseeko (catalog[0].f, 0, SEEK_SET)) {
+    perror ("fseeko: ");
+    return FALSE;
+  }
+  
   return TRUE;
 }
@@ -177 +293 @@
 }
 
-int delete_duplicate_measures_catalog (Catalog *catalog) {
+DeleteMeasureResult delete_duplicate_measures_catalog (Catalog *catalog) {
 
   off_t i, j, n, m, N, D, currentAve;
@@ -257 +373 @@
   }
     
+# if (0)
   FILE *fsave = NULL;
   if (SAVE_DUPLICATES) {
@@ -274 +391 @@
     }
   }    
-
+# endif
+
+  DeleteMeasureResult result;
+  result.NdelWarp = 0;
+  result.NdelChip = 0;
+  result.NdelStack = 0;
+  result.NdelOther = 0;
+  result.NdelAves = 0;
+  result.NdelMeas = 0;
+  
   // mark the measures to be dropped
   for (i = 0; i < Nmeasure; i++) {
@@ -282 +408 @@
     off_t N = measure[j].averef;
     if (VERBOSE) fprintf (stderr, "0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[j].imageID, measure[j].detID, average[N].R, average[N].D, measure[j].photcode);
-    if (fsave) {
-      fprintf (fsave, "0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[j].imageID, measure[j].detID, average[N].R, average[N].D, measure[j].photcode);
-    }
+//  if (fsave) {
+//    fprintf (fsave, "0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[j].imageID, measure[j].detID, average[N].R, average[N].D, measure[j].photcode);
+//  }
+    if (isGPC1chip(measure[j].photcode)) {
+      result.NdelChip ++;
+      continue;
+    } 
+    if (isGPC1warp(measure[j].photcode)) {
+      result.NdelWarp ++;
+      continue;
+    } 
+    if (isGPC1stack(measure[j].photcode)) {
+      result.NdelStack ++;
+      continue;
+    } 
+    result.NdelOther ++;
   }
   
-  if (fsave) fclose (fsave);
+  // if (fsave) fclose (fsave);
 
   // set up the measure sequence lists
@@ -311 +450 @@
   // n = measureRefOut[i] : measureOut[n] = measure[i]
   ALLOCATE (measureAveOut, off_t, NmeasOut);
-
-
 
   // count the number of measures for each averef
@@ -438 +575 @@
   if (VERBOSE) fprintf (stderr, "ending with Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT"\n",  catalog[0].Naverage,  catalog[0].Nmeasure);
 
-  off_t NdelAves = Naverage - catalog[0].Naverage;
-  off_t NdelMeas = Nmeasure - catalog[0].Nmeasure;
-
-  if (NdelAves || NdelMeas) {
-    fprintf (stderr, "deleting "OFF_T_FMT" measures and "OFF_T_FMT" averages : %s\n",  NdelMeas, NdelAves, catalog[0].filename);
-  }
+  result.NdelAves = Naverage - catalog[0].Naverage;
+  result.NdelMeas = Nmeasure - catalog[0].Nmeasure;
+
+  fprintf (stderr, "deleting from %s : "OFF_T_FMT" meas, "OFF_T_FMT" aves : "OFF_T_FMT" chip, "OFF_T_FMT" stack, "OFF_T_FMT" warp, "OFF_T_FMT" other\n", catalog[0].filename, result.NdelMeas, result.NdelAves, result.NdelChip, result.NdelStack, result.NdelWarp, result.NdelOther);
 
   FREE (fullID);
@@ -460 +595 @@
   FREE (measureAveOut);
 
-  if (NdelAves || NdelMeas) return TRUE;
-  return FALSE;
+  return result;
 }
 
@@ -480 +614 @@
 }
 
+// for now (20140710) I need to identify gpc1 chips explicitly.  generalize in the future
+int isGPC1chip (int photcode) {
+
+  if ((photcode > 10000) && (photcode < 10077)) return TRUE; // g-band
+  if ((photcode > 10100) && (photcode < 10177)) return TRUE; // r-band
+  if ((photcode > 10200) && (photcode < 10277)) return TRUE; // i-band
+  if ((photcode > 10300) && (photcode < 10377)) return TRUE; // z-band
+  if ((photcode > 10400) && (photcode < 10477)) return TRUE; // y-band
+  if ((photcode > 10500) && (photcode < 10577)) return TRUE; // w-band
+
+  return FALSE;
+}
+
+// for now (20140710) I need to identify gpc1 stacks explicitly.  generalize in the future
+int isGPC1stack (int photcode) {
+
+  if (photcode == 11000) return TRUE; // g-band
+  if (photcode == 11100) return TRUE; // r-band
+  if (photcode == 11200) return TRUE; // i-band
+  if (photcode == 11300) return TRUE; // z-band
+  if (photcode == 11400) return TRUE; // y-band
+  if (photcode == 11500) return TRUE; // w-band
+
+  return FALSE;
+}
+
+// for now (20140710) I need to identify gpc1 stacks explicitly.  generalize in the future
+int isGPC1warp (int photcode) {
+
+  if (photcode == 12000) return TRUE; // g-band
+  if (photcode == 12100) return TRUE; // r-band
+  if (photcode == 12200) return TRUE; // i-band
+  if (photcode == 12300) return TRUE; // z-band
+  if (photcode == 12400) return TRUE; // y-band
+  if (photcode == 12500) return TRUE; // w-band
+
+  return FALSE;
+}