Index: /branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src/dvomergeHistory.c
===================================================================
--- /branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src/dvomergeHistory.c	(revision 34356)
+++ /branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src/dvomergeHistory.c	(revision 34356)
@@ -0,0 +1,322 @@
+# include "dvomerge.c"
+
+// We track the dvomerge history (dmh) in the headers of the Image table (Image.dat) and
+// each of the Object tables (CPT files).  In the Image table, we track the IDs of the dvo
+// databases which have been merged.  In the object tables, we track the date and size of
+// the CPT files already merged.  When we merge a new dvodb, we block against re-merging
+// any Object table if the current one matches one of the already ingested tables.
+
+// XXX should I require all IDs to be in sequence?  The sequence has no meaning except to
+// ensure the keyword numbering.  If we require the sequence, then it simplifies the code
+// somewhat
+
+// struct to describe a sequence of dvomerges (Image.dat header)
+typedef struct {
+  int Nmerge;
+  char **IDs;
+} dmhImage;
+
+// struct to describe a sequence of dvomerges (Object table : cpt header)
+typedef struct {
+  int   Nmerge;
+  off_t  *size;
+  time_t *time;
+  char  **date;
+} dmhObject;
+
+// data on a single table, populated when a new file is merged
+typedef struct {
+  off_t  size;
+  time_t time;
+  char  *date;
+} dmhObjectStats;
+
+// struct to describe the current status of a single output file:
+// is it on this machine (valid)? have we already merged or not (missed)?
+// what is the collection of past merges (history)?  what is the real filename?
+typedef struct {
+  int valid;		      // is this object table on this machine?
+  int missed;		      // did we fail to merge into this table yet?
+  dmhTable *history;	      // complete sequence of previous merges
+  char *filename;	      // true filename on disk 
+} OutputStatus;
+
+// we have a sequence of dvomerge events.  
+// in the image table header, we want to track the IDs of the databases 
+// which have been merged
+
+// some (possible?) functions needed to track the history
+
+// given a dvo image db, add an ID to the header
+// XXX move this to libdvo and generate tools to update a database
+int dvoCreateID (char *catdir) {
+  
+  // open and read header
+  char filename[DVO_MAX_PATH];
+  snprintf (filename, DVO_MAX_PATH, "%s/Images.dat", catdir);
+
+  // open for update
+  FILE *file = fopen (filename, "r+");
+  if (!file) {
+    fprintf (stderr, "problem opening output file to read header %s\n", filename);
+    perror ("stats error message:");
+    exit (2);
+    // XXX should we generate an file with an empty table here?  
+  }
+
+  Header header;
+  if (!gfits_fread_header (file, &header)) {
+    fprintf (stderr, "problem reading header for output file %s\n", filename);
+    exit (2);
+  }
+      
+  // I have a header, I want to add ONE line.  double check there is enough space
+  char *p = gfits_header_field (&header, "END", 1);
+  if (p == NULL) {
+    fprintf (stderr, "header is missing END\n");
+    exit (1);
+  }
+  if (p - header.buffer + 80 == header.datasize) {
+    fprintf (stderr, "no free space in block, can't insert keyword\n");
+    exit (1);
+  }
+
+  int start_size = header.datasize;
+
+  char dbID[33];
+  long A = time(NULL);
+  srand48(A);
+  
+  for (i = 0; i < 32; i++) {
+    sprintf (&dbID[i], "%1x", (int)(16.0*drand48()));
+  }
+
+  gfits_modify (&header, "DVO_DB_ID", "%32s", 1, dbID);      
+  if (start_size != header.datasize) {
+    fprintf (stderr, "error: header buffer grew? (%d to %d bytes)\n", start_size, header.datasize);
+    exit (1);
+  }
+
+  fclose (file);
+  return TRUE;
+}
+
+// user must pass in a char * of length DVO_DB_ID_LEN
+char *dmhImageReadID (FITS_DB *db) {
+
+  char dbID[33];
+  status = gfits_scan (&db->header, "DVO_DB_ID", "%s", 1, dbID);      
+  if (!status) {
+    return NULL;
+  }
+
+  char *ID = strcreate (dbID);
+  return ID;
+}
+
+dmhImage *dmhImageRead (FITS_DB *db) {
+
+  if (!db) return FALSE;
+  if (!db->header) return FALSE;
+
+  dmhImage *history = NULL;
+  ALLOCATE (history, dmhImage, 1);
+
+  // should track the number in the header (would overwrite)
+  status = gfits_scan (&db->header, "DM_NMERGE", "%d", 1, &history->Nmerge); 
+  if (!status) {
+    fprintf (stderr, "failed to read DM_NMERGE\n");
+    exit (5);
+  }
+
+  // ALLOCATE full list
+  ALLOCATE (history->IDs, char *, history->Nmerge);
+
+  for (i = 0; i < history->Nmerge; i++) {
+    snprintf (name, 16, "DM_%05d", i);
+    status = gfits_scan (&header, name, "%s", ID);
+    if (!status) {
+      fprintf (stderr, "failed to read %s\n", name);
+      exit (6);
+    }
+    history->IDs[i] = strcreate (ID);
+  }
+  return history;
+}
+
+int dmhImageAdd (FITS_DB *db, dmhImage *history, char *dbID) {
+
+  // update the arrays
+  last = history->Nmerge;
+
+  history->Nmerge ++;
+  REALLOCATE (history->IDs, char *, history->Nmerge);
+
+  history->IDs[last] = strcreate (ID);
+
+  snprintf (name, 16, "DM_%05d", last);
+  gfits_modify (&db->header, name, "%s", ID);
+
+  return TRUE;
+}
+
+dmhObjectStats *dmhObjectStatsRead (char *filename) {
+
+  // instats.st_size & instats.st_mtime
+
+  // get the stats on this input file (for comparison with the output headers)
+  struct stat instats;
+  int stat_result = stat (filename, &instats);
+  if (stat_result) {
+    if (errno == ENOENT) return NULL
+    fprintf (stderr, "cannot read stats on input file %s\n", inlist[0].filename[i]);
+    perror ("stats error message:");
+    exit (2);
+  }
+  
+  ALLOCATE (stats, dmhStatsType, 1);
+  stats->size = instats.st_size;
+  stats->time = instats.st_mtime;
+  stats->date = ohana_sec_to_date (instats.st_mtime);
+  return stats;
+}
+
+void dmhObjectStatsFree (dmhObjectStats *stats) {
+  if (stats->date) free (stats->date);
+  free (stats);
+}
+
+// read the array of merged history information from this file's header
+dmhObject *dmhObjectRead (char *filename) {
+
+  // if the file does not exist, return NULL (means 'missed')
+  struct stat outstats;
+  stat_result = stat (filename, &outstats);
+  if (stat_result) {
+    if (errno == ENOENT) return NULL;
+    fprintf (stderr, "cannot read stats on output file %s\n", filename);
+    perror ("stats error message:");
+    exit (2);
+  }
+
+  FILE *fout = fopen (filename, "r");
+  if (!fout) {
+    fprintf (stderr, "problem opening output file to read header %s\n", filename);
+    perror ("stats error message:");
+    exit (2);
+  }
+
+  Header outheader;
+  if (!gfits_fread_header (fout, &outheader)) {
+    fprintf (stderr, "problem reading header for output file %s\n", filename);
+    exit (2);
+  }
+  fclose (fout);
+
+  dmhObject *history = NULL;
+  ALLOCATE (history, dmhObject, 1);
+
+  status = gfits_scan (&outheader, "DM_NMERGE", "%d", 1, &history->Nmerge);      
+  if (!status) {
+    fprintf (stderr, "failed to read DM_NMERGE\n");
+    exit (5);
+  }
+
+  // ALLOCATE full list
+  ALLOCATE (history->size, off_t,  history->Nmerge);
+  ALLOCATE (history->time, time_t, history->Nmerge);
+  ALLOCATE (history->date, char *, history->Nmerge);
+
+  for (i = 0; i < history->Nmerge; i++) {
+    off_t  size;
+    time_t time;
+    char   date[80];
+
+    snprintf (name, 16, "MS_%05d", i);
+    status = gfits_scan (&header, name, OFF_T_FMT, &size);
+    if (!status) {
+      fprintf (stderr, "failed to read %s\n", name);
+      exit (6);
+    }
+    history->size[i] = size;
+
+    snprintf (name, 16, "MT_%05d", i);
+    status = gfits_scan (&header, name, "%s", date);
+    if (!status) {
+      fprintf (stderr, "failed to read %s\n", name);
+      exit (7);
+    }
+    history->date[i] = strcreate (date);
+    history->time[i] = ohana_date_to_sec (date);
+  }
+
+  fclose (fout);
+
+  return history;
+}
+  
+// compare stats for the input to the full history for the output
+int dmhObjectCheck (dmhObject *history, dmhObjectStats *inStats) {
+
+  for (i = 0; i < history->Nmerge; i++) {
+    
+    // do we match this size?
+    if (history->size[i] != inStats->size) continue;
+
+    // do we match this date?
+    if (history->time[i] != inStats->time) continue;
+
+    return TRUE;  // we match (already found)
+  }
+  return FALSE;
+}
+
+// add the stats for the given input file to the output structure and header
+int dmhObjectAdd (dmhObject *history, Header *header, dmhObjectStats inStats) {
+
+  // update the arrays
+  last = history->Nmerge;
+
+  history->Nmerge ++;
+  REALLOCATE (history->size, off_t,  history->Nmerge);
+  REALLOCATE (history->time, time_t, history->Nmerge);
+  REALLOCATE (history->date, char *, history->Nmerge);
+
+  history->size[last] = inStats->size;
+  history->time[last] = inStats->time;
+  history->date[last] = inStats->date;
+
+  snprintf (name, 16, "MS_%05d", last);
+  status = gfits_modify (&header, name, OFF_T_FMT, inStats->size);
+  if (!status) { error out};
+
+  snprintf (name, 16, "MT_%05d", last);
+  status = gfits_modify (&header, name, "%s", inStats->date);
+  if (!status) { error out};
+
+  gfits_modify (&db->header, "DM_NMERGE", "%d", history->Nmerge);
+  return TRUE;
+}
+
+OutputStatus *OutputStatusInit (int N) {
+
+  OutputStatus *outstat = NULL;
+  ALLOCATE (outstat, OutputStatus, N);
+  for (i = 0; i < N; i++) {
+    outstat[i].valid = FALSE;
+    outstat[i].missed = FALSE;
+    outstat[i].history = NULL;
+    outstat[i].filename = NULL;
+  }
+
+}
+
+int OutputStatusFree (OutputStatus *outstat, int N) {
+
+  for (i = 0; i < N; i++) {
+    if (outstat[i].history)  { free (outstat[i].history); }
+    if (outstat[i].filename) { free (outstat[i].filename); }
+  }
+  free (outstat);
+}
+
Index: /branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src/dvomergeImageIDs.c
===================================================================
--- /branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src/dvomergeImageIDs.c	(revision 34355)
+++ /branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src/dvomergeImageIDs.c	(revision 34356)
@@ -21,7 +21,14 @@
     return TRUE;
   }
+  // this operation reads the PHU header (inDB.header)
   if (!dvo_image_load (&inDB, VERBOSE, TRUE)) {
     Shutdown ("can't read input image catalog %s", inDB.filename);
   }
+
+  // read the header for the database ID?
+  char *indbID = dvomergeHistoryReadID (&inDB);
+  if (!indbID) { 
+    Shutdown ("this database is missing a DVO database ID; please generate one before merging\n"); 
+  } 
 
   /*** load output/Images.dat ***/
@@ -41,7 +48,20 @@
   }
 
+  dmhImage *history = dmhImageRead (&outDB);
+  if (!history) { 
+    Shutdown ("error reading history for output database\n"); 
+  }
+
   // convert database table to internal structure & add to output image db
   dvo_image_merge_dbs(IDmap, &outDB, &inDB);
   dvo_image_unlock (&inDB); // unlock input
+
+  // add the new image db to merge history
+  // (updates header as well as history structure
+  if (!dmhImageAdd (&outDB, history, indbID)) { 
+    Shutdown ("error reading history for output database\n"); 
+  }
+
+  // XXX I need to pass the history structure back out somehow...
 
   SetProtect (TRUE);
Index: /branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c
===================================================================
--- /branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c	(revision 34355)
+++ /branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c	(revision 34356)
@@ -31,33 +31,29 @@
     outlist = SkyListByBounds (outsky, -1, inlist[0].regions[i][0].Rmin + dPos, inlist[0].regions[i][0].Rmax - dPos, inlist[0].regions[i][0].Dmin + dPos, inlist[0].regions[i][0].Dmax - dPos);
 
-    // there may be more than one output region for a given input region.  are any of these output regions relevant to this machine?
+    OutputStatus *outstat = OutputStatusInit (outlist->Nregions);
+
+    // there may be more than one output region for a given input region.  
+    // are any of these output regions relevant to this machine?
     int found = FALSE;
     for (j = 0; !found && (j < outlist[0].Nregions); j++) {
-      found = (found || HostTableTestHost(outlist[0].regions[j], HOST_ID));
+      outstat[j].valid = HostTableTestHost(outlist[0].regions[j], HOST_ID);
+      found = (found || outstat[j].valid);
     }
 
     // skip this input table for if no output files are on this machine
     if (!found) {
+      OutputStatusFree (outstat, outlist->Nregions);
       SkyListFree (outlist); 
       continue; 
     }
 
-    // get the stats on this input file (for comparison with the output headers)
-    struct stat instats;
-    int stat_result = stat (inlist[0].filename[i], &instats);
-    if (stat_result) {
-      if (errno == ENOENT) continue;
-      fprintf (stderr, "cannot read stats on input file %s\n", inlist[0].filename[i]);
-      perror ("stats error message:");
-      exit (2);
-    }
-
-    // instats.st_size & instats.st_mtime
-
-    // check if any of the output files have NOT yet received data from this input file
-
+    // get stats for history check
+    dmhObjectStats *inStats = dmhObjectStatsRead (inlist[0].filename[i]);
+
+    // Check if any of the output files have NOT yet received data from this input file
+    // If none have been missed, we can skip the input file completely
     int missed = FALSE;
-    for (j = 0; !missed && (j < outlist[0].Nregions); j++) {
-      if (!HostTableTestHost(outlist[0].regions[j], HOST_ID)) continue;
+    for (j = 0; j < outlist[0].Nregions; j++) {
+      if (!outstat[j].valid) continue;
 
       // set the parameters which guide catalog open/load/create
@@ -65,54 +61,16 @@
       snprintf (hostfile, DVO_MAX_PATH, "%s/%s.cpt", HOSTDIR, outlist[0].regions[j]->name);
       char *filename = HOST_ID ? hostfile : outlist[0].filename[j];
-
-      // get the stats on this input file (for comparison with the output headers)
-      struct stat outstats;
-      stat_result = stat (filename, &outstats);
-      if (stat_result) {
-	if (errno == ENOENT) {
-	  missed = TRUE;
-	  break;
-	}
-	fprintf (stderr, "cannot read stats on output file %s\n", filename);
-	perror ("stats error message:");
-	exit (2);
-      }
-
-      FILE *fout = fopen (filename, "r");
-      if (!fout) {
-	fprintf (stderr, "problem opening output file to read header %s\n", filename);
-	perror ("stats error message:");
-	exit (2);
-      }
-
-      Header outheader;
-      if (!gfits_fread_header (fout, &outheader)) {
-	fprintf (stderr, "problem reading header for output file %s\n", filename);
-	exit (2);
-      }
-      
-      fclose (fout);
-
-      // XXX note that we are hardwired to v 2
-      // check the header of output catalog for an existing merge
-      long long last_size;
-      char last_moddate[80];
-      gfits_scan (&outheader, "LMRG_SZ2", "%lld", 1, &last_size);      
-      gfits_scan (&outheader, "LMRG_DT2", "%s", 1, last_moddate);      
-
-      time_t last_mod = ohana_date_to_sec (last_moddate);
-
-      if (last_size != instats.st_size) {
-	missed = TRUE;
-	break;
-      }
-      
-      if (last_mod != instats.st_mtime) {
-	missed = TRUE;
-	break;
-      }
+      outstat[j].filename = strcreate (filename);
+
+      outstat[j].history = dmhObjectRead (outstat[j].filename);
+
+      outstat[j].missed = dmhObjectCheck (outstat[j].history, inStats);
+      missed = (missed || outstat[j].missed);
     }
     if (!missed) {
       fprintf (stderr, "skipping %s, already merged\n", inlist[0].filename[i]);
+      OutputStatusFree (outstat, outlist->Nregions);
+      dmhObjectStatsFree (inStats);
+      SkyListFree (outlist); 
       continue;
     }
@@ -125,21 +83,22 @@
 	dvo_catalog_unlock (&incatalog);
 	dvo_catalog_free (&incatalog);
+	OutputStatusFree (outstat, outlist->Nregions);
+	dmhObjectStatsFree (inStats);
 	SkyListFree (outlist); 
 	continue;
     }
-
-    char *moddate = ohana_sec_to_date (instats.st_mtime);
 
     // merge input into the appropriate output tables
     for (j = 0; j < outlist[0].Nregions; j++) {
       // skip tables for which the output files are not on this machine
-      if (!HostTableTestHost(outlist[0].regions[j], HOST_ID)) continue; 
+      if (!outstat[j].valid) continue; 
+
+      // skip if we have already done the merge
+      if (!outstat[j].missed) continue; 
 
       if (VERBOSE) fprintf (stderr, "output : %s\n", outlist[0].regions[j][0].name);
 
-      // set the parameters which guide catalog open/load/create
-      char hostfile[DVO_MAX_PATH];
-      snprintf (hostfile, DVO_MAX_PATH, "%s/%s.cpt", HOSTDIR, outlist[0].regions[j]->name);
-      outcatalog.filename  = HOST_ID ? hostfile : outlist[0].filename[j];
+      // the real filename
+      outcatalog.filename = outstat[j].filename; 
 
       // load input catalog
@@ -151,26 +110,5 @@
       outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
 
-# if (0)
-      // get the LMRG data from the previous pass
-      long long last_size;
-      char last_moddate[80];
-      int status_size = gfits_scan (&outcatalog.header, "LMRG_SZ", "%lld", 1, &last_size);      
-      int status_date = gfits_scan (&outcatalog.header, "LMRG_DT", "%s", 1, last_moddate);      
-
-      // save the LMRG data from the previous pass
-      if (status_size && status_date) {
-	gfits_modify (&outcatalog.header, "LMRG_SZ1", "%lld", 1, (long long) last_size);      
-	gfits_modify (&outcatalog.header, "LMRG_DT1", "%s", 1, last_moddate);      
-      }
-
-      // update header of output catalog
-      gfits_modify (&outcatalog.header, "LMRG_SZ", "%lld", 1, (long long) instats.st_size);      
-      gfits_modify (&outcatalog.header, "LMRG_DT", "%s", 1, moddate);      
-# endif
-
-      // update header of output catalog
-      // XXX note that we are hardwired to v 2
-      gfits_modify (&outcatalog.header, "LMRG_SZ2", "%lld", 1, (long long) instats.st_size);      
-      gfits_modify (&outcatalog.header, "LMRG_DT2", "%s", 1, moddate);      
+      dmhObjectAdd (outstat[j].history, &outcatalog.header, inStats);
 
       if (!dvo_catalog_backup (&outcatalog, TRUE)) {
@@ -196,6 +134,8 @@
       fprintf (stderr, "merged %s into %s\n", inlist[0].regions[i][0].name, outlist[0].regions[j][0].name);
     }
+
+    OutputStatusFree (outstat, outlist->Nregions);
     SkyListFree (outlist);
-    free (moddate);
+    dmhObjectStatsFree (inStats);
 
     dvo_catalog_unlock (&incatalog);