Changeset 34356

Timestamp:

Aug 25, 2012, 11:45:32 AM (14 years ago)

Author:

eugene

Message:

adding dvomerge history code to dvomerge

Location:

branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src

Files:

• dvomergeHistory.c (added)
• dvomergeImageIDs.c (modified) (2 diffs)
• dvomergeUpdate_catalogs.c (modified) (5 diffs)

branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src/dvomergeImageIDs.c

@@ -21 +21 @@
     return TRUE;
   }
+  // this operation reads the PHU header (inDB.header)
   if (!dvo_image_load (&inDB, VERBOSE, TRUE)) {
     Shutdown ("can't read input image catalog %s", inDB.filename);
   }
+
+  // read the header for the database ID?
+  char *indbID = dvomergeHistoryReadID (&inDB);
+  if (!indbID) { 
+    Shutdown ("this database is missing a DVO database ID; please generate one before merging\n"); 
+  } 
 
   /*** load output/Images.dat ***/
@@ -41 +48 @@
   }
 
+  dmhImage *history = dmhImageRead (&outDB);
+  if (!history) { 
+    Shutdown ("error reading history for output database\n"); 
+  }
+
   // convert database table to internal structure & add to output image db
   dvo_image_merge_dbs(IDmap, &outDB, &inDB);
   dvo_image_unlock (&inDB); // unlock input
+
+  // add the new image db to merge history
+  // (updates header as well as history structure
+  if (!dmhImageAdd (&outDB, history, indbID)) { 
+    Shutdown ("error reading history for output database\n"); 
+  }
+
+  // XXX I need to pass the history structure back out somehow...
 
   SetProtect (TRUE);

branches/eam_branches/ipp-20120805/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c

@@ -31 +31 @@
     outlist = SkyListByBounds (outsky, -1, inlist[0].regions[i][0].Rmin + dPos, inlist[0].regions[i][0].Rmax - dPos, inlist[0].regions[i][0].Dmin + dPos, inlist[0].regions[i][0].Dmax - dPos);
 
-    // there may be more than one output region for a given input region.  are any of these output regions relevant to this machine?
+    OutputStatus *outstat = OutputStatusInit (outlist->Nregions);
+
+    // there may be more than one output region for a given input region.  
+    // are any of these output regions relevant to this machine?
     int found = FALSE;
     for (j = 0; !found && (j < outlist[0].Nregions); j++) {
-      found = (found || HostTableTestHost(outlist[0].regions[j], HOST_ID));
+      outstat[j].valid = HostTableTestHost(outlist[0].regions[j], HOST_ID);
+      found = (found || outstat[j].valid);
     }
 
     // skip this input table for if no output files are on this machine
     if (!found) {
+      OutputStatusFree (outstat, outlist->Nregions);
       SkyListFree (outlist); 
       continue; 
     }
 
-    // get the stats on this input file (for comparison with the output headers)
-    struct stat instats;
-    int stat_result = stat (inlist[0].filename[i], &instats);
-    if (stat_result) {
-      if (errno == ENOENT) continue;
-      fprintf (stderr, "cannot read stats on input file %s\n", inlist[0].filename[i]);
-      perror ("stats error message:");
-      exit (2);
-    }
-
-    // instats.st_size & instats.st_mtime
-
-    // check if any of the output files have NOT yet received data from this input file
-
+    // get stats for history check
+    dmhObjectStats *inStats = dmhObjectStatsRead (inlist[0].filename[i]);
+
+    // Check if any of the output files have NOT yet received data from this input file
+    // If none have been missed, we can skip the input file completely
     int missed = FALSE;
-    for (j = 0; !missed && (j < outlist[0].Nregions); j++) {
-      if (!HostTableTestHost(outlist[0].regions[j], HOST_ID)) continue;
+    for (j = 0; j < outlist[0].Nregions; j++) {
+      if (!outstat[j].valid) continue;
 
       // set the parameters which guide catalog open/load/create
@@ -65 +61 @@
       snprintf (hostfile, DVO_MAX_PATH, "%s/%s.cpt", HOSTDIR, outlist[0].regions[j]->name);
       char *filename = HOST_ID ? hostfile : outlist[0].filename[j];
-
-      // get the stats on this input file (for comparison with the output headers)
-      struct stat outstats;
-      stat_result = stat (filename, &outstats);
-      if (stat_result) {
-        if (errno == ENOENT) {
-          missed = TRUE;
-          break;
-        }
-        fprintf (stderr, "cannot read stats on output file %s\n", filename);
-        perror ("stats error message:");
-        exit (2);
-      }
-
-      FILE *fout = fopen (filename, "r");
-      if (!fout) {
-        fprintf (stderr, "problem opening output file to read header %s\n", filename);
-        perror ("stats error message:");
-        exit (2);
-      }
-
-      Header outheader;
-      if (!gfits_fread_header (fout, &outheader)) {
-        fprintf (stderr, "problem reading header for output file %s\n", filename);
-        exit (2);
-      }
-      
-      fclose (fout);
-
-      // XXX note that we are hardwired to v 2
-      // check the header of output catalog for an existing merge
-      long long last_size;
-      char last_moddate[80];
-      gfits_scan (&outheader, "LMRG_SZ2", "%lld", 1, &last_size);      
-      gfits_scan (&outheader, "LMRG_DT2", "%s", 1, last_moddate);      
-
-      time_t last_mod = ohana_date_to_sec (last_moddate);
-
-      if (last_size != instats.st_size) {
-        missed = TRUE;
-        break;
-      }
-      
-      if (last_mod != instats.st_mtime) {
-        missed = TRUE;
-        break;
-      }
+      outstat[j].filename = strcreate (filename);
+
+      outstat[j].history = dmhObjectRead (outstat[j].filename);
+
+      outstat[j].missed = dmhObjectCheck (outstat[j].history, inStats);
+      missed = (missed || outstat[j].missed);
     }
     if (!missed) {
       fprintf (stderr, "skipping %s, already merged\n", inlist[0].filename[i]);
+      OutputStatusFree (outstat, outlist->Nregions);
+      dmhObjectStatsFree (inStats);
+      SkyListFree (outlist); 
       continue;
     }
@@ -125 +83 @@
         dvo_catalog_unlock (&incatalog);
         dvo_catalog_free (&incatalog);
+        OutputStatusFree (outstat, outlist->Nregions);
+        dmhObjectStatsFree (inStats);
         SkyListFree (outlist); 
         continue;
     }
-
-    char *moddate = ohana_sec_to_date (instats.st_mtime);
 
     // merge input into the appropriate output tables
     for (j = 0; j < outlist[0].Nregions; j++) {
       // skip tables for which the output files are not on this machine
-      if (!HostTableTestHost(outlist[0].regions[j], HOST_ID)) continue; 
+      if (!outstat[j].valid) continue; 
+
+      // skip if we have already done the merge
+      if (!outstat[j].missed) continue; 
 
       if (VERBOSE) fprintf (stderr, "output : %s\n", outlist[0].regions[j][0].name);
 
-      // set the parameters which guide catalog open/load/create
-      char hostfile[DVO_MAX_PATH];
-      snprintf (hostfile, DVO_MAX_PATH, "%s/%s.cpt", HOSTDIR, outlist[0].regions[j]->name);
-      outcatalog.filename  = HOST_ID ? hostfile : outlist[0].filename[j];
+      // the real filename
+      outcatalog.filename = outstat[j].filename; 
 
       // load input catalog
@@ -151 +110 @@
       outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
 
-# if (0)
-      // get the LMRG data from the previous pass
-      long long last_size;
-      char last_moddate[80];
-      int status_size = gfits_scan (&outcatalog.header, "LMRG_SZ", "%lld", 1, &last_size);      
-      int status_date = gfits_scan (&outcatalog.header, "LMRG_DT", "%s", 1, last_moddate);      
-
-      // save the LMRG data from the previous pass
-      if (status_size && status_date) {
-        gfits_modify (&outcatalog.header, "LMRG_SZ1", "%lld", 1, (long long) last_size);      
-        gfits_modify (&outcatalog.header, "LMRG_DT1", "%s", 1, last_moddate);      
-      }
-
-      // update header of output catalog
-      gfits_modify (&outcatalog.header, "LMRG_SZ", "%lld", 1, (long long) instats.st_size);      
-      gfits_modify (&outcatalog.header, "LMRG_DT", "%s", 1, moddate);      
-# endif
-
-      // update header of output catalog
-      // XXX note that we are hardwired to v 2
-      gfits_modify (&outcatalog.header, "LMRG_SZ2", "%lld", 1, (long long) instats.st_size);      
-      gfits_modify (&outcatalog.header, "LMRG_DT2", "%s", 1, moddate);      
+      dmhObjectAdd (outstat[j].history, &outcatalog.header, inStats);
 
       if (!dvo_catalog_backup (&outcatalog, TRUE)) {
@@ -196 +134 @@
       fprintf (stderr, "merged %s into %s\n", inlist[0].regions[i][0].name, outlist[0].regions[j][0].name);
     }
+
+    OutputStatusFree (outstat, outlist->Nregions);
     SkyListFree (outlist);
-    free (moddate);
+    dmhObjectStatsFree (inStats);
 
     dvo_catalog_unlock (&incatalog);