Changeset 33944

Timestamp:

May 29, 2012, 4:18:58 PM (14 years ago)

Author:

eugene

Message:

split out time from csystem inits in dbExtractAverageInit to dbExtractAverageInitAve; allow option to skip result output for parallel ops; fix remote -reload; flush, fsync, fsetlockfile on remote command to ensure all machines see it (and report errors caught); handle equal value bins in threshold function; set global variable to hold the scrip name

Location:

branches/eam_branches/ipp-20120405/Ohana/src/opihi

Files:

• cmd.data/threshold.c (modified) (2 diffs)
• dvo/avextract.c (modified) (4 diffs)
• dvo/avmatch.c (modified) (2 diffs)
• dvo/dvo_host_utils.c (modified) (4 diffs)
• dvo/mextract.c (modified) (1 diff)
• dvo/remote.c (modified) (2 diffs)
• lib.shell/VectorIO.c (modified) (1 diff)
• lib.shell/startup.c (modified) (1 diff)

branches/eam_branches/ipp-20120405/Ohana/src/opihi/cmd.data/threshold.c

@@ -69 +69 @@
     x0 = isFltX ? vecx[0].elements.Flt[Nhi] : vecy[0].elements.Int[Nhi];
     x1 = isFltX ? vecx[0].elements.Flt[Nhi+1]   : vecy[0].elements.Int[Nhi+1];
-    Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
+    if (y0 == y1) {
+      Xvalue = 0.5*(x0 + x1);
+    } else {
+      Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
+    }    
   } else {
     // interpolate to value:
@@ -76 +80 @@
     x0 = isFltX ? vecx[0].elements.Flt[Nhi-1] : vecy[0].elements.Int[Nhi-1];
     x1 = isFltX ? vecx[0].elements.Flt[Nhi]   : vecy[0].elements.Int[Nhi];
-    Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
+    if (y0 == y1) {
+      Xvalue = 0.5*(x0 + x1);
+    } else {
+      Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
+    }
   }
 

branches/eam_branches/ipp-20120405/Ohana/src/opihi/dvo/avextract.c

@@ -74 +74 @@
   }
 
+  // this is used to NOT save the results in the results file
+  // use this option when mextract is used in a script which does its
+  // own job of packaging the results
+  int SKIP_RESULTS = FALSE;
+  if ((N = get_argument (argc, argv, "-skip-results"))) {
+    remove_argument (N, &argc, argv);
+    SKIP_RESULTS = TRUE;
+  }
+
   dvo_catalog_init (&catalog, TRUE);
 
@@ -87 +96 @@
     goto escape; 
   }
+
+  // init locally static variables (time refs)
+  dbExtractAveragesInit (); 
 
   // command-line is of the form: avextract field,field, field [where (field op value)...]
@@ -198 +210 @@
       m = catalog.average[j].measureOffset;
 
-      // reset counters for saved fields, extract fields
-      dbExtractAveragesInit (); 
+      dbExtractAveragesInitAve ();  // reset counters for saved fields (costs very little)
+
       for (n = 0; n < Nfields; n++) {
         values[n] = dbExtractAverages (&catalog.average[j], &catalog.secfilt[j*Nsecfilt], &catalog.measure[m], &fields[n]);
@@ -234 +246 @@
 
   // write vectors to a table (this is used by parallel dvo operations, but can be used elsewhere)
-  if (RESULT_FILE) {
+  if (RESULT_FILE && !SKIP_RESULTS) {
     int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, Nreturn, FALSE, NULL);
-    if (!status) goto escape;
+    if (!status) {
+      goto escape;
+    }
   }
 

branches/eam_branches/ipp-20120405/Ohana/src/opihi/dvo/avmatch.c

@@ -66 +66 @@
   if (!InitPhotcodes ()) goto escape;
   Nsecfilt = GetPhotcodeNsecfilt ();
+
+  // init locally static variables (time refs)
+  dbExtractAveragesInit (); 
 
   // this does all the work of re-packaging the command, calling it on the remote machines, then loading in the results
@@ -206 +209 @@
 
       // reset counters for saved fields, extract fields
-      dbExtractAveragesInit (); 
+      dbExtractAveragesInitAve (); 
       for (n = 0; n < Nfields; n++) {
         values[n] = dbExtractAverages (&catalog.average[Ncat], &catalog.secfilt[Ncat*Nsecfilt], &catalog.measure[m], &fields[n]);

branches/eam_branches/ipp-20120405/Ohana/src/opihi/dvo/dvo_host_utils.c

@@ -6 +6 @@
 # define DVO_MAX_PATH 1024
 
+# define DIE(WHO,MSG) { perror(WHO); myAbort(MSG); }
+
 int HostTableLaunchJobs (HostTable *table, char *basecmd, char *options, int VERBOSE) {
 
@@ -33 +35 @@
     FILE *f = fopen (commandFile, "w");
     fprintf (f, "%s\n", basecmd);
-    fclose (f);
+    if (fflush (f)) DIE("flush", "failed to flush");
+
+    int fd = fileno (f);
+    if (fsync (fd)) DIE("fsync", "failed to fsync");
+
+    if (fclose (f)) DIE("close", "failed to close");
+
+    // force NFS to write the file to disk
+    int state;
+    f = fsetlockfile (commandFile, 0.5, LCK_XCLD, &state);
+    fclearlockfile (commandFile, f, LCK_XCLD, &state);
 
     char command[1024];
@@ -216 +228 @@
   Vector **vec = NULL;
   for (i = 0; i < table->Nhosts; i++) {
-    if (table->hosts[i].status) continue; 
-
     // ensure that the paths are absolute path names
     char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
@@ -240 +250 @@
     table->hosts[i].results = NULL;
 
+    // fprintf (stderr, "%s : %d\n", table->hosts[i].pathname, invec[0]->Nelements);
+
     vec = MergeVectors (vec, &Nvec, invec, Ninvec);
     if (vec != invec) {

branches/eam_branches/ipp-20120405/Ohana/src/opihi/dvo/mextract.c

@@ -189 +189 @@
       }
 
-      dbExtractMeasuresInitAve (); // reset counters for saved fields (costs very little
+      dbExtractMeasuresInitAve (); // reset counters for saved fields (costs very little)
 
       for (k = 0; (k < catalog.average[j].Nmeasure); k++, m++) {

branches/eam_branches/ipp-20120405/Ohana/src/opihi/dvo/remote.c

@@ -31 +31 @@
 
   if ((N = get_argument (argc, argv, "-reload"))) {
-    if (argc != N) {
+    remove_argument (N, &argc, argv);
+    if (argc != 2) {
       gprint (GP_ERR, "USAGE: remote -reload (uniquer)\n");
       gprint (GP_ERR, " (uniquer) is the element in the middle of the results file\n");
@@ -37 +38 @@
       return FALSE;
     }
-    remove_argument (N, &argc, argv);
     int status = HostTableReloadResults (argv[1], VERBOSE);
     return status;

branches/eam_branches/ipp-20120405/Ohana/src/opihi/lib.shell/VectorIO.c

@@ -20 +20 @@
   }
   if (f == (FILE *) NULL) {
-    gprint (GP_ERR, "can't open file for write\n");
+    gprint (GP_ERR, "can't open file for write : %s\n", filename);
     return (FALSE);
   }

branches/eam_branches/ipp-20120405/Ohana/src/opihi/lib.shell/startup.c

@@ -138 +138 @@
     if (is_script) {
       /* first argument in input script, rest are argv */
+      set_str_variable ("SCRIPT_NAME", argv[1]);
       list[Nlist] = strcreate (argv[1]);
       Nlist ++;