Changeset 33600

Timestamp:

Mar 23, 2012, 2:19:56 PM (14 years ago)

Author:

eugene

Message:

improve verbosity; clear some memory leaks in parallel ops

Location:

branches/eam_branches/ipp-20111122/Ohana/src

Files:

• libdvo/include/dvo.h (modified) (1 diff)
• libdvo/src/HostTable.c (modified) (8 diffs)
• opihi/cmd.data/Makefile (modified) (1 diff)
• opihi/cmd.data/cumulative.c (modified) (2 diffs)
• opihi/cmd.data/init.c (modified) (2 diffs)
• opihi/cmd.data/line.c (modified) (2 diffs)
• opihi/dvo/Makefile (modified) (1 diff)
• opihi/dvo/avextract.c (modified) (5 diffs)
• opihi/dvo/avmatch.c (modified) (1 diff)
• opihi/dvo/dvo_host_utils.c (modified) (3 diffs)
• opihi/dvo/hosts.c (added)
• opihi/dvo/init.c (modified) (2 diffs)
• opihi/dvo/mextract.c (modified) (1 diff)
• opihi/include/dvoshell.h (modified) (1 diff)
• opihi/lib.shell/VectorOps.c (modified) (2 diffs)
• photdbc/src/photdbc_catalogs.c (modified) (2 diffs)
• relastro/src/UpdateObjectOffsets.c (modified) (1 diff)
• relastro/src/high_speed_catalogs.c (modified) (1 diff)
• relastro/src/load_catalogs.c (modified) (1 diff)
• relastro/src/relastro_objects.c (modified) (1 diff)
• relphot/src/load_catalogs.c (modified) (1 diff)
• relphot/src/reload_catalogs.c (modified) (1 diff)
• relphot/src/relphot_objects.c (modified) (1 diff)
• uniphot/src/update_dvo_setphot.c (modified) (1 diff)

branches/eam_branches/ipp-20111122/Ohana/src/libdvo/include/dvo.h

@@ -628 +628 @@
 HostTable    *HostTableLoad (char *catdir, char *rootname);
 int HostTableWaitJobs (HostTable *table, char *file, int lineno);
-int HostTableWaitJobsGetIO (HostTable *table, char *file, int lineno);
+int HostTableWaitJobsGetIO (HostTable *table, char *file, int lineno, int VERBOSE);
 int HostTableTestHost (SkyRegion *region, int hostID);
 

branches/eam_branches/ipp-20111122/Ohana/src/libdvo/src/HostTable.c

@@ -29 +29 @@
 }
 
+void FreeTable (HostTable *table) {
+
+  if (!table) return;
+  if (table->hosts) {
+    FreeHosts (table->hosts, table->Nhosts);
+  }
+  if (table->index) free (table->index);
+  free (table);
+  return;
+}
+
 HostTable *HostTableLoad (char *catdir, char *rootname) {
 
@@ -41 +52 @@
   if (!f) {
     fprintf (stderr, "failed to open host table %s\n", filename);
+    free (filename);
     return NULL;
   }
@@ -73 +85 @@
       fprintf (stderr, "error reading line %d of host table %s\n", Nline, filename);
       FreeHosts (hosts, Nhosts);
+      free (filename);
       return NULL;
     }
@@ -119 +132 @@
   }
 
+  free (filename);
   return table;
 }
@@ -196 +210 @@
 
 // wait for all children to complete, report output to stdout
-int HostTableWaitJobsGetIO (HostTable *table, char *file, int lineno) {
+int HostTableWaitJobsGetIO (HostTable *table, char *file, int lineno, int VERBOSE) {
 
   // we have launched table->Nhosts jobs; wait for all of them to complete...
@@ -257 +271 @@
         // if (table->hosts[i].stdout.Nbuffer > 0x10000) {
         if (table->hosts[i].stdout.Nbuffer > 0x1000) {
-          fprintf (stdout, "--- stdout from %s ---\n", table->hosts[i].hostname);
+          int printHead = VERBOSE || table->hosts[i].stdout.Nbuffer;
+          if (printHead) fprintf (stdout, "--- stdout from %s ---\n", table->hosts[i].hostname);
           write (STDOUT_FILENO, table->hosts[i].stdout.buffer, table->hosts[i].stdout.Nbuffer);
           FlushIOBuffer (&table->hosts[i].stdout);
-          fprintf (stdout, "\n");
+          if (printHead) fprintf (stdout, "\n");
         }
       }
@@ -269 +284 @@
         // if (table->hosts[i].stderr.Nbuffer > 0x10000) {
         if (table->hosts[i].stderr.Nbuffer > 0x1000) {
-          fprintf (stdout, "--- stderr from %s ---\n", table->hosts[i].hostname);
+          int printHead = VERBOSE || table->hosts[i].stderr.Nbuffer;
+          if (printHead) fprintf (stdout, "--- stderr from %s ---\n", table->hosts[i].hostname);
           write (STDOUT_FILENO, table->hosts[i].stderr.buffer, table->hosts[i].stderr.Nbuffer);
           FlushIOBuffer (&table->hosts[i].stderr);
-          fprintf (stdout, "\n");
+          if (printHead) fprintf (stdout, "\n");
         }
       }
@@ -301 +317 @@
 
         // check on the status of this and report any output?
-        fprintf (stdout, "job finished for %s (%d)\n", host->hostname, pid);
+        if (VERBOSE) fprintf (stdout, "job finished for %s (%d)\n", host->hostname, pid);
 
         // read stdout
+        int printHead;
+        printHead = VERBOSE || host->stdout.Nbuffer;
         EmptyIOBuffer (&host->stdout, 100, host->stdio[HOST_STDOUT]);
-        fprintf (stdout, "--- stdout from %s ---\n", host->hostname);
+        if (printHead) fprintf (stdout, "--- stdout from %s ---\n", host->hostname);
         write (STDOUT_FILENO, host->stdout.buffer, host->stdout.Nbuffer);
         FlushIOBuffer (&host->stdout);
-        fprintf (stdout, "\n");
+        if (printHead) fprintf (stdout, "\n");
             
         // read stderr
+        printHead = VERBOSE || host->stderr.Nbuffer;
         EmptyIOBuffer (&host->stderr, 100, host->stdio[HOST_STDERR]);
-        fprintf (stdout, "--- stderr from %s ---\n", host->hostname);
+        if (printHead) fprintf (stdout, "--- stderr from %s ---\n", host->hostname);
         write (STDOUT_FILENO, host->stderr.buffer, host->stderr.Nbuffer);
         FlushIOBuffer (&host->stderr);
-        fprintf (stdout, "\n");
+        if (printHead) fprintf (stdout, "\n");
 
         if (WIFEXITED(status)) {
-          fprintf (stdout, "normal completion, exit status is %d\n", WEXITSTATUS(status));
+          if (VERBOSE) fprintf (stdout, "normal completion, exit status is %d\n", WEXITSTATUS(status));
           host->status = WEXITSTATUS(status);
           if (host->status) {

branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/Makefile

@@ -123 +123 @@
 $(SRC)/swapbytes.$(ARCH).o         \
 $(SRC)/textline.$(ARCH).o          \
+$(SRC)/threshold.$(ARCH).o              \
 $(SRC)/tv.$(ARCH).o                \
 $(SRC)/tvchannel.$(ARCH).o         \

branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/cumulative.c

@@ -23 +23 @@
     opihi_flt *Vi = ivec[0].elements.Flt;
     *Vo = *Vi;
+    Vi++;
+    Vo++;
     for (i = 1; i < ivec[0].Nelements; i++, Vi++, Vo++) {
       *Vo = Vo[-1] + *Vi;
@@ -29 +31 @@
     opihi_int *Vi = ivec[0].elements.Int;
     *Vo = *Vi;
+    Vi++;
+    Vo++;
     for (i = 1; i < ivec[0].Nelements; i++, Vi++, Vo++) {
       *Vo = Vo[-1] + *Vi;

branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/init.c

@@ -112 +112 @@
 int swapbytes        PROTO((int, char **));
 int textline         PROTO((int, char **));
+int threshold        PROTO((int, char **));
 int tv               PROTO((int, char **));
 int tvchannel        PROTO((int, char **));
@@ -262 +263 @@
   {1, "swapbytes",    swapbytes,        "byte swap thing"},
   {1, "textline",     textline,         "write text line on graph"},
+  {1, "threshold",    threshold,        "find (interpolate) location of transition"},
   {1, "tv",           tv,               "display an image on the Kii window"},
   {1, "tvchannel",    tvchannel,        "set the current tv channel"},

branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/line.c

@@ -3 +3 @@
 int line (int argc, char **argv) {
   
-  int kapa;
+  int kapa, N;
   Graphdata graphmode;
   float x[2], y[2];
+
+  /* FracPositions uses coordinates of 0-1 relative to axis range */
+  int FracPositions = FALSE;
+  if ((N = get_argument (argc, argv, "-frac"))) {
+    remove_argument (N, &argc, argv);
+    FracPositions = TRUE;
+  } 
 
   if (!style_args (&graphmode, &argc, argv, &kapa)) return FALSE;
@@ -18 +25 @@
   y[1] = atof(argv[5]);
 
+  if (FracPositions) {
+    x[0] =  x[0] * (graphmode.xmax - graphmode.xmin) + graphmode.xmin;
+    y[0] =  y[0] * (graphmode.ymax - graphmode.ymin) + graphmode.ymin;
+    x[1] =  x[1] * (graphmode.xmax - graphmode.xmin) + graphmode.xmin;
+    y[1] =  y[1] * (graphmode.ymax - graphmode.ymin) + graphmode.ymin;
+  }    
+
   /* set point style and errorbar mode (these are NOT sticky) */
   graphmode.style = 0;

branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/Makefile

@@ -52 +52 @@
 $(SRC)/gimages.$(ARCH).o                \
 $(SRC)/gstar.$(ARCH).o                  \
+$(SRC)/hosts.$(ARCH).o          \
 $(SRC)/images.$(ARCH).o         \
 $(SRC)/imbox.$(ARCH).o                  \

branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/avextract.c

@@ -41 +41 @@
     remove_argument (N, &argc, argv);
     PARALLEL = TRUE;
+  }
+
+  // use remote tables, but not dvo_client..
+  int PARALLEL_LOCAL = FALSE;
+  HostTable *table = NULL;
+  if ((N = get_argument (argc, argv, "-parallel-local"))) {
+    remove_argument (N, &argc, argv);
+    PARALLEL_LOCAL = TRUE;
+
+    char *CATDIR = GetCATDIR();
+    if (!CATDIR) {
+      gprint (GP_ERR, "CATDIR is not set\n");
+      return FALSE;
+    }
+    SkyTable *sky = GetSkyTable();
+    if (!sky) {
+      gprint (GP_ERR, "failed to load sky table for database\n");
+      return FALSE;
+    }
+    table = HostTableLoad (CATDIR, sky->hosts);
+    if (!table) {
+      gprint (GP_ERR, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
+      return FALSE;
+    }    
   }
 
@@ -96 +120 @@
   // this does all the work of re-packaging the command, calling it on the remote machines, then loading in the results
   if (PARALLEL && !HOST_ID) {
-    int status = HostTableParallelOps (argc, argv, RESULT_FILE, 0);
+    int status = HostTableParallelOps (argc, argv, RESULT_FILE, 0, VERBOSE);
 
     dbFreeFields (fields, Nfields);
@@ -141 +165 @@
     /* lock, load, unlock catalog */
     char hostfile[1024];
+    if (PARALLEL_LOCAL) {
+      int hostID = skylist[0].regions[i]->hostID;
+      int seq = table->index[hostID];
+      HOSTDIR = table->hosts[seq].pathname;
+    }
     snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
-    catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i];
+    catalog.filename = (HOST_ID || PARALLEL_LOCAL) ? hostfile : skylist[0].filename[i];
     catalog.catflags = LOAD_AVES | LOAD_SECF;
     if (needMeasures) {
@@ -205 +234 @@
   }
 
+  if (table) free (table);
   if (vec) free (vec);
   if (values) free (values);
@@ -215 +245 @@
 
  escape:
+  if (table) free (table);
   if (vec) free (vec);
   if (values) free (values);

branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/avmatch.c

@@ -92 +92 @@
 
     // I need to pass the RA & DEC vectors to the remote clients...
-    int status = HostTableParallelOps (argc, argv, RESULT_FILE, RAvec->Nelements);
+    int status = HostTableParallelOps (argc, argv, RESULT_FILE, RAvec->Nelements, VERBOSE);
     if (vec) free (vec);
     

branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/dvo_host_utils.c

@@ -69 +69 @@
 // concatenated, but for avmatch, the vectors are merged by index into a pre-known
 // length.  this is probably not a solution to a general problem..
-int HostTableParallelOps (int argc, char **argv, char *ResultFile, int Nelements) {
+int HostTableParallelOps (int argc, char **argv, char *ResultFile, int Nelements, int VERBOSE) {
 
   int i;
@@ -135 +135 @@
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
   }
 
@@ -146 +146 @@
     int    Ninvec = 0;
     Vector **invec = ReadVectorTableFITS (table->hosts[i].results, "RESULT", &Ninvec);
+    if (!invec) {
+      // failed to read the file, now what?
+      gprint (GP_ERR, "failed to read remote result file : %s\n", table->hosts[i].results);
+      free (table->hosts[i].results);
+      table->hosts[i].results = NULL;
+      continue;
+    }
     free (table->hosts[i].results);
     table->hosts[i].results = NULL;

branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/init.c

@@ -27 +27 @@
 int gstar           PROTO((int, char **));
 int gtypes          PROTO((int, char **));
+int hosts           PROTO((int, char **));
 int images          PROTO((int, char **));
 int imbox           PROTO((int, char **));
@@ -79 +80 @@
   {1, "gimages",     gimages,      "get images at location"},
   {1, "gstar",       gstar,        "get star statistics"},
+  {1, "hosts",       hosts,        "remote host support functions"},
   {1, "images",      images,       "plot image boxes"},
   {1, "imbox",       imbox,        "plot expected image box"},

branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/mextract.c

@@ -98 +98 @@
   // this does all the work of re-packaging the command, calling it on the remote machines, then loading in the results
   if (PARALLEL && !HOST_ID) {
-    int status = HostTableParallelOps (argc, argv, RESULT_FILE, 0);
+    int status = HostTableParallelOps (argc, argv, RESULT_FILE, 0, VERBOSE);
 
     dbFreeFields (fields, Nfields);

branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvoshell.h

@@ -100 +100 @@
 
 int          HostTableLaunchJobs    PROTO((HostTable *table, char *basecmd, char *options));
-int          HostTableParallelOps   PROTO((int argc, char **argv, char *ResultFile, int Nelements));
+int          HostTableParallelOps   PROTO((int argc, char **argv, char *ResultFile, int Nelements, int VERBOSE));
 # endif

branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/VectorOps.c

@@ -122 +122 @@
   /* is an old vector */
   if (mode == NEWVECTOR) goto error;
+  if (vectors[i]) {
+    if (vectors[i][0].elements.Ptr) free (vectors[i][0].elements.Ptr);
+    free (vectors[i]);
+  }
   vectors[i] = vec;
   return TRUE;
@@ -351 +355 @@
   }
 
-  myAssert (*Nvec == Ninvec, "programming error (1)");
+  myAssert (*Nvec == Ninvec, "programming error (1) %d vs %d", *Nvec, Ninvec);
 
   for (i = 0; i < Ninvec; i++) {
-    myAssert (!strcmp(vec[i]->name, invec[i]->name), "programming error (2)");
-    myAssert (vec[i]->type == invec[i]->type, "programming error (2)");
+    myAssert (!strcmp(vec[i]->name, invec[i]->name), "programming error (2) %s vs %s", vec[i]->name, invec[i]->name);
+    myAssert (vec[i]->type == invec[i]->type, "programming error (3), %d vs %d", vec[i]->type, invec[i]->type);
 
     int N = vec[i]->Nelements;

branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/photdbc_catalogs.c

@@ -121 +121 @@
     if (PHOTCODE_DROP_LIST) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -photcode-drop %s", command, PHOTCODE_DROP_LIST);         strcpy (command, tmpline); }
     if (PHOTCODE_SKIP_LIST) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -photcode-skip %s", command, PHOTCODE_SKIP_LIST);         strcpy (command, tmpline); }
-    if (CATFORMAT)          { snprintf (tmpline, MAX_PATH_LENGTH, "%s -catformat %s",     command, CATFORMAT);                  strcpy (command, tmpline); }
-    if (CATMODE)            { snprintf (tmpline, MAX_PATH_LENGTH, "%s -catmode %s",       command, CATMODE);                    strcpy (command, tmpline); }
+    if (CATFORMAT)          { snprintf (tmpline, MAX_PATH_LENGTH, "%s -set-format %s",    command, CATFORMAT);                  strcpy (command, tmpline); }
+    if (CATMODE)            { snprintf (tmpline, MAX_PATH_LENGTH, "%s -set-mode %s",      command, CATMODE);                    strcpy (command, tmpline); }
 
     fprintf (stderr, "command: %s\n", command);
@@ -151 +151 @@
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
   }
 

branches/eam_branches/ipp-20111122/Ohana/src/relastro/src/UpdateObjectOffsets.c

@@ -159 +159 @@
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
   }
 

branches/eam_branches/ipp-20111122/Ohana/src/relastro/src/high_speed_catalogs.c

@@ -196 +196 @@
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
   }
 

branches/eam_branches/ipp-20111122/Ohana/src/relastro/src/load_catalogs.c

@@ -201 +201 @@
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
   }
 

branches/eam_branches/ipp-20111122/Ohana/src/relastro/src/relastro_objects.c

@@ -187 +187 @@
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
   }
 

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/load_catalogs.c

@@ -190 +190 @@
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
   }
 

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/reload_catalogs.c

@@ -201 +201 @@
   } 
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
   }
   return (TRUE);

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/relphot_objects.c

@@ -181 +181 @@
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
   }
 

branches/eam_branches/ipp-20111122/Ohana/src/uniphot/src/update_dvo_setphot.c

@@ -147 +147 @@
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__);
+    int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
     if (!status) return FALSE;
   }