Changeset 34654

Timestamp:

Nov 5, 2012, 5:47:27 AM (14 years ago)

Author:

eugene

Message:

option to remote command to avoid loading results (but save the remote result names)

Location:

branches/eam_branches/ipp-20120905/Ohana/src/opihi

Files:

• cmd.astro (modified) (1 prop)
• dvo/avextract.c (modified) (1 diff)
• dvo/avmatch.c (modified) (1 diff)
• dvo/dvo_host_utils.c (modified) (4 diffs)
• dvo/mextract.c (modified) (1 diff)
• dvo/mmatch.c (modified) (1 diff)
• dvo/remote.c (modified) (2 diffs)
• include/dvoshell.h (modified) (1 diff)

branches/eam_branches/ipp-20120905/Ohana/src/opihi/dvo/avextract.c

@@ -137 +137 @@
   // this does all the work of re-packaging the command, calling it on the remote machines, then loading in the results
   if (PARALLEL && !HOST_ID) {
-    int status = HostTableParallelOps (argc, argv, RESULT_FILE, 0, VERBOSE);
+    int status = HostTableParallelOps (argc, argv, RESULT_FILE, TRUE, 0, VERBOSE);
 
     dbFreeFields (fields, Nfields);

branches/eam_branches/ipp-20120905/Ohana/src/opihi/dvo/avmatch.c

@@ -95 +95 @@
 
     // I need to pass the RA & DEC vectors to the remote clients...
-    int status = HostTableParallelOps (argc, argv, RESULT_FILE, RAvec->Nelements, VERBOSE);
+    int status = HostTableParallelOps (argc, argv, RESULT_FILE, TRUE, RAvec->Nelements, VERBOSE);
     if (vec) free (vec);
     

branches/eam_branches/ipp-20120905/Ohana/src/opihi/dvo/dvo_host_utils.c

@@ -79 +79 @@
 }
 
-// bundle the arguments into a command and pass to dvo_client.  this implementation
-// expects there to be a result file from the clients, and to load this into vectors in
-// the main shell.  'Nelements' is a temp hack : for most commands, the result vectors are
-// concatenated, but for avmatch, the vectors are merged by index into a pre-known
-// length.  this is probably not a solution to a general problem..
-int HostTableParallelOps (int argc, char **argv, char *ResultFile, int Nelements, int VERBOSE) {
+// bundle the arguments into a command and pass to dvo_client.  
+
+// the normal ending step expects there to be a result file from the clients, and to load
+// this into vectors in the main shell.  'Nelements' is a temp hack : for most commands,
+// the result vectors are concatenated, but for avmatch, the vectors are merged by index
+// into a pre-known length.  this is probably not a solution to a general problem..
+
+// an alternative ending step ignores the result files and instead saves the names into
+// the list 'result:n' for the user to access as desired
+int HostTableParallelOps (int argc, char **argv, char *ResultFile, int ReadVectors, int Nelements, int VERBOSE) {
 
   int i;
@@ -153 +157 @@
   }
 
+  // create the result file list
+  char name[256];
+  snprintf (name, 256, "RESULT_FILE:n");
+  set_int_variable (name, table->Nhosts);
+  snprintf (name, 256, "RESULT_DATA:n");
+  set_int_variable (name, table->Nhosts);
+  snprintf (name, 256, "RESULT_STATUS:n");
+  set_int_variable (name, table->Nhosts);
+
   // load fields from file
   int    Nvec = 0;
   Vector **vec = NULL;
   for (i = 0; i < table->Nhosts; i++) {
+
+    snprintf (name, 256, "RESULT_FILE:%d", i);
+    set_str_variable (name, table->hosts[i].results);
+
+    // DATA : 0 (unread), 1 (read)
+    snprintf (name, 256, "RESULT_DATA:%d", i);
+    set_int_variable (name, 0);
+
+    // STATUS : 0 (normal exit), -1 (crash), N (failure exit status)
+    snprintf (name, 256, "RESULT_STATUS:%d", i);
+    set_int_variable (name, table->hosts[i].status);
+
     if (table->hosts[i].status) continue; 
+
+    if (ReadVectors) {
+      int    Ninvec = 0;
+      Vector **invec = ReadVectorTableFITS (table->hosts[i].results, "RESULT", &Ninvec);
+      if (!invec) {
+        // failed to read the file, now what?
+        gprint (GP_ERR, "failed to read remote result file : %s\n", table->hosts[i].results);
+        free (table->hosts[i].results);
+        table->hosts[i].results = NULL;
+        continue;
+      }
+      free (table->hosts[i].results);
+      table->hosts[i].results = NULL;
+      set_int_variable (name, 1); // result file has been read
+
+      if (Nelements == 0) {
+        vec = MergeVectors (vec, &Nvec, invec, Ninvec);
+        if (vec != invec) {
+          FreeVectorArray (invec, Ninvec);
+        }
+      } else {
+        vec = MergeVectorsByIndex (vec, &Nvec, invec, Ninvec, Nelements);
+        FreeVectorArray (invec, Ninvec);
+      }
+    }
+  }
+
+  // write vectors to a table (this is used by parallel dvo operations, but can be used elsewhere)
+  if (ResultFile) {
+    int status = WriteVectorTableFITS (ResultFile, "RESULT", vec, Nvec, FALSE, NULL);
+    if (!status) {
+      gprint (GP_ERR, "failed to write result file %s\n", ResultFile);
+      return FALSE;
+    }
+  }
+
+  for (i = 0; i < Nvec; i++) {
+    AssignVector (vec[i], vec[i]->name, ANYVECTOR, TRUE);
+  }
+  free (vec);
+
+  free (table);
+  return TRUE;
+}
+
+// re-gather the remote results files: this can be used in case one of the clients failed,
+// and has since been re-run
+int HostTableReloadResults (char *uniquer, int VERBOSE) {
+
+  int i;
+
+  // load the list of hosts
+  SkyTable *sky = GetSkyTable();
+  if (!sky) {
+    gprint (GP_ERR, "failed to load sky table for database\n");
+    return FALSE;
+  }
+
+  char *CATDIR = GetCATDIR ();
+  if (!CATDIR) {
+    gprint (GP_ERR, "failed to get CATDIR for database\n");
+    return FALSE;
+  }
+
+  HostTable *table = HostTableLoad (CATDIR, sky->hosts);
+  if (!table) {
+    gprint (GP_ERR, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
+    return FALSE;
+  }    
+
+  // load fields from file
+  int    Nvec = 0;
+  Vector **vec = NULL;
+  for (i = 0; i < table->Nhosts; i++) {
+    // ensure that the paths are absolute path names
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
+    free (table->hosts[i].pathname);
+    table->hosts[i].pathname = tmppath;
+
+    // need to save the results filename with the uniquer
+    // XXX a bit of a waste (but only 1024 * 60 bytes or so
+    ALLOCATE (table->hosts[i].results, char, DVO_MAX_PATH);
+    snprintf (table->hosts[i].results, DVO_MAX_PATH, "%s/dvo.results.%s.fits", table->hosts[i].pathname, uniquer);
 
     int    Ninvec = 0;
@@ -171 +279 @@
     table->hosts[i].results = NULL;
 
-    if (Nelements == 0) {
-      vec = MergeVectors (vec, &Nvec, invec, Ninvec);
-      if (vec != invec) {
-        FreeVectorArray (invec, Ninvec);
-      }
-    } else {
-      vec = MergeVectorsByIndex (vec, &Nvec, invec, Ninvec, Nelements);
+    // fprintf (stderr, "%s : %d\n", table->hosts[i].pathname, invec[0]->Nelements);
+
+    vec = MergeVectors (vec, &Nvec, invec, Ninvec);
+    if (vec != invec) {
       FreeVectorArray (invec, Ninvec);
-    }
-  }
-
-  // write vectors to a table (this is used by parallel dvo operations, but can be used elsewhere)
-  if (ResultFile) {
-    int status = WriteVectorTableFITS (ResultFile, "RESULT", vec, Nvec, FALSE, NULL);
-    if (!status) {
-      gprint (GP_ERR, "failed to write result file %s\n", ResultFile);
-      return FALSE;
     }
   }
@@ -199 +295 @@
   return TRUE;
 }
-
-// re-gather the remote results files: this can be used in case one of the clients failed,
-// and has since been re-run
-int HostTableReloadResults (char *uniquer, int VERBOSE) {
-
-  int i;
-
-  // load the list of hosts
-  SkyTable *sky = GetSkyTable();
-  if (!sky) {
-    gprint (GP_ERR, "failed to load sky table for database\n");
-    return FALSE;
-  }
-
-  char *CATDIR = GetCATDIR ();
-  if (!CATDIR) {
-    gprint (GP_ERR, "failed to get CATDIR for database\n");
-    return FALSE;
-  }
-
-  HostTable *table = HostTableLoad (CATDIR, sky->hosts);
-  if (!table) {
-    gprint (GP_ERR, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
-    return FALSE;
-  }    
-
-  // load fields from file
-  int    Nvec = 0;
-  Vector **vec = NULL;
-  for (i = 0; i < table->Nhosts; i++) {
-    // ensure that the paths are absolute path names
-    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
-    free (table->hosts[i].pathname);
-    table->hosts[i].pathname = tmppath;
-
-    // need to save the results filename with the uniquer
-    // XXX a bit of a waste (but only 1024 * 60 bytes or so
-    ALLOCATE (table->hosts[i].results, char, DVO_MAX_PATH);
-    snprintf (table->hosts[i].results, DVO_MAX_PATH, "%s/dvo.results.%s.fits", table->hosts[i].pathname, uniquer);
-
-    int    Ninvec = 0;
-    Vector **invec = ReadVectorTableFITS (table->hosts[i].results, "RESULT", &Ninvec);
-    if (!invec) {
-      // failed to read the file, now what?
-      gprint (GP_ERR, "failed to read remote result file : %s\n", table->hosts[i].results);
-      free (table->hosts[i].results);
-      table->hosts[i].results = NULL;
-      continue;
-    }
-    free (table->hosts[i].results);
-    table->hosts[i].results = NULL;
-
-    // fprintf (stderr, "%s : %d\n", table->hosts[i].pathname, invec[0]->Nelements);
-
-    vec = MergeVectors (vec, &Nvec, invec, Ninvec);
-    if (vec != invec) {
-      FreeVectorArray (invec, Ninvec);
-    }
-  }
-
-  for (i = 0; i < Nvec; i++) {
-    AssignVector (vec[i], vec[i]->name, ANYVECTOR, TRUE);
-  }
-  free (vec);
-
-  free (table);
-  return TRUE;
-}

branches/eam_branches/ipp-20120905/Ohana/src/opihi/dvo/mextract.c

@@ -108 +108 @@
   // this does all the work of re-packaging the command, calling it on the remote machines, then loading in the results
   if (PARALLEL && !HOST_ID) {
-    int status = HostTableParallelOps (argc, argv, RESULT_FILE, 0, VERBOSE);
+    int status = HostTableParallelOps (argc, argv, RESULT_FILE, TRUE, 0, VERBOSE);
 
     dbFreeFields (fields, Nfields);

branches/eam_branches/ipp-20120905/Ohana/src/opihi/dvo/mmatch.c

@@ -124 +124 @@
 
     // call the remote client
-    int status = HostTableParallelOps (targc, targv, RESULT_FILE, 0, VERBOSE);
+    int status = HostTableParallelOps (targc, targv, RESULT_FILE, TRUE, 0, VERBOSE);
     if (vec) free (vec);
     

branches/eam_branches/ipp-20120905/Ohana/src/opihi/dvo/remote.c

@@ -12 +12 @@
     remove_argument (N, &argc, argv);
     VERBOSE = TRUE;
+  }
+
+  int ReadVectors = TRUE;
+  if ((N = get_argument (argc, argv, "-skip-result"))) {
+    remove_argument (N, &argc, argv);
+    ReadVectors = FALSE;
   }
 
@@ -43 +49 @@
 
   // strip of the 'remote' and send the remaining arguments to the remote machine
-  int status = HostTableParallelOps (argc - 1, &argv[1], NULL, 0, VERBOSE);
+  int status = HostTableParallelOps (argc - 1, &argv[1], NULL, ReadVectors, 0, VERBOSE);
   return status;
 }

branches/eam_branches/ipp-20120905/Ohana/src/opihi/include/dvoshell.h

@@ -100 +100 @@
 
 int          HostTableLaunchJobs    PROTO((HostTable *table, char *basecmd, char *options, int VERBOSE));
-int          HostTableParallelOps   PROTO((int argc, char **argv, char *ResultFile, int Nelements, int VERBOSE));
+int          HostTableParallelOps   PROTO((int argc, char **argv, char *ResultFile, int ReadVectors, int Nelements, int VERBOSE));
 int          HostTableReloadResults PROTO((char *uniquer, int VERBOSE));