Changeset 33301

Timestamp:

Feb 17, 2012, 6:20:27 AM (14 years ago)

Author:

eugene

Message:

add ohana version of realpath, fix rconnect version of relphot, update testing

Location:

branches/eam_branches/ipp-20111122/Ohana/src

Files:

• addstar/test/relphot.parallel.dvo (modified) (3 diffs)
• libohana/include/ohana.h (modified) (1 diff)
• libohana/src/findexec.c (modified) (1 diff)
• relphot/include/relphot.h (modified) (2 diffs)
• relphot/src/ConfigInit.c (modified) (1 diff)
• relphot/src/load_catalogs.c (modified) (2 diffs)
• relphot/src/reload_catalogs.c (modified) (4 diffs)
• relphot/src/relphot_client.c (modified) (3 diffs)

branches/eam_branches/ipp-20111122/Ohana/src/addstar/test/relphot.parallel.dvo

@@ -132 +132 @@
 
   # run relphot on the parallel db and check that the images now match the expected values
-  exec relphot -parallel-serial g,r,i -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir.p0 -D STAR_TOOFEW 1 -D SIGMA_LIM 0.07 -statmode WT_MEAN -cloud-limit 0.5 -keep-ubercal -D IMAGE_OFFSET 0.5 -update >& log.relphot.p0
-
-  exec dvodist -in $catdir.p0
+  exec relphot -parallel g,r,i -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir.p0 -D STAR_TOOFEW 1 -D SIGMA_LIM 0.07 -statmode WT_MEAN -cloud-limit 0.5 -keep-ubercal -D IMAGE_OFFSET 0.5 -update >& log.relphot.p0
+
+  exec dvodist -in $catdir.p0 >& log.dvodist.in
 
   for i 0 mjd_nc[]
@@ -432 +432 @@
   $catdir = $tmp1/$tmp2
 
-  exec rsync -auv $catdir/ $catdir.p0/
+  exec rsync -auv $catdir/ $catdir.p0/ >& log.rsync
   mkdir $catdir.d1
   mkdir $catdir.d2
@@ -442 +442 @@
   exec echo "3 $hostname $catdir.d3" >> $catdir.p0/HostTable.dat
 
-  exec dvodist -out $catdir.p0
-end
-
+  exec dvodist -out $catdir.p0 >& log.dvodist.out
+end
+

branches/eam_branches/ipp-20111122/Ohana/src/libohana/include/ohana.h

@@ -264 +264 @@
 int     check_dir_access       PROTO((char *path, int verbose));
 int     check_file_exec        PROTO((char *filename));
+char   *abspath                PROTO((char *oldpath, int maxlength));
 
 /* in glockfile.c */

branches/eam_branches/ipp-20111122/Ohana/src/libohana/src/findexec.c

@@ -404 +404 @@
   return TRUE;
 }
+
+// ohana-based equivalent to 'realpath'.  realpath does not guarantee an absolute path (eg on Solaris), 
+// just a 'cannonical path'.  I don't care about resolving out links, just ensuring an absolute path
+// also, realpath is a bit ambiguous on the maxlength argument
+char *abspath (char *oldpath, int maxlength) {
+
+  if (!oldpath) return NULL;
+
+  if (oldpath[0] == '/') {
+    char *newpath = strcreate (oldpath);
+    return newpath;
+  }
+
+  char *cwd  = getcwd(NULL, maxlength);
+  if (cwd == NULL) {
+    // XXX need an error reporting function...
+    fprintf (stderr, "error getting cwd (longer than %d chars?)\n", maxlength);
+    return NULL;
+  }
+
+  char *newpath = NULL;
+
+  int Nbytes = strlen(cwd) + strlen(oldpath) + 2;
+  ALLOCATE (newpath, char, Nbytes);
+  snprintf (newpath, Nbytes, "%s/%s", cwd, oldpath);
+  return newpath;
+}

branches/eam_branches/ipp-20111122/Ohana/src/relphot/include/relphot.h

@@ -71 +71 @@
 
 /* global variables set in parameter file */
-char   ImageCat[256];
-char   ImageTemplate[256];
-char   CatTemplate[256];
-char   GSCFILE[256];
-char   CATDIR[256];
+# define MAX_PATH_LENGTH 1024
+char   ImageCat[MAX_PATH_LENGTH];
+char   ImageTemplate[MAX_PATH_LENGTH];
+char   CatTemplate[MAX_PATH_LENGTH];
+char   GSCFILE[MAX_PATH_LENGTH];
+char  *CATDIR;
 char   CATMODE[16];    /* raw, mef, split, mysql */
 char   CATFORMAT[16];  /* internal, elixir, loneos, panstarrs */
-char   CameraConfig[256];
-char   SKY_TABLE[256];
+char   CameraConfig[MAX_PATH_LENGTH];
+char   SKY_TABLE[MAX_PATH_LENGTH];
 int    SKY_DEPTH;  /** XXX EAM : depth of catalog tables, fix usage */
 
@@ -314 +315 @@
 int CopyMeasureTiny (MeasureTiny *measureT, Measure *measure);
 
+int HostTableWaitJobs (HostTable *table, char *file, int lineno);

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/ConfigInit.c

@@ -33 +33 @@
   GetConfig (config, "IMAGE_GOOD_FRACTION",    "%lf", 0, &IMAGE_GOOD_FRACTION);
 
+  // force CATDIR to be absolute (so parallel mode will work)
+  char *tmpcatdir = NULL;
+  ALLOCATE (tmpcatdir, char, MAX_PATH_LENGTH);
+  GetConfig (config, "CATDIR",                 "%s",  0, tmpcatdir);
+  CATDIR = abspath (tmpcatdir, MAX_PATH_LENGTH);
+  free (tmpcatdir);
+
   GetConfig (config, "GSCFILE",                "%s",  0, GSCFILE);
-  GetConfig (config, "CATDIR",                 "%s",  0, CATDIR);
   ScanConfig (config, "CATMODE",                "%s",  0, CATMODE);
   ScanConfig (config, "CATFORMAT",              "%s",  0, CATFORMAT);

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/load_catalogs.c

@@ -108 +108 @@
   }    
 
-  char *cwd;
-  if ((cwd = getcwd(NULL, 1024)) == NULL) {
-    fprintf (stderr, "error getting cwd (longer than 1024 chars?)\n");
-    exit (1);
-  }
-
   int i;
   for (i = 0; i < table->Nhosts; i++) {
 
-    // if table->hosts[i].pathname is a relative path (pathname[0] is not '/'), then 
-    // we need to prepend our current path
-
-    if (table->hosts[i].pathname[0] != '/') {
-      int Nbytes = strlen(cwd) + strlen(table->hosts[i].pathname) + 2;
-      char *newpath = NULL;
-      ALLOCATE (newpath, char, Nbytes);
-      snprintf (newpath, Nbytes, "%s/%s", cwd, table->hosts[i].pathname);
-      free (table->hosts[i].pathname);
-      table->hosts[i].pathname = newpath;
-    }
+    // ensure that the paths are absolute path names
+    char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH);
+    free (table->hosts[i].pathname);
+    table->hosts[i].pathname = tmppath;
 
     char catalogFile[512];
@@ -190 +177 @@
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    // we have launched table->Nhosts jobs; wait for all of them to complete...
-    for (i = 0; i < table->Nhosts; i++) {
-      int status = 0;
-      int pid = waitpid (-1, &status, 0);
-
-      // XXX we'll need to pass in WNOHANG and keep retrying if we want to have a timeout...
-      // find the host which has finished
-      int j;
-      for (j = 0; j < table->Nhosts; j++) {
-        if (table->hosts[j].pid != pid) continue;
-        // check on the status of this and report any output?
-        fprintf (stderr, "job finished for %s (%d)\n", table->hosts[j].hostname, pid);
-        if (WIFEXITED(status)) {
-          fprintf (stderr, "normal completion, exit status is %d\n", WEXITSTATUS(status));
-          if (WEXITSTATUS(status)) {
-            fprintf (stderr, "job failed on %s\n", table->hosts[j].hostname);
-          }
-        }
-        // read the stderr and stdout
-        IOBuffer buffer;
-        InitIOBuffer (&buffer, 100);
-        EmptyIOBuffer (&buffer, 100, table->hosts[j].stdio[1]);
-        fprintf (stderr, "--- stdout from %s ---\n", table->hosts[j].hostname);
-        write (STDOUT_FILENO, buffer.buffer, buffer.Nbuffer);
-        fprintf (stderr, "\n");
-          
-        InitIOBuffer (&buffer, 100);
-        EmptyIOBuffer (&buffer, 100, table->hosts[j].stdio[2]);
-        fprintf (stderr, "--- stderr from %s ---\n", table->hosts[j].hostname);
-        write (STDOUT_FILENO, buffer.buffer, buffer.Nbuffer);
-        fprintf (stderr, "\n");
-      }
-    }
+    HostTableWaitJobs (table, __FILE__, __LINE__);
   }
 

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/reload_catalogs.c

@@ -1 +1 @@
 # include "relphot.h"
+# include <sys/types.h>
+# include <sys/wait.h>
 
 # define DEBUG 1
@@ -132 +134 @@
   int i;
   for (i = 0; i < table->Nhosts; i++) {
+
+    // ensure that the paths are absolute path names
+    char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH);
+    free (table->hosts[i].pathname);
+    table->hosts[i].pathname = tmppath;
 
     char command[1024];
@@ -179 +186 @@
       }
       table->hosts[i].pid = pid; // save for future reference
-      // check that all hosts started OK?
     }
   }
@@ -186 +192 @@
     fprintf (stderr, "run the relphot_client commands above.  when these are done, hit return\n");
     getchar();
-  } else {
-    // watch for stdout / stderr from those jobs...
-    // wait for all of them to complete...
-  }
-
+  } 
+  if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
+    HostTableWaitJobs (table, __FILE__, __LINE__);
+  }
   return (TRUE);
 }      
+
+// wait for all children to complete, report output to stdout
+int HostTableWaitJobs (HostTable *table, char *file, int lineno) {
+
+  int i;
+
+  // we have launched table->Nhosts jobs; wait for all of them to complete...
+  // if one (N) failed to launch, we will get an ECHILD error from the last (N) calls
+  int done = FALSE;
+  for (i = 0; !done  && (i < table->Nhosts); i++) {
+    int status = 0;
+
+    // XXX we'll need to pass in WNOHANG and keep retrying if we want to have a timeout...
+    int pid = waitpid (-1, &status, 0);
+    if (!pid) {
+      // this should only occur if we called waitpid with the WNOHANG option
+      fprintf (stderr, "programming error (1)? %s %d", file, lineno);
+      exit (2);
+    }
+    if (pid == -1) {
+      switch (errno) {
+        case ECHILD:
+          done = TRUE;
+          break;
+        default:
+          fprintf (stderr, "programming error (2)? %s %d", file, lineno);
+          exit (2);
+      }
+    }
+
+    // find the host which has finished
+    int j;
+    int found = FALSE;
+    for (j = 0; j < table->Nhosts; j++) {
+      if (table->hosts[j].pid != pid) continue;
+      found = TRUE;
+      // check on the status of this and report any output?
+      fprintf (stderr, "job finished for %s (%d)\n", table->hosts[j].hostname, pid);
+      // read the stderr and stdout
+      IOBuffer buffer;
+      InitIOBuffer (&buffer, 100);
+      EmptyIOBuffer (&buffer, 100, table->hosts[j].stdio[1]);
+      fprintf (stderr, "--- stdout from %s ---\n", table->hosts[j].hostname);
+      write (STDOUT_FILENO, buffer.buffer, buffer.Nbuffer);
+      fprintf (stderr, "\n");
+          
+      InitIOBuffer (&buffer, 100);
+      EmptyIOBuffer (&buffer, 100, table->hosts[j].stdio[2]);
+      fprintf (stderr, "--- stderr from %s ---\n", table->hosts[j].hostname);
+      write (STDOUT_FILENO, buffer.buffer, buffer.Nbuffer);
+      fprintf (stderr, "\n");
+      if (WIFEXITED(status)) {
+        fprintf (stderr, "normal completion, exit status is %d\n", WEXITSTATUS(status));
+        if (WEXITSTATUS(status)) {
+          fprintf (stderr, "job failed on %s\n", table->hosts[j].hostname);
+          exit (1);
+        }
+      } else {
+        fprintf (stderr, "job exited abnormally on %s\n", table->hosts[j].hostname);
+        exit (1);
+      }
+    }
+    if (!found) {
+      fprintf (stderr, "failed to matched finished job to known host!\n");
+      exit (2);
+    }
+  }
+  return TRUE;
+}

branches/eam_branches/ipp-20111122/Ohana/src/relphot/src/relphot_client.c

@@ -22 +22 @@
   // load the current sky table (layout of all SkyRegions) 
   SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, TRUE, -1, VERBOSE);
+  if (!sky) {
+      fprintf (stderr, "ERROR running loading sky table from %s\n", CATDIR);
+      exit (2);
+  }
   SkyTableSetFilenames (sky, CATDIR, "cpt");
 
   SkyList *skylist = SkyListByPatch (sky, -1, &UserPatch);
+  if (!skylist) {
+      fprintf (stderr, "ERROR setting up skylist for %s\n", CATDIR);
+      exit (2);
+  }
   
   switch (MODE) {
@@ -30 +38 @@
       int Ncatalog;
       Catalog *catalog = load_catalogs (skylist, &Ncatalog, HOST_ID, HOSTDIR);
+      if (!catalog) {
+          fprintf (stderr, "ERROR loading catalogs from %s\n", CATDIR);
+          exit (2);
+      }
       BrightCatalog *bcatalog = BrightCatalogMerge (catalog, Ncatalog);
       if (!BrightCatalogSave (BCATALOG, bcatalog)) {
-        abort();
+          fprintf (stderr, "ERROR saving bright catalog from %s\n", CATDIR);
+          exit (2);
       }
       break;
@@ -41 +54 @@
       off_t Nimage;
       ImageSubset *image = ImageSubsetLoad (IMAGES, &Nimage);
+      if (!image) {
+          fprintf (stderr, "ERROR loading image subset %s\n", CATDIR);
+          exit (2);
+      }
       
       // save the available image information in the static array in ImageOps.c