Changeset 35416 for trunk/Ohana/src/opihi

Timestamp:

Apr 19, 2013, 4:31:05 PM (13 years ago)

Author:

eugene

Message:

merge changes from eam_branches/ipp-20130307 : parallelize dvosecfilt, use dvo_average,measure,secfilt_init functions more broadly; move image calibration to relphot_images; add synthetic photometry referece (w-band); put relphot MARKTIME & INITTIME in macros; in dvo_clients, mextract fields which need image data use a subset image metadata table (instead of loading full Images.dat table); create FIELD and MOSAIC fields for mextract; somewhat better rules for photcode:ave & similar selections in mextract; add hpm_* concept in relastro (high-speed proper motions); fix precision errors in dvopsps; check for dvopsps exit conditions; error-bar clipping to limits of plotting window; parallelize delstar -dup-images; do NOT delete parents (because parent IDs are broken after dvomerge); minor handshake when setting up KAPA connection; avextract and related do not need to launch remote jobs on all clients if region does not include tables from all hosts; add fitplx and xsection functions to mana; enable addstar of diff cmfs (at least PS1_DV3)

Location:

trunk/Ohana

Files:

• . (modified) (1 prop)
• src/opihi (modified) (1 prop)
• src/opihi/cmd.astro (modified) (1 prop)
• src/opihi/cmd.astro/Makefile (modified) (1 diff)
• src/opihi/cmd.astro/fitplx.c (copied) (copied from branches/eam_branches/ipp-20130307/Ohana/src/opihi/cmd.astro/fitplx.c )
• src/opihi/cmd.astro/init.c (modified) (2 diffs)
• src/opihi/cmd.data (modified) (1 prop)
• src/opihi/cmd.data/Makefile (modified) (1 diff)
• src/opihi/cmd.data/init.c (modified) (2 diffs)
• src/opihi/cmd.data/xsection.c (copied) (copied from branches/eam_branches/ipp-20130307/Ohana/src/opihi/cmd.data/xsection.c )
• src/opihi/dvo/avextract.c (modified) (1 diff)
• src/opihi/dvo/avmatch.c (modified) (1 diff)
• src/opihi/dvo/dvo_host_utils.c (modified) (5 diffs)
• src/opihi/dvo/mextract.c (modified) (2 diffs)
• src/opihi/dvo/mmatch.c (modified) (3 diffs)
• src/opihi/dvo/photometry.c (modified) (3 diffs)
• src/opihi/dvo/remote.c (modified) (1 diff)
• src/opihi/dvo/skyregion.c (modified) (2 diffs)
• src/opihi/include/dvoshell.h (modified) (1 diff)

trunk/Ohana/src/opihi/cmd.astro/Makefile

@@ -30 +30 @@
 $(SRC)/drizzle.$(ARCH).o           \
 $(SRC)/flux.$(ARCH).o              \
+$(SRC)/fitplx.$(ARCH).o    \
 $(SRC)/fixwrap.$(ARCH).o           \
 $(SRC)/fixcols.$(ARCH).o           \

trunk/Ohana/src/opihi/cmd.astro/init.c

@@ -14 +14 @@
 int drizzle                 PROTO((int, char **));
 int flux                    PROTO((int, char **));
+int fitplx                  PROTO((int, char **));
 int fixwrap                 PROTO((int, char **));
 int fiximage                PROTO((int, char **));
@@ -71 +72 @@
   {1, "drizzle",     drizzle,      "transform image to image"},
   {1, "flux",        flux,         "flux in a convex contour"},
+  {1, "fitplx",      fitplx,       "fit proper motion and parallax"},
   {1, "fixwrap",     fixwrap,      "fix megacam over-wrapped pixels"},
   {1, "fiximage",    fiximage,     "fix pixels in an image by interpolation"},

trunk/Ohana/src/opihi/cmd.data/Makefile

@@ -150 +150 @@
 $(SRC)/vsmooth.$(ARCH).o           \
 $(SRC)/vstats.$(ARCH).o            \
+$(SRC)/xsection.$(ARCH).o          \
 $(SRC)/wd.$(ARCH).o                \
 $(SRC)/write_vectors.$(ARCH).o     \

trunk/Ohana/src/opihi/cmd.data/init.c

@@ -141 +141 @@
 int wd               PROTO((int, char **));
 int write_vectors    PROTO((int, char **));
+int xsection         PROTO((int, char **));
 int zap              PROTO((int, char **));
 int zplot            PROTO((int, char **));
@@ -299 +300 @@
   {1, "wd",           wd,               "write an image to a file"},
   {1, "write",        write_vectors,    "write vectors to datafile"},
+  {1, "xsection",     xsection,         "generate cross-section histogram for an image (or region)"},
   {1, "zap",          zap,              "assign values to pixel regions"},
   {1, "zplot",        zplot,            "plot x y with size scaled by z"},

trunk/Ohana/src/opihi/dvo/avextract.c

@@ -137 +137 @@
   // this does all the work of re-packaging the command, calling it on the remote machines, then loading in the results
   if (PARALLEL && !HOST_ID) {
-    int status = HostTableParallelOps (argc, argv, RESULT_FILE, TRUE, 0, VERBOSE);
+      int status = HostTableParallelOps (skylist, argc, argv, RESULT_FILE, TRUE, 0, VERBOSE);
 
     dbFreeFields (fields, Nfields);

trunk/Ohana/src/opihi/dvo/avmatch.c

@@ -95 +95 @@
 
     // I need to pass the RA & DEC vectors to the remote clients...
-    int status = HostTableParallelOps (argc, argv, RESULT_FILE, TRUE, RAvec->Nelements, VERBOSE);
+    int status = HostTableParallelOps (skylist, argc, argv, RESULT_FILE, TRUE, RAvec->Nelements, VERBOSE);
     if (vec) free (vec);
     

trunk/Ohana/src/opihi/dvo/dvo_host_utils.c

@@ -8 +8 @@
 # define DIE(WHO,MSG) { perror(WHO); myAbort(MSG); }
 
-int HostTableLaunchJobs (HostTable *table, char *basecmd, char *options, int VERBOSE) {
+int HostTableLaunchJobs (SkyList *sky, HostTable *table, char *basecmd, char *options, int VERBOSE) {
 
   char uniquer[12];
@@ -35 +35 @@
 
   int top_status = TRUE;
-  int i;
+  int i, j;
   for (i = 0; i < table->Nhosts; i++) {
+
+      if (sky && (sky->Nregions < table->Nhosts)) {
+      // do any of the regions want this host?
+      int wantThisHost = FALSE;
+      for (j = 0; j < sky->Nregions; j++) {
+        if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) {
+          wantThisHost = TRUE;
+          break;
+        }
+      }
+      if (!wantThisHost) {
+        // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname);
+        continue;
+      }
+      // fprintf (stderr, "not skip host %s\n", table->hosts[i].hostname);
+    }
 
     // ensure that the paths are absolute path names
@@ -88 +104 @@
 // an alternative ending step ignores the result files and instead saves the names into
 // the list 'result:n' for the user to access as desired
-int HostTableParallelOps (int argc, char **argv, char *ResultFile, int ReadVectors, int Nelements, int VERBOSE) {
+int HostTableParallelOps (SkyList *sky, int argc, char **argv, char *ResultFile, int ReadVectors, int Nelements, int VERBOSE) {
 
   int i;
 
-  // load the list of hosts
-  SkyTable *sky = GetSkyTable();
-  if (!sky) {
-    gprint (GP_ERR, "failed to load sky table for database\n");
-    return FALSE;
-  }
+  // XX // load the list of hosts
+  // XX SkyTable *sky = GetSkyTable();
+  // XX if (!sky) {
+  // XX   gprint (GP_ERR, "failed to load sky table for database\n");
+  // XX   return FALSE;
+  // XX }
 
   char *tmppath = GetCATDIR ();
@@ -145 +161 @@
 
   // launch this command remotely
-  HostTableLaunchJobs (table, basecmd, options, VERBOSE);
+  HostTableLaunchJobs (sky, table, basecmd, options, VERBOSE);
   free (options);
   free (basecmd);
@@ -183 +199 @@
 
     if (table->hosts[i].status) continue; 
+    if (!table->hosts[i].pid) continue;
 
     if (ReadVectors) {

trunk/Ohana/src/opihi/dvo/mextract.c

@@ -16 +16 @@
     if (field->ID == MEAS_CENTER_OFFSET)  return TRUE; // 0.5*NX, 0.5*NY
     if (field->ID == MEAS_EXPNAME_AS_INT) return TRUE; // expname (or as int)
-    if (field->ID == MEAS_AIRMASS)        return TRUE; // airmass
+    if (field->ID == MEAS_MEAN_AIRMASS)   return TRUE; // airmass
     return FALSE;
 }
@@ -171 +171 @@
 
     // call the remote client
-    int status = HostTableParallelOps (targc, targv, RESULT_FILE, TRUE, 0, VERBOSE);
+    int status = HostTableParallelOps (skylist, targc, targv, RESULT_FILE, TRUE, 0, VERBOSE);
 
     dbFreeFields (fields, Nfields);

trunk/Ohana/src/opihi/dvo/mmatch.c

@@ -106 +106 @@
   RAvec  = NULL;
   DECvec = NULL;
-  if (PARALLEL && !HOST_ID) {
-
-    // We need to copy the args to a temp array and modify them so that we send the
-    // correct set to the remote client.  The args list looks like this:
-    // if (!CoordsFile) : mmatch (RA) (DEC) (RADIUS) field, ...
-    // if ( CoordsFile) : mmatch (RADIUS) field, ... [because we stripped off the -coords filename elements]
-
-    // allocate the temp array and copy all but (RA) (DEC)
-    int targc = 0;
-    char **targv = NULL;
-    ALLOCATE (targv, char *, argc + 2);
-    for (i = 0; i < argc; i++) {
-      if (!CoordsFile && (i == 1)) continue;
-      if (!CoordsFile && (i == 2)) continue;
-      targv[targc] = strcreate (argv[i]);
-      targc ++;
-    }
-
-    // if not specified, create the coords.fits input file
-    if (!CoordsFile) {
-      // get vectors corresponding to coordinates of interest
-      if ((RAvec  = SelectVector (argv[1], ANYVECTOR, TRUE)) == NULL) goto help;
-      if ((DECvec = SelectVector (argv[2], ANYVECTOR, TRUE)) == NULL) goto help;
-      
-      ALLOCATE (vec, Vector *, 2);
-      vec[0] = RAvec;
-      vec[1] = DECvec;
-
-      CoordsFile = abspath("coords.fits", 1024);
-      int status = WriteVectorTableFITS (CoordsFile, "COORDS", vec, 2, FALSE, NULL);
-      if (!status) goto escape;
-    }
-
-    // add the coords file to the args list
-    targv[targc+0] = strcreate ("-coords");
-    targv[targc+1] = CoordsFile; // this gets freed with targv
-    targc += 2;
-    
-    // if needed, add the index vector to the args list
-    if (IDXvec) {
-      REALLOCATE (targv, char *, targc + 2);
-      targv[targc+0] = strcreate ("-index");
-      targv[targc+1] = strcreate (IDXvec[0].name);
-      targc += 2;
-    }      
-
-    if (loadImages) {
-      Image *image;
-      off_t Nimage;
-      if ((image = LoadImagesDVO (&Nimage)) == NULL) goto escape;
-
-      char *filename = abspath("image.metadata.fits", DVO_MAX_PATH);
-      ImageMetadataSave (filename, image, Nimage);
-
-      REALLOCATE (targv, char *, targc + 2);
-      targv[targc+0] = strcreate ("-image-metadata");
-      targv[targc+1] = strcreate (filename);
-      targc += 2;
-    }
-
-    // call the remote client
-    int status = HostTableParallelOps (targc, targv, RESULT_FILE, TRUE, 0, VERBOSE);
-    if (vec) free (vec);
-    
-    // free up targv
-    for (i = 0; i < targc; i++) {
-      free (targv[i]);
-    }
-    free (targv);
-
-    return status;
-  }
 
   // get vectors corresponding to coordinates of interest
@@ -193 +121 @@
     remove_argument (1, &argc, argv);
   }
+
+  /* load regions which contain all supplied RA,DEC coordinates */
+  if ((skylist = SelectRegionsByCoordVectors (RAvec, DECvec)) == NULL) goto escape;
+
+  if (PARALLEL && !HOST_ID) {
+
+    // We need to copy the args to a temp array and modify them so that we send the
+    // correct set to the remote client.  The args list looks like this:
+    // if (!CoordsFile) : mmatch (RADIUS) field, ... [we removed RA & DEC above]
+    // if ( CoordsFile) : mmatch (RADIUS) field, ... [because we stripped off the -coords filename elements]
+
+    // allocate the temp array and copy all but (RA) (DEC)
+    int targc = 0;
+    char **targv = NULL;
+    ALLOCATE (targv, char *, argc + 2);
+    for (i = 0; i < argc; i++) {
+      targv[targc] = strcreate (argv[i]);
+      targc ++;
+    }
+
+    // if not specified, create the coords.fits input file
+    // NOTE: RAvec, DECvec were set above
+    if (!CoordsFile) {
+      ALLOCATE (vec, Vector *, 2);
+      vec[0] = RAvec;
+      vec[1] = DECvec;
+
+      // XXX this is now set for both cases...
+      CoordsFile = abspath("coords.fits", 1024);
+      int status = WriteVectorTableFITS (CoordsFile, "COORDS", vec, 2, FALSE, NULL);
+      if (!status) goto escape;
+    }
+
+    // add the coords file to the args list
+    targv[targc+0] = strcreate ("-coords");
+    targv[targc+1] = CoordsFile; // this gets freed with targv
+    targc += 2;
+    
+    // if needed, add the index vector to the args list
+    if (IDXvec) {
+      REALLOCATE (targv, char *, targc + 2);
+      targv[targc+0] = strcreate ("-index");
+      targv[targc+1] = strcreate (IDXvec[0].name);
+      targc += 2;
+    }      
+
+    if (loadImages) {
+      Image *image;
+      off_t Nimage;
+      if ((image = LoadImagesDVO (&Nimage)) == NULL) goto escape;
+
+      char *filename = abspath("image.metadata.fits", DVO_MAX_PATH);
+      ImageMetadataSave (filename, image, Nimage);
+
+      REALLOCATE (targv, char *, targc + 2);
+      targv[targc+0] = strcreate ("-image-metadata");
+      targv[targc+1] = strcreate (filename);
+      targc += 2;
+    }
+
+    // call the remote client
+    int status = HostTableParallelOps (skylist, targc, targv, RESULT_FILE, TRUE, 0, VERBOSE);
+    if (vec) free (vec);
+    
+    // free up targv
+    for (i = 0; i < targc; i++) {
+      free (targv[i]);
+    }
+    free (targv);
+
+    return status;
+  } // END of remote call section
+
   RADIUS = atof (argv[1]);
   remove_argument (1, &argc, argv);
@@ -209 +210 @@
   }
 
-  /* load regions which contain all supplied RA,DEC coordinates */
-  if ((skylist = SelectRegionsByCoordVectors (RAvec, DECvec)) == NULL) goto escape;
 
   /* create output storage vectors */

trunk/Ohana/src/opihi/dvo/photometry.c

@@ -1127 +1127 @@
   double value;
   Image *image;
-  Coords *mosaic;
+  // Coords *mosaic;
 
   value = 0;
@@ -1193 +1193 @@
 # endif
       break;
+# if 0
     case MEAS_XMOSAIC: /* OK */
       ra  = average[0].R - measure[0].dR / 3600.0;
       dec = average[0].D - measure[0].dD / 3600.0;
-      mosaic = MatchMosaic (measure[0].t, measure[0].photcode);
+      mosaic = MatchMosaic (measure[0].t, measure[0].photcode); // XXX not used anymore
       if (mosaic == NULL) break;
       RD_to_XY (&x, &y, ra, dec, mosaic);
@@ -1204 +1205 @@
       ra  = average[0].R - measure[0].dR / 3600.0;
       dec = average[0].D - measure[0].dD / 3600.0;
-      mosaic = MatchMosaic (measure[0].t, measure[0].photcode);
+      mosaic = MatchMosaic (measure[0].t, measure[0].photcode); // XXX not used anymore
       if (mosaic == NULL) break;
       RD_to_XY (&x, &y, ra, dec, mosaic);
       value = y;
       break;
+# endif
   }
   return (value);

trunk/Ohana/src/opihi/dvo/remote.c

@@ -61 +61 @@
   }
 
+  // load the list of hosts
+  SkyTable *sky = GetSkyTable();
+  if (!sky) {
+    gprint (GP_ERR, "failed to load sky table for database\n");
+    return FALSE;
+  }
+  SkyList *skylist = NULL;
+  ALLOCATE (skylist, SkyList, 1);
+  skylist[0].Nregions = sky[0].Nregions;
+  strcpy (skylist[0].hosts, sky[0].hosts);
+
   // strip of the 'remote' and send the remaining arguments to the remote machine
-  int status = HostTableParallelOps (argc - 1, &argv[1], NULL, ReadVectors, 0, VERBOSE);
+  int status = HostTableParallelOps (skylist, argc - 1, &argv[1], NULL, ReadVectors, 0, VERBOSE);
+  free (skylist);
   return status;
 }

trunk/Ohana/src/opihi/dvo/skyregion.c

@@ -98 +98 @@
   new[0].Nregions = 0;
   new[0].ownElements = FALSE; // this list is only holding a view to the elements
+  strcpy (new[0].hosts, sky[0].hosts);
 
   // output list
@@ -107 +108 @@
   list[0].Nregions = 0;
   list[0].ownElements = FALSE; // this list is only holding a view to the elements
+  strcpy (list[0].hosts, sky[0].hosts);
 
   for (i = 0; i < Npts; i++) {

trunk/Ohana/src/opihi/include/dvoshell.h

@@ -99 +99 @@
 dbValue      dbExtractImages        PROTO((Image *image, off_t Nimage, off_t N, dbField *field));
 
-int          HostTableLaunchJobs    PROTO((HostTable *table, char *basecmd, char *options, int VERBOSE));
-int          HostTableParallelOps   PROTO((int argc, char **argv, char *ResultFile, int ReadVectors, int Nelements, int VERBOSE));
+int          HostTableLaunchJobs    PROTO((SkyList *sky, HostTable *table, char *basecmd, char *options, int VERBOSE));
+int          HostTableParallelOps   PROTO((SkyList *sky, int argc, char **argv, char *ResultFile, int ReadVectors, int Nelements, int VERBOSE));
 int          HostTableReloadResults PROTO((char *uniquer, int VERBOSE));
 int          HostTableGetResults    PROTO((char *uniquer, int VERBOSE));