Changeset 37745 for branches/eam_branches/ipp-20140904

Timestamp:

Dec 14, 2014, 4:47:22 AM (12 years ago)

Author:

eugene

Message:

define parallel output; copy astrom map table

Location:

branches/eam_branches/ipp-20140904/Ohana/src/photdbc

Files:

• include/photdbc.h (modified) (1 diff)
• src/args.c (modified) (3 diffs)
• src/copy_images.c (modified) (2 diffs)
• src/photdbc_catalogs.c (modified) (3 diffs)

branches/eam_branches/ipp-20140904/Ohana/src/photdbc/include/photdbc.h

@@ -31 +31 @@
 int    PARALLEL_MANUAL;
 int    PARALLEL_SERIAL;
+char  *PARALLEL_OUTHOSTS;
 
 int          HOST_ID;
 char        *HOSTDIR;
+char        *HOSTDIR_OUTPUT;
 
 // need to get RADIUS from Config 

branches/eam_branches/ipp-20140904/Ohana/src/photdbc/src/args.c

@@ -146 +146 @@
   }
 
+  // XXX for the moment, make this selection manual.  it needs to be automatic 
+  // based on the state of the SkyTable
+  HOSTDIR_OUTPUT = NULL;
+  PARALLEL_OUTHOSTS = NULL;
+  if ((N = get_argument (argc, argv, "-parallel-output"))) {
+    remove_argument (N, &argc, argv);
+    PARALLEL_OUTHOSTS = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
   if (argc != 2) usage();
 
@@ -186 +196 @@
   if (!HOSTDIR) usage();
 
+  HOSTDIR_OUTPUT = NULL;
+  if ((N = get_argument (argc, argv, "-hostdir-output"))) {
+    remove_argument (N, &argc, argv);
+    HOSTDIR_OUTPUT = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
   VERBOSE = FALSE;
   if ((N = get_argument (argc, argv, "-v"))) {
@@ -304 +321 @@
   fprintf (stderr, " photdbc (output)\n\n");
 
+  fprintf (stderr, " parallel options:\n");
+  fprintf (stderr, "  -parallel:\n");
+  fprintf (stderr, "  -parallel-output HostTable : HostTable should be relative path in OUTPUT directory\n");
+  fprintf (stderr, "  -parallel-manual\n");
+  fprintf (stderr, "  -parallel-serial\n\n");
+
   fprintf (stderr, " allowed filters / restrictions include:\n\n");
 

branches/eam_branches/ipp-20140904/Ohana/src/photdbc/src/copy_images.c

@@ -55 +55 @@
   // MARKTIME("build chip match: %f sec\n", dtime);
 
+  char mapfile[DVO_MAX_PATH];
+  snprintf (mapfile, DVO_MAX_PATH, "%s/AstroMap.fits", CATDIR);
+  AstromOffsetTable *table_src = AstromOffsetMapLoad (mapfile, VERBOSE);
+
+  // assign images.coords.offsetMap -> table->map[i]
+  if (table_src) {
+    AstromOffsetTableMatchChips (image, Nimage, table_src);
+  }
+
   subset = select_images (skylist, image, Nimage, &LineNumber, &Nsubset, FALSE);
+
+  if (table_src) {
+    AstromOffsetTable *table_tgt = AstromOffsetTableInit ();
+    
+    off_t i;
+    for (i = 0; i < Nsubset; i++) {
+      if (!subset[i].coords.offsetMap) continue;
+      if (subset[i].coords.Npolyterms != -1) continue; // assert on this?
+      AstromOffsetTableAddMapFromImage (table_tgt, &subset[i]);
+    }
+    snprintf (mapfile, DVO_MAX_PATH, "%s/AstroMap.fits", outdir);
+    AstromOffsetMapSave (table_tgt, mapfile);
+    free (table_tgt->imageIDtoTableSeq);
+    free (table_tgt->map);
+    free (table_tgt);
+  }
 
   dvo_image_addrows (&out, subset, Nsubset);
@@ -76 +101 @@
   SavePhotcodesFITS (filename);
 
+  // free the astrom offset tables
+  AstromOffsetTableFree (table_src);
+
   return (TRUE);
 }

branches/eam_branches/ipp-20140904/Ohana/src/photdbc/src/photdbc_catalogs.c

@@ -42 +42 @@
     snprintf (outfile, DVO_MAX_PATH, "%s/%s.cpt", outroot, skylist[0].regions[i]->name);
     outcatalog.filename = outfile;
+
+    if (HOSTDIR_OUTPUT) {
+      // parallel-ouput puts the output in a per-host directory (keeps the source sky partition)
+      snprintf (outfile, DVO_MAX_PATH, "%s/%s.cpt", HOSTDIR_OUTPUT, skylist[0].regions[i]->name);
+    }
+
     if (outcatalog.filename == NULL) Shutdown ("error with input catalog name");
 
@@ -95 +101 @@
   }    
 
+  HostTable *table_output = NULL;
+  if (PARALLEL_OUTHOSTS) {
+    table_output = HostTableLoad (outroot, PARALLEL_OUTHOSTS);
+    if (!table_output) {
+      fprintf (stderr, "%s not found in %s, please create\n", PARALLEL_OUTHOSTS, outroot);
+    }
+  }
+
   int i;
   for (i = 0; i < table->Nhosts; i++) {
@@ -123 +137 @@
     if (CATFORMAT)          { snprintf (tmpline, DVO_MAX_PATH, "%s -set-format %s",    command, CATFORMAT);                     strcpy (command, tmpline); }
     if (CATMODE)            { snprintf (tmpline, DVO_MAX_PATH, "%s -set-mode %s",      command, CATMODE);                       strcpy (command, tmpline); }
+
+    if (PARALLEL_OUTHOSTS) {
+      tmppath = abspath (table_output->hosts[i].pathname, DVO_MAX_PATH);
+      free  (table_output->hosts[i].pathname);
+      table_output->hosts[i].pathname = tmppath;
+      snprintf (tmpline, DVO_MAX_PATH, "%s -hostdir-output %s", command, table_output->hosts[i].pathname); strcpy (command, tmpline); 
+    }
 
     fprintf (stderr, "command: %s\n", command);