Changeset 33214

Timestamp:

Feb 8, 2012, 7:28:41 AM (14 years ago)

Author:

eugene

Message:

major cleanups: a new output directory may be specified (or split files stay in local catdir); format and mode are inherited from input (override with -set-format and -set-mode options)

Location:

branches/eam_branches/ipp-20111122/Ohana/src/dvosplit

Files:

• include/dvosplit.h (modified) (2 diffs)
• src/ConfigInit.c (modified) (2 diffs)
• src/args.c (modified) (4 diffs)
• src/dvosplit.c (modified) (6 diffs)
• src/open_output_catalogs.c (modified) (2 diffs)

branches/eam_branches/ipp-20111122/Ohana/src/dvosplit/include/dvosplit.h

@@ -23 +23 @@
 
 int    VERBOSE;
-char   CATDIR[256];
-char   CATMODE[16];    /* raw, mef, split, mysql */
-char   CATFORMAT[16];  /* internal, elixir, loneos, panstarrs */
+char  *OUTDIR;
+char  *CATMODE;    /* raw, mef, split, mysql */
+char  *CATFORMAT;  /* internal, elixir, loneos, panstarrs */
 
 SkyRegion UserPatch;  // used by MODE CAT
@@ -38 +38 @@
 int        args                   PROTO((int argc, char **argv));
 
-Catalog   *open_output_catalogs   PROTO((SkyList *outlist));
+Catalog   *open_output_catalogs   PROTO((SkyList *outlist, int catformat, int catmode));
 AveLinks  *split_averages         PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs));
 int        split_measures         PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs, AveLinks *avelinks));

branches/eam_branches/ipp-20111122/Ohana/src/dvosplit/src/ConfigInit.c

@@ -5 +5 @@
   double ZERO_POINT;
   char *config, *file;
-  char CatdirPhotcodeFile[256];
-  char MasterPhotcodeFile[256];
 
   /*** load configuration info ***/
@@ -18 +16 @@
   if (VERBOSE) fprintf (stderr, "loaded config file: %s\n", file);
 
-  GetConfig (config, "CATDIR",                  "%s",  0, CATDIR);
-  GetConfig (config, "PHOTCODE_FILE",           "%s",  0, MasterPhotcodeFile);
-
-  ScanConfig (config, "CATMODE",                "%s",  0, CATMODE);
-  ScanConfig (config, "CATFORMAT",              "%s",  0, CATFORMAT);
-
-  /* default mode, format, if not specified */
-  if (*CATMODE == 0) strcpy (CATMODE, "RAW");
-  if (*CATFORMAT == 0) strcpy (CATFORMAT, "ELIXIR");
-
+  // XXX this really needs to be deprecated.  It is always 25.0, and should just be hard-wired
+  // (the purpose of this value is now completely pointless...)
   GetConfig (config, "ZERO_PT",                "%lf", 0, &ZERO_POINT);
   SetZeroPoint (ZERO_POINT);
-
-  /* XXX this does not yet write out the master photcode table */
-  sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR);
-  if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) {
-    fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile);
-    exit (1);
-  }
 
   free (config);

branches/eam_branches/ipp-20111122/Ohana/src/dvosplit/src/args.c

@@ -34 +34 @@
   }
 
+  OUTDIR = NULL;
+  if ((N = get_argument (argc, argv, "-outdir"))) {
+    remove_argument (N, &argc, argv);
+    OUTDIR = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
+  // override input catalog format (PS1_V1, PS1_REF, etc)
+  CATFORMAT = NULL;
+  if ((N = get_argument (argc, argv, "-set-format"))) {
+    remove_argument (N, &argc, argv);
+    CATFORMAT = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
+  // override input catalog mode (raw, mef, split, mysql)
+  CATMODE = NULL;
+  if ((N = get_argument (argc, argv, "-set-mode"))) {
+    remove_argument (N, &argc, argv);
+    CATMODE = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
   /* extra error messages */
   VERBOSE = FALSE;
@@ -41 +64 @@
   }
 
-  if (argc == 2) {
+  if (argc == 3) {
     if (CONFIRM) {
       fprintf (stderr, "you are splitting the entire sky in one pass\n");
@@ -54 +77 @@
   }
 
-  fprintf (stderr, "USAGE: dvosplit (newlevel) [-region (Rmin) (Rmax) (Dmin) (Dmax)]\n");
+  fprintf (stderr, "USAGE: dvosplit (catdir) (newlevel) [-outdir outdir] [-region (Rmin) (Rmax) (Dmin) (Dmax)]\n");
   exit (2);
 }
@@ -61 +84 @@
 
   fprintf (stderr, "USAGE\n");
-  fprintf (stderr, "  dvosplit (newlevel)\n\n");
+  fprintf (stderr, "  dvosplit (catdir) (newlevel)\n\n");
   fprintf (stderr, "  optional flags:\n");
+  fprintf (stderr, "  -outdir (outdir)            : copy to a new location (does not copy Images.dat)\n");
   fprintf (stderr, "  -region ra ra dec dec       : migrate catalogs in specified region\n");
   fprintf (stderr, "  -v                          : verbose mode\n");
+  fprintf (stderr, "  -set-format (format)        : set format of output catalog (or inherit from input)\n");
+  fprintf (stderr, "  -set-mode (mode)            : set mode of output catalog (or inherit from input)\n");
   fprintf (stderr, "  -help                       : this list\n");
   fprintf (stderr, "  -h                          : this list\n\n");

branches/eam_branches/ipp-20111122/Ohana/src/dvosplit/src/dvosplit.c

@@ -1 +1 @@
 # include "dvosplit.h"
 
-// XXX : go to this : dvosplit (catdir) (outlevel) [-region Rmin Rmax Dmin Dmax]
+// dvosplit (catdir) (outlevel) [-outdir (outcat)} [-region Rmin Rmax Dmin Dmax]
 // dvosplit (outlevel) [-region Rmin Rmax Dmin Dmax]
+// (inherites the input catalog's format and mode, unless -set-format or -set-mode are used)
 int main (int argc, char **argv) {
 
@@ -10 +11 @@
   Catalog incatalog, *outcatalogs;
   AveLinks *avelinks;
-  char filename[256];
+  char *filename, *CATDIR;
 
   SetSignals ();
@@ -16 +17 @@
   args (argc, argv);
 
-  // XXX : OUTCAT = strcreate (argv[1]);
-  // XXX : OUT_DEPTH = atoi (argv[2]);
+  CATDIR = strcreate (argv[1]);
+  OUT_DEPTH = atoi (argv[2]);
+  if (!OUTDIR) {
+    OUTDIR = strcreate (CATDIR);
+  }
 
-  OUT_DEPTH = atoi (argv[1]);
+  // load the photcode table (for Nsecfilt and related)
+  char photcodeFile[1024];
+  snprintf (photcodeFile, 1024, "%s/Photcodes.dat", CATDIR);
+  if (!LoadPhotcodes (photcodeFile, NULL, FALSE)) {
+    fprintf (stderr, "error loading photcode table %s\n", photcodeFile);
+    exit (1);
+  }
+
+  // XXX should strip off any trailing slashes
+  int LocalCopy = !strcmp (CATDIR, OUTDIR);
+
+  // if not local, save the photcode table (for Nsecfilt and related)
+  if (!LocalCopy) {
+    if (!check_dir_access (OUTDIR, VERBOSE)) {
+      fprintf (stderr, "failed to create output directory %s\n", OUTDIR);
+      exit (1);
+    }
+    snprintf (photcodeFile, 1024, "%s/Photcodes.dat", OUTDIR);
+    if (!SavePhotcodesFITS (photcodeFile)) {
+      fprintf (stderr, "error loading photcode table %s\n", photcodeFile);
+      exit (1);
+    }
+  }
 
   // load the sky table for the existing database
@@ -29 +55 @@
   
   for (i = 0; i < skylist[0].Nregions; i++) {
-    fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
+    if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
 
+    // if !LocalCopy, always copy the file
     // if (current level >  out level) skip: cannot currently merge catalogs
     // if (current level == out level) skip: no action is needed
-    // XXX if outdir != indir, copy the tables across
-    if (skylist[0].regions[i][0].depth >= OUT_DEPTH) continue;
+
+    if (skylist[0].regions[i][0].depth > OUT_DEPTH) {
+      if (VERBOSE) fprintf (stderr, "WARNING, cannot merge deeper catalog %s (%d vs %d)\n", skylist[0].regions[i][0].name, skylist[0].regions[i][0].depth, OUT_DEPTH);
+      continue;
+    }
+
+    if (LocalCopy && (skylist[0].regions[i][0].depth == OUT_DEPTH)) continue;
 
     // set the parameters which guide catalog open/load/create
     incatalog.filename = skylist[0].filename[i];
     incatalog.Nsecfilt = GetPhotcodeNsecfilt ();
-    incatalog.catflags = LOAD_NONE;
+    incatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF | LOAD_MISS;
 
     // an error exit status here is a significant error
@@ -56 +88 @@
     // change sky.regions[i].depth for these regions
     outlist = SkyListByPatch (sky, OUT_DEPTH, skylist[0].regions[i]);
-    // XXX update the filenames for outlist given the new outdir
 
-    outcatalogs = open_output_catalogs (outlist);
+    // Modify the outlist filenames to match the output directory.  Note that the
+    // filenames are now owned by the list (and are not freed by SkyListFree)
+    SkyListSetFilenames (outlist, OUTDIR, "cpt");
+
+    // inherit the input catalog's format and mode (unless overridden)
+    outcatalogs = open_output_catalogs (outlist, incatalog.catformat, incatalog.catmode);
 
     avelinks = split_averages (&incatalog, outlist, outcatalogs); 
@@ -81 +117 @@
       outlist[0].regions[j][0].table = TRUE;
     }
+
+    // free the newly allocated filenames
+    for (j = 0; j < outlist[0].Nregions; j++) {
+      free (outlist[0].filename[j]);
+    }
+
+    // free the rest of the list
+    SkyListFree (outlist);
   }
 
   // save sky table copy
-  sprintf (filename, "%s/SkyTable.fits", CATDIR);
+  filename = SkyTableFilename (OUTDIR);
   check_file_access (filename, TRUE, TRUE, VERBOSE);
   if (!SkyTableSave (sky, filename)) {
-    fprintf (stderr, "ERROR: failed to save sky table for %s\n", CATDIR);
+    fprintf (stderr, "ERROR: failed to save sky table for %s\n", OUTDIR);
     exit (1);
   }

branches/eam_branches/ipp-20111122/Ohana/src/dvosplit/src/open_output_catalogs.c

@@ -1 +1 @@
 # include "dvosplit.h"
 
-Catalog *open_output_catalogs (SkyList *outlist) {
+Catalog *open_output_catalogs (SkyList *outlist, int catformat, int catmode) {
 
   int i;
@@ -15 +15 @@
     outcatalogs[i].Nsecfilt  = GetPhotcodeNsecfilt ();
     outcatalogs[i].catflags  = LOAD_NONE;
-    outcatalogs[i].catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
-    outcatalogs[i].catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
+    outcatalogs[i].catformat = CATFORMAT ? dvo_catalog_catformat (CATFORMAT) : catformat;  // set the default catformat from config data
+    outcatalogs[i].catmode   = CATMODE   ? dvo_catalog_catmode (CATMODE)     : catmode;    // set the default catmode from config data
+
+    // the user options -set-format and -set-mode assign values to the globals CATFORMAT
+    // and CATMODE (in args.c); otherwise the values from the input catalogs are inherited
 
     if (!dvo_catalog_open (&outcatalogs[i], outlist[0].regions[i], VERBOSE, "w")) {