Index: trunk/Ohana/src/dvosplit/include/dvosplit.h =================================================================== --- trunk/Ohana/src/dvosplit/include/dvosplit.h (revision 33655) +++ trunk/Ohana/src/dvosplit/include/dvosplit.h (revision 33656) @@ -23,7 +23,7 @@ int VERBOSE; -char CATDIR[256]; -char CATMODE[16]; /* raw, mef, split, mysql */ -char CATFORMAT[16]; /* internal, elixir, loneos, panstarrs */ +char *OUTDIR; +char *CATMODE; /* raw, mef, split, mysql */ +char *CATFORMAT; /* internal, elixir, loneos, panstarrs */ SkyRegion UserPatch; // used by MODE CAT @@ -38,5 +38,5 @@ int args PROTO((int argc, char **argv)); -Catalog *open_output_catalogs PROTO((SkyList *outlist)); +Catalog *open_output_catalogs PROTO((SkyList *outlist, int catformat, int catmode)); AveLinks *split_averages PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs)); int split_measures PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs, AveLinks *avelinks)); Index: trunk/Ohana/src/dvosplit/src/ConfigInit.c =================================================================== --- trunk/Ohana/src/dvosplit/src/ConfigInit.c (revision 33655) +++ trunk/Ohana/src/dvosplit/src/ConfigInit.c (revision 33656) @@ -5,6 +5,4 @@ double ZERO_POINT; char *config, *file; - char CatdirPhotcodeFile[256]; - char MasterPhotcodeFile[256]; /*** load configuration info ***/ @@ -18,23 +16,8 @@ if (VERBOSE) fprintf (stderr, "loaded config file: %s\n", file); - GetConfig (config, "CATDIR", "%s", 0, CATDIR); - GetConfig (config, "PHOTCODE_FILE", "%s", 0, MasterPhotcodeFile); - - ScanConfig (config, "CATMODE", "%s", 0, CATMODE); - ScanConfig (config, "CATFORMAT", "%s", 0, CATFORMAT); - - /* default mode, format, if not specified */ - if (*CATMODE == 0) strcpy (CATMODE, "RAW"); - if (*CATFORMAT == 0) strcpy (CATFORMAT, "ELIXIR"); - + // XXX this really needs to be deprecated. It is always 25.0, and should just be hard-wired + // (the purpose of this value is now completely pointless...) GetConfig (config, "ZERO_PT", "%lf", 0, &ZERO_POINT); SetZeroPoint (ZERO_POINT); - - /* XXX this does not yet write out the master photcode table */ - sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { - fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); - exit (1); - } free (config); Index: trunk/Ohana/src/dvosplit/src/args.c =================================================================== --- trunk/Ohana/src/dvosplit/src/args.c (revision 33655) +++ trunk/Ohana/src/dvosplit/src/args.c (revision 33656) @@ -34,4 +34,27 @@ } + OUTDIR = NULL; + if ((N = get_argument (argc, argv, "-outdir"))) { + remove_argument (N, &argc, argv); + OUTDIR = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // override input catalog format (PS1_V1, PS1_REF, etc) + CATFORMAT = NULL; + if ((N = get_argument (argc, argv, "-set-format"))) { + remove_argument (N, &argc, argv); + CATFORMAT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // override input catalog mode (raw, mef, split, mysql) + CATMODE = NULL; + if ((N = get_argument (argc, argv, "-set-mode"))) { + remove_argument (N, &argc, argv); + CATMODE = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + /* extra error messages */ VERBOSE = FALSE; @@ -41,5 +64,5 @@ } - if (argc == 2) { + if (argc == 3) { if (CONFIRM) { fprintf (stderr, "you are splitting the entire sky in one pass\n"); @@ -54,5 +77,5 @@ } - fprintf (stderr, "USAGE: dvosplit (newlevel) [-region (Rmin) (Rmax) (Dmin) (Dmax)]\n"); + fprintf (stderr, "USAGE: dvosplit (catdir) (newlevel) [-outdir outdir] [-region (Rmin) (Rmax) (Dmin) (Dmax)]\n"); exit (2); } @@ -61,8 +84,11 @@ fprintf (stderr, "USAGE\n"); - fprintf (stderr, " dvosplit (newlevel)\n\n"); + fprintf (stderr, " dvosplit (catdir) (newlevel)\n\n"); fprintf (stderr, " optional flags:\n"); + fprintf (stderr, " -outdir (outdir) : copy to a new location (does not copy Images.dat)\n"); fprintf (stderr, " -region ra ra dec dec : migrate catalogs in specified region\n"); fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -set-format (format) : set format of output catalog (or inherit from input)\n"); + fprintf (stderr, " -set-mode (mode) : set mode of output catalog (or inherit from input)\n"); fprintf (stderr, " -help : this list\n"); fprintf (stderr, " -h : this list\n\n"); Index: trunk/Ohana/src/dvosplit/src/dvosplit.c =================================================================== --- trunk/Ohana/src/dvosplit/src/dvosplit.c (revision 33655) +++ trunk/Ohana/src/dvosplit/src/dvosplit.c (revision 33656) @@ -1,5 +1,7 @@ # include "dvosplit.h" +// dvosplit (catdir) (outlevel) [-outdir (outcat)} [-region Rmin Rmax Dmin Dmax] // dvosplit (outlevel) [-region Rmin Rmax Dmin Dmax] +// (inherites the input catalog's format and mode, unless -set-format or -set-mode are used) int main (int argc, char **argv) { @@ -9,5 +11,5 @@ Catalog incatalog, *outcatalogs; AveLinks *avelinks; - char filename[256]; + char *filename, *CATDIR; SetSignals (); @@ -15,5 +17,34 @@ args (argc, argv); - OUT_DEPTH = atoi (argv[1]); + CATDIR = strcreate (argv[1]); + OUT_DEPTH = atoi (argv[2]); + if (!OUTDIR) { + OUTDIR = strcreate (CATDIR); + } + + // load the photcode table (for Nsecfilt and related) + char photcodeFile[1024]; + snprintf (photcodeFile, 1024, "%s/Photcodes.dat", CATDIR); + if (!LoadPhotcodes (photcodeFile, NULL, FALSE)) { + fprintf (stderr, "error loading photcode table %s\n", photcodeFile); + exit (1); + } + + // dvosplit can be run locally (no -outdir specified); otherwise, copy Photcode.dat, + // Images.dat, SkyTable.fits + int LocalCopy = !strcmp (CATDIR, OUTDIR); + + // if not local, save the photcode table (for Nsecfilt and related) + if (!LocalCopy) { + if (!check_dir_access (OUTDIR, VERBOSE)) { + fprintf (stderr, "failed to create output directory %s\n", OUTDIR); + exit (1); + } + snprintf (photcodeFile, 1024, "%s/Photcodes.dat", OUTDIR); + if (!SavePhotcodesFITS (photcodeFile)) { + fprintf (stderr, "error loading photcode table %s\n", photcodeFile); + exit (1); + } + } // load the sky table for the existing database @@ -25,9 +56,16 @@ for (i = 0; i < skylist[0].Nregions; i++) { - fprintf (stderr, "%s\n", skylist[0].regions[i][0].name); + if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name); + // if !LocalCopy, always copy the file // if (current level > out level) skip: cannot currently merge catalogs // if (current level == out level) skip: no action is needed - if (skylist[0].regions[i][0].depth >= OUT_DEPTH) continue; + + if (skylist[0].regions[i][0].depth > OUT_DEPTH) { + if (VERBOSE) fprintf (stderr, "WARNING, cannot merge deeper catalog %s (%d vs %d)\n", skylist[0].regions[i][0].name, skylist[0].regions[i][0].depth, OUT_DEPTH); + continue; + } + + if (LocalCopy && (skylist[0].regions[i][0].depth == OUT_DEPTH)) continue; // set the parameters which guide catalog open/load/create @@ -52,5 +90,10 @@ outlist = SkyListByPatch (sky, OUT_DEPTH, skylist[0].regions[i]); - outcatalogs = open_output_catalogs (outlist); + // Modify the outlist filenames to match the output directory. Note that the + // filenames are now owned by the list (and are not freed by SkyListFree) + SkyListSetFilenames (outlist, OUTDIR, "cpt"); + + // inherit the input catalog's format and mode (unless overridden) + outcatalogs = open_output_catalogs (outlist, incatalog.catformat, incatalog.catmode); avelinks = split_averages (&incatalog, outlist, outcatalogs); @@ -75,12 +118,27 @@ outlist[0].regions[j][0].table = TRUE; } + + // free the newly allocated filenames + for (j = 0; j < outlist[0].Nregions; j++) { + free (outlist[0].filename[j]); + } + + // free the rest of the list + SkyListFree (outlist); } - // save sky table copy - sprintf (filename, "%s/SkyTable.fits", CATDIR); + // save sky table copy (Local or not Local : the depth has changed) + filename = SkyTableFilename (OUTDIR); check_file_access (filename, TRUE, TRUE, VERBOSE); if (!SkyTableSave (sky, filename)) { - fprintf (stderr, "ERROR: failed to save sky table for %s\n", CATDIR); + fprintf (stderr, "ERROR: failed to save sky table for %s\n", OUTDIR); exit (1); + } + + if (!LocalCopy) { + // copy the images table + char line[2048]; + snprintf (line, 2048, "cp %s/Images.dat %s/Images.dat", CATDIR, OUTDIR); + system (line); } Index: trunk/Ohana/src/dvosplit/src/open_output_catalogs.c =================================================================== --- trunk/Ohana/src/dvosplit/src/open_output_catalogs.c (revision 33655) +++ trunk/Ohana/src/dvosplit/src/open_output_catalogs.c (revision 33656) @@ -1,5 +1,5 @@ # include "dvosplit.h" -Catalog *open_output_catalogs (SkyList *outlist) { +Catalog *open_output_catalogs (SkyList *outlist, int catformat, int catmode) { int i; @@ -15,6 +15,9 @@ outcatalogs[i].Nsecfilt = GetPhotcodeNsecfilt (); outcatalogs[i].catflags = LOAD_NONE; - outcatalogs[i].catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data - outcatalogs[i].catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + outcatalogs[i].catformat = CATFORMAT ? dvo_catalog_catformat (CATFORMAT) : catformat; // set the default catformat from config data + outcatalogs[i].catmode = CATMODE ? dvo_catalog_catmode (CATMODE) : catmode; // set the default catmode from config data + + // the user options -set-format and -set-mode assign values to the globals CATFORMAT + // and CATMODE (in args.c); otherwise the values from the input catalogs are inherited if (!dvo_catalog_open (&outcatalogs[i], outlist[0].regions[i], VERBOSE, "w")) { Index: trunk/Ohana/src/dvosplit/src/split_averages.c =================================================================== --- trunk/Ohana/src/dvosplit/src/split_averages.c (revision 33655) +++ trunk/Ohana/src/dvosplit/src/split_averages.c (revision 33656) @@ -26,11 +26,22 @@ // split out the average & secfilt entries: incatalog[0].catflags = LOAD_AVES | LOAD_SECF; - Nblocks = incatalog[0].Naves_disk / NROWS; - if (incatalog[0].Naves_disk % NROWS) Nblocks ++; + + if (incatalog[0].catmode == DVO_MODE_SPLIT) { + Nblocks = incatalog[0].Naves_disk / NROWS; + if (incatalog[0].Naves_disk % NROWS) Nblocks ++; + } else { + Nblocks = 1; + } + for (block = 0; block < Nblocks; block++) { - // read up to NROWS at a time - dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS); - fprintf (stderr, "splitting %s (averages) .. %d of %d\n", incatalog[0].filename, block, Nblocks); + if (incatalog[0].catmode == DVO_MODE_SPLIT) { + // read up to NROWS at a time + dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS); + fprintf (stderr, "splitting %s (averages) .. %d of %d\n", incatalog[0].filename, block, Nblocks); + } else { + dvo_catalog_load (incatalog, VERBOSE); + fprintf (stderr, "splitting %s (averages)\n", incatalog[0].filename); + } assert (block*NROWS == incatalog[0].Naves_off); @@ -84,4 +95,6 @@ // double check the values of Naverage, Nsecf_mem? + // XXX for output.catformat == MEF, we probably need to skip this stuff and the free below + // write out the new values for (cat = 0; cat < outlist[0].Nregions; cat++) { Index: trunk/Ohana/src/dvosplit/src/split_measures.c =================================================================== --- trunk/Ohana/src/dvosplit/src/split_measures.c (revision 33655) +++ trunk/Ohana/src/dvosplit/src/split_measures.c (revision 33656) @@ -20,11 +20,27 @@ // split out the measure entries: incatalog[0].catflags = LOAD_MEAS; - Nblocks = incatalog[0].Nmeas_disk / NROWS; - if (incatalog[0].Nmeas_disk % NROWS) Nblocks ++; + // if ((incatalog[0].catformat == DVO_FORMAT_ELIXIR) || (incatalog[0].catformat == DVO_FORMAT_LONEOS)) { + // // for these two formats, we need the average and secfilt values around until we do the measures... + // // XXX I am loading these 2x -- perhaps I can make the API smarter about reloading? + // incatalog[0].catflags |= LOAD_AVES | LOAD_SECF; + // } + + if (incatalog[0].catmode == DVO_MODE_SPLIT) { + Nblocks = incatalog[0].Nmeas_disk / NROWS; + if (incatalog[0].Nmeas_disk % NROWS) Nblocks ++; + } else { + Nblocks = 1; + } + for (block = 0; block < Nblocks; block++) { // read up to NROWS at a time - dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS); - fprintf (stderr, "splitting %s (measures) .. %d of %d\n", incatalog[0].filename, block, Nblocks); + if (incatalog[0].catmode == DVO_MODE_SPLIT) { + dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS); + fprintf (stderr, "splitting %s (measures) .. %d of %d\n", incatalog[0].filename, block, Nblocks); + } else { + dvo_catalog_load (incatalog, VERBOSE); + fprintf (stderr, "splitting %s (measures)\n", incatalog[0].filename); + } assert (block*NROWS == incatalog[0].Nmeas_off);