IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 33656


Ignore:
Timestamp:
Apr 1, 2012, 3:08:33 PM (14 years ago)
Author:
eugene
Message:

support to set output mode and format; output database need not be local

Location:
trunk/Ohana/src/dvosplit
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/Ohana/src/dvosplit/include/dvosplit.h

    r27435 r33656  
    2323
    2424int    VERBOSE;
    25 char   CATDIR[256];
    26 char   CATMODE[16];    /* raw, mef, split, mysql */
    27 char   CATFORMAT[16];  /* internal, elixir, loneos, panstarrs */
     25char  *OUTDIR;
     26char  *CATMODE;    /* raw, mef, split, mysql */
     27char  *CATFORMAT;  /* internal, elixir, loneos, panstarrs */
    2828
    2929SkyRegion UserPatch;  // used by MODE CAT
    … …  
    3838int        args                   PROTO((int argc, char **argv));
    3939
    40 Catalog   *open_output_catalogs   PROTO((SkyList *outlist));
     40Catalog   *open_output_catalogs   PROTO((SkyList *outlist, int catformat, int catmode));
    4141AveLinks  *split_averages         PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs));
    4242int        split_measures         PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs, AveLinks *avelinks));

  • trunk/Ohana/src/dvosplit/src/ConfigInit.c

    r25757 r33656  
    55  double ZERO_POINT;
    66  char *config, *file;
    7   char CatdirPhotcodeFile[256];
    8   char MasterPhotcodeFile[256];
    97
    108  /*** load configuration info ***/
    … …  
    1816  if (VERBOSE) fprintf (stderr, "loaded config file: %s\n", file);
    1917
    20   GetConfig (config, "CATDIR",                  "%s",  0, CATDIR);
    21   GetConfig (config, "PHOTCODE_FILE",           "%s",  0, MasterPhotcodeFile);
    22 
    23   ScanConfig (config, "CATMODE",                "%s",  0, CATMODE);
    24   ScanConfig (config, "CATFORMAT",              "%s",  0, CATFORMAT);
    25 
    26   /* default mode, format, if not specified */
    27   if (*CATMODE == 0) strcpy (CATMODE, "RAW");
    28   if (*CATFORMAT == 0) strcpy (CATFORMAT, "ELIXIR");
    29 
     18  // XXX this really needs to be deprecated.  It is always 25.0, and should just be hard-wired
     19  // (the purpose of this value is now completely pointless...)
    3020  GetConfig (config, "ZERO_PT",                "%lf", 0, &ZERO_POINT);
    3121  SetZeroPoint (ZERO_POINT);
    32 
    33   /* XXX this does not yet write out the master photcode table */
    34   sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR);
    35   if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) {
    36     fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile);
    37     exit (1);
    38   }
    3922
    4023  free (config);

  • trunk/Ohana/src/dvosplit/src/args.c

    r15743 r33656  
    3434  }
    3535
     36  OUTDIR = NULL;
     37  if ((N = get_argument (argc, argv, "-outdir"))) {
     38    remove_argument (N, &argc, argv);
     39    OUTDIR = strcreate (argv[N]);
     40    remove_argument (N, &argc, argv);
     41  }
     42
     43  // override input catalog format (PS1_V1, PS1_REF, etc)
     44  CATFORMAT = NULL;
     45  if ((N = get_argument (argc, argv, "-set-format"))) {
     46    remove_argument (N, &argc, argv);
     47    CATFORMAT = strcreate (argv[N]);
     48    remove_argument (N, &argc, argv);
     49  }
     50
     51  // override input catalog mode (raw, mef, split, mysql)
     52  CATMODE = NULL;
     53  if ((N = get_argument (argc, argv, "-set-mode"))) {
     54    remove_argument (N, &argc, argv);
     55    CATMODE = strcreate (argv[N]);
     56    remove_argument (N, &argc, argv);
     57  }
     58
    3659  /* extra error messages */
    3760  VERBOSE = FALSE;
    … …  
    4164  }
    4265
    43   if (argc == 2) {
     66  if (argc == 3) {
    4467    if (CONFIRM) {
    4568      fprintf (stderr, "you are splitting the entire sky in one pass\n");
    … …  
    5477  }
    5578
    56   fprintf (stderr, "USAGE: dvosplit (newlevel) [-region (Rmin) (Rmax) (Dmin) (Dmax)]\n");
     79  fprintf (stderr, "USAGE: dvosplit (catdir) (newlevel) [-outdir outdir] [-region (Rmin) (Rmax) (Dmin) (Dmax)]\n");
    5780  exit (2);
    5881}
    … …  
    6184
    6285  fprintf (stderr, "USAGE\n");
    63   fprintf (stderr, "  dvosplit (newlevel)\n\n");
     86  fprintf (stderr, "  dvosplit (catdir) (newlevel)\n\n");
    6487  fprintf (stderr, "  optional flags:\n");
     88  fprintf (stderr, "  -outdir (outdir)            : copy to a new location (does not copy Images.dat)\n");
    6589  fprintf (stderr, "  -region ra ra dec dec       : migrate catalogs in specified region\n");
    6690  fprintf (stderr, "  -v                          : verbose mode\n");
     91  fprintf (stderr, "  -set-format (format)        : set format of output catalog (or inherit from input)\n");
     92  fprintf (stderr, "  -set-mode (mode)            : set mode of output catalog (or inherit from input)\n");
    6793  fprintf (stderr, "  -help                       : this list\n");
    6894  fprintf (stderr, "  -h                          : this list\n\n");

  • trunk/Ohana/src/dvosplit/src/dvosplit.c

    r25757 r33656  
    11# include "dvosplit.h"
    22
     3// dvosplit (catdir) (outlevel) [-outdir (outcat)} [-region Rmin Rmax Dmin Dmax]
    34// dvosplit (outlevel) [-region Rmin Rmax Dmin Dmax]
     5// (inherites the input catalog's format and mode, unless -set-format or -set-mode are used)
    46int main (int argc, char **argv) {
    57
    … …  
    911  Catalog incatalog, *outcatalogs;
    1012  AveLinks *avelinks;
    11   char filename[256];
     13  char *filename, *CATDIR;
    1214
    1315  SetSignals ();
    … …  
    1517  args (argc, argv);
    1618
    17   OUT_DEPTH = atoi (argv[1]);
     19  CATDIR = strcreate (argv[1]);
     20  OUT_DEPTH = atoi (argv[2]);
     21  if (!OUTDIR) {
     22    OUTDIR = strcreate (CATDIR);
     23  }
     24
     25  // load the photcode table (for Nsecfilt and related)
     26  char photcodeFile[1024];
     27  snprintf (photcodeFile, 1024, "%s/Photcodes.dat", CATDIR);
     28  if (!LoadPhotcodes (photcodeFile, NULL, FALSE)) {
     29    fprintf (stderr, "error loading photcode table %s\n", photcodeFile);
     30    exit (1);
     31  }
     32
     33  // dvosplit can be run locally (no -outdir specified); otherwise, copy Photcode.dat,
     34  // Images.dat, SkyTable.fits
     35  int LocalCopy = !strcmp (CATDIR, OUTDIR);
     36
     37  // if not local, save the photcode table (for Nsecfilt and related)
     38  if (!LocalCopy) {
     39    if (!check_dir_access (OUTDIR, VERBOSE)) {
     40      fprintf (stderr, "failed to create output directory %s\n", OUTDIR);
     41      exit (1);
     42    }
     43    snprintf (photcodeFile, 1024, "%s/Photcodes.dat", OUTDIR);
     44    if (!SavePhotcodesFITS (photcodeFile)) {
     45      fprintf (stderr, "error loading photcode table %s\n", photcodeFile);
     46      exit (1);
     47    }
     48  }
    1849
    1950  // load the sky table for the existing database
    … …  
    2556 
    2657  for (i = 0; i < skylist[0].Nregions; i++) {
    27     fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
     58    if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
    2859
     60    // if !LocalCopy, always copy the file
    2961    // if (current level >  out level) skip: cannot currently merge catalogs
    3062    // if (current level == out level) skip: no action is needed
    31     if (skylist[0].regions[i][0].depth >= OUT_DEPTH) continue;
     63
     64    if (skylist[0].regions[i][0].depth > OUT_DEPTH) {
     65      if (VERBOSE) fprintf (stderr, "WARNING, cannot merge deeper catalog %s (%d vs %d)\n", skylist[0].regions[i][0].name, skylist[0].regions[i][0].depth, OUT_DEPTH);
     66      continue;
     67    }
     68
     69    if (LocalCopy && (skylist[0].regions[i][0].depth == OUT_DEPTH)) continue;
    3270
    3371    // set the parameters which guide catalog open/load/create
    … …  
    5290    outlist = SkyListByPatch (sky, OUT_DEPTH, skylist[0].regions[i]);
    5391
    54     outcatalogs = open_output_catalogs (outlist);
     92    // Modify the outlist filenames to match the output directory.  Note that the
     93    // filenames are now owned by the list (and are not freed by SkyListFree)
     94    SkyListSetFilenames (outlist, OUTDIR, "cpt");
     95
     96    // inherit the input catalog's format and mode (unless overridden)
     97    outcatalogs = open_output_catalogs (outlist, incatalog.catformat, incatalog.catmode);
    5598
    5699    avelinks = split_averages (&incatalog, outlist, outcatalogs);
    … …  
    75118      outlist[0].regions[j][0].table = TRUE;
    76119    }
     120
     121    // free the newly allocated filenames
     122    for (j = 0; j < outlist[0].Nregions; j++) {
     123      free (outlist[0].filename[j]);
     124    }
     125
     126    // free the rest of the list
     127    SkyListFree (outlist);
    77128  }
    78129
    79   // save sky table copy
    80   sprintf (filename, "%s/SkyTable.fits", CATDIR);
     130  // save sky table copy (Local or not Local : the depth has changed)
     131  filename = SkyTableFilename (OUTDIR);
    81132  check_file_access (filename, TRUE, TRUE, VERBOSE);
    82133  if (!SkyTableSave (sky, filename)) {
    83     fprintf (stderr, "ERROR: failed to save sky table for %s\n", CATDIR);
     134    fprintf (stderr, "ERROR: failed to save sky table for %s\n", OUTDIR);
    84135    exit (1);
     136  }
     137
     138  if (!LocalCopy) {
     139    // copy the images table
     140    char line[2048];
     141    snprintf (line, 2048, "cp %s/Images.dat %s/Images.dat", CATDIR, OUTDIR);
     142    system (line);
    85143  }
    86144

  • trunk/Ohana/src/dvosplit/src/open_output_catalogs.c

    r15743 r33656  
    11# include "dvosplit.h"
    22
    3 Catalog *open_output_catalogs (SkyList *outlist) {
     3Catalog *open_output_catalogs (SkyList *outlist, int catformat, int catmode) {
    44
    55  int i;
    … …  
    1515    outcatalogs[i].Nsecfilt  = GetPhotcodeNsecfilt ();
    1616    outcatalogs[i].catflags  = LOAD_NONE;
    17     outcatalogs[i].catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
    18     outcatalogs[i].catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
     17    outcatalogs[i].catformat = CATFORMAT ? dvo_catalog_catformat (CATFORMAT) : catformat;  // set the default catformat from config data
     18    outcatalogs[i].catmode   = CATMODE   ? dvo_catalog_catmode (CATMODE)     : catmode;    // set the default catmode from config data
     19
     20    // the user options -set-format and -set-mode assign values to the globals CATFORMAT
     21    // and CATMODE (in args.c); otherwise the values from the input catalogs are inherited
    1922
    2023    if (!dvo_catalog_open (&outcatalogs[i], outlist[0].regions[i], VERBOSE, "w")) {

  • trunk/Ohana/src/dvosplit/src/split_averages.c

    r15746 r33656  
    2626  // split out the average & secfilt entries:
    2727  incatalog[0].catflags = LOAD_AVES | LOAD_SECF;
    28   Nblocks = incatalog[0].Naves_disk / NROWS;
    29   if (incatalog[0].Naves_disk % NROWS) Nblocks ++;
     28
     29  if (incatalog[0].catmode == DVO_MODE_SPLIT) {
     30    Nblocks = incatalog[0].Naves_disk / NROWS;
     31    if (incatalog[0].Naves_disk % NROWS) Nblocks ++;
     32  } else {
     33    Nblocks = 1;
     34  }
     35
    3036  for (block = 0; block < Nblocks; block++) {
    3137
    32     // read up to NROWS at a time
    33     dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS);
    34     fprintf (stderr, "splitting %s (averages) .. %d of %d\n", incatalog[0].filename, block, Nblocks);
     38    if (incatalog[0].catmode == DVO_MODE_SPLIT) {
     39      // read up to NROWS at a time
     40      dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS);
     41      fprintf (stderr, "splitting %s (averages) .. %d of %d\n", incatalog[0].filename, block, Nblocks);
     42    } else {
     43      dvo_catalog_load (incatalog, VERBOSE);
     44      fprintf (stderr, "splitting %s (averages)\n", incatalog[0].filename);
     45    }
    3546
    3647    assert (block*NROWS == incatalog[0].Naves_off);
    … …  
    8495    // double check the values of Naverage, Nsecf_mem?
    8596
     97    // XXX for output.catformat == MEF, we probably need to skip this stuff and the free below
     98
    8699    // write out the new values
    87100    for (cat = 0; cat < outlist[0].Nregions; cat++) {

  • trunk/Ohana/src/dvosplit/src/split_measures.c

    r15746 r33656  
    2020  // split out the measure entries:
    2121  incatalog[0].catflags = LOAD_MEAS;
    22   Nblocks = incatalog[0].Nmeas_disk / NROWS;
    23   if (incatalog[0].Nmeas_disk % NROWS) Nblocks ++;
     22  // if ((incatalog[0].catformat == DVO_FORMAT_ELIXIR) || (incatalog[0].catformat == DVO_FORMAT_LONEOS)) {
     23  //   // for these two formats, we need the average and secfilt values around until we do the measures...
     24  //   // XXX I am loading these 2x -- perhaps I can make the API smarter about reloading?
     25  //   incatalog[0].catflags |= LOAD_AVES | LOAD_SECF;
     26  // }
     27
     28  if (incatalog[0].catmode == DVO_MODE_SPLIT) {
     29    Nblocks = incatalog[0].Nmeas_disk / NROWS;
     30    if (incatalog[0].Nmeas_disk % NROWS) Nblocks ++;
     31  } else {
     32    Nblocks = 1;
     33  }
     34
    2435  for (block = 0; block < Nblocks; block++) {
    2536
    2637    // read up to NROWS at a time
    27     dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS);
    28     fprintf (stderr, "splitting %s (measures) .. %d of %d\n", incatalog[0].filename, block, Nblocks);
     38    if (incatalog[0].catmode == DVO_MODE_SPLIT) {
     39      dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS);
     40      fprintf (stderr, "splitting %s (measures) .. %d of %d\n", incatalog[0].filename, block, Nblocks);
     41    } else {
     42      dvo_catalog_load (incatalog, VERBOSE);
     43      fprintf (stderr, "splitting %s (measures)\n", incatalog[0].filename);
     44    }
    2945
    3046    assert (block*NROWS == incatalog[0].Nmeas_off);
Note: See TracChangeset for help on using the changeset viewer.