IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 39301 for trunk


Ignore:
Timestamp:
Jan 4, 2016, 1:12:14 PM (11 years ago)
Author:
eugene
Message:

changes to dvomerge to handle a cpt list restriction

Location:
trunk/Ohana/src
Files:
1 added
10 edited

Legend:

Unmodified
Added
Removed

  • trunk/Ohana/src/dvomerge/Makefile

    r39300 r39301  
    2525dvoutils        : $(BIN)/dvoutils.$(ARCH)
    2626
    27 # all: dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client dvoutils
    28 all: dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client dvoutils
    29 
    30 #  $(SRC)/dvomergeContinue.$(ARCH).o
     27all: dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client dvoutils
    3128
    3229DVOMERGE = \
    … …  
    3734$(SRC)/dvomergeUpdate_threaded.$(ARCH).o \
    3835$(SRC)/dvomergeHistory.$(ARCH).o \
    39 $(SRC)/dvomergeFromList.$(ARCH).o \
    4036$(SRC)/dvomergeImageIDs.$(ARCH).o \
    4137$(SRC)/dvo_image_merge_dbs.$(ARCH).o \
    … …  
    4743$(SRC)/args.$(ARCH).o \
    4844$(SRC)/psps_ids.$(ARCH).o \
     45$(SRC)/cptlist.$(ARCH).o \
    4946$(SRC)/SkyTablePopulatedRange.$(ARCH).o \
    5047$(SRC)/LoadCatalog.$(ARCH).o \
    … …  
    7269$(SRC)/args.$(ARCH).o \
    7370$(SRC)/psps_ids.$(ARCH).o \
     71$(SRC)/cptlist.$(ARCH).o \
    7472$(SRC)/SkyTablePopulatedRange.$(ARCH).o \
    7573$(SRC)/LoadCatalog.$(ARCH).o \
    … …  
    9189$(SRC)/help.$(ARCH).o \
    9290$(SRC)/args.$(ARCH).o \
     91$(SRC)/cptlist.$(ARCH).o \
    9392$(SRC)/SkyTablePopulatedRange.$(ARCH).o \
    9493$(SRC)/LoadCatalog.$(ARCH).o \
    … …  
    105104$(SRC)/Shutdown.$(ARCH).o \
    106105$(SRC)/help.$(ARCH).o \
     106$(SRC)/cptlist.$(ARCH).o \
    107107$(SRC)/args.$(ARCH).o
    108108
    … …  
    117117$(SRC)/Shutdown.$(ARCH).o \
    118118$(SRC)/help.$(ARCH).o \
     119$(SRC)/cptlist.$(ARCH).o \
    119120$(SRC)/args.$(ARCH).o
    120121

  • trunk/Ohana/src/dvomerge/include/dvomerge.h

    r39300 r39301  
    4545char  *UPDATE_CATFORMAT;
    4646char  *UPDATE_CATCOMPRESS;
    47 char  *RESTRICT_CPT;
     47
     48char  *CPTLIST_FILENAME;
     49char **CPTLIST;
     50int   NCPTLIST;
    4851
    4952int    MATCH_BY_EXTERN_ID;
    … …  
    208211int        dvomergeFromList       PROTO((int argc, char **argv));
    209212int        dvomergeUpdate_threaded PROTO((int argc, char **argv));
    210 int        dvomergeUpdate_catalogs PROTO((char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap));
     213int        dvomergeUpdate_catalogs PROTO((char *input, char *output, SkyList *inlist, SkyTable *outsky, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap));
    211214
    212215int        dvo_image_match_dbs_by_time_and_photcode PROTO((IDmapType *IDmap, FITS_DB *tgt, FITS_DB *src));
    … …  
    262265int dvorepairFixStackIDs (int argc, char **argv);
    263266
     267char **load_cptlist (char *filename, int *nlist);

  • trunk/Ohana/src/dvomerge/src/args.c

    r39300 r39301  
    7777  }
    7878
    79   RESTRICT_CPT = NULL;
     79  NCPTLIST = 0;
     80  CPTLIST = NULL;
     81  CPTLIST_FILENAME = NULL;
    8082  if ((N = get_argument (*argc, argv, "-restrict-cpt"))) {
    8183    remove_argument (N, argc, argv);
    82     RESTRICT_CPT = strcreate (argv[N]);
     84    CPTLIST_FILENAME = strcreate (argv[N]);
     85    CPTLIST = load_cptlist (CPTLIST_FILENAME, &NCPTLIST);
    8386    remove_argument (N, argc, argv);
    8487  }
    … …  
    122125    NTHREADS = MAX(0, atoi(argv[N]));
    123126    remove_argument (N, argc, argv);
     127    myAbort ("threads deprecated for now");
    124128  }
    125129
    … …  
    266270  }
    267271
    268   RESTRICT_CPT = NULL;
     272  NCPTLIST = 0;
     273  CPTLIST = NULL;
    269274  if ((N = get_argument (*argc, argv, "-restrict-cpt"))) {
    270275    remove_argument (N, argc, argv);
    271     RESTRICT_CPT = strcreate (argv[N]);
     276    CPTLIST = load_cptlist (argv[N], &NCPTLIST);
    272277    remove_argument (N, argc, argv);
    273278  }
    … …  
    311316    NTHREADS = MAX(0, atoi(argv[N]));
    312317    remove_argument (N, argc, argv);
     318    myAbort ("threads deprecated for now");
    313319  }
    314320

  • trunk/Ohana/src/dvomerge/src/dvomerge.c

    r38553 r39301  
    1111
    1212  if (argc == 6) {
    13     if (!strcasecmp (argv[2], "and")) {
    14       if (VERIFY) {
    15         fprintf (stderr, "WARNING / ERROR : dvomerge (input1) and (input2) into (output) : VERIFY mode not implemented\n");
    16         exit (3);
    17       }
    18       if (IMAGES_ONLY) {
    19         fprintf (stderr, "ERROR : dvomerge (input1) and (input2) into (output) : not compatible with '-images-only'\n");
    20         exit (5);
    21       }
    22       dvomergeCreate (argc, argv);
    23     } else {
    24       if (VERIFY) {
    25         fprintf (stderr, "WARNING / ERROR : dvomerge (input) into (output) from (list) : VERIFY mode not implemented\n");
    26         exit (3);
    27       }
    28       if (IMAGES_ONLY) {
    29         fprintf (stderr, "ERROR : dvomerge (input) input (output) from (list) : not compatible with '-images-only'\n");
    30         exit (5);
    31       }
    32       dvomergeFromList (argc, argv);
     13    if (strcasecmp (argv[2], "and")) dvomerge_usage();
     14    if (VERIFY) {
     15      fprintf (stderr, "WARNING / ERROR : dvomerge (input1) and (input2) into (output) : VERIFY mode not implemented\n");
     16      exit (3);
    3317    }
     18    if (IMAGES_ONLY) {
     19      fprintf (stderr, "ERROR : dvomerge (input1) and (input2) into (output) : not compatible with '-images-only'\n");
     20      exit (5);
     21    }
     22    dvomergeCreate (argc, argv);
    3423  }
     24
    3525  if ((argc == 4) || (argc == 5)) {
    3626    if (NTHREADS) {
    … …  
    4030    }
    4131  }
     32
    4233  dvomerge_args_free ();
    4334  ohana_memcheck (TRUE);
    … …  
    5344   we also have varients on Update:
    5445   Continue  : dvomerge (in) into (out) continue -- merge only unmerged tables into the existing db
    55    From List : dvomerge (in) into (out) from (list) -- merge only specified tables into the existing db
    5646
    5747   neither of the 2 above modes update the image table

  • trunk/Ohana/src/dvomerge/src/dvomergeFromList.c

    r39300 r39301  
    11# include "dvomerge.h"
    2 
    3 char **load_cptlist (char *filename, int *nlist);
    42
    53// merge from list implies 'continue' (ie, we must have already done a partial merge)
    … …  
    2523  listname = argv[5];
    2624
     25  int Ncptlist;
    2726  char *cptlist = load_cptlist (listname, &Ncptlist);
    2827
    … …  
    8988
    9089  // loop over the populated input regions
    91   for (i = 0; i < Nlist; i++) {
    92     if (VERBOSE) fprintf (stderr, "input: %s\n", list[i]);
     90  for (i = 0; i < Ncptlist; i++) {
     91    if (VERBOSE) fprintf (stderr, "input: %s\n", cptlist[i]);
    9392
    94     sprintf (inputfile,  "%s/%s", input,  list[i]);
    95     sprintf (outputfile, "%s/%s", output, list[i]);
     93    sprintf (inputfile,  "%s/%s", input,  cptlist[i]);
     94    sprintf (outputfile, "%s/%s", output, cptlist[i]);
    9695
    9796    OutputStatus *outstat = OutputStatusInit (1);
    … …  
    161160}
    162161
    163 char **load_cptlist (char *filename, int *nlist) {
    164 
    165   int NLIST = 100;
    166  
    167   char **list = NULL;
    168   ALLOCATE(list, char *, NLIST);
    169 
    170   int i;
    171   for (i = 0; i < NLIST; i++) {
    172     ALLOCATE(list[i], char, DVO_MAX_PATH);
    173     memset(list[i], 0, DVO_MAX_PATH);
    174   }
    175  
    176   FILE *listfile = fopen(filename, "r");
    177   myAssert(listfile, "error opening list");
    178 
    179   for (i = 0; fscanf (listfile, "%s", list[i]) != EOF; i++) {
    180     if (i == NLIST - 1) {
    181       NLIST += 100;
    182       REALLOCATE(list, char *, NLIST);
    183       for (j = i + 1; j < NLIST; j++) {
    184         ALLOCATE(list[j], char, DVO_MAX_PATH);
    185         memset(list[j], 0, DVO_MAX_PATH);
    186       }
    187     }
    188   }     
    189   *nlist = i;
    190   fclose(listfile);
    191 
    192   return list;
    193 }

  • trunk/Ohana/src/dvomerge/src/dvomergeUpdate.c

    r39300 r39301  
    9696  SkyTableSetFilenames (insky, input, "cpt");
    9797
    98   // XXX apply this...generate the subset matching the user-selected region
     98  // generate the subset matching the user-selected region
    9999  inlist = SkyListByPatch (insky, -1, &UserPatch);
    100100
    101101  // modify the list if we are restricting:
    102   inlist = SkyListSubset (inlist, cptlist);
     102  if (CPTLIST) {
     103    inlist = SkyListMatchList (inlist, CPTLIST, NCPTLIST);
     104  }
    103105
    104106  // generate an output table populated at the desired depth
    … …  
    117119  SetPhotcodeTable(NULL);
    118120
    119   int status = dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, &IDmap, secfiltMap);
     121  int status = dvomergeUpdate_catalogs (input, output, inlist, outsky, NsecfiltInput, NsecfiltOutput, &IDmap, secfiltMap);
    120122
    121123  // save the output sky table copy

  • trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c

    r39299 r39301  
    1212int dvomergeUpdate_parallel_group (HostTableGroup *group, char *absinput, char *absoutput);
    1313
    14 int dvomergeUpdate_catalogs (char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap) {
     14int dvomergeUpdate_catalogs (char *input, char *output, SkyList *inlist, SkyTable *outsky, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap) {
    1515
    1616  off_t i, j;
    … …  
    7676      outlist = SkyListByBounds (outsky, -1, inlist[0].regions[i][0].Rmin - dPos, inlist[0].regions[i][0].Rmax + dPos, inlist[0].regions[i][0].Dmin - dPos, inlist[0].regions[i][0].Dmax + dPos);
    7777    }
     78
     79    if (CPTLIST) {
     80      outlist = SkyListMatchList (outlist, CPTLIST, NCPTLIST);
     81    }
     82   
     83    // XXX validate MATCHED_TABLES
     84    myAssert (outlist->Nregions == inlist->Nregions, "oops");
    7885
    7986    OutputStatus *outstat = OutputStatusInit (outlist->Nregions);
    … …  
    303310    if (UPDATE_CATFORMAT)    { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); }
    304311    if (UPDATE_CATCOMPRESS)  { strextend (&command, "-update-catcompress %s", UPDATE_CATCOMPRESS); }
     312    if (CPTLIST_FILENAME)    { strextend (&command, "-restrict-cpt %s", CPTLIST_FILENAME); }
    305313    if (REPAIR_BY_OBJID)     { strextend (&command, "-repair-by-objid"); }
    306314

  • trunk/Ohana/src/dvomerge/src/dvomerge_client.c

    r38986 r39301  
    7575  inlist = SkyListByPatch (insky, -1, &UserPatch);
    7676
     77  // modify the list if we are restricting:
     78  if (CPTLIST) {
     79    inlist = SkyListMatchList (inlist, CPTLIST, NCPTLIST);
     80  }
     81
    7782  // generate an output table populated at the desired depth
    7883  outsky = SkyTableLoadOptimal (output, NULL, GSCFILE, TRUE, SKY_DEPTH, VERBOSE);
    … …  
    8691  if (REPLACE_TYCHO) replace_tycho_init();
    8792
    88   dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, IDmap, secfiltMap);
     93  dvomergeUpdate_catalogs (input, output, inlist, outsky, NsecfiltInput, NsecfiltOutput, IDmap, secfiltMap);
    8994
    9095  exit (0);

  • trunk/Ohana/src/libdvo/include/dvo.h

    r39293 r39301  
    10711071SkyTable  *SkyTableLoadOptimal     PROTO((char *catdir, char *SKYFILE, char *GSCFILE, int readwrite, int depth, int VERBOSE));
    10721072int        SkyTableSetDepth        PROTO((SkyTable *sky, int depth));
     1073SkyList   *SkyListMatchList        PROTO((SkyList *inlist, char **cptlist, int Ncptlist));
    10731074SkyList   *SkyRegionByIndex        PROTO((SkyTable *table, int index));
    10741075SkyList   *SkyRegionByCPT          PROTO((SkyTable *table, char *filename));

  • trunk/Ohana/src/libdvo/src/skyregion_ops.c

    r38553 r39301  
    154154    }
    155155  }
     156  list[0].Nregions = N;
     157  return (list);
     158}
     159
     160/* find regions at all levels which match list of names */
     161SkyList *SkyListMatchList (SkyList *inlist, char **cptlist, int Ncptlist) {
     162
     163  int i, j;
     164  SkyList *list;
     165 
     166  int N = 0;
     167  int NREGIONS = 10;
     168  ALLOCATE (list, SkyList, 1);
     169  ALLOCATE (list[0].regions, SkyRegion *, NREGIONS);
     170  ALLOCATE (list[0].filename, char *, NREGIONS);
     171  list[0].Nregions = N;
     172  list[0].ownElements = FALSE; // this list is only holding a view to the elements
     173  strcpy (list[0].hosts, inlist[0].hosts);
     174
     175  SkyRegion **regions = inlist[0].regions;
     176
     177  for (i = 0; i < inlist[0].Nregions; i++) {
     178    int found = FALSE;
     179    for (j = 0; !found && (j < Ncptlist); j++) {
     180      int Nchar = strlen (cptlist[j]);
     181      if (strncasecmp (regions[i][0].name, cptlist[j], Nchar)) continue;
     182
     183      list[0].regions[N] = regions[i];
     184      list[0].filename[N] = inlist[0].filename[i];
     185      N++;
     186      if (N >= NREGIONS) {
     187        NREGIONS += 10;
     188        REALLOCATE (list[0].regions, SkyRegion *, NREGIONS);
     189        REALLOCATE (list[0].filename, char *, NREGIONS);
     190      }
     191      found = TRUE;
     192    }
     193    // we do not care if one of the regions does not match.  do we care if any of the
     194    // cptfile do not match?
     195  }
     196  SkyListFree (inlist);
     197
    156198  list[0].Nregions = N;
    157199  return (list);
Note: See TracChangeset for help on using the changeset viewer.