IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 30, 2012, 1:46:12 PM (14 years ago)
Author:
eugene
Message:

merge changes from eam_branches/ipp-20120405

Location:
trunk
Files:
13 edited
9 copied

Legend:

Unmodified
Added
Removed

  • trunk

  • trunk/Ohana/src/dvomerge/Makefile

    r31496 r33963  
    11default: all
    22help:
    3         @echo "make options: dvomerge (default)"
     3        @echo "make options: dvomerge dvomerge_client dvoconvert (default)"
    44
    55include ../../Makefile.System
    … …  
    1717FULL_LDFLAGS  = -lkapa -ldvo -lFITS -lohana $(BASE_LDFLAGS)
    1818
    19 dvomerge     : $(BIN)/dvomerge.$(ARCH)
    20 dvomerge     : $(BIN)/dvoconvert.$(ARCH)
     19dvomerge        : $(BIN)/dvomerge.$(ARCH)
     20dvomerge_client : $(BIN)/dvomerge_client.$(ARCH)
     21dvoconvert      : $(BIN)/dvoconvert.$(ARCH)
    2122
    22 all: dvomerge dvoconvert dvosecfilt
     23all: dvomerge dvomerge_client dvoconvert dvosecfilt
    2324
    2425#  $(SRC)/dvomergeContinue.$(ARCH).o
    … …  
    2829$(SRC)/dvomergeUpdate.$(ARCH).o \
    2930$(SRC)/dvomergeCreate.$(ARCH).o \
     31$(SRC)/dvomergeUpdate_catalogs.$(ARCH).o \
    3032$(SRC)/dvomergeUpdate_threaded.$(ARCH).o \
    3133$(SRC)/dvomergeFromList.$(ARCH).o \
    3234$(SRC)/dvomergeImageIDs.$(ARCH).o \
    3335$(SRC)/dvo_image_merge_dbs.$(ARCH).o \
     36$(SRC)/IDmapIO.$(ARCH).o \
    3437$(SRC)/SetSignals.$(ARCH).o \
    3538$(SRC)/ConfigInit.$(ARCH).o \
    … …  
    4144$(SRC)/LoadCatalog.$(ARCH).o \
    4245$(SRC)/build_links.$(ARCH).o \
     46$(SRC)/replace_match.$(ARCH).o \
    4347$(SRC)/merge_catalogs_new.$(ARCH).o \
    4448$(SRC)/merge_catalogs_old.$(ARCH).o
    … …  
    4751
    4852$(BIN)/dvomerge.$(ARCH) : $(DVOMERGE)
     53
     54DVOMERGE_CLIENT = \
     55$(SRC)/dvomerge_client.$(ARCH).o \
     56$(SRC)/dvomergeUpdate_catalogs.$(ARCH).o \
     57$(SRC)/dvomergeImageIDs.$(ARCH).o \
     58$(SRC)/dvo_image_merge_dbs.$(ARCH).o \
     59$(SRC)/IDmapIO.$(ARCH).o \
     60$(SRC)/SetSignals.$(ARCH).o \
     61$(SRC)/ConfigInit.$(ARCH).o \
     62$(SRC)/Shutdown.$(ARCH).o \
     63$(SRC)/help.$(ARCH).o \
     64$(SRC)/args.$(ARCH).o \
     65$(SRC)/psps_ids.$(ARCH).o \
     66$(SRC)/SkyTablePopulatedRange.$(ARCH).o \
     67$(SRC)/LoadCatalog.$(ARCH).o \
     68$(SRC)/build_links.$(ARCH).o \
     69$(SRC)/replace_match.$(ARCH).o \
     70$(SRC)/merge_catalogs_new.$(ARCH).o \
     71$(SRC)/merge_catalogs_old.$(ARCH).o
     72
     73$(DVOMERGE_CLIENT) : $(INC)/dvomerge.h
     74
     75$(BIN)/dvomerge_client.$(ARCH) : $(DVOMERGE_CLIENT)
    4976
    5077DVOCONVERT = \
    … …  
    99126$(BIN)/dvoverify.$(ARCH) : $(DVOVERIFY)
    100127
    101 INSTALL = dvomerge dvoconvert dvosecfilt dvorepair dvoverify
     128INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvorepair dvoverify
    102129
    103130# dependancy rules for binary code #########################

  • trunk/Ohana/src/dvomerge/include/dvomerge.h

    r33657 r33963  
    1717# include <glob.h>
    1818
     19# define DVO_MAX_PATH 1024
     20
     21int    PARALLEL;
     22int    PARALLEL_MANUAL;
     23int    PARALLEL_SERIAL;
     24
     25int    HOST_ID;
     26char  *HOSTDIR;
     27
     28
    1929int    VERBOSE;
    2030char   CATDIR[256];
    … …  
    2636int    SKY_DEPTH;
    2737int    NTHREADS;
    28 char   *ALTERNATE_PHOTCODE_FILE;
     38char  *ALTERNATE_PHOTCODE_FILE;
     39int    REPLACE_BY_PHOTCODE;
    2940
    3041SkyRegion UserPatch;  // used by MODE CAT
    3142
    3243typedef struct {
    33   off_t Nmap;
    34   off_t *old;
    35   off_t *new;
     44  unsigned int Nmap;
     45  unsigned int *old;
     46  unsigned int *new;
    3647} IDmapType;
    3748
    … …  
    5162void       dvomerge_help          PROTO((int argc, char **argv));
    5263int        dvomerge_args          PROTO((int *argc, char **argv));
     64
     65void       dvomerge_client_usage  PROTO((void));
     66void       dvomerge_client_help   PROTO((int argc, char **argv));
     67int        dvomerge_client_args   PROTO((int *argc, char **argv));
    5368
    5469void       dvoconvert_usage       PROTO((void));
    … …  
    110125int        dvomergeFromList       PROTO((int argc, char **argv));
    111126int        dvomergeUpdate_threaded PROTO((int argc, char **argv));
     127int        dvomergeUpdate_catalogs PROTO((char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap));
     128
     129int        replace_match           PROTO((Average *average_out, Measure *measure_out, Average *average_in, Measure *measure_in));
     130
     131int        IDmapSave               PROTO((char *filename, IDmapType *IDmap));
     132IDmapType *IDmapLoad               PROTO((char *filename));

  • trunk/Ohana/src/dvomerge/src/args.c

    r33657 r33963  
    1515    remove_argument (N, argc, argv);
    1616    ALTERNATE_PHOTCODE_FILE = strcreate(argv[N]);
     17    remove_argument (N, argc, argv);
     18  }
     19
     20  /* replace measurement, don't duplicate */
     21  REPLACE_BY_PHOTCODE = FALSE;
     22  if ((N = get_argument (*argc, argv, "-replace"))) {
     23    REPLACE_BY_PHOTCODE = TRUE;
    1724    remove_argument (N, argc, argv);
    1825  }
    … …  
    4350  }
    4451
     52  // XXX for the moment, make this selection manual.  it needs to be automatic
     53  // based on the state of the SkyTable
     54  PARALLEL = FALSE;
     55  if ((N = get_argument (*argc, argv, "-parallel"))) {
     56    PARALLEL = TRUE;
     57    remove_argument (N, argc, argv);
     58  }
     59  // this is a test mode : rather than launching the remote jobs and waiting for completion,
     60  // relphot will simply list the remote command and wait for the user to signal completion
     61  PARALLEL_MANUAL = FALSE;
     62  if ((N = get_argument (*argc, argv, "-parallel-manual"))) {
     63    PARALLEL = TRUE; // -parallel-manual implies -parallel
     64    PARALLEL_MANUAL = TRUE;
     65    remove_argument (N, argc, argv);
     66  }
     67  // this is a test mode : rather than launching the relphot_client jobs remotely, they are
     68  // run in serial via 'system'
     69  PARALLEL_SERIAL = FALSE;
     70  if ((N = get_argument (*argc, argv, "-parallel-serial"))) {
     71    if (PARALLEL_MANUAL) {
     72      fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n");
     73      exit (1);
     74    }
     75    PARALLEL = TRUE; // -parallel-serial implies -parallel
     76    PARALLEL_SERIAL = TRUE;
     77    remove_argument (N, argc, argv);
     78  }
     79
    4580  if ((*argc < 4) || (*argc > 6)) dvomerge_usage();
     81  return TRUE;
     82}
     83
     84/*** check for command line options ***/
     85int dvomerge_client_args (int *argc, char **argv) {
     86 
     87  int N;
     88
     89  // by definition, the client is not parallel
     90  PARALLEL = FALSE;
     91  PARALLEL_MANUAL = FALSE;
     92  PARALLEL_SERIAL = FALSE;
     93
     94  HOST_ID = 0;
     95  if ((N = get_argument (*argc, argv, "-hostID"))) {
     96    remove_argument (N, argc, argv);
     97    HOST_ID = atoi (argv[N]);
     98    remove_argument (N, argc, argv);
     99  }
     100  if (!HOST_ID) dvomerge_client_usage();
     101
     102  HOSTDIR = NULL;
     103  if ((N = get_argument (*argc, argv, "-hostdir"))) {
     104    remove_argument (N, argc, argv);
     105    HOSTDIR = strcreate (argv[N]);
     106    remove_argument (N, argc, argv);
     107  }
     108  if (!HOSTDIR) dvomerge_client_usage();
     109
     110  /* extra error messages */
     111  VERBOSE = FALSE;
     112  if ((N = get_argument (*argc, argv, "-v"))) {
     113    VERBOSE = TRUE;
     114    remove_argument (N, argc, argv);
     115  }
     116
     117  /* replace measurement, don't duplicate */
     118  REPLACE_BY_PHOTCODE = FALSE;
     119  if ((N = get_argument (*argc, argv, "-replace"))) {
     120    REPLACE_BY_PHOTCODE = TRUE;
     121    remove_argument (N, argc, argv);
     122  }
     123
     124  NTHREADS = 0;
     125  if ((N = get_argument (*argc, argv, "-threads"))) {
     126    remove_argument (N, argc, argv);
     127    NTHREADS = MAX(0, atoi(argv[N]));
     128    remove_argument (N, argc, argv);
     129  }
     130
     131  /*** provide additional data ***/
     132  /* restrict to a portion of the sky? */
     133  UserPatch.Rmin = 0;
     134  UserPatch.Rmax= 360;
     135  UserPatch.Dmin = -90;
     136  UserPatch.Dmax = +90;
     137  if ((N = get_argument (*argc, argv, "-region"))) {
     138    remove_argument (N, argc, argv);
     139    UserPatch.Rmin = atof (argv[N]);
     140    remove_argument (N, argc, argv);
     141    UserPatch.Rmax = atof (argv[N]);
     142    remove_argument (N, argc, argv);
     143    UserPatch.Dmin = atof (argv[N]);
     144    remove_argument (N, argc, argv);
     145    UserPatch.Dmax = atof (argv[N]);
     146    remove_argument (N, argc, argv);
     147  }
     148
     149  if ((*argc < 4) || (*argc > 6)) dvomerge_client_usage();
    46150  return TRUE;
    47151}

  • trunk/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c

    r31450 r33963  
    11# include "dvomerge.h"
    22
     3// utility functions
    34void sort_IDmap (IDmapType *IDmap);
    4 
    5 off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt) {
    6 
    7   // use bisection to find the starting entry by time
    8 
    9   off_t Nlo, Nhi, N;
    10 
    11   // find the last TGT before start
    12   Nlo = 0; Nhi = NimagesTgt;
    13   while (Nhi - Nlo > 10) {
    14     N = 0.5*(Nlo + Nhi);
    15     if (TgtTimes[N] < start) {
    16       Nlo = MAX(N, 0);
    17     } else {
    18       Nhi = MIN(N + 1, NimagesTgt);
    19     }
    20   }
    21 
    22   // check for the matched mosaic starting from Nlo
    23   // we may have to go much beyond Nlo since stop is not sorted
    24   // can we use a sorted version of stop to check when we are beyond the valid range??
    25   for (N = Nlo; N < NimagesTgt; N++) {
    26     if (TgtTimes[N] < start) continue;
    27     if (TgtTimes[N] > stop) return (-1);
    28     if (TgtCodes[N] != photcode) continue;
    29     return (TgtIndex[N]);
    30   }
    31   return (-1);
    32 }
    33 
    34 // sort two times vectors and an index by first time vector
    35 void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N) {
    36 
    37 # define SWAPFUNC(A,B){ e_time tmp_t; off_t tmp_i; short tmp_c;         \
    38   tmp_t = S[A]; S[A] = S[B]; S[B] = tmp_t; \
    39   tmp_i = I[A]; I[A] = I[B]; I[B] = tmp_i; \
    40   tmp_c = C[A]; C[A] = C[B]; C[B] = tmp_c; \
    41 }
    42 # define COMPARE(A,B)(S[A] < S[B])
    43 
    44   OHANA_SORT (N, COMPARE, SWAPFUNC);
    45 
    46 # undef SWAPFUNC
    47 # undef COMPARE
    48 
    49 }
     5void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N);
     6off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt);
    507
    518// we have two tables; 'tgt' contains some exposures from 'src' : find them and match a map
    … …  
    7128  }
    7229
    73   ALLOCATE (IDmap->old, off_t, NimagesSrc);
    74   ALLOCATE (IDmap->new, off_t, NimagesSrc);
     30  ALLOCATE (IDmap->old, unsigned int, NimagesSrc);
     31  ALLOCATE (IDmap->new, unsigned int, NimagesSrc);
    7532  IDmap->Nmap = NimagesSrc;
    7633
    … …  
    11673  int status;
    11774 
     75  IDmap->old = NULL;
     76  IDmap->new = NULL;
     77
     78  // it is OK if there are no images in the database, but there should be no imageIDs to map...
     79  if (in->dbstate == LCK_EMPTY) {
     80    return TRUE;
     81  }
     82
    11883  images = gfits_table_get_Image (&in[0].ftable, &Nimages, &in[0].swapped);
    11984  if (!images) {
    … …  
    12287  }
    12388
    124   ALLOCATE (IDmap->old, off_t, Nimages);
    125   ALLOCATE (IDmap->new, off_t, Nimages);
     89  ALLOCATE (IDmap->old, unsigned int, Nimages);
     90  ALLOCATE (IDmap->new, unsigned int, Nimages);
    12691  IDmap->Nmap = Nimages;
    12792
    … …  
    157122}
    158123
    159 // XXX isn't the map just ID_new = ID_old + offset ??
     124// map this ID to the new table
     125// XXX isn't the map just ID_new = ID_old + offset ?? (probably not)
    160126off_t dvo_map_image_ID (IDmapType *IDmap, off_t oldID) {
    161127
    … …  
    191157}
    192158
     159// given a table of detections, update their image IDs based on the new map
    193160int dvo_update_image_IDs (IDmapType *IDmap, Catalog *catalog) {
    194161
    … …  
    198165    oldID = catalog[0].measure[i].imageID;
    199166    if (oldID == 0) continue;
     167
     168    if (!IDmap->old) {
     169      fprintf (stderr, "input database has image IDs, but no Image table\n");
     170    }
    200171
    201172    newID = dvo_map_image_ID (IDmap, oldID);
    … …  
    225196}
    226197
     198off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt) {
     199
     200  // use bisection to find the starting entry by time
     201
     202  off_t Nlo, Nhi, N;
     203
     204  // find the last TGT before start
     205  Nlo = 0; Nhi = NimagesTgt;
     206  while (Nhi - Nlo > 10) {
     207    N = 0.5*(Nlo + Nhi);
     208    if (TgtTimes[N] < start) {
     209      Nlo = MAX(N, 0);
     210    } else {
     211      Nhi = MIN(N + 1, NimagesTgt);
     212    }
     213  }
     214
     215  // check for the matched mosaic starting from Nlo
     216  // we may have to go much beyond Nlo since stop is not sorted
     217  // can we use a sorted version of stop to check when we are beyond the valid range??
     218  for (N = Nlo; N < NimagesTgt; N++) {
     219    if (TgtTimes[N] < start) continue;
     220    if (TgtTimes[N] > stop) return (-1);
     221    if (TgtCodes[N] != photcode) continue;
     222    return (TgtIndex[N]);
     223  }
     224  return (-1);
     225}
     226
     227// sort two times vectors and an index by first time vector
     228void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N) {
     229
     230# define SWAPFUNC(A,B){ e_time tmp_t; off_t tmp_i; short tmp_c;         \
     231  tmp_t = S[A]; S[A] = S[B]; S[B] = tmp_t; \
     232  tmp_i = I[A]; I[A] = I[B]; I[B] = tmp_i; \
     233  tmp_c = C[A]; C[A] = C[B]; C[B] = tmp_c; \
     234}
     235# define COMPARE(A,B)(S[A] < S[B])
     236
     237  OHANA_SORT (N, COMPARE, SWAPFUNC);
     238
     239# undef SWAPFUNC
     240# undef COMPARE
     241
     242}
     243

  • trunk/Ohana/src/dvomerge/src/dvomergeCreate.c

    r33657 r33963  
    124124  depth2 = insky2[0].regions[Ns].depth;
    125125
     126  SkyList *outlist = SkyListByPatch (outsky, -1, &UserPatch);
     127
    126128  // loop over the populatable output regions
    127   for (i = 0; i < outsky[0].Nregions; i++) {
    128     if (!outsky[0].regions[i].table) continue;
    129     if (VERBOSE) fprintf (stderr, "output: %s\n", outsky[0].regions[i].name);
     129  for (i = 0; i < outlist[0].Nregions; i++) {
     130    if (!outlist[0].regions[i][0].table) continue;
     131    if (VERBOSE) fprintf (stderr, "output: %s\n", outlist[0].regions[i][0].name);
    130132
    131133    // load / create output catalog
    132     LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w", NsecfiltOutput);
     134    LoadCatalog (&outcatalog, outlist[0].regions[i], outlist[0].filename[i], "w", NsecfiltOutput);
    133135
    134136    // combine only tables at equal or larger depth
    135137     
    136138    // load in all of the tables from input1 for this region
    137     inlist = SkyListByBounds (insky1, depth1, outsky[0].regions[i].Rmin, outsky[0].regions[i].Rmax, outsky[0].regions[i].Dmin, outsky[0].regions[i].Dmax);
     139    inlist = SkyListByBounds (insky1, depth1, outlist[0].regions[i][0].Rmin + 0.01, outlist[0].regions[i][0].Rmax - 0.01, outlist[0].regions[i][0].Dmin + 0.01, outlist[0].regions[i][0].Dmax - 0.01);
    138140    for (j = 0; j < inlist[0].Nregions; j++) {
    139141      if (VERBOSE) fprintf (stderr, "input 1: %s\n", inlist[0].regions[j][0].name);
    … …  
    149151      }
    150152      dvo_update_image_IDs (&IDmap1, &incatalog);
    151       merge_catalogs_new (&outsky[0].regions[i], &outcatalog, &incatalog, secfiltMap1);
     153      merge_catalogs_new (outlist[0].regions[i], &outcatalog, &incatalog, secfiltMap1);
    152154      dvo_catalog_unlock (&incatalog);
    153155      dvo_catalog_free (&incatalog);
    … …  
    156158
    157159    // load in all of the tables from input2 for this region
    158     inlist = SkyListByBounds (insky2, depth2, outsky[0].regions[i].Rmin, outsky[0].regions[i].Rmax, outsky[0].regions[i].Dmin, outsky[0].regions[i].Dmax);
     160    inlist = SkyListByBounds (insky2, depth2, outlist[0].regions[i][0].Rmin + 0.01, outlist[0].regions[i][0].Rmax - 0.01, outlist[0].regions[i][0].Dmin + 0.01, outlist[0].regions[i][0].Dmax - 0.01);
    159161    for (j = 0; j < inlist[0].Nregions; j++) {
    160162      if (VERBOSE) fprintf (stderr, "input 2: %s\n", inlist[0].regions[j][0].name);
    … …  
    170172      }
    171173      dvo_update_image_IDs (&IDmap2, &incatalog);
    172       merge_catalogs_old (&outsky[0].regions[i], &outcatalog, &incatalog, RADIUS, secfiltMap2);
     174      merge_catalogs_old (outlist[0].regions[i], &outcatalog, &incatalog, RADIUS, secfiltMap2);
    173175      dvo_catalog_unlock (&incatalog);
    174176      dvo_catalog_free (&incatalog);
    … …  
    219221    in1DB.mode   = dvo_catalog_catmode (CATMODE);
    220222    in1DB.format = dvo_catalog_catformat (CATFORMAT);
    221     status       = dvo_image_lock (&in1DB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
     223    status       = dvo_image_lock (&in1DB, ImageCat, 3600.0, LCK_SOFT);  // shorter timeout?
    222224    if (!status) Shutdown ("ERROR: failure to lock image catalog %s", in1DB.filename);
    223225
    224226    // load the image table
    225     if (in1DB.dbstate == LCK_EMPTY) {
    226       Shutdown ("can't find input (1) image catalog %s", in1DB.filename);
    227     }
    228     if (!dvo_image_load (&in1DB, VERBOSE, TRUE)) {
    229       Shutdown ("can't read input (1) image catalog %s", in1DB.filename);
    230     }
    231 
     227    if (in1DB.dbstate != LCK_EMPTY) {
     228      if (!dvo_image_load (&in1DB, VERBOSE, TRUE)) {
     229        Shutdown ("can't read input (1) image catalog %s", in1DB.filename);
     230      }
     231    }
     232     
    232233    // convert database table to internal structure & add to output image db
     234    // if in1DB has no images, we will (later) insist that there are no image IDs to map
    233235    dvo_image_merge_dbs(IDmap1, &outDB, &in1DB);
    234236    dvo_image_unlock (&in1DB); // unlock input1
    235 
    236237
    237238    /*** load input2/Images.dat ***/
    … …  
    239240    in2DB.mode   = dvo_catalog_catmode (CATMODE);
    240241    in2DB.format = dvo_catalog_catformat (CATFORMAT);
    241     status       = dvo_image_lock (&in2DB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
     242    status       = dvo_image_lock (&in2DB, ImageCat, 3600.0, LCK_SOFT);  // shorter timeout?
    242243    if (!status) Shutdown ("ERROR: failure to lock image catalog %s", in2DB.filename);
    243244
    244245    /* load the image table */
    245     if (in2DB.dbstate == LCK_EMPTY) {
    246       Shutdown ("can't find input (2) image catalog %s", in2DB.filename);
    247     }
    248     if (!dvo_image_load (&in2DB, VERBOSE, TRUE)) {
    249       Shutdown ("can't read input (2) image catalog %s", in2DB.filename);
     246    if (in2DB.dbstate != LCK_EMPTY) {
     247      if (!dvo_image_load (&in2DB, VERBOSE, TRUE)) {
     248        Shutdown ("can't read input (2) image catalog %s", in2DB.filename);
     249      }
    250250    }
    251251
    252252    // convert database table to internal structure & add to output image db
     253    // if in1DB has no images, we will (later) insist that there are no image IDs to map
    253254    dvo_image_merge_dbs(IDmap2, &outDB, &in2DB);
    254255    dvo_image_unlock (&in2DB); // unlock input2

  • trunk/Ohana/src/dvomerge/src/dvomergeImageIDs.c

    r29938 r33963  
    11# include "dvomerge.h"
    22
    3 /*** update the image table ***/
     3/*** generate the IDmap for the given databases, and update the output image table ***/
    44int dvomergeImagesUpdate (IDmapType *IDmap, char *input, char *output) {
    55
    … …  
    1212  inDB.mode   = dvo_catalog_catmode (CATMODE);
    1313  inDB.format = dvo_catalog_catformat (CATFORMAT);
    14   status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
     14  status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_SOFT);  // shorter timeout?
    1515  if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
    1616
    1717  // load the image table
    1818  if (inDB.dbstate == LCK_EMPTY) {
     19    dvo_image_unlock (&inDB); // unlock input
    1920    IDmap->Nmap = 0;
    2021    return TRUE;
    … …  
    5253}
    5354
    54 /*** update the image table ***/
     55/*** generate the map for the given 2 databases ***/
    5556int dvomergeImagesGetMap (IDmapType *IDmap, char *input, char *output) {
    5657
    … …  
    6364  inDB.mode   = dvo_catalog_catmode (CATMODE);
    6465  inDB.format = dvo_catalog_catformat (CATFORMAT);
    65   status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
     66  status       = dvo_image_lock (&inDB, ImageCat, 5400.0, LCK_XCLD);  // shorter timeout?
    6667  if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
    6768
    6869  // load the image table
    6970  if (inDB.dbstate == LCK_EMPTY) {
     71    dvo_image_unlock (&inDB); // unlock input
    7072    IDmap->Nmap = 0;
    7173    return TRUE;
    … …  
    7678    Shutdown ("can't read input image catalog %s", inDB.filename);
    7779  }
     80  dvo_image_unlock (&inDB); // unlock input
    7881
    7982  /*** load output/Images.dat ***/
    … …  
    8184  outDB.mode   = dvo_catalog_catmode (CATMODE);
    8285  outDB.format = dvo_catalog_catformat (CATFORMAT);
    83   status       = dvo_image_lock (&outDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
     86  status       = dvo_image_lock (&outDB, ImageCat, 5400.0, LCK_XCLD);  // shorter timeout?
    8487  if (!status) Shutdown ("ERROR: failure to lock image catalog %s", outDB.filename);
    8588
    8689  /* load the image table */
    8790  if (outDB.dbstate == LCK_EMPTY) {
    88     Shutdown ("only use -continue for an existing, partially merged database");
     91    Shutdown ("for dvomerge -continue or dvomerge_client, the output database must exist");
    8992  }
    9093  if (!dvo_image_load (&outDB, VERBOSE, TRUE)) {
    9194    Shutdown ("can't read output image catalog %s", outDB.filename);
    9295  }
     96  dvo_image_unlock (&outDB); // unlock output
    9397
    9498  // convert database table to internal structure & add to output image db
    9599  dvo_image_match_dbs(IDmap, &outDB, &inDB);
    96   dvo_image_unlock (&inDB); // unlock input
    97   dvo_image_unlock (&outDB); // unlock output
    98100
    99101  return TRUE;

  • trunk/Ohana/src/dvomerge/src/dvomergeUpdate.c

    r33657 r33963  
    44
    55  int CONTINUE;
    6   off_t i, j;
    76  SkyTable *outsky, *insky;
    8   SkyList *outlist, *inlist;
    9   Catalog incatalog, outcatalog;
     7  SkyList *inlist;
    108  char filename[256], *input, *output;
    119  IDmapType IDmap;
    … …  
    107105  SetPhotcodeTable(NULL);
    108106
    109   // loop over the populated input regions
    110   for (i = 0; i < inlist[0].Nregions; i++) {
    111     if (!inlist[0].regions[i][0].table) continue;
    112     if (VERBOSE) fprintf (stderr, "input: %s\n", inlist[0].regions[i][0].name);
    113 
    114     // load / create output catalog (if catalog does not exist, it will be created)
    115     LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput);
    116     // skip empty input catalogs
    117     if (!incatalog.Naves_disk) {
    118         dvo_catalog_unlock (&incatalog);
    119         dvo_catalog_free (&incatalog);
    120         continue;
    121     }
    122 
    123     // combine only tables at equal or larger depth
    124      
    125     // load in all of the tables from input for this region
    126     // SkyListByBounds will return neighbor catalogs if the boundaries exactly match (due to rounding).  Since the regions are not infinitely small,
    127     // compare to a slightly reduced footprint
    128     float dPos = 2.0/3600.0;
    129     outlist = SkyListByBounds (outsky, inlist[0].regions[i][0].depth, inlist[0].regions[i][0].Rmin + dPos, inlist[0].regions[i][0].Rmax - dPos, inlist[0].regions[i][0].Dmin + dPos, inlist[0].regions[i][0].Dmax - dPos);
    130     for (j = 0; j < outlist[0].Nregions; j++) {
    131       if (VERBOSE) fprintf (stderr, "output : %s\n", outlist[0].regions[j][0].name);
    132 
    133       // load input catalog
    134       LoadCatalog (&outcatalog, outlist[0].regions[j], outlist[0].filename[j], "w", NsecfiltOutput);
    135 
    136       dvo_update_image_IDs (&IDmap, &incatalog);
    137       merge_catalogs_old (&outsky[0].regions[j], &outcatalog, &incatalog, RADIUS, secfiltMap);
    138 
    139       outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
    140 
    141       // if we receive a signal which would cause us to exit, wait until the full catalog is written
    142       SetProtect (TRUE);
    143       if (!dvo_catalog_save (&outcatalog, VERBOSE)) {
    144         fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]);
    145         exit (1);
    146       }
    147       SetProtect (FALSE);
    148 
    149       dvo_catalog_unlock (&outcatalog);
    150       dvo_catalog_free (&outcatalog);
    151 
    152       fprintf (stderr, "merged %s into %s\n", inlist[0].regions[i][0].name, outlist[0].regions[j][0].name);
    153     }
    154     SkyListFree (outlist);
    155 
    156     dvo_catalog_unlock (&incatalog);
    157     dvo_catalog_free (&incatalog);
    158   }
     107  dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, &IDmap, secfiltMap);
    159108
    160109  // save the output sky table copy

  • trunk/Ohana/src/dvomerge/src/dvoverify.c

    r33657 r33963  
    4444    CHECKSORTED = TRUE;
    4545    remove_argument (N, &argc, argv);
     46  }
     47
     48  if ((N = get_argument (argc, argv, "-cpt"))) {
     49    remove_argument (N, &argc, argv);
     50    char *filename = strcreate (argv[N]);
     51    remove_argument (N, &argc, argv);
     52
     53    int isBad = FALSE;
     54
     55    if (!VerifyTableFile (filename)) {
     56      fprintf (stderr, "bad average table %s\n", filename);
     57      isBad = TRUE;
     58    }
     59
     60    // change last 't' to 's':
     61    int Nlast;
     62    Nlast = strlen(filename) - 1;
     63    filename[Nlast] = 's';
     64    if (!VerifyTableFile (filename)) {
     65      fprintf (stderr, "bad secfilt table %s\n", filename);
     66      isBad = TRUE;
     67    }
     68
     69    // change last 't' to 's':
     70    Nlast = strlen(filename) - 1;
     71    filename[Nlast] = 'm';
     72    if (!VerifyTableFile (filename)) {
     73      fprintf (stderr, "bad measure table %s\n", filename);
     74      isBad = TRUE;
     75    }
     76
     77    if (isBad) exit (1);
     78    exit (0);
    4679  }
    4780

  • trunk/Ohana/src/dvomerge/src/help.c

    r30615 r33963  
    66  fprintf (stderr, "   OR: dvomerge (input) into (output) continue\n");
    77  fprintf (stderr, "   OR: dvomerge (input) into (output) from (list)\n");
     8  fprintf (stderr, "   [-region Rmin Rmax Dmin Dmax]\n");
     9  exit (2);
     10}
     11
     12void dvomerge_client_usage (void) {
     13  fprintf (stderr, "USAGE: dvomerge_client (input) into (output)\n");
    814  fprintf (stderr, "   [-region Rmin Rmax Dmin Dmax]\n");
    915  exit (2);
    … …  
    4551  fprintf (stderr, "  -h                          : this list\n\n");
    4652  fprintf (stderr, "  -region Rmin Rmax Dmin Dmax : region to merge\n\n");
     53  fprintf (stderr, "  -replace                    : replace existing detections of the same photcode\n");
     54  fprintf (stderr, "                                (this should only be used to merge reference dbs)\n\n");
     55  exit (2);
     56}
     57
     58void dvomerge_client_help (int argc, char **argv) {
     59
     60  /* check for help request */
     61  if (get_argument (argc, argv, "-help")) goto show_help;
     62  if (get_argument (argc, argv, "-h"))    goto show_help;
     63  return;
     64
     65show_help:
     66
     67  fprintf (stderr, "USAGE\n");
     68  fprintf (stderr, "  dvomerge_client (input) into (output)\n");
     69  fprintf (stderr, "  merge DVO databases\n");
     70  fprintf (stderr, "  optional flags:\n");
     71  fprintf (stderr, "  -v                          : verbose mode\n");
     72  fprintf (stderr, "  -help                       : this list\n");
     73  fprintf (stderr, "  -h                          : this list\n\n");
     74  fprintf (stderr, "  -region Rmin Rmax Dmin Dmax : region to merge\n\n");
     75  fprintf (stderr, "  -replace                    : replace existing detections of the same photcode\n");
     76  fprintf (stderr, "                                (this should only be used to merge reference dbs)\n\n");
    4777  exit (2);
    4878}

  • trunk/Ohana/src/dvomerge/src/merge_catalogs_old.c

    r30761 r33963  
    111111  if (Nave > 1) sort_coords_index (X2, Y2, N2, Nave);
    112112
     113  if (REPLACE_BY_PHOTCODE && output[0].Nmeasure && !output[0].sorted) {
     114    fprintf (stderr, "ERROR: attempt to merge with replace into an unsorted database\n");
     115    exit (3);
     116  }
     117
     118
    113119  /* set up pointers for linked list of measure */
    114120  if (output[0].sorted && (output[0].Nmeasure >= output[0].Nmeas_disk)) {
    … …  
    186192      offset = input[0].average[N].measureOffset + Nin;
    187193
     194      if (REPLACE_BY_PHOTCODE) {
     195        // index to first measure for this object
     196        int Mout = output[0].average[n].measureOffset; 
     197        if (replace_match(&output[0].average[n], &output[0].measure[Mout], &input[0].average[N], &input[0].measure[offset])) {
     198          input[0].found[N] = Mout;
     199          continue;
     200        }
     201      }
    188202      /* add to end of measurement list */
    189203      add_meas_link (&output[0].average[n], next_meas, Nmeas, NMEAS);

  • trunk/Ohana/src/dvomerge/test

    • Property svn:ignore set to
      catdir.test

  • trunk/Ohana/src/dvomerge/test/catdir.2mass

    • Property svn:ignore set to
      s8230
      s3730
      s6730
      n3000
      s0000
      n1500
      n6000
      n2230
      s3000
      n4500
      n5230
      s6000
      s1500
      n7500
      s2230
      n8230
      n3730
      s4500
      s0730
      Images.dat
      s5230
      n6730
      s7500
Note: See TracChangeset for help on using the changeset viewer.