IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 39329 for trunk


Ignore:
Timestamp:
Feb 4, 2016, 5:07:08 PM (10 years ago)
Author:
eugene
Message:

adding options to reset the starpar tables; option to skip images in the src db which are not in the tgt database

Location:
trunk/Ohana/src/dvomerge
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/Ohana/src/dvomerge/include/dvomerge.h

    r39314 r39329  
    4141int    REPLACE_TYCHO;
    4242int    FORCE_MERGE;
     43int    ALLOW_MISSING_INPUT_IMAGES;
     44
    4345int    MAX_CLIENTS;
    4446char  *ALTERNATE_PHOTCODE_FILE;
    … …  
    5355
    5456int    MATCHED_TABLES;
     57int    RESET_STARPAR;
     58
    5559int    REPAIR_BY_OBJID;
    5660
    … …  
    7478  unsigned int *old;
    7579  unsigned int *new;
    76   char *notFound;
     80  unsigned int  oldIDmax;
     81  int *notFoundMeasure;
     82  int *notFoundLensing;
    7783} IDmapType;
    7884
    … …  
    222228IDmapType *IDmapLoad               PROTO((char *filename));
    223229int        create_IDmap_lookup     PROTO((IDmapType *IDmap));
     230void       dvo_report_image_IDs    PROTO((IDmapType *IDmap));
    224231
    225232// dvomerge history functions
    … …  
    267274
    268275char **load_cptlist (char *filename, int *nlist);
     276int ResetStarPar (Catalog *catalog);

  • trunk/Ohana/src/dvomerge/src/LoadCatalog.c

    r39139 r39329  
    11# include "dvomerge.h"
    22# define SKIP_GALPHOT 0
     3# define SKIP_STARPAR 0
    34
    45int LoadCatalog (Catalog *catalog, SkyRegion *region, char *filename, char *mode, int Nsecfilt) {
    … …  
    1213  // XXXX TEMP HACK : skip GALPHOT
    1314
     15  catalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ;
    1416# if (SKIP_GALPHOT) 
    15   catalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR;
     17  catalog[0].catflags = catalog[0].catflags | DVO_SKIP_GALPHOT;
    1618# else
    17   catalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT;
     19  catalog[0].catflags = catalog[0].catflags | DVO_LOAD_GALPHOT;
     20# endif
     21
     22# if (SKIP_STARPAR) 
     23  catalog[0].catflags = catalog[0].catflags | DVO_SKIP_STARPAR;
     24# else
     25  catalog[0].catflags = catalog[0].catflags | DVO_LOAD_STARPAR;
    1826# endif
    1927 
    … …  
    2735  return (TRUE);
    2836}
     37
     38int ResetStarPar (Catalog *catalog) {
     39
     40  off_t i;
     41
     42  // we are going to delete the starpar values:
     43
     44  catalog[0].Nstarpar = 0;
     45  catalog[0].Nstarpar_disk = 0;
     46  catalog[0].Nstarpar_off = 0;
     47
     48  REALLOCATE (catalog[0].starpar, StarPar, 1);
     49
     50  for (i = 0; i < catalog[0].Naverage; i++) {
     51    catalog[0].average[i].Nstarpar = 0;
     52    catalog[0].average[i].starparOffset = -1;
     53  }
     54  return TRUE;
     55}

  • trunk/Ohana/src/dvomerge/src/args.c

    r39314 r39329  
    4848
    4949  /* extra error messages */
     50  RESET_STARPAR = FALSE;
     51  if ((N = get_argument (*argc, argv, "-reset-starpar"))) {
     52    RESET_STARPAR = TRUE;
     53    remove_argument (N, argc, argv);
     54  }
     55
     56  /* extra error messages */
    5057  MATCH_BY_EXTERN_ID = FALSE;
    5158  if ((N = get_argument (*argc, argv, "-match-by-extern-id"))) {
    … …  
    104111  if ((N = get_argument (*argc, argv, "-replace"))) {
    105112    REPLACE_BY_PHOTCODE = TRUE;
     113    remove_argument (N, argc, argv);
     114  }
     115
     116  /* extra error messages */
     117  ALLOW_MISSING_INPUT_IMAGES = FALSE;
     118  if ((N = get_argument (*argc, argv, "-allow-missing-input-images"))) {
     119    ALLOW_MISSING_INPUT_IMAGES = TRUE;
    106120    remove_argument (N, argc, argv);
    107121  }
    … …  
    271285
    272286  /* extra error messages */
     287  RESET_STARPAR = FALSE;
     288  if ((N = get_argument (*argc, argv, "-reset-starpar"))) {
     289    RESET_STARPAR = TRUE;
     290    remove_argument (N, argc, argv);
     291  }
     292
     293  /* extra error messages */
    273294  MATCH_BY_EXTERN_ID = FALSE;
    274295  if ((N = get_argument (*argc, argv, "-match-by-extern-id"))) {
    275296    MATCH_BY_EXTERN_ID = TRUE;
     297    remove_argument (N, argc, argv);
     298  }
     299
     300  /* extra error messages */
     301  ALLOW_MISSING_INPUT_IMAGES = FALSE;
     302  if ((N = get_argument (*argc, argv, "-allow-missing-input-images"))) {
     303    ALLOW_MISSING_INPUT_IMAGES = TRUE;
    276304    remove_argument (N, argc, argv);
    277305  }

  • trunk/Ohana/src/dvomerge/src/build_links.c

    r39143 r39329  
    403403  N = 0;
    404404
     405  // NOTE that is we choose DVO_SKIP_STARPAR, catalog.starpar is NULL.
     406  // this code will let merge_catalogs_old.c do nothing for starpar
    405407  ALLOCATE (next_star, off_t, Nstarpar);
     408  if (!starpar) return next_star;
     409
    406410  for (i = 0; i < Naverage; i++) {
    407411    if (!average[i].Nstarpar) continue;
    … …  
    442446
    443447  ALLOCATE (next_star, off_t, Nstarpar);
     448  if (!starpar) return next_star;
    444449
    445450  /* reset the Nm, offset values for average */
    … …  
    514519  off_t i, k, n, np, N;
    515520  StarPar *tmpstarpar;
     521
     522  if (!starpar) return NULL;
    516523
    517524  /* fix order of StarPar (memory intensive, but fast) */

  • trunk/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c

    r39299 r39329  
    1414  IDmap->old = NULL;
    1515  IDmap->new = NULL;
    16   IDmap->notFound = NULL;
     16  IDmap->notFoundMeasure = NULL;
     17  IDmap->notFoundLensing = NULL;
    1718  IDmap->Nmap = 0;
     19  IDmap->oldIDmax = 0;
    1820}
    1921
    … …  
    2224  FREE (IDmap->old);
    2325  FREE (IDmap->new);
    24   FREE (IDmap->notFound);
     26  FREE (IDmap->notFoundMeasure);
     27  FREE (IDmap->notFoundLensing);
    2528}
    2629
    … …  
    6568  for (iSrc = 0; iSrc < NimagesSrc; iSrc++) {
    6669    iTgt = getTgtIndexByExtID (imagesSrc[iSrc].externID, TgtIndex, TgtExtID, NimagesTgt);
    67     if (iTgt < 0) Shutdown ("failure to find matching image: %s\n", imagesSrc[iSrc].name);
     70    if ((iTgt < 0) && (!ALLOW_MISSING_INPUT_IMAGES)) {
     71      Shutdown ("failure to find matching image: %s\n", imagesSrc[iSrc].name);
     72    }
    6873    IDmap[0].old[iSrc] = imagesSrc[iSrc].imageID;
    69     IDmap[0].new[iSrc] = imagesTgt[iTgt].imageID;
     74    if (iTgt < 0) {
     75      IDmap[0].new[iSrc] = 0;
     76    } else {
     77      IDmap[0].new[iSrc] = imagesTgt[iTgt].imageID;
     78    }
    7079  }
    7180
    … …  
    124133  for (iSrc = 0; iSrc < NimagesSrc; iSrc++) {
    125134    iTgt = getTgtIndexByTimeAndPhotcode (imagesSrc[iSrc].tzero, imagesSrc[iSrc].tzero + (int) imagesSrc[iSrc].exptime, imagesSrc[iSrc].photcode, TgtIndex, TgtTimes, TgtCodes, NimagesTgt);
    126     if (iTgt < 0) Shutdown ("failure to match images: %s\n", imagesSrc[iSrc].name);
     135    if ((iTgt < 0) && (!ALLOW_MISSING_INPUT_IMAGES)) {
     136      Shutdown ("failure to find matching image: %s\n", imagesSrc[iSrc].name);
     137    }
    127138    IDmap[0].old[iSrc] = imagesSrc[iSrc].imageID;
    128139    IDmap[0].new[iSrc] = imagesTgt[iTgt].imageID;
    … …  
    197208}
    198209
     210# if (0)
     211// merge db2 into db1, skipping any from 'src' with the same EXTERN_ID as 'tgt'
     212int dvo_image_merge_dbs_skip_duplicates (IDmapType *IDmap, FITS_DB *tgt, FITS_DB *stc) {
     213
     214  off_t NimagesSrc, NimagesTgt;
     215  off_t Ntgt;
     216  off_t i, IDstart;
     217 
     218  IDmap->old = NULL;
     219  IDmap->new = NULL;
     220
     221  // it is OK if there are no images in the database, but there should be no imageIDs to map...
     222  if (src->dbstate == LCK_EMPTY) {
     223    return TRUE;
     224  }
     225
     226  // load both image tables
     227  Images *imagesSrc = gfits_table_get_Image (&src[0].ftable, &NimagesSrc, &src[0].scaledValue, &src[0].nativeOrder);
     228  if (!imagesSrc) {
     229    fprintf (stderr, "ERROR: failed to read images (src)\n");
     230    exit (2);
     231  }
     232
     233  Images *imagesTgt = gfits_table_get_Image (&tgt[0].ftable, &NimagesTgt, &tgt[0].scaledValue, &tgt[0].nativeOrder);
     234  if (!imagesTgt) {
     235    fprintf (stderr, "ERROR: failed to read images (tgt)\n");
     236    exit (2);
     237  }
     238
     239  // generate an index for target extern IDs:
     240  off_t  *TgtIndex;
     241  unsigned int *TgtExtID;
     242  ALLOCATE (TgtIndex, off_t,        NimagesTgt);
     243  ALLOCATE (TgtExtID, unsigned int, NimagesTgt);
     244
     245  // save the Index & EXTERN_ID for all TGT images
     246  off_t iSrc, iTgt;
     247  for (iTgt = 0; iTgt < NimagesTgt; iTgt++) {
     248    TgtIndex[iTgt] = iTgt;
     249    TgtExtID[iTgt] = imagesTgt[iTgt].externID;
     250  }
     251
     252  // sort the index, start, and stop by the start times:
     253  SortTgtByExtID (TgtExtID, TgtIndex, NimagesTgt);
     254
     255  // identify the src images which are NOT in tgt
     256  off_t *keepSrc = NULL;
     257  ALLOCATE (keepSrc, off_t, NimagesSrc);
     258  memset (keepSrc, 0, NimagesSrc*sizeof(off_t));
     259
     260  off_t Nkeep = 0;
     261  for (iSrc = 0; iSrc < NimagesSrc; iSrc++) {
     262    iTgt = getTgtIndexByExtID (imagesSrc[iSrc].externID, TgtIndex, TgtExtID, NimagesTgt);
     263    if (iTgt >= 0) continue;
     264    keepSrc[iSrc] = 1;
     265    Nkeep ++;
     266  }
     267
     268  // append the new image data to the end of the tgt structure, adjusting imageID as we go:
     269  int status = gfits_scan (&tgt[0].header, "IMAGEID", OFF_T_FMT, 1,  &IDstart);
     270  if (!status) {
     271    IDstart = 1;
     272  }
     273
     274  off_t jSrc = 0;
     275  REALLOCATE (imagesTgt, Image, NimagesTgt + Nkeep);
     276
     277  for (iSrc = 0; iSrc < NimagesSrc; iSrc++) {
     278    if (!keepSrc[iSrc]) continue; // skip images already in the image table
     279    imagesTgt[NimagesTgt + jSrc] = imagesSrc[iSrc];
     280    imagesTgt[NimagesTgt + jSrc].imageID = jSrc + IDstart;
     281    jSrc ++;
     282  }
     283  myAssert (jSrc == Nkeep, "oops");
     284
     285  Ntgt += Nkeep;
     286  IDstart += Nkeep;
     287  gfits_modify (&tgt[0].header, "NIMAGES", OFF_T_FMT, 1,  Ntgt);
     288  gfits_modify (&tgt[0].header, "IMAGEID", OFF_T_FMT, 1,  IDstart);
     289
     290  // XXX need to update ftable with the full Image table
     291  // gfits_add_rows (&tgt[0].ftable, (char *) images, Nimages, sizeof(Image));
     292  return TRUE;
     293}
     294# endif
     295
    199296// map this ID to the new table
    200297// XXX isn't the map just ID_new = ID_old + offset ?? (probably not)
    … …  
    257354    if (newID == 0) {
    258355      if (oldID > lastID) {
    259         fprintf (stderr, "problem with image IDs : input out of range\n");
     356        // this is a case where the measure->imageIDs are not consistent with the image->imageIDs
     357        fprintf (stderr, "problem with image IDs (measure) : input out of range\n");
    260358        fprintf (stderr, "old ID: "OFF_T_FMT", last ID: "OFF_T_FMT"\n", oldID, lastID);
    261359        exit (2);
    262360      }
    263       if (!IDmap->notFound[oldID]) {
    264         fprintf (stderr, "cannot find image ID "OFF_T_FMT"\n",  oldID);
    265         IDmap->notFound[oldID] = TRUE;
     361      if (!IDmap->notFoundMeasure[oldID]) {
     362        // this is a case where the measure->imageIDs are not found in the OUTPUT image->imageIDs
    266363        // once we discover an imageID is not found, record that fact so we do not complain for every detection
     364        fprintf (stderr, "cannot find image ID "OFF_T_FMT" with at least one measure\n",  oldID);
    267365      }
     366      IDmap->notFoundMeasure[oldID]++;
    268367      // optionally exit here? or wait until end to report an error?
    269368      // exit (2);
    … …  
    280379    if (newID == 0) {
    281380      if (oldID > lastID) {
    282         fprintf (stderr, "problem with image IDs : input out of range\n");
     381        // this is a case where the measure->imageIDs are not consistent with the image->imageIDs
     382        fprintf (stderr, "problem with image IDs (lensing) : input out of range\n");
    283383        fprintf (stderr, "old ID: "OFF_T_FMT", last ID: "OFF_T_FMT"\n", oldID, lastID);
    284384        exit (2);
    285385      }
    286       if (!IDmap->notFound[oldID]) {
     386      if (!IDmap->notFoundLensing[oldID]) {
     387        // once we discover an imageID is not found, record that fact so we do not complain for every detection
    287388        fprintf (stderr, "cannot find image ID "OFF_T_FMT"\n",  oldID);
    288         IDmap->notFound[oldID] = TRUE;
    289         // once we discover an imageID is not found, record that fact so we do not complain for every detection
    290389      }
     390      IDmap->notFoundLensing[oldID] ++;
    291391      // optionally exit here? or wait until end to report an error?
    292392      // exit (2);
    … …  
    298398
    299399  return TRUE;
     400}
     401
     402void dvo_report_image_IDs (IDmapType *IDmap) {
     403
     404  off_t i;
     405
     406  for (i = 0; i < IDmap->oldIDmax + 1; i++) {
     407    if (!IDmap->notFoundMeasure[i]) continue;
     408    fprintf (stderr, "SKIP IMAGE MEASURE: "OFF_T_FMT", %d measures\n", i, IDmap->notFoundMeasure[i]);
     409  }
     410  for (i = 0; i < IDmap->oldIDmax + 1; i++) {
     411    if (!IDmap->notFoundLensing[i]) continue;
     412    fprintf (stderr, "SKIP IMAGE LENSING: "OFF_T_FMT", %d lensings\n", i, IDmap->notFoundLensing[i]);
     413  }
     414  return;
    300415}
    301416
    … …  
    415530    oldIDmax = IDmap->old[i];
    416531  }
    417   ALLOCATE (IDmap->notFound, char, oldIDmax + 1);
    418   memset (IDmap->notFound, 0, sizeof(char) * (oldIDmax + 1));
     532  IDmap->oldIDmax = oldIDmax;
     533
     534  ALLOCATE_ZERO (IDmap->notFoundMeasure, int, oldIDmax + 1);
     535  ALLOCATE_ZERO (IDmap->notFoundLensing, int, oldIDmax + 1);
    419536  return TRUE;
    420537}

  • trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c

    r39319 r39329  
    181181      LoadCatalog (&outcatalog, outlist[0].regions[j], outcatalog.filename, "w", NsecfiltOutput);
    182182
     183      if (RESET_STARPAR) {
     184        myAssert (MATCHED_TABLES, "must use -matched-tables to reset starpar");
     185        ResetStarPar (&outcatalog);
     186      }
     187
    183188      if (UPDATE_CATFORMAT) {
    184189        outcatalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT);
    … …  
    239244    dvo_catalog_free (&incatalog);
    240245  }
     246
     247  dvo_report_image_IDs (IDmap);
     248
    241249  return TRUE;
    242250}
    … …  
    306314               UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, RADIUS);
    307315
    308     if (VERBOSE)             { strextend (&command, "-v"); }
    309     if (VERIFY)              { strextend (&command, "-verify"); }
    310     if (VERIFY_CATALOG_ONLY) { strextend (&command, "-verify-catalogs"); }
    311     if (REPLACE_BY_PHOTCODE) { strextend (&command, "-replace"); }
    312     if (REPLACE_TYCHO)       { strextend (&command, "-replace-tycho"); }
    313     if (PARALLEL_INPUT)      { strextend (&command, "-parallel-input"); }
    314     if (FORCE_MERGE)         { strextend (&command, "-force-merge"); }
    315     if (MATCHED_TABLES)      { strextend (&command, "-matched-tables"); }
    316     if (MATCH_BY_EXTERN_ID)  { strextend (&command, "-match-by-extern-id"); }
    317     if (UPDATE_CATFORMAT)    { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); }
    318     if (UPDATE_CATCOMPRESS)  { strextend (&command, "-update-catcompress %s", UPDATE_CATCOMPRESS); }
    319     if (CPTLIST_FILENAME)    { strextend (&command, "-restrict-cpt %s", CPTLIST_FILENAME); }
    320     if (REPAIR_BY_OBJID)     { strextend (&command, "-repair-by-objid"); }
     316    if (VERBOSE)                    { strextend (&command, "-v"); }
     317    if (VERIFY)                     { strextend (&command, "-verify"); }
     318    if (VERIFY_CATALOG_ONLY)        { strextend (&command, "-verify-catalogs"); }
     319    if (REPLACE_BY_PHOTCODE)        { strextend (&command, "-replace"); }
     320    if (REPLACE_TYCHO)              { strextend (&command, "-replace-tycho"); }
     321    if (PARALLEL_INPUT)             { strextend (&command, "-parallel-input"); }
     322    if (FORCE_MERGE)                { strextend (&command, "-force-merge"); }
     323    if (MATCHED_TABLES)             { strextend (&command, "-matched-tables"); }
     324    if (MATCH_BY_EXTERN_ID)         { strextend (&command, "-match-by-extern-id"); }
     325    if (RESET_STARPAR)              { strextend (&command, "-reset-starpar"); }
     326    if (ALLOW_MISSING_INPUT_IMAGES) { strextend (&command, "-allow-missing-input-images"); }
     327    if (UPDATE_CATFORMAT)           { strextend (&command, "-update-catformat %s", UPDATE_CATFORMAT); }
     328    if (UPDATE_CATCOMPRESS)         { strextend (&command, "-update-catcompress %s", UPDATE_CATCOMPRESS); }
     329    if (CPTLIST_FILENAME)           { strextend (&command, "-restrict-cpt %s", CPTLIST_FILENAME); }
     330    if (REPAIR_BY_OBJID)            { strextend (&command, "-repair-by-objid"); }
    321331
    322332    // add some config variables:

  • trunk/Ohana/src/dvomerge/src/dvorepairFixWarpIDs.c

    r39323 r39329  
    2626    exit (2);
    2727  }
    28 
    29   fprintf (stderr, "is this mode tested?\n");
    30   exit (2);
    3128
    3229  char *catdir_list  = argv[1];
Note: See TracChangeset for help on using the changeset viewer.