IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 39323 for trunk


Ignore:
Timestamp:
Jan 27, 2016, 11:16:51 AM (10 years ago)
Author:
eugene
Message:

update dvorepair to delete detections in a parallel context

Location:
trunk/Ohana/src/dvomerge
Files:
4 added
14 edited

Legend:

Unmodified
Added
Removed

  • trunk/Ohana/src/dvomerge/Makefile

    r39301 r39323  
    2525dvoutils        : $(BIN)/dvoutils.$(ARCH)
    2626
    27 all: dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client dvoutils
     27all: dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvorepair_client dvoverify dvoverify_client dvoutils
    2828
    2929DVOMERGE = \
    … …  
    125125DVOREPAIR = \
    126126$(SRC)/dvorepair.$(ARCH).o \
     127$(SRC)/dvorepair_args.$(ARCH).o \
    127128$(SRC)/dvorepairFixCPT.$(ARCH).o \
    128129$(SRC)/dvorepairFixWarpIDs.$(ARCH).o \
    … …  
    137138$(SRC)/psps_ids.$(ARCH).o \
    138139$(SRC)/LoadImages.$(ARCH).o \
     140$(SRC)/DeleteImagesOps.$(ARCH).o \
    139141$(SRC)/ReadDeleteList.$(ARCH).o \
    140142$(SRC)/ReadDeleteListExternID.$(ARCH).o \
    … …  
    146148$(SRC)/ImageOps.$(ARCH).o
    147149
    148 $(DVOREPAIR)  : $(INC)/dvomerge.h
     150$(DVOREPAIR)  : $(INC)/dvorepair.h
    149151$(BIN)/dvorepair.$(ARCH) : $(DVOREPAIR)
     152
     153DVOREPAIR_CLIENT = \
     154$(SRC)/dvorepair_client.$(ARCH).o \
     155$(SRC)/dvorepair_args.$(ARCH).o \
     156$(SRC)/dvorepairDeleteImagesByExternID.$(ARCH).o \
     157$(SRC)/psps_ids.$(ARCH).o \
     158$(SRC)/LoadImages.$(ARCH).o \
     159$(SRC)/DeleteImagesOps.$(ARCH).o \
     160$(SRC)/ReadDeleteListExternID.$(ARCH).o \
     161$(SRC)/SetSignals.$(ARCH).o \
     162$(SRC)/Shutdown.$(ARCH).o \
     163$(SRC)/myIndex.$(ARCH).o \
     164$(SRC)/help.$(ARCH).o
     165
     166$(DVOREPAIR_CLIENT)  : $(INC)/dvorepair.h
     167$(BIN)/dvorepair_client.$(ARCH) : $(DVOREPAIR_CLIENT)
    150168
    151169DVOVERIFY = \
    … …  
    184202$(BIN)/dvoutils.$(ARCH) : $(DVOUTILS)
    185203
    186 INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client dvoverify dvoutils
     204INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvorepair_client dvoverify dvoverify_client dvoverify dvoutils
    187205
    188206# dependancy rules for binary code #########################

  • trunk/Ohana/src/dvomerge/src/dvomerge_client.c

    r39319 r39323  
    110110
    111111  dvomerge_client_args_free ();
    112   ohana_memcheck (TRUE);
    113   ohana_memdump (TRUE);
     112  // ohana_memcheck (TRUE);
     113  // ohana_memdump (TRUE);
    114114
    115115  exit (0);

  • trunk/Ohana/src/dvomerge/src/dvorepair.c

    r39281 r39323  
    1 # include "dvomerge.h"
     1# include "dvorepair.h"
     2
     3// I have updated this program to use a common arg-parsing function, with operation more similar to other ohana programs
     4// most of the modes here are not necessarily tested for current databases (parallel, compressed) and are disabled for now
    25
    36int main (int argc, char **argv) {
     7
     8  SetSignals ();
     9  dvorepair_help(argc, argv);
     10  dvorepair_args(&argc, argv);
     11 
     12  switch (MODE) {
     13    case DVOREPAIR_MODE_FixWarpIDs:
     14      dvorepairFixWarpIDs(argc, argv);
     15      break;
     16    case DVOREPAIR_MODE_FixStackIDs:
     17      dvorepairFixStackIDs(argc, argv);
     18      break;
     19    case DVOREPAIR_MODE_FixCPT:
     20      dvorepairFixCPT(argc, argv);
     21      break;
     22    case DVOREPAIR_MODE_BY_OBJ_ID:
     23      dvorepair_by_objID(argc, argv);
     24      break;
     25    case DVOREPAIR_MODE_ImagesVsMeasures:
     26      dvorepairImagesVsMeasures(argc, argv);
     27      break;
     28    case DVOREPAIR_MODE_DeleteImageList:
     29      dvorepairDeleteImageList(argc, argv);
     30      break;
     31    case DVOREPAIR_MODE_DeleteImagesByExternID:
     32      dvorepairDeleteImagesByExternID(argc, argv);
     33      break;
     34    case DVOREPAIR_MODE_DeleteImagesByExternID_v2:
     35      dvorepairDeleteImagesByExternID_v2(argc, argv);
     36      break;
     37    case DVOREPAIR_MODE_FixImages:
     38      dvorepairFixImages(argc, argv);
     39      break;
     40    default:
     41      fprintf (stderr, "unknown mode\n");
     42      exit (1);
     43  }
     44
     45  exit (2);
     46}
    447
    548  // fprintf (stderr, "this program needs to be updated to load old format Measure tables (pre PV1_V5) in which dR,dD are saved, not R,D\n");
    … …  
    750  // exit (2);
    851
    9   SetSignals ();
    10   dvorepair_help(argc, argv);
    11  
    12   if (!strcmp(argv[1], "-fix-warp-ids")) dvorepairFixWarpIDs(argc, argv);
    13   if (!strcmp(argv[1], "-fix-stack-ids")) dvorepairFixStackIDs(argc, argv);
    14 
    15   if (!strcmp(argv[1], "-fix-cpt")) dvorepairFixCPT(argc, argv);
    16   if (!strcmp(argv[1], "-fix-cpt-by-objID")) dvorepair_by_objID(argc, argv);
    17   // if (!strcmp(argv[1], "-fix-tables")) dvorepairFixTables(argc, argv);
    18   if (!strcmp(argv[1], "-images-vs-measures")) dvorepairImagesVsMeasures(argc, argv);
    19   if (!strcmp(argv[1], "-delete-image-list")) dvorepairDeleteImageList(argc, argv);
    20   if (!strcmp(argv[1], "-delete-images-by-extern-id")) dvorepairDeleteImagesByExternID(argc, argv);
    21   if (!strcmp(argv[1], "-delete-images-by-extern-id-v2")) dvorepairDeleteImagesByExternID_v2(argc, argv);
    22   if (!strcmp(argv[1], "-fix-images")) dvorepairFixImages(argc, argv);
    23   dvorepair_help(0, NULL);
    24   exit (2);
    25 }

  • trunk/Ohana/src/dvomerge/src/dvorepairDeleteImageList.c

    r38441 r39323  
    1919
    2020  off_t i, j, Nmeasure, NmeasureNew, Ndelete, Nvalid, Nimage, Nindex, *imageIdx, index;
    21   int N, seq, Nbytes, NbytesPerRow, Ncheck, nPass, raPass;
     21  int seq, Nbytes, NbytesPerRow, Ncheck, nPass, raPass;
    2222  double Rthis, Dthis, Rmin, Rmax, Dmin, Dmax, Qthis, Qmin, Qmax, dR, dQ;
    2323
    … …  
    4848  DVOCatFormat catformat;
    4949
    50   N = get_argument (argc, argv, "-delete-image-list");
    51   myAssert(N == 1, "programming error: -delete-image-list must be first from main");
    52   remove_argument (N, &argc, argv);
    53 
    5450  if (argc != 3) {
    5551    fprintf (stderr, "USAGE: dvorepair -delete-image-list (catdir) (deleteList)\n");
    … …  
    5854    exit (2);
    5955  }
     56
     57  fprintf (stderr, "is this mode tested?\n");
     58  exit (2);
    6059
    6160  char *catdir = argv[1];

  • trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c

    r39295 r39323  
    1 # include "dvomerge.h"
     1# include "dvorepair.h"
     2# define DEBUG 1
    23
    34// delete images based on a text table of the target IDs
    … …  
    2021int RepairAverage (Catalog *catalog);
    2122
     23typedef struct {
     24  double Rmin;
     25  double Rmax;
     26  double Qmin;
     27  double Qmax;
     28  double Dmin;
     29  double Dmax;
     30} DeleteRegion;
     31
     32
    2233int dvorepairDeleteImagesByExternID (int argc, char **argv) {
    2334
    2435  FITS_DB db;  // database handle pointing to input image table
    2536 
    26   int i, N;
     37  int i;
    2738  off_t Nimage;
    28   int nPass, raPass;
    2939
    3040  Image *image;
    … …  
    3242  char filename[DVO_MAX_PATH];
    3343
    34   N = get_argument (argc, argv, "-delete-images-by-extern-id");
    35   myAssert(N == 1, "programming error: -delete-images-by-extern-id must be first argument");
    36   remove_argument (N, &argc, argv);
    37 
    38   VERBOSE = FALSE;
    39   if ((N = get_argument (argc, argv, "-v"))) {
    40     VERBOSE = TRUE;
    41     remove_argument (N, &argc, argv);
    42   }
    43 
    44   if (argc != 3) {
    45     fprintf (stderr, "USAGE: dvorepair -delete-images-by-extern-id (catdir) (deleteList)\n");
    46     fprintf (stderr, "  catdir : database of interest\n");
    47     fprintf (stderr, "  deleteList : table from dvorepair giving images to be deleted\n");
    48     exit (2);
    49   }
    50 
    51   char *catdir = argv[1];
     44  // CATDIR from first arg, not -D CATDIR
     45  CATDIR = argv[1];
    5246  char *delList = argv[2];
    5347
    54   sprintf (filename, "%s/Photcodes.dat", catdir);
     48  sprintf (filename, "%s/Photcodes.dat", CATDIR);
    5549  if (!LoadPhotcodes (filename, NULL, FALSE)) {
    56     fprintf (stderr, "error reading photcodes from %s\n", catdir);
     50    fprintf (stderr, "error reading photcodes from %s\n", CATDIR);
    5751    exit (1);
    5852  }     
    5953
    6054  // load the image data
    61   ALLOCATE(imageFilename, char, strlen(catdir) + 12);
    62   sprintf (imageFilename, "%s/Images.dat", catdir);
     55  ALLOCATE(imageFilename, char, strlen(CATDIR) + 12);
     56  sprintf (imageFilename, "%s/Images.dat", CATDIR);
    6357  if ((image = LoadImages (&db, imageFilename, &Nimage)) == NULL) return (FALSE);
    6458  BuildChipMatch (image, Nimage); // needed for astrometric tranformations
    … …  
    125119  }
    126120
    127   // find the RA & DEC range of the images we want to delete
    128   double Rmin = 360.0;
    129   double Rmax = 0.0;
    130   double Dmin = +90.0;
    131   double Dmax = -90.0;
    132   double Qmin = 360.0;
    133   double Qmax =   0.0;
     121  SkyRegion UserPatch[2]; // 1 or 2 UserPatch areas are set by FindDeleteRegion
     122  int nUserPatch = FindDeleteRegion (UserPatch, image, Nimage, deleteImage);
     123
     124  // delete detections in the external catalog.  things I need:
     125  // UserPatch, nUserPatch : definition of the region
     126  // deleteImage : boolean vector of length Nimage (delete or not?)
     127  // imageID : vector of length Nimage, with the image IDs in the same order as deleteImage
     128
     129  dvorepairDeleteImagesByExternID_catalogs (UserPatch, nUserPatch, image, Nimage, deleteImage, imageIDindex);
     130
     131  off_t NnewImage;
     132  Image *newImage = DeleteSelectedImages (image, Nimage, deleteImage, imageIDindex, &NnewImage);
     133
     134  SaveImageTable (newImage, NnewImage, CATDIR, &db);
     135
     136  myIndexFree (externIDindex);
     137  myIndexFree (imageIDindex);
     138
     139  free (imageFilename);
     140  free (deleteImage);
     141  free (deleteIDs);
     142
     143  gfits_db_free (&db);
     144
     145  ohana_memcheck (TRUE);
     146
     147  exit (0);
     148}
     149
     150Image *DeleteSelectedImages (Image *image, off_t Nimage, int *deleteImage, myIndexType *imageIDindex, off_t *NnewImage) {
     151
     152  int i;
     153
     154  // we need to remove the (WRP) images, but we also want to remove the DIS [PHU] image entries if they have
     155  // had all of their children deleted
     156
     157  // first, generate an array of the number of children for each parent
     158
     159  int *Nchildren = NULL;
     160  ALLOCATE (Nchildren, int, Nimage);
     161  memset (Nchildren, 0, Nimage*sizeof(int));
     162
     163  // parentID is not necessarily reliable (dvomerge breaks it, for example).  rebuild it
     164  // here by using BuildChipMatch (libdvo/src/mosaic_astrom.c):
     165
     166  BuildChipMatch (image, Nimage);
     167
     168  // reassign parentID values
     169  int NbadParentIDs  = 0;
     170  for (i = 0; i < Nimage; i++) {
     171    if (!image[i].parent) continue;
     172   
     173    if (image[i].parentID != image[i].parent[0].imageID) {
     174      NbadParentIDs ++;
     175      if (NbadParentIDs < 100) {
     176        int n = myIndexGetEntry (imageIDindex, image[i].parentID);
     177        fprintf (stderr, "bad parent ID: %s matchs %s not %s\n", image[i].name, image[i].parent[0].name, image[n].name);
     178      }
     179      image[i].parentID = image[i].parent[0].imageID;
     180    }
     181  }
     182
     183  // count up the children for each parent
     184  for (i = 0; i < Nimage; i++) {
     185    if (!image[i].parentID) continue;
     186    if (strcmp(&image[i].coords.ctype[4], "-WRP")) {
     187      fprintf (stderr, "warning: child with non WRP coords : %s\n", image[i].name);
     188    }
     189
     190    int n = myIndexGetEntry (imageIDindex, image[i].parentID);
     191    myAssert (n != -1, "invalid parentID?");
     192    myAssert (n < Nimage, "invalid parentID??");
     193    Nchildren[n] ++;
     194  }
     195
     196  // now subtract the children which need to be deleted
    134197  for (i = 0; i < Nimage; i++) {
    135198    if (!deleteImage[i]) continue;
     199    if (!image[i].parentID) continue; // should I warn on this?
     200
     201    int n = myIndexGetEntry (imageIDindex, image[i].parentID);
     202    myAssert (n != -1, "invalid parentID?");
     203    myAssert (n < Nimage, "invalid parentID??");
     204    Nchildren[n] --;
     205    myAssert (Nchildren[n] >= 0, "over deleted???");
     206  }
     207 
     208  // now check for any parent and delete it
     209  for (i = 0; i < Nimage; i++) {
     210    if (strcmp(&image[i].coords.ctype[4], "-DIS")) continue;
     211    myAssert (!image[i].parentID, "parent ID for DIS image??");
     212    if (!Nchildren[i]) deleteImage[i] = TRUE;
     213  }
     214
     215  fprintf (stderr, "deleting the following images:\n");
     216
     217  off_t Nnew = 0;
     218  Image *newImage = NULL;
     219  ALLOCATE (newImage, Image, Nimage);
     220  for (i = 0; i < Nimage; i++) {
     221    if (deleteImage[i]) {
     222      fprintf (stderr, "DELETE: %s : %d : %d : %d\n", image[i].name, image[i].imageID, image[i].parentID, image[i].externID);
     223      continue;
     224    }
     225    newImage[Nnew] = image[i];
     226    Nnew ++;
     227  }
     228
     229  REALLOCATE (newImage, Image, Nnew);
     230
     231  *NnewImage = Nnew;
     232  return newImage;
     233}
     234
     235int SaveImageTable (Image *image, off_t Nimage, char *catdir, FITS_DB *oldDB) {
     236
     237  FITS_DB db;
     238
     239  char ImageCatSrc[DVO_MAX_PATH];
     240  char ImageCatTgt[DVO_MAX_PATH];
     241
     242  // we are creating a new image table (what about db ID?)
     243  snprintf (ImageCatSrc, DVO_MAX_PATH, "%s/Images.dat", catdir);
     244  snprintf (ImageCatTgt, DVO_MAX_PATH, "%s/Images.dat.broken", catdir);
     245
     246  // rename the old cpt file:
     247  if (rename (ImageCatSrc, ImageCatTgt)) {
     248    perror ("tried to rename file");
     249    exit (2);
     250  }
     251
     252  /* setup image table format and lock */
     253  gfits_db_init (&db);
     254  db.mode    = dvo_catalog_catmode ("SPLIT");
     255  db.format  = dvo_catalog_catformat ("PS1_V5");
     256  int status = dvo_image_lock (&db, ImageCatSrc, 3600.0, LCK_XCLD);  // shorter timeout?
     257  if (!status) {
     258    fprintf (stderr, "ERROR: failure to lock image catalog %s", db.filename);
     259    exit (2);
     260  }
     261
     262  /* load or create the image table */
     263  if (db.dbstate != LCK_EMPTY) {
     264    fprintf (stderr, "image table %s exists, exiting", db.filename);
     265    exit (2);
     266  }
     267  dvo_image_create (&db, GetZeroPoint());
     268
     269  // I want to keep the old DVO_DBID
     270  char dbID[33];
     271  int imageIDmax;
     272  if (!gfits_scan (&oldDB->header, "DVO_DBID", "%s", 1, dbID)) {
     273    fprintf (stderr, "source image table is missing DVO_DBID, exiting\n");
     274    exit (3);
     275  }
     276  if (!gfits_scan (&oldDB->header, "IMAGEID", "%d", 1, &imageIDmax)) {
     277    fprintf (stderr, "max image ID is missing, exiting\n");
     278    exit (3);
     279  }
     280  gfits_modify (&db.header, "IMAGEID", "%d", 1, imageIDmax);     
     281  gfits_modify (&db.header, "DVO_DBID", "%s", 1, dbID);     
     282
     283  // also save the merge history:
     284  int Nmerge;
     285  if (gfits_scan (&oldDB->header, "NMERGE", "%d", 1, &Nmerge)) {
     286    int i;
     287    gfits_modify (&db.header, "NMERGE", "%d", 1, Nmerge);
     288    for (i = 0; i < Nmerge; i++) {
     289      char field[80], value[80];
     290      snprintf (field, 80, "DM_%05d", i);
     291      if (!gfits_scan (&oldDB->header, field, "%s", 1, value)) {
     292        fprintf (stderr, "missing merge info %s\n", field);
     293        continue;
     294      }
     295      gfits_modify (&db.header, field, "%s", 1, value);
     296    }
     297  }
     298
     299  /* add the new image and save */
     300  dvo_image_addrows (&db, image, Nimage);
     301  dvo_image_update (&db, VERBOSE);
     302  dvo_image_unlock (&db); /* unlock? */
     303
     304  return TRUE;
     305}
     306
     307int RepairTableCPT_V1(char *cptFilenameSrc, char *cptFilenameTgt, char *cpsFilenameSrc, char *cpsFilenameTgt, Measure *measure, off_t Nmeasure, Image *image, off_t Nimage, myIndexType *imageIDindex, char catformat) {
     308
     309  off_t *averefMatch;
     310  off_t i, NaveMax, Naverage, NAVERAGE, NaverageOut, Nave, Nout, Nold;
     311  int *found, Nsecfilt;
     312
     313  Average *average, *averageOut;
     314
     315  Matrix matrix;
     316
     317  Header cptHeaderPHU;
     318  Header cptHeaderTBL;
     319  FTable cptFtable;
     320
     321  cptFtable.header = &cptHeaderTBL;
     322
     323  NaveMax = 0;
     324  NAVERAGE = 1000;
     325  ALLOCATE (average, Average, NAVERAGE);
     326  memset (average, 0, NAVERAGE*sizeof(Average));
     327
     328  ALLOCATE (found, int, NAVERAGE);
     329  memset (found, 0, NAVERAGE*sizeof(int));
     330
     331  // examine all measurements and create new objects as needed
     332  // we start with a valid, sorted dvo database, so we use averef to link objects
     333  for (i = 0; i < Nmeasure; i++) {
     334    Nave = measure[i].averef;
     335
     336    // we only allocate as many as we need
     337    if (Nave >= NAVERAGE) {
     338      Nold = NAVERAGE;
     339      NAVERAGE = MAX(Nave + 1000, NAVERAGE + 1000);
     340      REALLOCATE (average, Average, NAVERAGE);
     341      memset (&average[Nold], 0, (NAVERAGE - Nold)*sizeof(Average));
     342
     343      REALLOCATE (found, int, NAVERAGE);
     344      memset (&found[Nold], 0, (NAVERAGE - Nold)*sizeof(int));
     345    }
     346
     347    // this measure matches an existing average, just check it is OK and bump Nmeasure
     348    if (found[Nave]) {
     349      average[Nave].Nmeasure ++;
     350      myAssert(average[Nave].objID == measure[i].objID, "objIDs do not match!");
     351      myAssert(average[Nave].catID == measure[i].catID, "catIDs do not match!");
     352      continue;
     353    }
     354
     355    NaveMax = MAX(Nave, NaveMax);
     356
     357    found[Nave] = TRUE;
     358
     359    // we are going to leave most of the elements of average unset: they are the result of
     360    // the relastro analysis for this object and can be recreated with a call to relastro
     361
     362    // need to find image so we can use ccd coordinates to determine RA & DEC
     363    int n = myIndexGetEntry (imageIDindex, measure[i].imageID);
     364    myAssert (n > -1, "impossible!");
     365   
     366    dvo_average_init (&average[Nave]);
     367
     368    // I actually have measure->R,D, I could just use those...
     369    XY_to_RD (&average[Nave].R, &average[Nave].D, measure[i].Xccd, measure[i].Yccd, &image[n].coords);
     370
     371    average[Nave].Nmeasure = 1;
     372    average[Nave].Nmissing = 0;
     373
     374    // assume the resulting table set is unsorted
     375    average[Nave].measureOffset = -1;
     376    average[Nave].missingOffset = -1;
     377
     378    average[Nave].objID = measure[i].objID;
     379    average[Nave].catID = measure[i].catID;
     380    average[Nave].extID = CreatePSPSObjectID(average[Nave].R, average[Nave].D);
     381  }
     382  Naverage = NaveMax + 1;
     383
     384  // we now have an average table, but there will be holes due to deleted measurements
     385  // create a new average table with only existing entries
     386
     387  ALLOCATE (averageOut, Average, Naverage);
     388  memset (averageOut, 0, Naverage*sizeof(Average));
     389
     390  ALLOCATE (averefMatch, off_t, Naverage);
     391  memset (averefMatch, 0, Naverage*sizeof(int));
     392
     393  Nave = 0;
     394  for (i = 0; i < Naverage; i++) {
     395    if (!found[i]) continue;
     396    averageOut[Nave] = average[i]; // use a memcpy?
     397    averefMatch[i] = Nave;
     398    Nave ++;
     399  }
     400  NaverageOut = Nave;
     401
     402  // modify measure.averef to match the new sequence
     403  for (i = 0; i < Nmeasure; i++) {
     404    Nave = measure[i].averef;
     405    Nout = averefMatch[Nave];
     406    myAssert(Nout < NaverageOut, "output averef is wrong");
     407   
     408    myAssert(average[Nave].objID == measure[i].objID, "objIDs do not match");
     409    myAssert(average[Nave].catID == measure[i].catID, "objIDs do not match");
     410    myAssert(averageOut[Nout].objID == measure[i].objID, "objIDs do not match");
     411    myAssert(averageOut[Nout].catID == measure[i].catID, "objIDs do not match");
     412
     413    measure[i].averef = Nout;
     414  }
     415
     416  fprintf (stderr, "cpt file : %d obj -> %d obj (%s -> %s)\n", (int) Naverage, (int) NaverageOut, cptFilenameSrc, cptFilenameTgt);
     417
     418  // open source cpt file
     419  FILE *cptFile = fopen(cptFilenameSrc, "r");
     420  myAssert(cptFile, "failed to open cpt file");
     421
     422  // load the cpt header (use for CATID, RA, DEC range, filenames)
     423  if (!gfits_fread_header (cptFile, &cptHeaderPHU)) {
     424    myAbort("failure to cpt header");
     425  }
     426
     427  // update the output header
     428  gfits_modify (&cptHeaderPHU, "NSTARS",     OFF_T_FMT, 1,  NaverageOut);
     429  gfits_modify (&cptHeaderPHU, "NMEAS",      OFF_T_FMT, 1,  Nmeasure);
     430  gfits_modify (&cptHeaderPHU, "NMISS",      "%d",      1,  0);
     431  gfits_modify_alt (&cptHeaderPHU, "SORTED", "%t",      1,  FALSE);
     432
     433  gfits_scan (&cptHeaderPHU, "NSECFILT",     "%d",      1,  &Nsecfilt);
     434
     435  if (0) {
     436    char compressMode[256];
     437    if (gfits_scan (&cptHeaderTBL, "DVO_CMP", "%s", 1, compressMode)) {
     438      if (strcmp (compressMode, "NONE")) {
     439        myAbort ("fix compression");
     440      }
     441    }
     442  }
     443
     444  /* convert internal to external format */
     445  if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL, TRUE)) {
     446    myAbort("trouble converting format");
     447  }
     448  fclose(cptFile);
     449 
     450  // rename the old cpt file:
     451  if (rename (cptFilenameSrc, cptFilenameTgt)) {
     452    perror ("tried to rename file");
     453    exit (2);
     454  }
     455
     456  // create and write the output file
     457  cptFile = fopen(cptFilenameSrc, "w");
     458  myAssert(cptFile, "failed to open cpt file");
     459   
     460  // write PHU header
     461  if (!gfits_fwrite_header (cptFile, &cptHeaderPHU)) {
     462    myAbort("can't write primary header");
     463  }
     464
     465  // write the PHU matrix; this is probably a NOP, do I have to keep it in?
     466  gfits_create_matrix (&cptHeaderPHU, &matrix);
     467  if (!gfits_fwrite_matrix  (cptFile, &matrix)) {
     468    myAbort("can't write primary matrix");
     469  }
     470  gfits_free_matrix (&matrix);
     471
     472  // write the table data
     473  if (!gfits_fwrite_ftable_range (cptFile, &cptFtable, 0, NaverageOut, 0, NaverageOut)) {
     474    myAbort("can't write table data");
     475  }
     476  fclose(cptFile);
     477
     478  gfits_free_table (&cptFtable);
     479  gfits_free_header (&cptHeaderPHU);
     480  gfits_free_header (&cptHeaderTBL);
     481  free (average);
     482  free (averageOut);
     483
     484  free (found);
     485  free (averefMatch);
     486
     487  {
     488    Header cpsHeaderPHU;
     489    Header cpsHeaderTBL;
     490    FTable cpsFtable;
     491
     492    SecFilt *secfilt = NULL;
     493
     494    cpsFtable.header = &cpsHeaderTBL;
     495
     496    // open source cpt file
     497    FILE *cpsFile = fopen(cpsFilenameSrc, "r");
     498    myAssert(cpsFile, "failed to open cps file");
     499   
     500    // load the cps header (use for CATID, RA,DEC range, filenames)
     501    if (!gfits_fread_header (cpsFile, &cpsHeaderPHU)) {
     502      myAbort("failure to cps header");
     503    }
     504
     505    int Nrows = Nsecfilt*NaverageOut;
     506    ALLOCATE (secfilt, SecFilt, Nrows);
     507
     508    for (i = 0; i < Nrows; i++) {
     509      dvo_secfilt_init (&secfilt[i], SECFILT_RESET_ALL);
     510    }
     511
     512    /* convert internal to external format */
     513    if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat, TRUE)) {
     514      myAbort("trouble converting format");
     515    }
     516    fclose(cpsFile);
     517
     518    // rename the old cpt file:
     519    if (rename (cpsFilenameSrc, cpsFilenameTgt)) {
     520      perror ("tried to rename file");
     521      exit (2);
     522    }
     523
     524    // create and write the output file
     525    cpsFile = fopen(cpsFilenameSrc, "w");
     526    myAssert(cpsFile, "failed to open cps file");
     527   
     528    // write PHU header
     529    if (!gfits_fwrite_header (cpsFile, &cpsHeaderPHU)) {
     530      myAbort("can't write primary header");
     531    }
     532
     533    // write the PHU matrix; this is probably a NOP, do I have to keep it in?
     534    gfits_create_matrix (&cpsHeaderPHU, &matrix);
     535    if (!gfits_fwrite_matrix  (cpsFile, &matrix)) {
     536      myAbort("can't write primary matrix");
     537    }
     538    gfits_free_matrix (&matrix);
     539
     540    // write the table data
     541    if (!gfits_fwrite_ftable_range (cpsFile, &cpsFtable, 0, Nrows, 0, Nrows)) {
     542      myAbort("can't write table data");
     543    }
     544    fclose(cpsFile);
     545
     546    gfits_free_table (&cpsFtable);
     547    gfits_free_header (&cpsHeaderPHU);
     548    gfits_free_header (&cpsHeaderTBL);
     549    free (secfilt);
     550  }
     551
     552  return (TRUE);
     553}
     554
     555int RepairAverage (Catalog *catalog) {
     556
     557  off_t i, j, Nave;
     558
     559  // average is sorted so averef is valid
     560  // secfilt is also sorted so sequence is valid (but this is kind of moot since it will be re-calculated)
     561  // average.measureOffset is NOT valid
     562  // average.lensingOffset is NOT valid
     563
     564  Average *average = catalog->average;
     565  Measure *measure = catalog->measure;
     566  Lensing *lensing = catalog->lensing;
     567 
     568  // reset the values of average.Nmeasure, average.Nlensing
     569  for (i = 0; i < catalog->Naverage; i++) {
     570    average[i].Nmeasure = 0;
     571    average[i].Nlensing = 0;
     572  }
     573
     574  // re-calculate average.Nmeasure
     575  for (i = 0; i < catalog->Nmeasure; i++) {
     576    Nave = measure[i].averef;
     577    myAssert (measure[i].objID == average[Nave].objID, "invalid measure:average match");
     578    average[Nave].Nmeasure ++;
     579  }
     580   
     581  // re-calculate average.Nlensing
     582  for (i = 0; i < catalog->Nlensing; i++) {
     583    Nave = lensing[i].averef;
     584    myAssert (lensing[i].objID == average[Nave].objID, "invalid lensing:average match");
     585    average[Nave].Nlensing ++;
     586  }
     587
     588  // create a copy of the average table, keeping only the entries with Nmeasure & Nlensing > 0
     589  ALLOCATE_PTR (averageNew, Average, catalog->Naverage);
     590  ALLOCATE_PTR (averefNew,  int,     catalog->Naverage);
     591
     592  Nave = 0;
     593  for (i = 0; i < catalog->Naverage; i++) {
     594    averefNew[i] = -1;
     595    if (!average[i].Nmeasure && !average[i].Nlensing) continue;
     596
     597    averageNew[Nave] = average[i];
     598    averefNew[i] = Nave;
     599    Nave ++;
     600  }
     601  off_t NaverageNew = Nave;
     602
     603  // update measure.averef values
     604  for (i = 0; i < catalog->Nmeasure; i++) {
     605    Nave = averefNew[measure[i].averef];
     606    myAssert (Nave >= 0, "oops");
     607    measure[i].averef = Nave;
     608  }
     609   
     610  // update lensing.averef values
     611  for (i = 0; i < catalog->Nlensing; i++) {
     612    Nave = averefNew[lensing[i].averef];
     613    myAssert (Nave >= 0, "oops");
     614    lensing[i].averef = Nave;
     615  }
     616 
     617  // XXX need to update measureOffset and lensingOffset
     618
     619  // measure[] should be blocked and sequential:
     620  // measure[i+1].averef >= measure[i].averef
     621
     622  // update average.measureOffset values
     623  Nave = -1;
     624  for (i = 0; i < catalog->Nmeasure; i++) {
     625    if (measure[i].averef == Nave) continue;
     626    Nave = measure[i].averef;
     627    averageNew[Nave].measureOffset = i;
     628  }
     629
     630  // update average.lensingOffset values
     631  Nave = -1;
     632  for (i = 0; i < catalog->Nlensing; i++) {
     633    if (lensing[i].averef == Nave) continue;
     634    Nave = lensing[i].averef;
     635    averageNew[Nave].lensingOffset = i;
     636  }
     637
     638  // check the result (measure -> average)
     639  for (i = 0; i < catalog->Nmeasure; i++) {
     640    Nave = measure[i].averef;
     641    myAssert(averageNew[Nave].objID == measure[i].objID, "objIDs do not match");
     642    myAssert(averageNew[Nave].catID == measure[i].catID, "catIDs do not match");
     643  }
     644  // check the result (average -> measure)
     645  for (i = 0; i < NaverageNew; i++) {
     646    int m = averageNew[i].measureOffset;
     647    for (j = 0; j < averageNew[i].Nmeasure; j++) {
     648      myAssert(averageNew[i].objID == measure[j+m].objID, "objIDs do not match");
     649      myAssert(averageNew[i].catID == measure[j+m].catID, "catIDs do not match");
     650      myAssert(measure[j+m].averef == i, "averef broken");
     651    }
     652  }
     653
     654  // check the result (lensing -> average)
     655  for (i = 0; i < catalog->Nlensing; i++) {
     656    Nave = lensing[i].averef;
     657    myAssert(averageNew[Nave].objID == lensing[i].objID, "objIDs do not match");
     658    myAssert(averageNew[Nave].catID == lensing[i].catID, "catIDs do not match");
     659  }
     660  // check the result (average -> lensing)
     661  for (i = 0; i < NaverageNew; i++) {
     662    int m = averageNew[i].lensingOffset;
     663    for (j = 0; j < averageNew[i].Nlensing; j++) {
     664      myAssert(averageNew[i].objID == lensing[j+m].objID, "objIDs do not match");
     665      myAssert(averageNew[i].catID == lensing[j+m].catID, "catIDs do not match");
     666      myAssert(lensing[j+m].averef == i, "averef broken");
     667    }
     668  }
     669
     670  free (catalog->secfilt);
     671  ALLOCATE (catalog->secfilt, SecFilt, NaverageNew*catalog->Nsecfilt);
     672  for (i = 0; i < NaverageNew*catalog->Nsecfilt; i++) {
     673    dvo_secfilt_init (&catalog->secfilt[i], SECFILT_RESET_ALL);
     674  }
     675
     676  free (averefNew);
     677  free (catalog->average);
     678  catalog->average = averageNew;
     679  catalog->Naverage = NaverageNew;
     680  catalog->Naverage_disk = NaverageNew;
     681
     682  catalog->Nsecfilt_disk = NaverageNew*catalog->Nsecfilt;
     683
     684  return (TRUE);
     685}
     686
     687// delete measure entries
     688int DeleteMeasure (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) {
     689
     690  int j;
     691
     692  Measure *measure = catalog->measure;
     693
     694  // allocate an output array of measures (to replace, if needed)
     695  ALLOCATE_PTR (measureNew, Measure, catalog->Nmeasure);
     696
     697  int NmeasureNew = 0;
     698  int NmeasureDel = 0;
     699
     700  // examine all measurements: find ones that need to be deleted
     701  for (j = 0; j < catalog->Nmeasure; j++) {
     702    int imageID = measure[j].imageID;
     703    if (!imageID) continue;
     704    // myAssert(imageID, "measure is missing an image ID");
     705    // this case is valid if we have REF detections (no associated image)
     706
     707    int N = myIndexGetEntry(imageIDindex, imageID);
     708    if (N < 0) {
     709      // this detection comes from a non-existant image; delete
     710      NmeasureDel ++;
     711      continue;
     712    }
     713
     714    // measure matches a bad image; delete
     715    if (deleteImage[N]) {
     716      NmeasureDel ++;
     717      continue;
     718    }
     719
     720    // keep this measure
     721    measureNew[NmeasureNew] = measure[j];
     722    NmeasureNew ++;
     723  }
     724
     725  free (catalog->measure);
     726  catalog->measure = measureNew;
     727  catalog->Nmeasure = NmeasureNew;
     728  catalog->Nmeasure_disk = NmeasureNew;
     729
     730  *nDelete = NmeasureDel;
     731
     732  return TRUE;
     733}
     734
     735// delete lensing entries
     736int DeleteLensing (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) {
     737
     738  int j;
     739
     740  Lensing *lensing = catalog->lensing;
     741
     742  // allocate an output array of measures (to replace, if needed)
     743  ALLOCATE_PTR (lensingNew, Lensing, catalog->Nlensing);
     744
     745  int NlensingNew = 0;
     746  int NlensingDel = 0;
     747
     748  // examine all lensing: find ones that need to be deleted
     749  for (j = 0; j < catalog->Nlensing; j++) {
     750    int imageID = lensing[j].imageID;
     751    myAssert(imageID, "lensing is missing an image ID");
     752
     753    int N = myIndexGetEntry(imageIDindex, imageID);
     754    if (N < 0) {
     755      // this detection comes from a non-existant image; delete
     756      NlensingDel ++;
     757      continue;
     758    }
     759
     760    // lensing matches a bad image; delete
     761    if (deleteImage[N]) {
     762      NlensingDel ++;
     763      continue;
     764    }
     765
     766    // keep this lensing
     767    lensingNew[NlensingNew] = lensing[j];
     768    NlensingNew ++;
     769  }
     770
     771  free (catalog->lensing);
     772  catalog->lensing = lensingNew;
     773  catalog->Nlensing = NlensingNew;
     774  catalog->Nlensing_disk = NlensingNew;
     775
     776  *nDelete = NlensingDel;
     777
     778  return TRUE;
     779}
     780
     781// return number of SkyRegions which get set
     782int FindDeleteRegion (SkyRegion *UserPatch, Image *image, off_t Nimage, int *deleteImage) {
     783
     784  // Rmin,Rmax run from 0 - 360.0; for Rmin < 180.0, Rmin points Qmin,Qmax run from -180.0 - +180.0
     785
     786  // find the RA & DEC range of the images we want to delete
     787  double Dmin =  +90.0;
     788  double Dmax =  -90.0;
     789  double Rmin = +360.0;
     790  double Rmax =    0.0;
     791  double Qmin = +180.0;
     792  double Qmax = -180.0;
     793
     794  int i;
     795  for (i = 0; i < Nimage; i++) {
     796    if (!deleteImage[i]) continue;
    136797
    137798    double Rthis, Dthis, Qthis;
    138799    XY_to_RD(&Rthis, &Dthis, 0, 0, &image[i].coords);
     800    Rthis = ohana_normalize_angle_to_midpoint (Rthis, 180.0); // Rthis:    0.0 - 360.0
     801    Qthis = ohana_normalize_angle_to_midpoint (Rthis,   0.0); // Qthis: -180.0 - 180.0
    139802    Rmin = MIN(Rthis, Rmin);
    140803    Rmax = MAX(Rthis, Rmax);
    141804    Dmin = MIN(Dthis, Dmin);
    142805    Dmax = MAX(Dthis, Dmax);
    143     Qthis = (Rthis < 180.0) ? Rthis : Rthis - 360.0;
    144806    Qmin = MIN(Qthis, Qmin);
    145807    Qmax = MAX(Qthis, Qmax);
    146808
    147     // fprintf (stderr, "image @ %f,%f\n", Rthis, Dthis);
    148 
    149809    XY_to_RD(&Rthis, &Dthis, image[i].NX, 0, &image[i].coords);
     810    Rthis = ohana_normalize_angle_to_midpoint (Rthis, 180.0); // Rthis:    0.0 - 360.0
     811    Qthis = ohana_normalize_angle_to_midpoint (Rthis,   0.0); // Qthis: -180.0 - 180.0
    150812    Rmin = MIN(Rthis, Rmin);
    151813    Rmax = MAX(Rthis, Rmax);
    152814    Dmin = MIN(Dthis, Dmin);
    153815    Dmax = MAX(Dthis, Dmax);
    154     Qthis = (Rthis < 180.0) ? Rthis : Rthis - 360.0;
    155816    Qmin = MIN(Qthis, Qmin);
    156817    Qmax = MAX(Qthis, Qmax);
    157818
    158819    XY_to_RD(&Rthis, &Dthis, 0, image[i].NY, &image[i].coords);
     820    Rthis = ohana_normalize_angle_to_midpoint (Rthis, 180.0); // Rthis:    0.0 - 360.0
     821    Qthis = ohana_normalize_angle_to_midpoint (Rthis,   0.0); // Qthis: -180.0 - 180.0
    159822    Rmin = MIN(Rthis, Rmin);
    160823    Rmax = MAX(Rthis, Rmax);
    161824    Dmin = MIN(Dthis, Dmin);
    162825    Dmax = MAX(Dthis, Dmax);
    163     Qthis = (Rthis < 180.0) ? Rthis : Rthis - 360.0;
    164826    Qmin = MIN(Qthis, Qmin);
    165827    Qmax = MAX(Qthis, Qmax);
    166828
    167829    XY_to_RD(&Rthis, &Dthis, image[i].NX, image[i].NY, &image[i].coords);
     830    Rthis = ohana_normalize_angle_to_midpoint (Rthis, 180.0); // Rthis:    0.0 - 360.0
     831    Qthis = ohana_normalize_angle_to_midpoint (Rthis,   0.0); // Qthis: -180.0 - 180.0
    168832    Rmin = MIN(Rthis, Rmin);
    169833    Rmax = MAX(Rthis, Rmax);
    170834    Dmin = MIN(Dthis, Dmin);
    171835    Dmax = MAX(Dthis, Dmax);
    172     Qthis = (Rthis < 180.0) ? Rthis : Rthis - 360.0;
    173836    Qmin = MIN(Qthis, Qmin);
    174837    Qmax = MAX(Qthis, Qmax);
    … …  
    178841  double dR = Rmax - Rmin;
    179842
    180   // load the sky table for the existing database
    181   SkyTable *insky = SkyTableLoadOptimal (catdir, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
    182   myAssert(insky, "can't read SkyTable");
    183   SkyTableSetFilenames (insky, catdir, "cpt");
    184  
    185   // XXX apply this...generate the subset matching the user-selected region
    186   SkyRegion UserPatch;
    187 
     843  UserPatch[0].Dmin = Dmin - 0.1;
     844  UserPatch[0].Dmax = Dmax + 0.1;
     845  UserPatch[1].Dmin = Dmin - 0.1;
     846  UserPatch[1].Dmax = Dmax + 0.1;
     847
     848  int nPass;
    188849  if (dR < dQ + 0.1) {
     850    UserPatch[0].Rmin = Rmin - 0.1;
     851    UserPatch[0].Rmax = Rmax + 0.1;
    189852    fprintf (stderr, "R,D range: %f - %f, %f - %f\n", Rmin, Rmax, Dmin, Dmax);
    190853    nPass = 1;
    191854  } else {
     855    // Qmax is close to (but above) 0.0
     856    // Qmin + 360.0 maps to a point close to (but below) 360.0
     857    UserPatch[0].Rmin = 0.0;
     858    UserPatch[0].Rmax = Qmax + 0.1;
     859    UserPatch[1].Rmin = Qmin + 360.0 - 0.1;
     860    UserPatch[1].Rmax = 360.0;
    192861    fprintf (stderr, "R,D range: %f - %f, %f - %f\n", Qmin, Qmax, Dmin, Dmax);
    193862    nPass = 2;
    194863  }
     864
     865  return nPass;
     866}
     867
     868int dvorepairDeleteImagesByExternID_catalogs (SkyRegion *UserPatch, int nUserPatch, Image *image, off_t Nimage, int *deleteImage, myIndexType *imageIDindex) {
     869
     870  if (PARALLEL && !HOST_ID) {
     871    int status = dvorepairDeleteImagesByExternID_parallel (UserPatch, nUserPatch, image, Nimage, deleteImage);
     872    return status;
     873  }
     874
     875  // load the sky table for the existing database
     876  SkyTable *insky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
     877  myAssert(insky, "can't read SkyTable");
     878  SkyTableSetFilenames (insky, CATDIR, "cpt");
    195879 
    196880  int NmeasureDelTotal = 0;
    197881  int NlensingDelTotal = 0;
    198882
    199   for (raPass = 0; raPass < nPass; raPass++) {
    200    
    201     UserPatch.Dmin = Dmin - 0.1;
    202     UserPatch.Dmax = Dmax + 0.1;
    203 
    204     if (dR < dQ + 0.1) {
    205       UserPatch.Rmin = Rmin - 0.1;
    206       UserPatch.Rmax = Rmax + 0.1;
    207     } else {
    208       if (raPass == 0) {
    209         UserPatch.Rmin = 0.0;
    210         UserPatch.Rmax = Qmax + 0.1;
    211       } else {
    212         UserPatch.Rmin = Qmin + 360.0 - 0.1;
    213         UserPatch.Rmax = 360.0;
    214       }
    215     }
    216 
    217     SkyList *inlist = SkyListByPatch (insky, -1, &UserPatch);
     883  int patch;
     884  for (patch = 0; patch < nUserPatch; patch++) {
     885    SkyList *inlist = SkyListByPatch (insky, -1, &UserPatch[patch]);
    218886 
    219887    fprintf (stderr, "%d cpt regions affected\n", (int) inlist->Nregions);
    220888
    221     // SkyListPopulatedRange (&Ns, &Ne, inlist, 0);
    222     // depth = inlist[0].regions[Ns][0].depth;
    223  
    224889    // loop over the populated input regions
     890    int i;
    225891    for (i = 0; i < inlist[0].Nregions; i++) {
    226892      if (!inlist[0].regions[i][0].table) continue;
    227893
    228       snprintf (filename, DVO_MAX_PATH, "%s/%s.cpt", catdir, inlist[0].regions[i][0].name);
     894      // XXX deal with parallel db filenames
     895      // snprintf (filename, DVO_MAX_PATH, "%s/%s.cpt", CATDIR, inlist[0].regions[i][0].name);
     896
     897      // does this host ID match the desired location for the table?
     898      if (!HostTableTestHost(inlist[0].regions[i], HOST_ID)) continue;
     899
     900      // set the parameters which guide catalog open/load/create
     901      char hostfile[DVO_MAX_PATH];
     902      snprintf (hostfile, DVO_MAX_PATH, "%s/%s.cpt", HOSTDIR, inlist[0].regions[i][0].name);
     903      char *filename = HOST_ID ? hostfile : inlist[0].filename[i];
    229904
    230905      Catalog catalog;
    … …  
    232907      // set up the basic catalog info
    233908      dvo_catalog_init (&catalog, TRUE);
    234       catalog.filename  = filename; // XXX only allow non-parallel dbs for now
     909      catalog.filename  = filename;
    235910      catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT | DVO_LOAD_LENSING;
    236911      catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
    … …  
    288963
    289964  fprintf (stderr, "Deleted %d measure, %d lensing detections\n", NmeasureDelTotal, NlensingDelTotal);
    290 
    291   off_t NnewImage;
    292   Image *newImage = DeleteSelectedImages (image, Nimage, deleteImage, imageIDindex, &NnewImage);
    293 
    294   SaveImageTable (newImage, NnewImage, catdir, &db);
    295 
    296   myIndexFree (externIDindex);
    297   myIndexFree (imageIDindex);
    298 
    299   free (imageFilename);
    300   free (deleteImage);
    301   free (deleteIDs);
    302 
    303   SkyTableFree(insky);
    304 
    305   gfits_db_free (&db);
    306 
    307   ohana_memcheck (TRUE);
    308 
    309   exit (0);
    310 }
    311 
    312 Image *DeleteSelectedImages (Image *image, off_t Nimage, int *deleteImage, myIndexType *imageIDindex, off_t *NnewImage) {
    313 
    314   // we need to remove the (WRP) images, but we also want to remove the DIS [PHU] image entries if they have
    315   // had all of their children deleted
    316 
    317   // first, generate an array of the number of children for each parent
    318 
    319   int *Nchildren = NULL;
    320   ALLOCATE (Nchildren, int, Nimage);
    321   memset (Nchildren, 0, Nimage*sizeof(int));
    322 
    323   int i;
    324   for (i = 0; i < Nimage; i++) {
    325     if (!image[i].parentID) continue;
    326     if (strcmp(&image[i].coords.ctype[4], "-WRP")) {
    327       fprintf (stderr, "warning: child with non WRP coords : %s\n", image[i].name);
    328     }
    329 
    330     int n = myIndexGetEntry (imageIDindex, image[i].parentID);
    331     myAssert (n != -1, "invalid parentID?");
    332     myAssert (n < Nimage, "invalid parentID??");
    333     Nchildren[n] ++;
    334   }
    335 
    336   // now subtract the children which need to be deleted
    337   for (i = 0; i < Nimage; i++) {
    338     if (!deleteImage[i]) continue;
    339     if (!image[i].parentID) continue; // should I warn on this?
    340 
    341     int n = myIndexGetEntry (imageIDindex, image[i].parentID);
    342     myAssert (n != -1, "invalid parentID?");
    343     myAssert (n < Nimage, "invalid parentID??");
    344     Nchildren[n] --;
    345     myAssert (Nchildren[n] >= 0, "over deleted???");
    346   }
    347  
    348   // now check for any parent and delete it
    349   for (i = 0; i < Nimage; i++) {
    350     if (strcmp(&image[i].coords.ctype[4], "-DIS")) continue;
    351     myAssert (!image[i].parentID, "parent ID for DIS image??");
    352     if (!Nchildren[i]) deleteImage[i] = TRUE;
    353   }
    354 
    355   fprintf (stderr, "deleting the following images:\n");
    356 
    357   off_t Nnew = 0;
    358   Image *newImage = NULL;
    359   ALLOCATE (newImage, Image, Nimage);
    360   for (i = 0; i < Nimage; i++) {
    361     if (deleteImage[i]) {
    362       fprintf (stderr, "DELETE: %s : %d : %d : %d\n", image[i].name, image[i].imageID, image[i].parentID, image[i].externID);
    363       continue;
    364     }
    365     newImage[Nnew] = image[i];
    366     Nnew ++;
    367   }
    368 
    369   REALLOCATE (newImage, Image, Nnew);
    370 
    371   *NnewImage = Nnew;
    372   return newImage;
    373 }
    374 
    375 int SaveImageTable (Image *image, off_t Nimage, char *catdir, FITS_DB *oldDB) {
    376 
    377   FITS_DB db;
    378 
    379   char ImageCatSrc[DVO_MAX_PATH];
    380   char ImageCatTgt[DVO_MAX_PATH];
    381 
    382   // we are creating a new image table (what about db ID?)
    383   snprintf (ImageCatSrc, DVO_MAX_PATH, "%s/Images.dat", catdir);
    384   snprintf (ImageCatTgt, DVO_MAX_PATH, "%s/Images.dat.broken", catdir);
    385 
    386   // rename the old cpt file:
    387   if (rename (ImageCatSrc, ImageCatTgt)) {
    388     perror ("tried to rename file");
    389     exit (2);
    390   }
    391 
    392   /* setup image table format and lock */
    393   gfits_db_init (&db);
    394   db.mode    = dvo_catalog_catmode ("SPLIT");
    395   db.format  = dvo_catalog_catformat ("PS1_V5");
    396   int status = dvo_image_lock (&db, ImageCatSrc, 3600.0, LCK_XCLD);  // shorter timeout?
    397   if (!status) {
    398     fprintf (stderr, "ERROR: failure to lock image catalog %s", db.filename);
    399     exit (2);
    400   }
    401 
    402   /* load or create the image table */
    403   if (db.dbstate != LCK_EMPTY) {
    404     fprintf (stderr, "image table %s exists, exiting", db.filename);
    405     exit (2);
    406   }
    407   dvo_image_create (&db, GetZeroPoint());
    408 
    409   // I want to keep the old DVO_DBID
    410   char dbID[33];
    411   int imageIDmax;
    412   if (!gfits_scan (&oldDB->header, "DVO_DBID", "%s", 1, dbID)) {
    413     fprintf (stderr, "source image table is missing DVO_DBID, exiting\n");
    414     exit (3);
    415   }
    416   if (!gfits_scan (&oldDB->header, "IMAGEID", "%d", 1, &imageIDmax)) {
    417     fprintf (stderr, "max image ID is missing, exiting\n");
    418     exit (3);
    419   }
    420   gfits_modify (&db.header, "IMAGEID", "%d", 1, imageIDmax);     
    421   gfits_modify (&db.header, "DVO_DBID", "%s", 1, dbID);     
    422 
    423   // also save the merge history:
    424   int Nmerge;
    425   if (gfits_scan (&oldDB->header, "NMERGE", "%d", 1, &Nmerge)) {
    426     int i;
    427     gfits_modify (&db.header, "NMERGE", "%d", 1, Nmerge);
    428     for (i = 0; i < Nmerge; i++) {
    429       char field[80], value[80];
    430       snprintf (field, 80, "DM_%05d", i);
    431       if (!gfits_scan (&oldDB->header, field, "%s", 1, value)) {
    432         fprintf (stderr, "missing merge info %s\n", field);
    433         continue;
    434       }
    435       gfits_modify (&db.header, field, "%s", 1, value);
    436     }
    437   }
    438 
    439   /* add the new image and save */
    440   dvo_image_addrows (&db, image, Nimage);
    441   dvo_image_update (&db, VERBOSE);
    442   dvo_image_unlock (&db); /* unlock? */
    443 
    444965  return TRUE;
    445966}
    446967
    447 int RepairTableCPT_V1(char *cptFilenameSrc, char *cptFilenameTgt, char *cpsFilenameSrc, char *cpsFilenameTgt, Measure *measure, off_t Nmeasure, Image *image, off_t Nimage, myIndexType *imageIDindex, char catformat) {
    448 
    449   off_t *averefMatch;
    450   off_t i, NaveMax, Naverage, NAVERAGE, NaverageOut, Nave, Nout, Nold;
    451   int *found, Nsecfilt;
    452 
    453   Average *average, *averageOut;
    454 
    455   Matrix matrix;
    456 
    457   Header cptHeaderPHU;
    458   Header cptHeaderTBL;
    459   FTable cptFtable;
    460 
    461   cptFtable.header = &cptHeaderTBL;
    462 
    463   NaveMax = 0;
    464   NAVERAGE = 1000;
    465   ALLOCATE (average, Average, NAVERAGE);
    466   memset (average, 0, NAVERAGE*sizeof(Average));
    467 
    468   ALLOCATE (found, int, NAVERAGE);
    469   memset (found, 0, NAVERAGE*sizeof(int));
    470 
    471   // examine all measurements and create new objects as needed
    472   // we start with a valid, sorted dvo database, so we use averef to link objects
    473   for (i = 0; i < Nmeasure; i++) {
    474     Nave = measure[i].averef;
    475 
    476     // we only allocate as many as we need
    477     if (Nave >= NAVERAGE) {
    478       Nold = NAVERAGE;
    479       NAVERAGE = MAX(Nave + 1000, NAVERAGE + 1000);
    480       REALLOCATE (average, Average, NAVERAGE);
    481       memset (&average[Nold], 0, (NAVERAGE - Nold)*sizeof(Average));
    482 
    483       REALLOCATE (found, int, NAVERAGE);
    484       memset (&found[Nold], 0, (NAVERAGE - Nold)*sizeof(int));
    485     }
    486 
    487     // this measure matches an existing average, just check it is OK and bump Nmeasure
    488     if (found[Nave]) {
    489       average[Nave].Nmeasure ++;
    490       myAssert(average[Nave].objID == measure[i].objID, "objIDs do not match!");
    491       myAssert(average[Nave].catID == measure[i].catID, "catIDs do not match!");
    492       continue;
    493     }
    494 
    495     NaveMax = MAX(Nave, NaveMax);
    496 
    497     found[Nave] = TRUE;
    498 
    499     // we are going to leave most of the elements of average unset: they are the result of
    500     // the relastro analysis for this object and can be recreated with a call to relastro
    501 
    502     // need to find image so we can use ccd coordinates to determine RA & DEC
    503     int n = myIndexGetEntry (imageIDindex, measure[i].imageID);
    504     myAssert (n > -1, "impossible!");
    505    
    506     dvo_average_init (&average[Nave]);
    507 
    508     // I actually have measure->R,D, I could just use those...
    509     XY_to_RD (&average[Nave].R, &average[Nave].D, measure[i].Xccd, measure[i].Yccd, &image[n].coords);
    510 
    511     average[Nave].Nmeasure = 1;
    512     average[Nave].Nmissing = 0;
    513 
    514     // assume the resulting table set is unsorted
    515     average[Nave].measureOffset = -1;
    516     average[Nave].missingOffset = -1;
    517 
    518     average[Nave].objID = measure[i].objID;
    519     average[Nave].catID = measure[i].catID;
    520     average[Nave].extID = CreatePSPSObjectID(average[Nave].R, average[Nave].D);
    521   }
    522   Naverage = NaveMax + 1;
    523 
    524   // we now have an average table, but there will be holes due to deleted measurements
    525   // create a new average table with only existing entries
    526 
    527   ALLOCATE (averageOut, Average, Naverage);
    528   memset (averageOut, 0, Naverage*sizeof(Average));
    529 
    530   ALLOCATE (averefMatch, off_t, Naverage);
    531   memset (averefMatch, 0, Naverage*sizeof(int));
    532 
    533   Nave = 0;
    534   for (i = 0; i < Naverage; i++) {
    535     if (!found[i]) continue;
    536     averageOut[Nave] = average[i]; // use a memcpy?
    537     averefMatch[i] = Nave;
    538     Nave ++;
    539   }
    540   NaverageOut = Nave;
    541 
    542   // modify measure.averef to match the new sequence
    543   for (i = 0; i < Nmeasure; i++) {
    544     Nave = measure[i].averef;
    545     Nout = averefMatch[Nave];
    546     myAssert(Nout < NaverageOut, "output averef is wrong");
    547    
    548     myAssert(average[Nave].objID == measure[i].objID, "objIDs do not match");
    549     myAssert(average[Nave].catID == measure[i].catID, "objIDs do not match");
    550     myAssert(averageOut[Nout].objID == measure[i].objID, "objIDs do not match");
    551     myAssert(averageOut[Nout].catID == measure[i].catID, "objIDs do not match");
    552 
    553     measure[i].averef = Nout;
    554   }
    555 
    556   fprintf (stderr, "cpt file : %d obj -> %d obj (%s -> %s)\n", (int) Naverage, (int) NaverageOut, cptFilenameSrc, cptFilenameTgt);
    557 
    558   // open source cpt file
    559   FILE *cptFile = fopen(cptFilenameSrc, "r");
    560   myAssert(cptFile, "failed to open cpt file");
    561 
    562   // load the cpt header (use for CATID, RA, DEC range, filenames)
    563   if (!gfits_fread_header (cptFile, &cptHeaderPHU)) {
    564     myAbort("failure to cpt header");
    565   }
    566 
    567   // update the output header
    568   gfits_modify (&cptHeaderPHU, "NSTARS",     OFF_T_FMT, 1,  NaverageOut);
    569   gfits_modify (&cptHeaderPHU, "NMEAS",      OFF_T_FMT, 1,  Nmeasure);
    570   gfits_modify (&cptHeaderPHU, "NMISS",      "%d",      1,  0);
    571   gfits_modify_alt (&cptHeaderPHU, "SORTED", "%t",      1,  FALSE);
    572 
    573   gfits_scan (&cptHeaderPHU, "NSECFILT",     "%d",      1,  &Nsecfilt);
    574 
    575   if (0) {
    576     char compressMode[256];
    577     if (gfits_scan (&cptHeaderTBL, "DVO_CMP", "%s", 1, compressMode)) {
    578       if (strcmp (compressMode, "NONE")) {
    579         myAbort ("fix compression");
     968int dvorepairDeleteImagesByExternID_parallel (SkyRegion *UserPatch, int nUserPatch, Image *image, off_t Nimage, int *deleteImage) {
     969
     970  // ensure that the paths are absolute path names
     971  char *abscatdir = abspath (CATDIR, DVO_MAX_PATH);
     972
     973  // load the sky table for the existing database
     974  SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
     975  myAssert(sky, "can't read SkyTable");
     976 
     977  // load the list of hosts
     978  HostTable *table = HostTableLoad (CATDIR, sky->hosts);
     979  if (!table) {
     980    fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
     981    exit (1);
     982  }   
     983
     984  char filename[DVO_MAX_PATH];
     985  snprintf (filename, DVO_MAX_PATH, "%s/DeleteImages.fits", CATDIR);
     986  if (!DeleteImagesSave (filename, image, Nimage, deleteImage)) {
     987    fprintf (stderr, "ERROR: failure to save delete image info\n");
     988    exit (2);
     989  }
     990
     991  int i;
     992  for (i = 0; i < table->Nhosts; i++) {
     993
     994    // ensure that the paths are absolute path names
     995    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
     996    free (table->hosts[i].pathname);
     997    table->hosts[i].pathname = tmppath;
     998
     999    // options / arguments that can affect relastro_client -update-objects:
     1000    char *command = NULL;
     1001    strextend (&command, "dvorepair_client -delete-images-by-extern-id %s -hostID %d -hostdir %s", abscatdir, table->hosts[i].hostID, table->hosts[i].pathname);
     1002
     1003    strextend (&command, "-region %f %f %f %f", UserPatch[0].Rmin, UserPatch[0].Rmax, UserPatch[0].Dmin, UserPatch[0].Dmax);
     1004    if (nUserPatch == 2) {
     1005      strextend (&command, "-region %f %f %f %f", UserPatch[1].Rmin, UserPatch[1].Rmax, UserPatch[1].Dmin, UserPatch[1].Dmax);
     1006    }
     1007
     1008    if (VERBOSE) { strextend (&command, "-v"); }
     1009
     1010    fprintf (stderr, "command: %s\n", command);
     1011
     1012    if (PARALLEL_MANUAL) continue;
     1013
     1014    if (PARALLEL_SERIAL) {
     1015      int status = system (command);
     1016      if (status) {
     1017        fprintf (stderr, "ERROR running dvorepair_client\n");
     1018        exit (2);
    5801019      }
    581     }
    582   }
    583 
    584   /* convert internal to external format */
    585   if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL, TRUE)) {
    586     myAbort("trouble converting format");
    587   }
    588   fclose(cptFile);
    589  
    590   // rename the old cpt file:
    591   if (rename (cptFilenameSrc, cptFilenameTgt)) {
    592     perror ("tried to rename file");
    593     exit (2);
    594   }
    595 
    596   // create and write the output file
    597   cptFile = fopen(cptFilenameSrc, "w");
    598   myAssert(cptFile, "failed to open cpt file");
    599    
    600   // write PHU header
    601   if (!gfits_fwrite_header (cptFile, &cptHeaderPHU)) {
    602     myAbort("can't write primary header");
    603   }
    604 
    605   // write the PHU matrix; this is probably a NOP, do I have to keep it in?
    606   gfits_create_matrix (&cptHeaderPHU, &matrix);
    607   if (!gfits_fwrite_matrix  (cptFile, &matrix)) {
    608     myAbort("can't write primary matrix");
    609   }
    610   gfits_free_matrix (&matrix);
    611 
    612   // write the table data
    613   if (!gfits_fwrite_ftable_range (cptFile, &cptFtable, 0, NaverageOut, 0, NaverageOut)) {
    614     myAbort("can't write table data");
    615   }
    616   fclose(cptFile);
    617 
    618   gfits_free_table (&cptFtable);
    619   gfits_free_header (&cptHeaderPHU);
    620   gfits_free_header (&cptHeaderTBL);
    621   free (average);
    622   free (averageOut);
    623 
    624   free (found);
    625   free (averefMatch);
    626 
    627   {
    628     Header cpsHeaderPHU;
    629     Header cpsHeaderTBL;
    630     FTable cpsFtable;
    631 
    632     SecFilt *secfilt = NULL;
    633 
    634     cpsFtable.header = &cpsHeaderTBL;
    635 
    636     // open source cpt file
    637     FILE *cpsFile = fopen(cpsFilenameSrc, "r");
    638     myAssert(cpsFile, "failed to open cps file");
    639    
    640     // load the cps header (use for CATID, RA,DEC range, filenames)
    641     if (!gfits_fread_header (cpsFile, &cpsHeaderPHU)) {
    642       myAbort("failure to cps header");
    643     }
    644 
    645     int Nrows = Nsecfilt*NaverageOut;
    646     ALLOCATE (secfilt, SecFilt, Nrows);
    647 
    648     for (i = 0; i < Nrows; i++) {
    649       dvo_secfilt_init (&secfilt[i], SECFILT_RESET_ALL);
    650     }
    651 
    652     /* convert internal to external format */
    653     if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat, TRUE)) {
    654       myAbort("trouble converting format");
    655     }
    656     fclose(cpsFile);
    657 
    658     // rename the old cpt file:
    659     if (rename (cpsFilenameSrc, cpsFilenameTgt)) {
    660       perror ("tried to rename file");
    661       exit (2);
    662     }
    663 
    664     // create and write the output file
    665     cpsFile = fopen(cpsFilenameSrc, "w");
    666     myAssert(cpsFile, "failed to open cps file");
    667    
    668     // write PHU header
    669     if (!gfits_fwrite_header (cpsFile, &cpsHeaderPHU)) {
    670       myAbort("can't write primary header");
    671     }
    672 
    673     // write the PHU matrix; this is probably a NOP, do I have to keep it in?
    674     gfits_create_matrix (&cpsHeaderPHU, &matrix);
    675     if (!gfits_fwrite_matrix  (cpsFile, &matrix)) {
    676       myAbort("can't write primary matrix");
    677     }
    678     gfits_free_matrix (&matrix);
    679 
    680     // write the table data
    681     if (!gfits_fwrite_ftable_range (cpsFile, &cpsFtable, 0, Nrows, 0, Nrows)) {
    682       myAbort("can't write table data");
    683     }
    684     fclose(cpsFile);
    685 
    686     gfits_free_table (&cpsFtable);
    687     gfits_free_header (&cpsHeaderPHU);
    688     gfits_free_header (&cpsHeaderTBL);
    689     free (secfilt);
    690   }
    691 
    692   return (TRUE);
    693 }
    694 
    695 int RepairAverage (Catalog *catalog) {
    696 
    697   off_t i, j, Nave;
    698 
    699   // average is sorted so averef is valid
    700   // secfilt is also sorted so sequence is valid (but this is kind of moot since it will be re-calculated)
    701   // average.measureOffset is NOT valid
    702   // average.lensingOffset is NOT valid
    703 
    704   Average *average = catalog->average;
    705   Measure *measure = catalog->measure;
    706   Lensing *lensing = catalog->lensing;
    707  
    708   // reset the values of average.Nmeasure, average.Nlensing
    709   for (i = 0; i < catalog->Naverage; i++) {
    710     average[i].Nmeasure = 0;
    711     average[i].Nlensing = 0;
    712   }
    713 
    714   // re-calculate average.Nmeasure
    715   for (i = 0; i < catalog->Nmeasure; i++) {
    716     Nave = measure[i].averef;
    717     myAssert (measure[i].objID == average[Nave].objID, "invalid measure:average match");
    718     average[Nave].Nmeasure ++;
    719   }
    720    
    721   // re-calculate average.Nlensing
    722   for (i = 0; i < catalog->Nlensing; i++) {
    723     Nave = lensing[i].averef;
    724     myAssert (lensing[i].objID == average[Nave].objID, "invalid lensing:average match");
    725     average[Nave].Nlensing ++;
    726   }
    727 
    728   // create a copy of the average table, keeping only the entries with Nmeasure & Nlensing > 0
    729   ALLOCATE_PTR (averageNew, Average, catalog->Naverage);
    730   ALLOCATE_PTR (averefNew,  int,     catalog->Naverage);
    731 
    732   Nave = 0;
    733   for (i = 0; i < catalog->Naverage; i++) {
    734     averefNew[i] = -1;
    735     if (!average[i].Nmeasure && !average[i].Nlensing) continue;
    736 
    737     averageNew[Nave] = average[i];
    738     averefNew[i] = Nave;
    739     Nave ++;
    740   }
    741   off_t NaverageNew = Nave;
    742 
    743   // update measure.averef values
    744   for (i = 0; i < catalog->Nmeasure; i++) {
    745     Nave = averefNew[measure[i].averef];
    746     myAssert (Nave >= 0, "oops");
    747     measure[i].averef = Nave;
    748   }
    749    
    750   // update lensing.averef values
    751   for (i = 0; i < catalog->Nlensing; i++) {
    752     Nave = averefNew[lensing[i].averef];
    753     myAssert (Nave >= 0, "oops");
    754     lensing[i].averef = Nave;
    755   }
    756  
    757   // XXX need to update measureOffset and lensingOffset
    758 
    759   // measure[] should be blocked and sequential:
    760   // measure[i+1].averef >= measure[i].averef
    761 
    762   // update average.measureOffset values
    763   Nave = -1;
    764   for (i = 0; i < catalog->Nmeasure; i++) {
    765     if (measure[i].averef == Nave) continue;
    766     Nave = measure[i].averef;
    767     averageNew[Nave].measureOffset = i;
    768   }
    769 
    770   // update average.lensingOffset values
    771   Nave = -1;
    772   for (i = 0; i < catalog->Nlensing; i++) {
    773     if (lensing[i].averef == Nave) continue;
    774     Nave = lensing[i].averef;
    775     averageNew[Nave].lensingOffset = i;
    776   }
    777 
    778   // check the result (measure -> average)
    779   for (i = 0; i < catalog->Nmeasure; i++) {
    780     Nave = measure[i].averef;
    781     myAssert(averageNew[Nave].objID == measure[i].objID, "objIDs do not match");
    782     myAssert(averageNew[Nave].catID == measure[i].catID, "catIDs do not match");
    783   }
    784   // check the result (average -> measure)
    785   for (i = 0; i < NaverageNew; i++) {
    786     int m = averageNew[i].measureOffset;
    787     for (j = 0; j < averageNew[i].Nmeasure; j++) {
    788       myAssert(averageNew[i].objID == measure[j+m].objID, "objIDs do not match");
    789       myAssert(averageNew[i].catID == measure[j+m].catID, "catIDs do not match");
    790       myAssert(measure[j+m].averef == i, "averef broken");
    791     }
    792   }
    793 
    794   // check the result (lensing -> average)
    795   for (i = 0; i < catalog->Nlensing; i++) {
    796     Nave = lensing[i].averef;
    797     myAssert(averageNew[Nave].objID == lensing[i].objID, "objIDs do not match");
    798     myAssert(averageNew[Nave].catID == lensing[i].catID, "catIDs do not match");
    799   }
    800   // check the result (average -> lensing)
    801   for (i = 0; i < NaverageNew; i++) {
    802     int m = averageNew[i].lensingOffset;
    803     for (j = 0; j < averageNew[i].Nlensing; j++) {
    804       myAssert(averageNew[i].objID == lensing[j+m].objID, "objIDs do not match");
    805       myAssert(averageNew[i].catID == lensing[j+m].catID, "catIDs do not match");
    806       myAssert(lensing[j+m].averef == i, "averef broken");
    807     }
    808   }
    809 
    810   free (catalog->secfilt);
    811   ALLOCATE (catalog->secfilt, SecFilt, NaverageNew*catalog->Nsecfilt);
    812   for (i = 0; i < NaverageNew*catalog->Nsecfilt; i++) {
    813     dvo_secfilt_init (&catalog->secfilt[i], SECFILT_RESET_ALL);
    814   }
    815 
    816   free (averefNew);
    817   free (catalog->average);
    818   catalog->average = averageNew;
    819   catalog->Naverage = NaverageNew;
    820   catalog->Naverage_disk = NaverageNew;
    821 
    822   catalog->Nsecfilt_disk = NaverageNew*catalog->Nsecfilt;
    823 
    824   return (TRUE);
    825 }
    826 
    827 // delete measure entries
    828 int DeleteMeasure (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) {
    829 
    830   int j;
    831 
    832   Measure *measure = catalog->measure;
    833 
    834   // allocate an output array of measures (to replace, if needed)
    835   ALLOCATE_PTR (measureNew, Measure, catalog->Nmeasure);
    836 
    837   int NmeasureNew = 0;
    838   int NmeasureDel = 0;
    839 
    840   // examine all measurements: find ones that need to be deleted
    841   for (j = 0; j < catalog->Nmeasure; j++) {
    842     int imageID = measure[j].imageID;
    843     myAssert(imageID, "measure is missing an image ID");
    844     // XXX this case is valid if we have REF detections (no associated image)
    845 
    846     int N = myIndexGetEntry(imageIDindex, imageID);
    847     if (N < 0) {
    848       // this detection comes from a non-existant image; delete
    849       NmeasureDel ++;
    850       continue;
    851     }
    852 
    853     // measure matches a bad image; delete
    854     if (deleteImage[N]) {
    855       NmeasureDel ++;
    856       continue;
    857     }
    858 
    859     // keep this measure
    860     measureNew[NmeasureNew] = measure[j];
    861     NmeasureNew ++;
    862   }
    863 
    864   free (catalog->measure);
    865   catalog->measure = measureNew;
    866   catalog->Nmeasure = NmeasureNew;
    867   catalog->Nmeasure_disk = NmeasureNew;
    868 
    869   *nDelete = NmeasureDel;
     1020    } else {
     1021      // launch the job on the remote machine (no handshake)
     1022      int errorInfo = 0;
     1023      int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
     1024      if (!pid) {
     1025        if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
     1026        exit (1);
     1027      }
     1028      table->hosts[i].pid = pid; // save for future reference
     1029    }
     1030    free (command);
     1031  }
     1032
     1033  if (PARALLEL_MANUAL) {
     1034    fprintf (stderr, "run the dvorepair_client commands above.  when these are done, hit return\n");
     1035    getchar();
     1036  }
     1037  if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
     1038    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
     1039  }
    8701040
    8711041  return TRUE;
    872 }
    873 
    874 // delete lensing entries
    875 int DeleteLensing (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) {
    876 
    877   int j;
    878 
    879   Lensing *lensing = catalog->lensing;
    880 
    881   // allocate an output array of measures (to replace, if needed)
    882   ALLOCATE_PTR (lensingNew, Lensing, catalog->Nlensing);
    883 
    884   int NlensingNew = 0;
    885   int NlensingDel = 0;
    886 
    887   // examine all lensingments: find ones that need to be deleted
    888   for (j = 0; j < catalog->Nlensing; j++) {
    889     int imageID = lensing[j].imageID;
    890     myAssert(imageID, "lensing is missing an image ID");
    891     // XXX this case is valid if we have REF detections (no associated image)
    892 
    893     int N = myIndexGetEntry(imageIDindex, imageID);
    894     if (N < 0) {
    895       // this detection comes from a non-existant image; delete
    896       NlensingDel ++;
    897       continue;
    898     }
    899 
    900     // lensing matches a bad image; delete
    901     if (deleteImage[N]) {
    902       NlensingDel ++;
    903       continue;
    904     }
    905 
    906     // keep this lensing
    907     lensingNew[NlensingNew] = lensing[j];
    908     NlensingNew ++;
    909   }
    910 
    911   free (catalog->lensing);
    912   catalog->lensing = lensingNew;
    913   catalog->Nlensing = NlensingNew;
    914   catalog->Nlensing_disk = NlensingNew;
    915 
    916   *nDelete = NlensingDel;
    917 
    918   return TRUE;
    919 }
    920 
     1042}     

  • trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID_v2.c

    r39281 r39323  
    1 # include "dvomerge.h"
     1# include "dvorepair.h"
    22
    33// delete images based on a text table of the target IDs
    … …  
    2424  FITS_DB db;  // database handle pointing to input image table
    2525 
    26   int i, N;
     26  int i;
    2727  off_t Nimage;
    2828
    … …  
    3030  char *imageFilename = NULL;
    3131  char filename[DVO_MAX_PATH];
    32 
    33   N = get_argument (argc, argv, "-delete-images-by-extern-id-v2");
    34   myAssert(N == 1, "programming error: -delete-images-by-extern-id must be first argument");
    35   remove_argument (N, &argc, argv);
    36 
    37   VERBOSE = FALSE;
    38   if ((N = get_argument (argc, argv, "-v"))) {
    39     VERBOSE = TRUE;
    40     remove_argument (N, &argc, argv);
    41   }
    4232
    4333  if (argc != 3) {

  • trunk/Ohana/src/dvomerge/src/dvorepairFixCPT.c

    r38441 r39323  
    1212int dvorepairFixCPT (int argc, char **argv) {
    1313
    14   int i, j, N, Nfiles, NFILES;
     14  int i, j, Nfiles, NFILES;
    1515  off_t Nimage;
    1616  FITS_DB db;  // database handle pointing to input image table
    … …  
    1919  char **files;
    2020
    21   N = get_argument (argc, argv, "-fix-cpt");
    22   myAssert(N == 1, "programming error: -fix-cpt must be first from main");
    23   remove_argument (N, &argc, argv);
    24 
    2521  if (argc != 3) {
    2622    fprintf (stderr, "USAGE: dvorepair -fix-cpt (images) (filelist)\n");
    2723    exit (2);
    2824  }
     25
     26  fprintf (stderr, "is this mode tested?\n");
     27  exit (2);
    2928
    3029  imageFilename  = argv[1];

  • trunk/Ohana/src/dvomerge/src/dvorepairFixImages.c

    r38986 r39323  
    2121  off_t i, Nimage, Nout, Nindex, *imageIdx, index;
    2222 
    23   int N;
    24 
    2523  Image *image, *imageOut;
    2624
    2725  char *imageFilenameOld = NULL;
    2826  char *imageFilenameNew = NULL;
    29 
    30   N = get_argument (argc, argv, "-fix-images");
    31   myAssert(N == 1, "programming error: -fix-images must be first from main");
    32   remove_argument (N, &argc, argv);
    3327
    3428  if (argc != 3) {
    … …  
    3832    exit (2);
    3933  }
     34
     35  fprintf (stderr, "is this mode tested?\n");
     36  exit (2);
    4037
    4138  char *catdir = argv[1];

  • trunk/Ohana/src/dvomerge/src/dvorepairFixStackIDs.c

    r39225 r39323  
    1111  Image *image;
    1212
    13   int N = get_argument (argc, argv, "-fix-stack-ids");
    14   myAssert(N == 1, "programming error: -fix-stack-ids must be first from main");
    15   remove_argument (N, &argc, argv);
    16 
    1713  if (argc != 2) {
    1814    fprintf (stderr, "USAGE: dvorepair -fix-stack-ids (catdir.list)\n");
    … …  
    2319    exit (2);
    2420  }
     21
     22  fprintf (stderr, "is this mode tested?\n");
     23  exit (2);
    2524
    2625  char *catdir_list  = argv[1];

  • trunk/Ohana/src/dvomerge/src/dvorepairFixWarpIDs.c

    r38986 r39323  
    1717  off_t Nimage;
    1818 
    19   int N;
    20 
    2119  Image *image;
    2220
    23   N = get_argument (argc, argv, "-fix-warp-ids");
    24   myAssert(N == 1, "programming error: -fix-images must be first from main");
    25   remove_argument (N, &argc, argv);
    26 
    2721  if (argc != 3) {
     22    // NOTE: mode has been stripped
    2823    fprintf (stderr, "USAGE: dvorepair -fix-warp-ids (catdir.list) (warp.ids)\n");
    2924    fprintf (stderr, "  catdir.list : list of databases of interest\n");
    … …  
    3126    exit (2);
    3227  }
     28
     29  fprintf (stderr, "is this mode tested?\n");
     30  exit (2);
    3331
    3432  char *catdir_list  = argv[1];

  • trunk/Ohana/src/dvomerge/src/dvorepairImagesVsMeasures.c

    r38441 r39323  
    1313
    1414  off_t i, j, Nmeasure, Nvalid, Nimage, Nindex, *imageIdx, index, imageSeq;
    15   int N, Nbytes, NbytesPerRow, Nbad, Ncheck, Ntol;
     15  int Nbytes, NbytesPerRow, Nbad, Ncheck, Ntol;
    1616  int *detCounts;
    1717
    … …  
    3434  DVOCatFormat catformat;
    3535
    36   N = get_argument (argc, argv, "-images-vs-measures");
    37   myAssert(N == 1, "programming error: -images-vs-measures must be first from main");
    38   remove_argument (N, &argc, argv);
    39 
    4036  if (argc != 3) {
    4137    fprintf (stderr, "USAGE: dvorepair -images-vs-measures (catdir) (Ntol)\n");
    … …  
    4440    exit (2);
    4541  }
     42
     43  fprintf (stderr, "is this mode tested?\n");
     44  exit (2);
    4645
    4746  catdir = argv[1];

  • trunk/Ohana/src/dvomerge/src/dvorepair_by_objID.c

    r38986 r39323  
    22
    33int dvorepair_by_objID (int argc, char **argv) {
    4 
    5   int N = get_argument (argc, argv, "-fix-cpt-by-objID");
    6   if (N != 1) {
    7     fprintf (stderr, "-fix-cpt must be first from main\n");
    8     exit (2);
    9   }
    10   remove_argument (N, &argc, argv);
    114
    125  if (argc != 2) {
    … …  
    147    exit (2);
    158  }
     9
     10  fprintf (stderr, "is this mode tested?\n");
     11  exit (2);
    1612
    1713  char *cptFile = argv[1];

  • trunk/Ohana/src/dvomerge/src/help.c

    r39309 r39323  
    159159}
    160160
    161 void dvorepair_help (int argc, char **argv) {
    162 
    163   /* check for help request */
    164   if (!argv) goto show_help;
    165   if (get_argument (argc, argv, "-help")) goto show_help;
    166   if (get_argument (argc, argv, "-h"))    goto show_help;
    167   if (argc < 2) goto show_help;
    168   return;
    169 
    170 show_help:
    171 
    172   fprintf (stderr, "USAGE\n");
    173   fprintf (stderr, "  dvorepair -fix-warp-ids (catdir.list) (idfile) - regenerate images.dat warp_skyfile_ids (EXTERN_ID) values\n");
    174   fprintf (stderr, "  dvorepair -fix-stack-ids (catdir.list) - regenerate images.dat stackID (EXTERN_ID) values\n");
    175   fprintf (stderr, "  dvorepair -fix-cpt (images) (rootlist) - regenerate cpt & cps files from the cpm files\n");
    176   fprintf (stderr, "  dvorepair -images-vs-measures (catdir) (Ntol) - find images with too many missing detections\n");
    177   fprintf (stderr, "  dvorepair -delete-image-list (catdir) (deleteList) - delete a set of images based on image IDs (output from -images-vs-measures)\n");
    178   fprintf (stderr, "  dvorepair -fix-images (catdir) (deleteList) - delete a set of images based on image IDs (output from -images-vs-measures)\n");
    179   fprintf (stderr, "\n");
    180 
    181   fprintf (stderr, "  optional flags:\n");
    182   fprintf (stderr, "  -help                       : this list\n");
    183   fprintf (stderr, "  -h                          : this list\n\n");
    184   exit (2);
    185 }
    186 

  • trunk/Ohana/src/dvomerge/src/myIndex.c

    r37807 r39323  
    5555
    5656  myAssert (n >= 0, "impossible!");
    57   myAssert(myIndex->index[n] == -1, "stomping on an earlier index");
     57
     58  if (myIndex->index[n] != -1) {
     59    fprintf (stderr, "skipping duplicate ID value: %d\n", value);
     60    return -1;
     61  }
    5862
    5963  myIndex->index[n] = entry;
Note: See TracChangeset for help on using the changeset viewer.