Index: /branches/eam_branches/ipp-20130509/Ohana/src/delstar/Makefile
===================================================================
--- /branches/eam_branches/ipp-20130509/Ohana/src/delstar/Makefile	(revision 35706)
+++ /branches/eam_branches/ipp-20130509/Ohana/src/delstar/Makefile	(revision 35707)
@@ -30,4 +30,5 @@
 $(SRC)/delete_imagefile.$(ARCH).o \
 $(SRC)/delete_duplicate_images.$(ARCH).o \
+$(SRC)/delete_duplicate_measures.$(ARCH).o \
 $(SRC)/delete_image_photcodes.$(ARCH).o \
 $(SRC)/delete_photcodes.$(ARCH).o \
@@ -48,4 +49,5 @@
 $(SRC)/args.$(ARCH).o	          \
 $(SRC)/delete_duplicate_images.$(ARCH).o \
+$(SRC)/delete_duplicate_measures.$(ARCH).o \
 $(SRC)/delete_photcodes.$(ARCH).o \
 $(SRC)/delete_photcodes_single.$(ARCH).o \
Index: /branches/eam_branches/ipp-20130509/Ohana/src/delstar/include/delstar.h
===================================================================
--- /branches/eam_branches/ipp-20130509/Ohana/src/delstar/include/delstar.h	(revision 35706)
+++ /branches/eam_branches/ipp-20130509/Ohana/src/delstar/include/delstar.h	(revision 35707)
@@ -52,4 +52,5 @@
 int          HOST_ID;
 char        *HOSTDIR;
+char        *SINGLE_CPT;
 
 int SKIP_IMAGES;
@@ -62,5 +63,5 @@
 
 int    MODE;
-enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES};
+enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES, MODE_DUP_MEASURES};
 
 char DateKeyword[64], DateMode[64], UTKeyword[64], MJDKeyword[64], JDKeyword[64];
@@ -109,5 +110,5 @@
 void SortAveMeasMatch (off_t *MEAS, off_t *AVE, off_t N);
 
-int delete_photcodes (int hostID, char *hostpath);
+int delete_photcodes ();
 int delete_photcodes_parallel (SkyList *sky);
 int delete_photcodes_catalog (Catalog *catalog, PhotCode **photcodes, int Nphotcodes);
@@ -115,6 +116,6 @@
 int delete_photcodes_single (char *cptname);
 
-int delete_duplicate_images (int hostID, char *hostpath, FITS_DB *db);
-int delete_duplicate_image_measures (int hostID, char *hostpath, IndexArray *imageID);
+int delete_duplicate_images (FITS_DB *db);
+int delete_duplicate_image_measures (IndexArray *imageID);
 int delete_duplicate_image_measures_parallel (SkyList *sky, IndexArray *imageID);
 int delete_duplicate_image_measures_catalog (Catalog *catalog, IndexArray *imageID);
@@ -125,2 +126,7 @@
 IndexArray *ImageIDLoad(char *filename);
 int ImageIDSave(char *filename, IndexArray *imageID);
+
+int delete_duplicate_measures ();
+int delete_duplicate_measures_parallel (SkyList *sky);
+int delete_duplicate_measures_catalog (Catalog *catalog);
+
Index: /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/args.c
===================================================================
--- /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/args.c	(revision 35706)
+++ /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/args.c	(revision 35707)
@@ -11,8 +11,9 @@
   fprintf (stderr, "  delstar -photcodes (list) : delete by photcode\n\n");
   fprintf (stderr, "  delstar -dup-images : delete duplicate images (by externID)\n\n");
+  fprintf (stderr, "  delstar -dup-measures : delete duplicate measures (by imageID + detID)\n\n");
   fprintf (stderr, "  optional flags:\n");
   fprintf (stderr, "  -v               : verbose mode\n");
   fprintf (stderr, "  -photcode (code) : restrict by photcode\n");
-  fprintf (stderr, "  -update : apply changes (-dup-images only)\n");
+  fprintf (stderr, "  -update : apply changes (-dup-images or -dup-measures only)\n");
   fprintf (stderr, "  -image-details : list info about the deleted images (-dup-images only)\n");
   fprintf (stderr, "  -image-only : only examine the image table (changes are NOT saved; -dup-images only)\n");
@@ -79,4 +80,11 @@
   }
 
+  SINGLE_CPT = NULL;
+  if ((N = get_argument (argc, argv, "-cpt"))) {
+    remove_argument (N, &argc, argv);
+    SINGLE_CPT = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
   // region of interest
   UserPatch.Rmin = 0;
@@ -104,4 +112,5 @@
   // XXX for the moment, make this selection manual.  it needs to be automatic 
   // based on the state of the SkyTable
+  HOST_ID = 0;
   PARALLEL = FALSE;
   if ((N = get_argument (argc, argv, "-parallel"))) {
@@ -159,4 +168,9 @@
     if (MODE != MODE_NONE) usage();
     MODE = MODE_DUP_IMAGES;
+    remove_argument (N, &argc, argv);
+  }
+  if ((N = get_argument (argc, argv, "-dup-measures"))) {
+    if (MODE != MODE_NONE) usage();
+    MODE = MODE_DUP_MEASURES;
     remove_argument (N, &argc, argv);
   }
@@ -303,4 +317,11 @@
   }
 
+  SINGLE_CPT = NULL;
+  if ((N = get_argument (argc, argv, "-cpt"))) {
+    remove_argument (N, &argc, argv);
+    SINGLE_CPT = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
   /* specify portion of the sky */
   UserPatch.Rmin = 0;
@@ -333,4 +354,9 @@
     remove_argument (N, &argc, argv);
   }
+  if ((N = get_argument (argc, argv, "-dup-measures"))) {
+    if (MODE != MODE_NONE) usage();
+    MODE = MODE_DUP_MEASURES;
+    remove_argument (N, &argc, argv);
+  }
 
   if (MODE == MODE_NONE) delstar_client_usage ();
Index: /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delete_duplicate_images.c
===================================================================
--- /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delete_duplicate_images.c	(revision 35706)
+++ /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delete_duplicate_images.c	(revision 35707)
@@ -6,5 +6,5 @@
 // find & delete duplicate images
 // duplicates based on externID (skip 0s)
-int delete_duplicate_images (int hostID, char *hostpath, FITS_DB *db) {
+int delete_duplicate_images (FITS_DB *db) {
 
   off_t i, Nimage;
@@ -28,5 +28,5 @@
 
   if (!IMAGE_ONLY) {
-    delete_duplicate_image_measures (hostID, hostpath, imageID);
+    delete_duplicate_image_measures (imageID);
   }
 
@@ -63,5 +63,5 @@
 }
 
-int delete_duplicate_image_measures (int hostID, char *hostpath, IndexArray *imageID) {
+int delete_duplicate_image_measures (IndexArray *imageID) {
 
   int i;
@@ -83,5 +83,5 @@
 
   // launch the remote jobs
-  if (PARALLEL && !hostID) {
+  if (PARALLEL && !HOST_ID) {
     int status = delete_duplicate_image_measures_parallel (skylist, imageID);
     return status;
@@ -92,10 +92,10 @@
 
     // does this host ID match the desired location for the table?
-    if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue;
+    if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue;
 
     // set up the basic catalog info
     char hostfile[1024];
-    snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name);
-    catalog.filename  = hostID ? hostfile : skylist[0].filename[i];
+    snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
+    catalog.filename  = HOST_ID ? hostfile : skylist[0].filename[i];
     catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
     catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
@@ -241,4 +241,5 @@
   // mark the measures to be dropped
   for (i = 0; i < Nmeasure; i++) {
+    measureDrop[i] = 0;
     if (measure[i].imageID == 0) continue;
     off_t Ni = measure[i].imageID - imageID->minID;
Index: /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delete_duplicate_measures.c
===================================================================
--- /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delete_duplicate_measures.c	(revision 35707)
+++ /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delete_duplicate_measures.c	(revision 35707)
@@ -0,0 +1,405 @@
+# include "delstar.h"
+
+// this function identifies detections to be deleted as being duplicates based on imageID + detID
+
+int delete_duplicate_measures () {
+
+  int i;
+  Catalog catalog;
+
+  // load the current sky table (layout of all SkyRegions) 
+  SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, TRUE, -1, VERBOSE);
+  if (!sky) {
+    fprintf (stderr, "ERROR running loading sky table from %s\n", CATDIR);
+    exit (2);
+  }
+  SkyTableSetFilenames (sky, CATDIR, "cpt");
+
+  // determine the populated SkyRegions overlapping the requested area (default depth)
+  SkyList *skylist = NULL;
+  if (SINGLE_CPT) {
+      skylist = SkyRegionByCPT (sky, SINGLE_CPT);
+  } else {
+      skylist = SkyListByPatch (sky, -1, &UserPatch);
+  }
+  if (!skylist) {
+    fprintf (stderr, "ERROR setting up skylist for %s\n", CATDIR);
+    exit (2);
+  }
+
+  // launch the remote jobs
+  if (PARALLEL && !HOST_ID) {
+    int status = delete_duplicate_measures_parallel (skylist);
+    return status;
+  }
+
+  // delete detections from region
+  for (i = 0; i < skylist[0].Nregions; i++) {
+
+    // does this host ID match the desired location for the table?
+    if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue;
+
+    // set up the basic catalog info
+    char hostfile[1024];
+    snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
+    catalog.filename  = HOST_ID ? hostfile : skylist[0].filename[i];
+    catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
+    catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
+
+    if (VERBOSE) fprintf (stderr, "deleting from %s\n", catalog.filename);
+
+    // an error exit status here is a significant error
+    if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE2, "a")) {
+      fprintf (stderr, "ERROR: failure to open catalog file %s\n", catalog.filename);
+      exit (2);
+    }
+    if (!catalog.Naves_disk) {
+      if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename);
+      dvo_catalog_unlock (&catalog);
+      dvo_catalog_free (&catalog);
+      continue;
+    }
+
+    delete_duplicate_measures_catalog (&catalog);
+
+    if (UPDATE) {
+      // XXX skip if nothing was deleted
+      dvo_catalog_save_complete (&catalog, VERBOSE2);
+    }
+
+    dvo_catalog_unlock (&catalog);
+    dvo_catalog_free (&catalog);
+  }
+  return TRUE;
+}
+
+// CATDIR is supplied globally
+int delete_duplicate_measures_parallel (SkyList *sky) {
+
+  // launch the delstar_client jobs to the parallel hosts
+
+  // load the list of hosts
+  HostTable *table = HostTableLoad (CATDIR, sky->hosts);
+  if (!table) {
+    fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
+    exit (1);
+  }    
+
+  int i;
+  for (i = 0; i < table->Nhosts; i++) {
+
+    // ensure that the paths are absolute path names
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
+    free (table->hosts[i].pathname);
+    table->hosts[i].pathname = tmppath;
+
+    char command[1024];
+    snprintf (command, 1024, "delstar_client -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -dup-measures", 
+	      table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, 
+	      UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
+
+    char tmpline[1024];
+    if (VERBOSE)       { snprintf (tmpline, 1024, "%s -v",              command);                    strcpy (command, tmpline); }
+    if (VERBOSE2)      { snprintf (tmpline, 1024, "%s -vv",             command);                    strcpy (command, tmpline); }
+
+    fprintf (stderr, "command: %s\n", command);
+
+    if (PARALLEL_MANUAL) continue;
+
+    if (PARALLEL_SERIAL) {
+      int status = system (command);
+      if (status) {
+	fprintf (stderr, "ERROR running delstar_client\n");
+	exit (2);
+      }
+    } else {
+      // launch the job on the remote machine (no handshake)
+      int errorInfo = 0;
+      int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
+      if (!pid) {
+	fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
+	exit (1);
+      }
+      table->hosts[i].pid = pid; // save for future reference
+    }
+  }
+
+  if (PARALLEL_MANUAL) {
+    fprintf (stderr, "run the client commands above.  when these are done, hit return\n");
+    getchar();
+  }
+  if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
+  }
+
+  return TRUE;
+}
+
+void sort_fullIDs (uint64_t *I, int *S, off_t N);
+
+int delete_duplicate_measures_catalog (Catalog *catalog) {
+
+  off_t i, j, n, m, N, D, currentAve;
+
+  Measure *measureOut = NULL;
+  Average *averageOut = NULL;
+  SecFilt *secfiltOut = NULL;
+
+  off_t *measureDrop, *measureSeqRaw, *measureSeqOut, *measureRefOut, *measureAveRaw, *averageNmeas, *averageDmeas, *averageSeqOut, *averageRefOut, *measureAveOut;
+  uint64_t *fullID;
+  int *seq, *duplicates;
+
+  int Nsecfilt = GetPhotcodeNsecfilt ();
+
+  /* internal counters */
+  off_t Nmeasure = catalog[0].Nmeasure;
+  off_t Naverage = catalog[0].Naverage;
+
+  Measure *measure = catalog[0].measure;
+  Average *average = catalog[0].average;
+  SecFilt *secfilt = catalog[0].secfilt;
+  
+  if (VERBOSE) fprintf (stderr, "starting with Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT"\n",  catalog[0].Naverage,  catalog[0].Nmeasure);
+
+  // we have a table of average objects and an unsorted table of measurements.  each measurement
+  // has a reference to the average object sequence (as well as an ID)
+  // measure[i].averef -> average[averef]
+  // measure[i].objID = average[averef].objID
+  // measure[i].catID = average[averef].catID
+
+  // we want a sorted measure array with all averef entries in sequence, skipping the entries which match our criteria
+
+  // arrays 
+  ALLOCATE (measureDrop,   off_t, Nmeasure);
+  ALLOCATE (measureSeqRaw, off_t, Nmeasure);
+  ALLOCATE (measureSeqOut, off_t, Nmeasure);
+  ALLOCATE (measureRefOut, off_t, Nmeasure);
+  ALLOCATE (measureAveRaw, off_t, Nmeasure);
+  ALLOCATE (averageNmeas,  off_t, Naverage);
+  ALLOCATE (averageDmeas,  off_t, Naverage);
+  ALLOCATE (averageSeqOut, off_t, Naverage);
+  ALLOCATE (averageRefOut, off_t, Naverage);
+
+  ALLOCATE (fullID,     uint64_t, Nmeasure);
+  ALLOCATE (seq,             int, Nmeasure);
+  ALLOCATE (duplicates,      int, Nmeasure);
+
+  memset (measureDrop, 0, sizeof(int) * Nmeasure);
+  memset (duplicates,  0, sizeof(int) * Nmeasure);
+
+  // make an array of the full IDs
+  for (i = 0; i < Nmeasure; i++) {
+    fullID[i] = ((uint64_t) (measure[i].imageID) << 32) | (measure[i].detID);
+    seq[i] = i;
+  }
+
+  // sort the array and sequence number
+  sort_fullIDs (fullID, seq, catalog[0].Nmeasure);
+
+  uint64_t current = fullID[0];	      // current unique value
+  int count = 0;		      // number of duplicates for the current unique value
+  duplicates[0] = count;	      // duplicate sequence
+
+  for (i = 1; i < catalog[0].Nmeasure; i++) {
+    off_t j = seq[i];
+    int newID = (measure[j].imageID == 0) || (fullID[i] != current);
+    if (newID) {
+      count = 0;
+      current = fullID[i];
+    } else {
+      count ++;
+    }
+    duplicates[i] = count;
+  }
+    
+  // mark the measures to be dropped
+  for (i = 0; i < Nmeasure; i++) {
+    if (!duplicates[i]) continue;
+    off_t j = seq[i];
+    measureDrop[j] = TRUE;
+  }
+
+  // set up the measure sequence lists
+  for (i = 0; i < Nmeasure; i++) {
+    measureSeqRaw[i] = i;
+    measureAveRaw[i] = measure[i].averef;
+  }
+  // sort measureSeqRaw and measureAveRaw in order of measureAveRaw
+  SortAveMeasMatch(measureSeqRaw, measureAveRaw, Nmeasure);
+
+  // generate the measureSeqOut array
+  off_t NmeasOut = 0;
+  for (i = 0; i < Nmeasure; i++) {
+    j = measureSeqRaw[i];
+    if (measureDrop[j]) continue;
+    measureSeqOut[NmeasOut] = j;
+    measureRefOut[j] = NmeasOut;
+    NmeasOut ++;
+  }
+  // n = measureSeqOut[i] : measureOut[i] = measure[n]  (i = 0 -- NmeasOut, n = 0 -- Nmeasure)
+  // n = measureRefOut[i] : measureOut[n] = measure[i]
+  ALLOCATE (measureAveOut, off_t, NmeasOut);
+
+  // count the number of measures for each averef
+  N =  0;
+  D = -1;
+  currentAve = measureAveRaw[0];
+  for (i = 0; i < Nmeasure; i++) {
+    if (measureAveRaw[i] != currentAve) {
+      // we have hit the next entry in the list
+      averageNmeas[currentAve] = N; // number of measures
+      averageDmeas[currentAve] = D; // first measure 
+      N =  0;
+      D = -1;
+      currentAve = measureAveRaw[i];
+    }
+    j = measureSeqRaw[i];
+    if (measureDrop[j]) continue;
+    N++;
+    if (D == -1) { 
+      // first valid measureOut for this averef
+      D = measureRefOut[j];
+    }
+  }
+  averageNmeas[currentAve] = N; // number of measures
+  averageDmeas[currentAve] = D; // first measure 
+
+  // generate the new average sequence, skipping entries with no measurements
+  off_t NaveOut = 0;
+  for (i = 0; i < Naverage; i++) {
+    if (averageNmeas[i] == 0) continue;
+    averageSeqOut[NaveOut] = i;
+    averageRefOut[i] = NaveOut;
+    NaveOut ++;
+  }
+  // n = averageSeqOut[i] : averageOut[i] = average[n]  (i = 0 -- NaveOut, n = 0 -- Naverage)
+  // n = averageRefOut[i] : averageOut[n] = average[i]
+
+  // generate the output averefs
+  for (i = 0; i < NmeasOut; i++) {
+    j = measureSeqOut[i]; // measure[j] = measureOut[i]
+    n = measureAveRaw[j]; // average[n] : measure[j]
+    N = averageRefOut[n]; // averageOut[N] = average[n];
+    measureAveOut[i] = N; // measureOut[i].averef = measureAveOut[i] 
+  }
+
+  // copy the (kept) measurements in the sorted order
+  ALLOCATE (measureOut, Measure, NmeasOut);
+  for (i = 0; i < NmeasOut; i++) {
+    j = measureSeqOut[i];
+    measureOut[i] = measure[j];
+    measureOut[i].averef = measureAveOut[i];
+  }
+
+  // copy the (kept) average entries
+  ALLOCATE (averageOut, Average, NaveOut);
+  for (i = 0; i < NaveOut; i++) {
+    j = averageSeqOut[i];
+    averageOut[i] = average[j];
+    averageOut[i].Nmeasure      = averageNmeas[j]; 
+    averageOut[i].measureOffset = averageDmeas[j];
+  }
+
+  // copy the secfilt entries for the (kept) average entries
+  ALLOCATE (secfiltOut, SecFilt, NaveOut*Nsecfilt);
+  for (i = 0; i < NaveOut; i++) {
+    j = averageSeqOut[i];
+    for (m = 0; m < Nsecfilt; m++) {
+      secfiltOut[i*Nsecfilt + m] = secfilt[j*Nsecfilt + m];
+    }
+  }
+
+  // update the values of average.measureOffset and average.Nmeasure
+  FREE(measure);
+  FREE(average);
+  FREE(secfilt);
+  catalog[0].measure = measureOut;
+  catalog[0].average = averageOut;
+  catalog[0].secfilt = secfiltOut;
+  catalog[0].Nmeasure = NmeasOut;
+  catalog[0].Naverage = NaveOut;
+
+  // update catalog.Nmeasure and catalog.Naverage, double check 
+  int NmeasureTotal = 0;
+  int measureOffsetOK = TRUE;
+  int averefOK = TRUE;
+  for (i = 0; i < NaveOut; i++) {
+    NmeasureTotal += catalog[0].average[i].Nmeasure;
+    if (VERBOSE2 && !(NmeasureTotal <= catalog[0].Nmeasure)) {
+      fprintf (stderr, "too few measurements: %d %d %d\n", (int) i, NmeasureTotal, (int) catalog[0].Nmeasure);
+    }
+    measureOffsetOK &= (catalog[0].average[i].measureOffset < catalog[0].Nmeasure);
+    if (VERBOSE2 && !(catalog[0].average[i].measureOffset < catalog[0].Nmeasure)) {
+      fprintf (stderr, "offset too large: %d %d %d\n", (int) i, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure);
+    }
+    measureOffsetOK &= (catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure);
+    if (VERBOSE2 && !(catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure)) {
+      fprintf (stderr, "orrset + Nmeasure too large: %d + %d > %d %d\n", (int) i, catalog[0].average[i].measureOffset, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure);
+    }
+    m = catalog[0].average[i].measureOffset;
+    for (j = 0; j < catalog[0].average[i].Nmeasure; j++) {
+      averefOK &= (catalog[0].measure[m+j].averef == i);
+      if (VERBOSE2 && !(catalog[0].measure[m+j].averef == i)) {
+	fprintf (stderr, "averef broken: %d vs %d (measure %d)\n", (int) i, catalog[0].measure[m+j].averef, (int) (m+j));
+      }
+    }
+  }
+
+  if (!measureOffsetOK) {
+    fprintf (stderr, "ERROR: catalog %s has an invalid measureOffset\n", catalog[0].filename);
+  }
+
+  if (!averefOK) {
+    fprintf (stderr, "ERROR: catalog %s has invalid averefs\n", catalog[0].filename);
+  }
+
+  if (NmeasureTotal != catalog[0].Nmeasure) {
+    fprintf (stderr, "ERROR: catalog %s has an invalid Nmeasure\n", catalog[0].filename);
+  }
+
+  // MARKTIME("  match time %9.4f sec for %7lld measures, %6lld average\n", dtime, (long long) Nmeasure, (long long) Naverage);
+
+  catalog[0].sorted = TRUE;
+
+  if (VERBOSE) fprintf (stderr, "ending with Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT"\n",  catalog[0].Naverage,  catalog[0].Nmeasure);
+
+  off_t NdelAves = Naverage - catalog[0].Naverage;
+  off_t NdelMeas = Nmeasure - catalog[0].Nmeasure;
+  if (NdelAves || NdelMeas) {
+    fprintf (stderr, "deleting "OFF_T_FMT" measures and "OFF_T_FMT" averages : %s\n",  NdelMeas, NdelAves, catalog[0].filename);
+  }
+
+  FREE (fullID);
+  FREE (seq);
+  FREE (duplicates);
+
+  FREE (measureDrop);
+  FREE (measureSeqRaw);
+  FREE (measureSeqOut);
+  FREE (measureRefOut);
+  FREE (measureAveRaw);
+  FREE (averageNmeas);
+  FREE (averageDmeas);
+  FREE (averageSeqOut);
+  FREE (averageRefOut);
+  FREE (measureAveOut);
+
+  return TRUE;
+}
+
+// sort fullId vector and an index vector
+void sort_fullIDs (uint64_t *I, int *S, off_t N) {
+
+# define SWAPFUNC(A,B){ uint64_t tmp_i; int tmp_s;	\
+  tmp_i = I[A]; I[A] = I[B]; I[B] = tmp_i; \
+  tmp_s = S[A]; S[A] = S[B]; S[B] = tmp_s; \
+}
+# define COMPARE(A,B)(I[A] < I[B])
+
+  OHANA_SORT (N, COMPARE, SWAPFUNC);
+
+# undef SWAPFUNC
+# undef COMPARE
+
+}
+
Index: /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delete_photcodes.c
===================================================================
--- /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delete_photcodes.c	(revision 35706)
+++ /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delete_photcodes.c	(revision 35707)
@@ -1,5 +1,5 @@
 # include "delstar.h"
 
-int delete_photcodes (int hostID, char *hostpath) {
+int delete_photcodes () {
 
   int i;
@@ -21,5 +21,5 @@
 
   // launch the remote jobs
-  if (PARALLEL && !hostID) {
+  if (PARALLEL && !HOST_ID) {
     int status = delete_photcodes_parallel (skylist);
     return status;
@@ -34,10 +34,10 @@
 
     // does this host ID match the desired location for the table?
-    if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue;
+    if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue;
 
     // set up the basic catalog info
     char hostfile[1024];
-    snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name);
-    catalog.filename  = hostID ? hostfile : skylist[0].filename[i];
+    snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
+    catalog.filename  = HOST_ID ? hostfile : skylist[0].filename[i];
     catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
     catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
Index: /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delstar.c
===================================================================
--- /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delstar.c	(revision 35706)
+++ /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delstar.c	(revision 35707)
@@ -21,5 +21,9 @@
   switch (MODE) {
     case MODE_DUP_IMAGES:
-      if (!delete_duplicate_images (0, NULL, &db)) exit (1);
+      if (!delete_duplicate_images (&db)) exit (1);
+      exit (0);
+      break;
+    case MODE_DUP_MEASURES:
+      if (!delete_duplicate_measures ()) exit (1);
       exit (0);
       break;
Index: /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delstar_client.c
===================================================================
--- /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delstar_client.c	(revision 35706)
+++ /branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delstar_client.c	(revision 35707)
@@ -22,5 +22,9 @@
 	exit (1);
       }
-      if (!delete_duplicate_image_measures (HOST_ID, HOSTDIR, imageID)) exit (1);
+      if (!delete_duplicate_image_measures (imageID)) exit (1);
+      exit (0);
+      break;
+    case MODE_DUP_MEASURES:
+      if (!delete_duplicate_measures ()) exit (1);
       exit (0);
       break;
@@ -32,5 +36,5 @@
       break;
     case MODE_PHOTCODES:
-      if (!delete_photcodes (HOST_ID, HOSTDIR)) {
+      if (!delete_photcodes ()) {
 	fprintf (stderr, "failure deleting measurements from %s\n", HOSTDIR);
 	exit (1);