Changeset 36767 for branches/eam_branches/ipp-20140423/Ohana

Timestamp:

May 23, 2014, 9:36:11 AM (12 years ago)

Author:

eugene

Message:

add option to delete duplicate images based on their obstime/photcode values (ignoring externID)

Location:

branches/eam_branches/ipp-20140423/Ohana/src/delstar

Files:

• include/delstar.h (modified) (2 diffs)
• src/args.c (modified) (2 diffs)
• src/delete_duplicate_images.c (modified) (3 diffs)

branches/eam_branches/ipp-20140423/Ohana/src/delstar/include/delstar.h

@@ -2 +2 @@
 # include <dvo.h>
 # include <signal.h>
+
+# define MARKTIME(MSG,...) {                    \
+    gettimeofday (&stopTimer, (void *) NULL);   \
+    float dtime = DTIME (stopTimer, startTimer);        \
+    fprintf (stderr, MSG, __VA_ARGS__); }
+
+# define INITTIME \
+  struct timeval startTimer, stopTimer; \
+  gettimeofday (&startTimer, (void *) NULL);
 
 // options for generating the IndexArray used to select images for deletion (delete_duplicate_images.c)
@@ -67 +76 @@
 int    UPDATE;
 int    IMAGE_DETAILS;
+int    IMAGE_DUPLICATES_BY_OBSTIME;
 int    IMAGE_ONLY;
 int    ORPHAN;

branches/eam_branches/ipp-20140423/Ohana/src/delstar/src/args.c

@@ -18 +18 @@
   fprintf (stderr, "  -image-details : list info about the deleted images (-dup-images only)\n");
   fprintf (stderr, "  -image-only : only examine the image table (changes are NOT saved; -dup-images only)\n");
+  fprintf (stderr, "  -image-only-force : modify only the image table (-dup-images only)\n");
+  fprintf (stderr, "  -image-by-obstime : use date/time and photcode (not externID) to find duplicates\n");
   fprintf (stderr, "  -region Rmin Rmax Dmin Dmax : apply changes to this part of the sky\n");
   
@@ -68 +70 @@
   if ((N = get_argument (argc, argv, "-image-details"))) {
     IMAGE_DETAILS = TRUE;
+    remove_argument (N, &argc, argv);
+  }
+
+  IMAGE_DUPLICATES_BY_OBSTIME = FALSE;
+  if ((N = get_argument (argc, argv, "-image-by-obstime"))) {
+    IMAGE_DUPLICATES_BY_OBSTIME = TRUE;
     remove_argument (N, &argc, argv);
   }

branches/eam_branches/ipp-20140423/Ohana/src/delstar/src/delete_duplicate_images.c

@@ -1 +1 @@
 # include "delstar.h"
+
+IndexArray *find_duplicates_obstime (Image *image, off_t Nimage, off_t *Nduplicates);
+off_t find_obstime_range (Image *image, off_t Nimage, off_t firstEntry);
+void sort_by_obstime (e_time *T, short *P, off_t *I, off_t N);
+void sort_by_photcode (short *P, off_t *I, off_t N);
 
 // this function identifies the images to be deleted based on duplication of the 
@@ -18 +23 @@
 
   off_t Nduplicates = 0;
-  IndexArray *imageID = find_duplicates (image, Nimage, &Nduplicates);
+  IndexArray *imageID = NULL;
+
+  if (IMAGE_DUPLICATES_BY_OBSTIME) {
+    imageID = find_duplicates_obstime (image, Nimage, &Nduplicates);
+  } else {
+    imageID = find_duplicates (image, Nimage, &Nduplicates);
+  }
 
   if (Nduplicates == 0) {
@@ -494 +505 @@
 }
 
+// sort by increasing obstime
+void sort_by_obstime (e_time *T, short *P, off_t *I, off_t N) {
+
+# define SWAPFUNC(A,B){ e_time tmpT; short tmpP; off_t tmpI; \
+  tmpT = T[A]; T[A] = T[B]; T[B] = tmpT; \
+  tmpP = P[A]; P[A] = P[B]; P[B] = tmpP; \
+  tmpI = I[A]; I[A] = I[B]; I[B] = tmpI; \
+}
+# define COMPARE(A,B)(T[A] < T[B])
+
+  OHANA_SORT (N, COMPARE, SWAPFUNC);
+
+# undef SWAPFUNC
+# undef COMPARE
+
+}
+
+// sort by increasing photcode
+void sort_by_photcode (short *P, off_t *I, off_t N) {
+
+# define SWAPFUNC(A,B){ short tmpP; off_t tmpI; \
+  tmpP = P[A]; P[A] = P[B]; P[B] = tmpP; \
+  tmpI = I[A]; I[A] = I[B]; I[B] = tmpI; \
+}
+# define COMPARE(A,B)(P[A] < P[B])
+
+  OHANA_SORT (N, COMPARE, SWAPFUNC);
+
+# undef SWAPFUNC
+# undef COMPARE
+
+}
+
+static e_time *obstime  = NULL;
+static short  *photcode = NULL;
+static off_t  *primary  = NULL;
+static off_t  *idx      = NULL;
+static char   *keep     = NULL;
+
+static short  *photcode_subset = NULL;
+static off_t  *idx_subset      = NULL;
+
+static off_t Nsubset = 0;
+static off_t NSUBSET = 300;
+
+off_t find_obstime_range (Image *image, off_t Nimage, off_t firstEntry) {
+
+  Nsubset = 0;
+
+  // find all entries with the same obstime:
+  e_time firstTime = obstime[firstEntry];
+
+  idx_subset[Nsubset] = idx[firstEntry];
+  photcode_subset[Nsubset] = photcode[firstEntry];
+  Nsubset++;
+
+  off_t i = firstEntry + Nsubset;
+  while ((i < Nimage) && (obstime[i] == firstTime)) {
+    idx_subset[Nsubset] = idx[i];
+    photcode_subset[Nsubset] = photcode[i];
+    Nsubset++;
+    i++;
+    if (Nsubset >= NSUBSET) {
+      NSUBSET += 1000;
+      REALLOCATE (photcode_subset, short, NSUBSET);
+      REALLOCATE (idx_subset, off_t, NSUBSET);
+    }
+  }      
+
+  sort_by_photcode (photcode_subset, idx_subset, Nsubset);
+
+  return i;
+}
+
+// alternative version to find duplicates based on obstime and photcode
+IndexArray *find_duplicates_obstime (Image *image, off_t Nimage, off_t *Nduplicates) {
+
+  if (Nimage < 1) return NULL;
+
+  // how to find duplicates:
+  // generate a set of arrays (idx, obstime, photcode, keep)
+  // sort the arrays by obstime
+  // loop over obstime.  
+  // for a given new obstime, scan through the entries with the same value
+  // create a sub-array of photcodes, idx
+  // sort by photcode
+  // loop over entries
+  // find matching photcodes
+  // mark duplicate enties
+
+  ALLOCATE (obstime, e_time, Nimage);
+  ALLOCATE (photcode, short, Nimage);
+  ALLOCATE (primary, off_t, Nimage);
+  ALLOCATE (idx, off_t, Nimage);
+  ALLOCATE (keep, char, Nimage);
+
+  off_t i;
+
+  INITTIME;
+
+  // skip entries with photcode == 0?
+  for (i = 0; i < Nimage; i++) {
+    idx[i] = i;
+    primary[i] = -1; // only duplicates get a value for primary
+    keep[i] = TRUE;
+    obstime[i] = image[i].tzero;
+    photcode[i] = image[i].photcode;
+  }
+  MARKTIME("  generate index arrays: %f sec\n", dtime);
+
+  // sort the 4 arrays
+  sort_by_obstime (obstime, photcode, idx, Nimage);
+  MARKTIME("  sort index arrays: %f sec\n", dtime);
+
+  // image[idx[i]].tzero = obstime[i]
+  // keep[i] -> keep image[i] ('keep' is NOT resorted)
+
+  // allocate arrays to store the subsets (these are global static)
+  // These get reallocated if necessary in find_obstime_range()
+  ALLOCATE (photcode_subset, short, NSUBSET);
+  ALLOCATE (idx_subset, off_t, NSUBSET);
+
+  // entries of idx_subset correspond to the original image sequence:
+  // image[idx_subset[i]].tzero = obstime[i]
+
+  off_t firstEntry = 0;
+  off_t nextEntry = 0;
+  
+  while (nextEntry < Nimage) {
+    if (firstEntry >= Nimage) {
+      fprintf (stderr, "error, too far?\n");
+    }
+    
+    // generate photcode_subset, idx_subset in order of photcode for this unique obstime[firstEntry]
+    // returned value is the first value of the next entry
+    nextEntry = find_obstime_range (image, Nimage, firstEntry);
+    
+    // step through the photcodes and find duplicates
+    int j;
+    int firstCodeEntry = 0;
+    short firstCode = photcode_subset[firstCodeEntry];
+    for (j = 1; j < Nsubset; j++) {
+      if (photcode_subset[j] == firstCode) {
+        // mark as duplicate
+        off_t dupIndex = idx_subset[j];
+        keep[dupIndex] = FALSE;
+        primary[dupIndex] = idx_subset[firstCodeEntry];
+      } else {
+        // new value of photcode, call if the first one
+        firstCodeEntry = j;
+        firstCode = photcode_subset[firstCodeEntry];
+      }
+    }
+    firstEntry = nextEntry;
+  }
+  MARKTIME("  find duplicates: %f sec\n", dtime);
+
+  IndexArray *imageID = make_index_array (image, Nimage, IMAGE_ID);
+  MARKTIME("  make index array: %f sec\n", dtime);
+
+  // set imageID->value to TRUE for images we want to delete
+  off_t Ndup = 0;
+  for (i = 0; i < Nimage; i++) {
+    if (keep[i]) continue;
+    off_t Ni = image[i].imageID - imageID->minID;
+    myAssert (Ni >= 0, "oops");
+    myAssert (Ni < imageID->range, "oops");
+    imageID->value[Ni] = TRUE;
+    Ndup ++;
+  }
+  MARKTIME("  mark duplicates: %f sec\n", dtime);
+
+  for (i = 0; IMAGE_DETAILS && (i < Nimage); i++) {
+    off_t Ni = image[i].imageID - imageID->minID;
+    if (!imageID->value[Ni]) continue;
+    
+    char *date = NULL;
+    date = ohana_sec_to_date (image[i].tzero);
+    fprintf (stderr, "delete image : (" OFF_T_FMT "), extID = %d : %30s : %20s %5d  ==  ", i, image[i].externID, image[i].name, date, image[i].photcode);
+    free (date);
+
+    off_t myPrimary = primary[i];
+    if (myPrimary < 0) {
+      fprintf (stderr, "ERROR: this should never happen\n");
+      abort();
+    }
+    date = ohana_sec_to_date (image[myPrimary].tzero);
+    fprintf (stderr, "parent image : (" OFF_T_FMT "), extID = %d : %30s : %20s %5d\n", myPrimary, image[myPrimary].externID, image[myPrimary].name, date, image[myPrimary].photcode);
+    free (date);
+  }
+
+  *Nduplicates = Ndup;
+
+  free (photcode_subset);
+  free (idx_subset);
+  
+  free (obstime);
+  free (photcode);
+  free (primary);
+  free (idx);
+  free (keep);
+
+  return imageID;;
+}
+
 // find the min & max values of the given ID (externID or imageID)
 // construct an empty array with length needed to fit IDs