Index: /branches/eam_branches/ohana.20170822/src/opihi/dvo/find_dups.c
===================================================================
--- /branches/eam_branches/ohana.20170822/src/opihi/dvo/find_dups.c	(revision 40171)
+++ /branches/eam_branches/ohana.20170822/src/opihi/dvo/find_dups.c	(revision 40171)
@@ -0,0 +1,105 @@
+# include "data.h"
+// NOTE: if there are only a few uniq values, the old algorithm is not bad.  
+// for 10000 uniq values, 30M points take ~20sec in the new algorithm, 
+// 3M points takess 45 sec in the old method.
+
+/* given imageID and detID, make a 64bit joint ID and sort it */ 
+int find_dups (int argc, char **argv) {
+
+  int Nnew, i, N;
+  Vector *ivec, *ovec;
+
+  int VERBOSE = FALSE;
+  if ((N = get_argument (argc, argv, "-v"))) {
+    remove_argument (N, &argc, argv);
+    VERBOSE = TRUE;
+  }
+
+  Vector *cvec = NULL;
+  if ((N = get_argument (argc, argv, "-c"))) {
+    remove_argument (N, &argc, argv);
+    if ((cvec = SelectVector (argv[N], ANYVECTOR, TRUE)) == NULL) {
+      gprint (GP_ERR, "invalid vector %s\n", argv[N]);
+      return FALSE;
+    }
+    remove_argument (N, &argc, argv);
+  }
+
+  if (argc != 3) {
+    gprint (GP_ERR, "USAGE: find_dups (ID1) (ID2) (sequence) (out) -c count\n");
+    return (FALSE);
+  }
+
+  if ((ID1vec = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE);
+  if ((ID2vec = SelectVector (argv[2], ANYVECTOR, TRUE)) == NULL) return (FALSE);
+  if ((SEQvec = SelectVector (argv[3], ANYVECTOR, TRUE)) == NULL) return (FALSE);
+
+  if ((ID1vec->type == OPIHI_FLT) || (ID2vec->type == OPIHI_FLT) || (ID1vec->type == OPIHI_FLT)) {
+    gprint (GP_ERR, "ERROR: ID and sequence vectors much be int\n");
+    return (FALSE);
+  }
+
+  /* allocate the maximum possible needed */
+  ResetVector (ovec, ivec->type, ivec->Nelements);
+  if (cvec) {
+    ResetVector (cvec, OPIHI_INT, ivec->Nelements);
+  }
+
+  Nnew = 0;
+
+  // generate the joined vector
+
+  
+  // copy the input data to a temporary array to avoid damaging it with sort
+  opihi_int *ID1 = ID1vec->elements.Int;
+  opihi_int *ID2 = ID2vec->elements.Int;
+
+  opihi_int *IDfull = NULL;
+  ALLOCATE (IDfull, opihi_int, ID1vec->Nelements);
+  for (i = 0; i < ID1vec->Nelements; i++) {
+    IDfull[i] = ID1[i] + (ID2[i] << 32);
+  }
+
+  llsorttwo (IDfull, SEQvec->elements.Int, ivec->Nelements);
+
+  Nnew = 0;
+  opihi_int *vtgt = ovec[0].elements.Int;
+
+  opihi_int *vsrc = IDfull;
+  opihi_int *vidx = SEQvec->elements.Int;
+
+  int onePercent = ID1vec->Nelements / 100;
+
+  /* ADD sequence to output
+     do not allocate all memory to start
+   */
+
+  struct sigaction *old_sigaction = SetInterrupt();
+  for (i = 0; (i < ID1vec->Nelements) && !interrupt; Nnew++) {
+    vtgt[Nnew] = *vsrc;
+    int Ndup = 0;
+    opihi_int lastValue = *vsrc;
+    while ((i < ID1vec->Nelements) && (*vsrc == lastValue)) {
+      i++;
+      vsrc ++;
+      Ndup ++;
+      if (VERBOSE && (i % onePercent == 0)) gprint (GP_ERR, ".");
+    }
+    if (cvec) {
+      cvec->elements.Int[Nnew] = Ndup;
+    }
+  }
+  ClearInterrupt (old_sigaction);
+  if (VERBOSE) gprint (GP_ERR, "\n");
+  free (indata);
+
+  // free up extra memory
+  ResetVector (ovec, ivec->type, Nnew);
+  if (cvec) ResetVector (cvec, OPIHI_INT, Nnew);
+
+  return (TRUE);
+}
+
+
+
+}