Changeset 35237 for trunk

Timestamp:

Mar 5, 2013, 2:33:50 PM (13 years ago)

Author:

eugene

Message:

merge changes from eam_branches/ipp-20130207

Location:

trunk/Ohana/src

Files:

• delstar/Makefile (modified) (2 diffs)
• delstar/include/delstar.h (modified) (5 diffs)
• delstar/src/args.c (modified) (4 diffs)
• delstar/src/delete_duplicate_images.c (copied) (copied from branches/eam_branches/ipp-20130207/Ohana/src/delstar/src/delete_duplicate_images.c )
• delstar/src/delstar.c (modified) (1 diff)
• kapa2/src/CheckColors.c (modified) (2 diffs)
• kapa2/src/CheckDisplayName.c (modified) (2 diffs)
• kapa2/src/CheckGeometry.c (modified) (1 diff)
• kapa2/src/CheckVisual.c (modified) (4 diffs)
• kapa2/src/Layout.c (modified) (1 diff)
• kapa2/src/LoadFont.c (modified) (2 diffs)
• kapa2/src/OpenDisplay.c (modified) (1 diff)
• kapa2/src/QuitX.c (modified) (1 diff)
• libdvo/include/dvodb.h (modified) (1 diff)
• libdvo/src/dbExtractMeasures.c (modified) (1 diff)
• libdvo/src/dbFields.c (modified) (1 diff)
• libkapa/src/KapaOpen.c (modified) (6 diffs)
• libohana/src (modified) (1 prop)
• opihi (modified) (1 prop)
• opihi/cmd.astro (modified) (1 prop)
• opihi/cmd.basic/Makefile (modified) (1 diff)
• opihi/cmd.basic/init.c (modified) (2 diffs)
• opihi/cmd.basic/scannext.c (copied) (copied from branches/eam_branches/ipp-20130207/Ohana/src/opihi/cmd.basic/scannext.c )
• opihi/cmd.data (modified) (1 prop)
• opihi/cmd.data/Makefile (modified) (1 diff)
• opihi/cmd.data/dbadmin.c (copied) (copied from branches/eam_branches/ipp-20130207/Ohana/src/opihi/cmd.data/dbadmin.c )
• opihi/cmd.data/dbupdate.c (copied) (copied from branches/eam_branches/ipp-20130207/Ohana/src/opihi/cmd.data/dbupdate.c )
• opihi/cmd.data/init.c (modified) (2 diffs)

trunk/Ohana/src/delstar/Makefile

@@ -29 +29 @@
 $(SRC)/delete_imagename.$(ARCH).o \
 $(SRC)/delete_imagefile.$(ARCH).o \
+$(SRC)/delete_duplicate_images.$(ARCH).o \
 $(SRC)/delete_image_photcodes.$(ARCH).o \
 $(SRC)/delete_photcodes.$(ARCH).o \
@@ -45 +46 @@
 $(SRC)/ConfigInit.$(ARCH).o       \
 $(SRC)/args.$(ARCH).o             \
+$(SRC)/delete_duplicate_images.$(ARCH).o \
 $(SRC)/delete_photcodes.$(ARCH).o \
 $(SRC)/delete_photcodes_single.$(ARCH).o \

trunk/Ohana/src/delstar/include/delstar.h

@@ -2 +2 @@
 # include <dvo.h>
 # include <signal.h>
+
+// options for generating the IndexArray used to select images for deletion (delete_duplicate_images.c)
+enum {NONE, EXTERN_ID, IMAGE_ID};
 
 typedef struct {
@@ -10 +13 @@
   double Area, density, spacing;
 } CatStats;
+
+typedef struct {
+  off_t minID;
+  off_t maxID;
+  off_t range;
+  off_t *value;
+} IndexArray;
 
 /* global variables set in parameter file */
@@ -28 +38 @@
 int    VERBOSE;
 int    VERBOSE2;
+int    UPDATE;
+int    IMAGE_DETAILS;
+int    IMAGE_ONLY;
 int    ORPHAN;
 int    MISSED;
@@ -49 +62 @@
 
 int    MODE;
-enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES};
+enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES};
 
 char DateKeyword[64], DateMode[64], UTKeyword[64], MJDKeyword[64], JDKeyword[64];
@@ -101 +114 @@
 int delete_image_photcodes (FITS_DB *db);
 int delete_photcodes_single (char *cptname);
+
+int delete_duplicate_images (int hostID, char *hostpath, FITS_DB *db);
+int delete_duplicate_image_measures (int hostID, char *hostpath, IndexArray *imageID);
+int delete_duplicate_image_measures_parallel (SkyList *sky);
+int delete_duplicate_image_measures_catalog (Catalog *catalog, IndexArray *imageID);
+
+IndexArray *find_duplicates (Image *image, off_t Nimage);
+IndexArray *make_index_array (Image *image, off_t Nimage, int mode);
+

trunk/Ohana/src/delstar/src/args.c

@@ -9 +9 @@
   fprintf (stderr, "  delstar -orphan (region)\n");
   fprintf (stderr, "  delstar -missed (region)\n\n");
+  fprintf (stderr, "  delstar -photcodes (list) : delete by photcode\n\n");
+  fprintf (stderr, "  delstar -dup-images : delete duplicate images (by externID)\n\n");
   fprintf (stderr, "  optional flags:\n");
   fprintf (stderr, "  -v               : verbose mode\n");
   fprintf (stderr, "  -photcode (code) : restrict by photcode\n");
+  fprintf (stderr, "  -update : apply changes (-dup-images only)\n");
+  fprintf (stderr, "  -image-details : list info about the deleted images (-dup-images only)\n");
+  fprintf (stderr, "  -image-only : only examine the image table (changes are NOT saved; -dup-images only)\n");
+  fprintf (stderr, "  -region Rmin Rmax Dmin Dmax : apply changes to this part of the sky\n");
+  
   fprintf (stderr, "\n"); 
   exit (2);
@@ -48 +55 @@
     VERBOSE = TRUE;
     VERBOSE2 = TRUE;
+    remove_argument (N, &argc, argv);
+  }
+
+  UPDATE = FALSE;
+  if ((N = get_argument (argc, argv, "-update"))) {
+    UPDATE = TRUE;
+    remove_argument (N, &argc, argv);
+  }
+
+  IMAGE_DETAILS = FALSE;
+  if ((N = get_argument (argc, argv, "-image-details"))) {
+    IMAGE_DETAILS = TRUE;
+    remove_argument (N, &argc, argv);
+  }
+
+  // we should never delete the images before the measures (or we won't know what to delete)
+  // -image-only is for testing
+  IMAGE_ONLY = FALSE;
+  if ((N = get_argument (argc, argv, "-image-only"))) {
+    IMAGE_ONLY = TRUE;
+    UPDATE = FALSE;
     remove_argument (N, &argc, argv);
   }
@@ -126 +154 @@
     if (MODE != MODE_NONE) usage();
     MODE = MODE_MISSED;
+    remove_argument (N, &argc, argv);
+  }
+  if ((N = get_argument (argc, argv, "-dup-images"))) {
+    if (MODE != MODE_NONE) usage();
+    MODE = MODE_DUP_IMAGES;
     remove_argument (N, &argc, argv);
   }
@@ -251 +284 @@
   }
 
+  UPDATE = FALSE;
+  if ((N = get_argument (argc, argv, "-update"))) {
+    UPDATE = TRUE;
+    remove_argument (N, &argc, argv);
+  }
+
+  IMAGE_DETAILS = FALSE;
+  if ((N = get_argument (argc, argv, "-image-details"))) {
+    IMAGE_DETAILS = TRUE;
+    remove_argument (N, &argc, argv);
+  }
+
+  IMAGE_ONLY = FALSE;
+  if ((N = get_argument (argc, argv, "-image-only"))) {
+    IMAGE_ONLY = TRUE;
+    UPDATE = FALSE;
+    remove_argument (N, &argc, argv);
+  }
+
   /* specify portion of the sky */
   UserPatch.Rmin = 0;

trunk/Ohana/src/delstar/src/delstar.c

@@ -20 +20 @@
 
   switch (MODE) {
+    case MODE_DUP_IMAGES:
+      delete_duplicate_images (0, NULL, &db);
+      break;
     case MODE_IMAGEFILE:
       delete_imagefile (&db);

trunk/Ohana/src/kapa2/src/CheckColors.c

@@ -12 +12 @@
     if (*argc <= N + 1) {
       fprintf (stderr, "error: usage is -fg color\n");
-      exit (0);
+      exit (2);
     }
     remove_argument (N, argc, argv);
@@ -27 +27 @@
     if (*argc <= N + 1) {
       fprintf (stderr, "error: usage is -bg color\n");
-      exit (0);
+      exit (2);
     }
     remove_argument (N, argc, argv);

trunk/Ohana/src/kapa2/src/CheckDisplayName.c

@@ -10 +10 @@
     if (*argc <= N + 1) {
       fprintf (stderr, "error: usage is [-display/-d] DisplayName\n");
-      exit (0);
+      exit (2);
     }
     remove_argument(N, argc, argv);
@@ -21 +21 @@
     if (*argc <= N + 1) {
       fprintf (stderr, "error: usage is [-display/-d] DisplayName\n");
-      exit (0);
+      exit (2);
     }
     remove_argument(N, argc, argv);

trunk/Ohana/src/kapa2/src/CheckGeometry.c

@@ -11 +11 @@
     if (*argc <= N + 1) {
       fprintf (stderr, "error: usage is -geom DisplayName\n");
-      exit (0);
+      exit (2);
     }
     remove_argument (N, argc, argv);

trunk/Ohana/src/kapa2/src/CheckVisual.c

@@ -31 +31 @@
   if (Nfound == 0) {
     fprintf (stderr, "error finding useful visual\n");
-    exit (0);
+    exit (2);
   }
   
@@ -157 +157 @@
         // We've already tried to allocate a colormap above...
         fprintf (stderr, "can't allocate enough cells in private colormap\n");
-        exit (0);
+        exit (2);
       }
       if (DEBUG) fprintf (stderr, "can't allocate enough cells, using private colormap\n");
@@ -173 +173 @@
       if ((N = get_argument (*argc, argv, "-colorcount"))) {
         fprintf (stderr, "kapa can grab %d colors\n", graphic[0].Npixels);
-        exit (0);
+        exit (1);
       } 
       if (graphic[0].Npixels < 16) {
         fprintf (stderr, "can't even allocate enough cells in private colormap\n");
-        exit (0);
+        exit (2);
       }
     }
@@ -196 +196 @@
   if ((N = get_argument (*argc, argv, "-colorcount"))) {
     fprintf (stderr, "kii can grab %d colors\n", graphic[0].Npixels);
-    exit (0);
+    exit (1);
   } 
   

trunk/Ohana/src/kapa2/src/Layout.c

@@ -36 +36 @@
   if (argc != 1) {
     fprintf (stderr, "USAGE: kapa\n");
-    exit (0);
+    exit (1);
   }
 

trunk/Ohana/src/kapa2/src/LoadFont.c

@@ -14 +14 @@
     if (*argc <= N + 1) {
       fprintf (stderr, "error: usage is -font fontname\n");
-      exit (0);
+      exit (2);
     }
     remove_argument(N, argc, argv);
@@ -23 +23 @@
     if (*argc <= N + 1) {
       fprintf (stderr, "error: usage is -fn fontname\n");
-      exit (0);
+      exit (2);
     }
     remove_argument(N, argc, argv);

trunk/Ohana/src/kapa2/src/OpenDisplay.c

@@ -9 +9 @@
   if (display == NULL) {
     fprintf (stderr, "Error could not open X display to %s\n", XDisplayName (name));
-    exit (0);
+    exit (2);
   }
   *screen = DefaultScreen (display);

trunk/Ohana/src/kapa2/src/QuitX.c

@@ -6 +6 @@
   fprintf (stderr, "Error: %s\n", error_message);
   XCloseDisplay (display);
-  exit (0);
+  exit (3);
 }

trunk/Ohana/src/libdvo/include/dvodb.h

@@ -141 +141 @@
       MEAS_EXT_NSIGMA, 
       MEAS_EXTERN_ID,
+      MEAS_EXPNAME_AS_INT,
       MEAS_MCAL_OFFSET,
       MEAS_FLAT,

trunk/Ohana/src/libdvo/src/dbExtractMeasures.c

@@ -523 +523 @@
       break;
 
+    case MEAS_EXPNAME_AS_INT:
+      {
+        Image *image;
+        image = MatchImageDVO (measure[0].t, measure[0].photcode, measure[0].imageID);
+        if (image == NULL) break;
+        // XXX very crude: if this matches oNNNNgNNNNo, then convert to an int
+        // otherwise, leave as 0
+        if ((image->name[0] == 'o') && (image->name[5] == 'g') && (image->name[10] == 'o')) {
+          int mjd  = atoi(&image->name[1]);
+          int Nexp = atoi(&image->name[6]);
+          value.Int = mjd * 10000 + Nexp;
+        }
+      }
+      break;
+
     case MEAS_MCAL_OFFSET: /* OK */
       value.Flt = measure[0].Mcal;

trunk/Ohana/src/libdvo/src/dbFields.c

@@ -231 +231 @@
   if (!strcasecmp (fieldName, "IMAGEID"))        ESCAPE (MEAS_IMAGE_ID,       MAG_NONE, OPIHI_INT);
   if (!strcasecmp (fieldName, "EXTERNID"))       ESCAPE (MEAS_EXTERN_ID,      MAG_NONE, OPIHI_INT);
+  if (!strcasecmp (fieldName, "EXPNAME"))        ESCAPE (MEAS_EXPNAME_AS_INT, MAG_NONE, OPIHI_INT);
   if (!strcasecmp (fieldName, "PSF_QF"))         ESCAPE (MEAS_PSF_QF,         MAG_NONE, OPIHI_FLT);
   if (!strcasecmp (fieldName, "PSF_QF_PERFECT")) ESCAPE (MEAS_PSF_QF_PERFECT, MAG_NONE, OPIHI_FLT);

trunk/Ohana/src/libkapa/src/KapaOpen.c

@@ -8 +8 @@
 static int Nvalid = 0;
 static int *VALID = NULL;
+
+char *_parse_nextword (char *string);
+int KapaLaunchCommand (char *line);
 
 int KapaServerInit (KapaSockAddress *Address) {
@@ -225 +228 @@
 
   if (kapa_name == NULL) {
-    sprintf (line, "%s &", kapa_exec);
+    sprintf (line, "%s", kapa_exec);
   } else {
-    sprintf (line, "%s -name '%s' &", kapa_exec, kapa_name);
-  }
-  if (system (line) == -1) {
+    sprintf (line, "%s -name '%s'", kapa_exec, kapa_name);
+  }
+
+  int pid = KapaLaunchCommand (line);
+  if (!pid) {
     fprintf (stderr, "failed to launch kapa\n");
-    return 0;
+    return (-1);
   }
 
@@ -241 +246 @@
     // if (errno != EAGAIN) break;
     if (errno != ECONNREFUSED) break;
+    // no connection yet. try again, but first check 
+    // if the kapa job has exited
+    int waitstatus;
+    int result = waitpid (pid, &waitstatus, WNOHANG);
+    if (result == -1) {
+      fprintf (stderr, "problem checking for kapa\n");
+      return (-1);
+    }
+    if (result > 0) {
+      fprintf (stderr, "kapa exited\n");
+      return (-1);
+    }
     usleep (10000);
     Ntry ++;
@@ -277 +294 @@
     sprintf (temp, "%s -socket %s -name %s &", kapa_exec, socket_name, name);
   }
-  if (system (temp) == -1) {
+
+  int pid = KapaLaunchCommand (temp);
+  if (!pid) {
     fprintf (stderr, "failed to launch kapa\n");
-    return 0;
+    return (-1);
   }
 
@@ -291 +310 @@
     if (fd != -1) break;
     if (errno != EAGAIN) break;
+    // no connection yet. try again, but first check 
+    // if the kapa job has exited
+    int waitstatus;
+    int result = waitpid (pid, &waitstatus, WNOHANG);
+    if (result == -1) {
+      fprintf (stderr, "problem checking for kapa\n");
+      return (-1);
+    }
+    if (result > 0) {
+      fprintf (stderr, "kapa exited\n");
+      return (-1);
+    }
     usleep (10000);
     Ntry ++;
@@ -322 +353 @@
   return (sock);
 }
+
+int KapaLaunchCommand (char *line) {
+
+  // use fork & exec to launch the command; return the PID for testing exit
+  int i, done, Nalloc, Nargv;
+  char **argv, *p, *q;
+
+  i = 0;
+  Nalloc = 10;
+  ALLOCATE (argv, char *, Nalloc);
+
+  // split out line into unique words
+  p = line;
+  done = FALSE;
+  while (!done) {
+    q = _parse_nextword (p);
+    if (q && *q) {
+      argv[i] = strncreate (p, q - p);
+      stripwhite (argv[i]);
+      p = q;
+    } else {
+      argv[i] = strcreate (p);
+      stripwhite (argv[i]);
+      done = TRUE;
+    }
+    i++;
+    if (i == Nalloc - 1) {
+      // need to keep one extra for the NULL pointer
+      Nalloc += 10;
+      REALLOCATE (argv, char *, Nalloc);
+    }
+  }
+  Nargv = i;
+  argv[Nargv] = NULL;
+
+  int pid = fork ();
+  if (!pid) { /* must be child process */
+    execvp (argv[0], argv); 
+    return (0);
+  }
+  for (i = 0; i < Nargv; i++) {
+    if (argv[i]) free (argv[i]);
+  }
+  free (argv);
+
+  if (pid == -1) {
+    return (0);
+  }
+
+  return (pid);
+}

trunk/Ohana/src/opihi/cmd.basic/Makefile

@@ -48 +48 @@
 $(SRC)/run_while.$(ARCH).o  \
 $(SRC)/scan.$(ARCH).o        \
+$(SRC)/scannext.$(ARCH).o            \
 $(SRC)/shell.$(ARCH).o       \
 $(SRC)/sprintf.$(ARCH).o    \

trunk/Ohana/src/opihi/cmd.basic/init.c

@@ -35 +35 @@
 int run_while       PROTO((int, char **));
 int scan            PROTO((int, char **));
+int scannext        PROTO((int, char **));
 int shell           PROTO((int, char **));
 int sprintf_opihi   PROTO((int, char **));
@@ -86 +87 @@
   {1, "while",         run_while,          "while loop"}, 
   {1, "scan",          scan,               "scan line from keyboard or file to variable *"},
+  {1, "scannext",      scannext,           "scan next line from file to variable (file stays open)"},
   {1, "!",             shell,              "system call"},
   {1, "exec",          shell,              "system call"},

trunk/Ohana/src/opihi/cmd.data/Makefile

@@ -43 +43 @@
 $(SRC)/dbselect.$(ARCH).o       \
 $(SRC)/dbinsert.$(ARCH).o       \
+$(SRC)/dbadmin.$(ARCH).o        \
+$(SRC)/dbupdate.$(ARCH).o       \
 $(SRC)/erase.$(ARCH).o          \
 $(SRC)/extract.$(ARCH).o        \

trunk/Ohana/src/opihi/cmd.data/init.c

@@ -25 +25 @@
 int dbselect         PROTO((int, char **));
 int dbinsert         PROTO((int, char **));
+int dbadmin          PROTO((int, char **));
+int dbupdate         PROTO((int, char **));
 int delete           PROTO((int, char **));
 int densify          PROTO((int, char **));
@@ -172 +174 @@
   {1, "dbselect",     dbselect,         "extract vectors from mysql database table"},
   {1, "dbinsert",     dbinsert,         "sql insert command"},
+  {1, "dbadmin",      dbadmin,          "sql admin command (create, drop)"},
+  {1, "dbupdate",     dbupdate,         "sql update command"},
   {1, "delete",       delete,           "delete vectors or images"},
   {1, "densify",      densify,          "create an image histogram from a set of vectors"},