Index: trunk/Ohana/src/delstar/Makefile =================================================================== --- trunk/Ohana/src/delstar/Makefile (revision 35233) +++ trunk/Ohana/src/delstar/Makefile (revision 35237) @@ -29,4 +29,5 @@ $(SRC)/delete_imagename.$(ARCH).o \ $(SRC)/delete_imagefile.$(ARCH).o \ +$(SRC)/delete_duplicate_images.$(ARCH).o \ $(SRC)/delete_image_photcodes.$(ARCH).o \ $(SRC)/delete_photcodes.$(ARCH).o \ @@ -45,4 +46,5 @@ $(SRC)/ConfigInit.$(ARCH).o \ $(SRC)/args.$(ARCH).o \ +$(SRC)/delete_duplicate_images.$(ARCH).o \ $(SRC)/delete_photcodes.$(ARCH).o \ $(SRC)/delete_photcodes_single.$(ARCH).o \ Index: trunk/Ohana/src/delstar/include/delstar.h =================================================================== --- trunk/Ohana/src/delstar/include/delstar.h (revision 35233) +++ trunk/Ohana/src/delstar/include/delstar.h (revision 35237) @@ -2,4 +2,7 @@ # include # include + +// options for generating the IndexArray used to select images for deletion (delete_duplicate_images.c) +enum {NONE, EXTERN_ID, IMAGE_ID}; typedef struct { @@ -10,4 +13,11 @@ double Area, density, spacing; } CatStats; + +typedef struct { + off_t minID; + off_t maxID; + off_t range; + off_t *value; +} IndexArray; /* global variables set in parameter file */ @@ -28,4 +38,7 @@ int VERBOSE; int VERBOSE2; +int UPDATE; +int IMAGE_DETAILS; +int IMAGE_ONLY; int ORPHAN; int MISSED; @@ -49,5 +62,5 @@ int MODE; -enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES}; +enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES}; char DateKeyword[64], DateMode[64], UTKeyword[64], MJDKeyword[64], JDKeyword[64]; @@ -101,2 +114,11 @@ int delete_image_photcodes (FITS_DB *db); int delete_photcodes_single (char *cptname); + +int delete_duplicate_images (int hostID, char *hostpath, FITS_DB *db); +int delete_duplicate_image_measures (int hostID, char *hostpath, IndexArray *imageID); +int delete_duplicate_image_measures_parallel (SkyList *sky); +int delete_duplicate_image_measures_catalog (Catalog *catalog, IndexArray *imageID); + +IndexArray *find_duplicates (Image *image, off_t Nimage); +IndexArray *make_index_array (Image *image, off_t Nimage, int mode); + Index: trunk/Ohana/src/delstar/src/args.c =================================================================== --- trunk/Ohana/src/delstar/src/args.c (revision 35233) +++ trunk/Ohana/src/delstar/src/args.c (revision 35237) @@ -9,7 +9,14 @@ fprintf (stderr, " delstar -orphan (region)\n"); fprintf (stderr, " delstar -missed (region)\n\n"); + fprintf (stderr, " delstar -photcodes (list) : delete by photcode\n\n"); + fprintf (stderr, " delstar -dup-images : delete duplicate images (by externID)\n\n"); fprintf (stderr, " optional flags:\n"); fprintf (stderr, " -v : verbose mode\n"); fprintf (stderr, " -photcode (code) : restrict by photcode\n"); + fprintf (stderr, " -update : apply changes (-dup-images only)\n"); + fprintf (stderr, " -image-details : list info about the deleted images (-dup-images only)\n"); + fprintf (stderr, " -image-only : only examine the image table (changes are NOT saved; -dup-images only)\n"); + fprintf (stderr, " -region Rmin Rmax Dmin Dmax : apply changes to this part of the sky\n"); + fprintf (stderr, "\n"); exit (2); @@ -48,4 +55,25 @@ VERBOSE = TRUE; VERBOSE2 = TRUE; + remove_argument (N, &argc, argv); + } + + UPDATE = FALSE; + if ((N = get_argument (argc, argv, "-update"))) { + UPDATE = TRUE; + remove_argument (N, &argc, argv); + } + + IMAGE_DETAILS = FALSE; + if ((N = get_argument (argc, argv, "-image-details"))) { + IMAGE_DETAILS = TRUE; + remove_argument (N, &argc, argv); + } + + // we should never delete the images before the measures (or we won't know what to delete) + // -image-only is for testing + IMAGE_ONLY = FALSE; + if ((N = get_argument (argc, argv, "-image-only"))) { + IMAGE_ONLY = TRUE; + UPDATE = FALSE; remove_argument (N, &argc, argv); } @@ -126,4 +154,9 @@ if (MODE != MODE_NONE) usage(); MODE = MODE_MISSED; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-dup-images"))) { + if (MODE != MODE_NONE) usage(); + MODE = MODE_DUP_IMAGES; remove_argument (N, &argc, argv); } @@ -251,4 +284,23 @@ } + UPDATE = FALSE; + if ((N = get_argument (argc, argv, "-update"))) { + UPDATE = TRUE; + remove_argument (N, &argc, argv); + } + + IMAGE_DETAILS = FALSE; + if ((N = get_argument (argc, argv, "-image-details"))) { + IMAGE_DETAILS = TRUE; + remove_argument (N, &argc, argv); + } + + IMAGE_ONLY = FALSE; + if ((N = get_argument (argc, argv, "-image-only"))) { + IMAGE_ONLY = TRUE; + UPDATE = FALSE; + remove_argument (N, &argc, argv); + } + /* specify portion of the sky */ UserPatch.Rmin = 0; Index: trunk/Ohana/src/delstar/src/delete_duplicate_images.c =================================================================== --- trunk/Ohana/src/delstar/src/delete_duplicate_images.c (revision 35237) +++ trunk/Ohana/src/delstar/src/delete_duplicate_images.c (revision 35237) @@ -0,0 +1,507 @@ +# include "delstar.h" + +// this function identifies the images to be deleted based on duplication of the +// externID values (ex 0). The result is an array of imageID values to be deleted + +// find & delete duplicate images +// duplicates based on externID (skip 0s) +int delete_duplicate_images (int hostID, char *hostpath, FITS_DB *db) { + + off_t i, Nimage; + Image *image, *outimage; + + image = gfits_table_get_Image (&db[0].ftable, &Nimage, &db[0].swapped); + if (!image) { + fprintf (stderr, "ERROR: failed to read images\n"); + exit (2); + } + + IndexArray *imageID = find_duplicates (image, Nimage); + + if (!IMAGE_ONLY) { + delete_duplicate_image_measures (hostID, hostpath, imageID); + } + + if (!UPDATE) return TRUE; + + /* delete the identified images */ + off_t Noutimage = 0; + ALLOCATE (outimage, Image, Nimage); + for (i = 0; i < Nimage; i++) { + if (image[i].imageID == 0) continue; + off_t Ni = image[i].imageID - imageID->minID; + myAssert (Ni >= 0, "oops"); + myAssert (Ni < imageID->range, "oops"); + + // imageID->value[Ni] is TRUE if we want to delete the image + if (imageID->value[Ni]) continue; + outimage[Noutimage] = image[i]; + Noutimage ++; + } + free (image); + + fprintf (stderr, "removing "OFF_T_FMT" images (leaving "OFF_T_FMT" of "OFF_T_FMT")\n", Nimage - Noutimage, Noutimage, Nimage); + // gfits_table_set_Image (&db[0].ftable, outimage, Noutimage); + + gfits_modify (&db[0].theader, "NAXIS2", OFF_T_FMT, 1, Noutimage); + gfits_modify (&db[0].header, "NIMAGES", OFF_T_FMT, 1, Noutimage); + db[0].theader.Naxis[1] = Noutimage; + db[0].ftable.buffer = (char *) outimage; + + if (!dvo_image_save (db, VERBOSE)) return FALSE; + if (!dvo_image_unlock (db)) return FALSE; + + return TRUE; +} + +int delete_duplicate_image_measures (int hostID, char *hostpath, IndexArray *imageID) { + + int i; + Catalog catalog; + + // load the current sky table (layout of all SkyRegions) + SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, TRUE, -1, VERBOSE); + if (!sky) { + fprintf (stderr, "ERROR running loading sky table from %s\n", CATDIR); + exit (2); + } + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + SkyList *skylist = SkyListByPatch (sky, -1, &UserPatch); + if (!skylist) { + fprintf (stderr, "ERROR setting up skylist for %s\n", CATDIR); + exit (2); + } + + // launch the remote jobs + if (PARALLEL && !hostID) { + int status = delete_duplicate_image_measures_parallel (skylist); + return status; + } + + // delete detections from region + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue; + + // set up the basic catalog info + char hostfile[1024]; + snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name); + catalog.filename = hostID ? hostfile : skylist[0].filename[i]; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + + if (VERBOSE) fprintf (stderr, "deleting from %s\n", catalog.filename); + + // an error exit status here is a significant error + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE2, "a")) { + fprintf (stderr, "ERROR: failure to open catalog file %s\n", catalog.filename); + exit (2); + } + if (!catalog.Naves_disk) { + if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + delete_duplicate_image_measures_catalog (&catalog, imageID); + + if (UPDATE) { + dvo_catalog_save_complete (&catalog, VERBOSE2); + } + + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + } + return TRUE; +} + +// CATDIR is supplied globally +int delete_duplicate_image_measures_parallel (SkyList *sky) { + + // launch the delstar_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + exit (1); + } + + int i; + for (i = 0; i < table->Nhosts; i++) { + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + char command[1024]; + snprintf (command, 1024, "delstar_client -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -dup-images", + table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, + UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + char tmpline[1024]; + if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } + if (VERBOSE2) { snprintf (tmpline, 1024, "%s -vv", command); strcpy (command, tmpline); } + + fprintf (stderr, "command: %s\n", command); + + fprintf (stderr, "parallel version is not ready (need to save / load the imageID array\n"); + + if (PARALLEL_MANUAL) continue; + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running delstar_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + } + + return TRUE; +} + +int delete_duplicate_image_measures_catalog (Catalog *catalog, IndexArray *imageID) { + + off_t i, j, n, m, N, D, currentAve; + + Measure *measureOut = NULL; + Average *averageOut = NULL; + SecFilt *secfiltOut = NULL; + + off_t *measureDrop, *measureSeqRaw, *measureSeqOut, *measureRefOut, *measureAveRaw, *averageNmeas, *averageDmeas, *averageSeqOut, *averageRefOut, *measureAveOut; + + int Nsecfilt = GetPhotcodeNsecfilt (); + + /* internal counters */ + off_t Nmeasure = catalog[0].Nmeasure; + off_t Naverage = catalog[0].Naverage; + + Measure *measure = catalog[0].measure; + Average *average = catalog[0].average; + SecFilt *secfilt = catalog[0].secfilt; + + if (VERBOSE) fprintf (stderr, "starting with Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT"\n", catalog[0].Naverage, catalog[0].Nmeasure); + + // we have a table of average objects and an unsorted table of measurements. each measurement + // has a reference to the average object sequence (as well as an ID) + // measure[i].averef -> average[averef] + // measure[i].objID = average[averef].objID + // measure[i].catID = average[averef].catID + + // we want a sorted measure array with all averef entries in sequence, skipping the entries which match our criteria + + // arrays + ALLOCATE (measureDrop, off_t, Nmeasure); + ALLOCATE (measureSeqRaw, off_t, Nmeasure); + ALLOCATE (measureSeqOut, off_t, Nmeasure); + ALLOCATE (measureRefOut, off_t, Nmeasure); + ALLOCATE (measureAveRaw, off_t, Nmeasure); + ALLOCATE (averageNmeas, off_t, Naverage); + ALLOCATE (averageDmeas, off_t, Naverage); + ALLOCATE (averageSeqOut, off_t, Naverage); + ALLOCATE (averageRefOut, off_t, Naverage); + + // mark the measures to be dropped + for (i = 0; i < Nmeasure; i++) { + if (measure[i].imageID == 0) continue; + off_t Ni = measure[i].imageID - imageID->minID; + myAssert (Ni >= 0, "oops"); + myAssert (Ni < imageID->range, "oops"); + measureDrop[i] = imageID->value[Ni]; + } + + // set up the measure sequence lists + for (i = 0; i < Nmeasure; i++) { + measureSeqRaw[i] = i; + measureAveRaw[i] = measure[i].averef; + } + // sort measureSeqRaw and measureAveRaw in order of measureAveRaw + SortAveMeasMatch(measureSeqRaw, measureAveRaw, Nmeasure); + + // generate the measureSeqOut array + off_t NmeasOut = 0; + for (i = 0; i < Nmeasure; i++) { + j = measureSeqRaw[i]; + if (measureDrop[j]) continue; + measureSeqOut[NmeasOut] = j; + measureRefOut[j] = NmeasOut; + NmeasOut ++; + } + // n = measureSeqOut[i] : measureOut[i] = measure[n] (i = 0 -- NmeasOut, n = 0 -- Nmeasure) + // n = measureRefOut[i] : measureOut[n] = measure[i] + ALLOCATE (measureAveOut, off_t, NmeasOut); + + // count the number of measures for each averef + N = 0; + D = -1; + currentAve = measureAveRaw[0]; + for (i = 0; i < Nmeasure; i++) { + if (measureAveRaw[i] != currentAve) { + // we have hit the next entry in the list + averageNmeas[currentAve] = N; // number of measures + averageDmeas[currentAve] = D; // first measure + N = 0; + D = -1; + currentAve = measureAveRaw[i]; + } + j = measureSeqRaw[i]; + if (measureDrop[j]) continue; + N++; + if (D == -1) { + // first valid measureOut for this averef + D = measureRefOut[j]; + } + } + averageNmeas[currentAve] = N; // number of measures + averageDmeas[currentAve] = D; // first measure + + // generate the new average sequence, skipping entries with no measurements + off_t NaveOut = 0; + for (i = 0; i < Naverage; i++) { + if (averageNmeas[i] == 0) continue; + averageSeqOut[NaveOut] = i; + averageRefOut[i] = NaveOut; + NaveOut ++; + } + // n = averageSeqOut[i] : averageOut[i] = average[n] (i = 0 -- NaveOut, n = 0 -- Naverage) + // n = averageRefOut[i] : averageOut[n] = average[i] + + // generate the output averefs + for (i = 0; i < NmeasOut; i++) { + j = measureSeqOut[i]; // measure[j] = measureOut[i] + n = measureAveRaw[j]; // average[n] : measure[j] + N = averageRefOut[n]; // averageOut[N] = average[n]; + measureAveOut[i] = N; // measureOut[i].averef = measureAveOut[i] + } + + // copy the (kept) measurements in the sorted order + ALLOCATE (measureOut, Measure, NmeasOut); + for (i = 0; i < NmeasOut; i++) { + j = measureSeqOut[i]; + measureOut[i] = measure[j]; + measureOut[i].averef = measureAveOut[i]; + } + + // copy the (kept) average entries + ALLOCATE (averageOut, Average, NaveOut); + for (i = 0; i < NaveOut; i++) { + j = averageSeqOut[i]; + averageOut[i] = average[j]; + averageOut[i].Nmeasure = averageNmeas[j]; + averageOut[i].measureOffset = averageDmeas[j]; + } + + // copy the secfilt entries for the (kept) average entries + ALLOCATE (secfiltOut, SecFilt, NaveOut*Nsecfilt); + for (i = 0; i < NaveOut; i++) { + j = averageSeqOut[i]; + for (m = 0; m < Nsecfilt; m++) { + secfiltOut[i*Nsecfilt + m] = secfilt[j*Nsecfilt + m]; + } + } + + // update the values of average.measureOffset and average.Nmeasure + FREE(measure); + FREE(average); + FREE(secfilt); + catalog[0].measure = measureOut; + catalog[0].average = averageOut; + catalog[0].secfilt = secfiltOut; + catalog[0].Nmeasure = NmeasOut; + catalog[0].Naverage = NaveOut; + + // update catalog.Nmeasure and catalog.Naverage, double check + int NmeasureTotal = 0; + int measureOffsetOK = TRUE; + int averefOK = TRUE; + for (i = 0; i < NaveOut; i++) { + NmeasureTotal += catalog[0].average[i].Nmeasure; + if (VERBOSE2 && !(NmeasureTotal <= catalog[0].Nmeasure)) { + fprintf (stderr, "too few measurements: %d %d %d\n", (int) i, NmeasureTotal, (int) catalog[0].Nmeasure); + } + measureOffsetOK &= (catalog[0].average[i].measureOffset < catalog[0].Nmeasure); + if (VERBOSE2 && !(catalog[0].average[i].measureOffset < catalog[0].Nmeasure)) { + fprintf (stderr, "offset too large: %d %d %d\n", (int) i, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure); + } + measureOffsetOK &= (catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure); + if (VERBOSE2 && !(catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure)) { + fprintf (stderr, "orrset + Nmeasure too large: %d + %d > %d %d\n", (int) i, catalog[0].average[i].measureOffset, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure); + } + m = catalog[0].average[i].measureOffset; + for (j = 0; j < catalog[0].average[i].Nmeasure; j++) { + averefOK &= (catalog[0].measure[m+j].averef == i); + if (VERBOSE2 && !(catalog[0].measure[m+j].averef == i)) { + fprintf (stderr, "averef broken: %d vs %d (measure %d)\n", (int) i, catalog[0].measure[m+j].averef, (int) (m+j)); + } + } + } + + if (!measureOffsetOK) { + fprintf (stderr, "ERROR: catalog %s has an invalid measureOffset\n", catalog[0].filename); + } + + if (!averefOK) { + fprintf (stderr, "ERROR: catalog %s has invalid averefs\n", catalog[0].filename); + } + + if (NmeasureTotal != catalog[0].Nmeasure) { + fprintf (stderr, "ERROR: catalog %s has an invalid Nmeasure\n", catalog[0].filename); + } + + // MARKTIME(" match time %9.4f sec for %7lld measures, %6lld average\n", dtime, (long long) Nmeasure, (long long) Naverage); + + catalog[0].sorted = TRUE; + + if (VERBOSE) fprintf (stderr, "ending with Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT"\n", catalog[0].Naverage, catalog[0].Nmeasure); + + off_t NdelAves = Naverage - catalog[0].Naverage; + off_t NdelMeas = Nmeasure - catalog[0].Nmeasure; + if (NdelAves || NdelMeas) { + fprintf (stderr, "deleting "OFF_T_FMT" measures and "OFF_T_FMT" averages : %s\n", NdelMeas, NdelAves, catalog[0].filename); + } + + FREE (measureDrop); + FREE (measureSeqRaw); + FREE (measureSeqOut); + FREE (measureRefOut); + FREE (measureAveRaw); + FREE (averageNmeas); + FREE (averageDmeas); + FREE (averageSeqOut); + FREE (averageRefOut); + FREE (measureAveOut); + + return TRUE; +} + +IndexArray *find_duplicates (Image *image, off_t Nimage) { + + off_t i, *Ndelete; + + // how to find duplicates: + // generate an index array of externID + // * find min & max value + // * array has length max - min + // generate the count of externID values + // mark those with N > 1 + + // index->value array is set to 0 initially + IndexArray *extID = make_index_array (image, Nimage, EXTERN_ID); + + ALLOCATE (Ndelete, off_t, extID->range); + for (i = 0; i < extID->range; i++) Ndelete[i] = 0; + + // set extID->value to have the number of images with a given externID + for (i = 0; i < Nimage; i++) { + if (image[i].externID == 0) continue; + off_t N = image[i].externID - extID->minID; + myAssert (N >= 0, "oops"); + myAssert (N < extID->range, "oops"); + extID->value[N] ++; + } + // count has the number for each image.externID + // now find things to be deleted: + + // now I know the externID values to delete, I need to get a list of imageID values + // more specifically, i need an index of the imageIDs against an array saying delete or not + + // somewhere downstream I want to be able to say: + // if (delete[measure.imageID]) delete_measure... + + IndexArray *imageID = make_index_array (image, Nimage, IMAGE_ID); + + // set imageID->value to TRUE for images we want to delete + for (i = 0; i < Nimage; i++) { + if (image[i].externID == 0) continue; + off_t Nx = image[i].externID - extID->minID; + myAssert (Nx >= 0, "oops"); + myAssert (Nx < extID->range, "oops"); + if (extID->value[Nx] < 2) continue; + if (Ndelete[Nx] == extID->value[Nx] - 1) continue; + + off_t Ni = image[i].imageID - imageID->minID; + myAssert (Ni >= 0, "oops"); + myAssert (Ni < imageID->range, "oops"); + imageID->value[Ni] = TRUE; + Ndelete[Nx] ++; + + // mark the parentID for deletion as well.. + // XXX this is probably bad : it assumes all chips + // should be deleted for a given exposure + off_t parentID = image[i].parentID; + if (parentID) { + off_t Np = parentID - imageID->minID; + myAssert (Np >= 0, "oops"); + myAssert (Np < imageID->range, "oops"); + imageID->value[Np] = TRUE; + } + } + + for (i = 0; IMAGE_DETAILS && (i < Nimage); i++) { + off_t Ni = image[i].imageID - imageID->minID; + if (!imageID->value[Ni]) continue; + fprintf (stderr, "delete image (i) " OFF_T_FMT ", extID = %d : %s\n", i, image[i].externID, image[i].name); + } + + free (extID->value); + free (extID); + + return imageID;; +} + +// find the min & max values of the given ID (externID or imageID) +// construct an empty array with length needed to fit IDs +IndexArray *make_index_array (Image *image, off_t Nimage, int mode) { + + off_t i, value; + + IndexArray *idxarray = NULL; + ALLOCATE (idxarray, IndexArray, 1); + + idxarray->minID = -1; + idxarray->maxID = -1; + for (i = 0; i < Nimage; i++) { + if (mode == EXTERN_ID) { + value = image[i].externID; + if (value == 0) continue; + } + if (mode == IMAGE_ID) { + value = image[i].imageID; + if (value == 0) continue; + } + if (idxarray->minID == -1) idxarray->minID = value; + idxarray->minID = MIN(idxarray->minID, value); + idxarray->maxID = MAX(idxarray->maxID, value); + } + idxarray->range = idxarray->maxID- idxarray->minID + 1; + + // check if this will be too large? + ALLOCATE (idxarray->value, off_t, idxarray->range); + + for (i = 0; i < idxarray->range; i++) idxarray->value[i] = 0; + + return idxarray; +} + Index: trunk/Ohana/src/delstar/src/delstar.c =================================================================== --- trunk/Ohana/src/delstar/src/delstar.c (revision 35233) +++ trunk/Ohana/src/delstar/src/delstar.c (revision 35237) @@ -20,4 +20,7 @@ switch (MODE) { + case MODE_DUP_IMAGES: + delete_duplicate_images (0, NULL, &db); + break; case MODE_IMAGEFILE: delete_imagefile (&db); Index: trunk/Ohana/src/kapa2/src/CheckColors.c =================================================================== --- trunk/Ohana/src/kapa2/src/CheckColors.c (revision 35233) +++ trunk/Ohana/src/kapa2/src/CheckColors.c (revision 35237) @@ -12,5 +12,5 @@ if (*argc <= N + 1) { fprintf (stderr, "error: usage is -fg color\n"); - exit (0); + exit (2); } remove_argument (N, argc, argv); @@ -27,5 +27,5 @@ if (*argc <= N + 1) { fprintf (stderr, "error: usage is -bg color\n"); - exit (0); + exit (2); } remove_argument (N, argc, argv); Index: trunk/Ohana/src/kapa2/src/CheckDisplayName.c =================================================================== --- trunk/Ohana/src/kapa2/src/CheckDisplayName.c (revision 35233) +++ trunk/Ohana/src/kapa2/src/CheckDisplayName.c (revision 35237) @@ -10,5 +10,5 @@ if (*argc <= N + 1) { fprintf (stderr, "error: usage is [-display/-d] DisplayName\n"); - exit (0); + exit (2); } remove_argument(N, argc, argv); @@ -21,5 +21,5 @@ if (*argc <= N + 1) { fprintf (stderr, "error: usage is [-display/-d] DisplayName\n"); - exit (0); + exit (2); } remove_argument(N, argc, argv); Index: trunk/Ohana/src/kapa2/src/CheckGeometry.c =================================================================== --- trunk/Ohana/src/kapa2/src/CheckGeometry.c (revision 35233) +++ trunk/Ohana/src/kapa2/src/CheckGeometry.c (revision 35237) @@ -11,5 +11,5 @@ if (*argc <= N + 1) { fprintf (stderr, "error: usage is -geom DisplayName\n"); - exit (0); + exit (2); } remove_argument (N, argc, argv); Index: trunk/Ohana/src/kapa2/src/CheckVisual.c =================================================================== --- trunk/Ohana/src/kapa2/src/CheckVisual.c (revision 35233) +++ trunk/Ohana/src/kapa2/src/CheckVisual.c (revision 35237) @@ -31,5 +31,5 @@ if (Nfound == 0) { fprintf (stderr, "error finding useful visual\n"); - exit (0); + exit (2); } @@ -157,5 +157,5 @@ // We've already tried to allocate a colormap above... fprintf (stderr, "can't allocate enough cells in private colormap\n"); - exit (0); + exit (2); } if (DEBUG) fprintf (stderr, "can't allocate enough cells, using private colormap\n"); @@ -173,9 +173,9 @@ if ((N = get_argument (*argc, argv, "-colorcount"))) { fprintf (stderr, "kapa can grab %d colors\n", graphic[0].Npixels); - exit (0); + exit (1); } if (graphic[0].Npixels < 16) { fprintf (stderr, "can't even allocate enough cells in private colormap\n"); - exit (0); + exit (2); } } @@ -196,5 +196,5 @@ if ((N = get_argument (*argc, argv, "-colorcount"))) { fprintf (stderr, "kii can grab %d colors\n", graphic[0].Npixels); - exit (0); + exit (1); } Index: trunk/Ohana/src/kapa2/src/Layout.c =================================================================== --- trunk/Ohana/src/kapa2/src/Layout.c (revision 35233) +++ trunk/Ohana/src/kapa2/src/Layout.c (revision 35237) @@ -36,5 +36,5 @@ if (argc != 1) { fprintf (stderr, "USAGE: kapa\n"); - exit (0); + exit (1); } Index: trunk/Ohana/src/kapa2/src/LoadFont.c =================================================================== --- trunk/Ohana/src/kapa2/src/LoadFont.c (revision 35233) +++ trunk/Ohana/src/kapa2/src/LoadFont.c (revision 35237) @@ -14,5 +14,5 @@ if (*argc <= N + 1) { fprintf (stderr, "error: usage is -font fontname\n"); - exit (0); + exit (2); } remove_argument(N, argc, argv); @@ -23,5 +23,5 @@ if (*argc <= N + 1) { fprintf (stderr, "error: usage is -fn fontname\n"); - exit (0); + exit (2); } remove_argument(N, argc, argv); Index: trunk/Ohana/src/kapa2/src/OpenDisplay.c =================================================================== --- trunk/Ohana/src/kapa2/src/OpenDisplay.c (revision 35233) +++ trunk/Ohana/src/kapa2/src/OpenDisplay.c (revision 35237) @@ -9,5 +9,5 @@ if (display == NULL) { fprintf (stderr, "Error could not open X display to %s\n", XDisplayName (name)); - exit (0); + exit (2); } *screen = DefaultScreen (display); Index: trunk/Ohana/src/kapa2/src/QuitX.c =================================================================== --- trunk/Ohana/src/kapa2/src/QuitX.c (revision 35233) +++ trunk/Ohana/src/kapa2/src/QuitX.c (revision 35237) @@ -6,4 +6,4 @@ fprintf (stderr, "Error: %s\n", error_message); XCloseDisplay (display); - exit (0); + exit (3); } Index: trunk/Ohana/src/libdvo/include/dvodb.h =================================================================== --- trunk/Ohana/src/libdvo/include/dvodb.h (revision 35233) +++ trunk/Ohana/src/libdvo/include/dvodb.h (revision 35237) @@ -141,4 +141,5 @@ MEAS_EXT_NSIGMA, MEAS_EXTERN_ID, + MEAS_EXPNAME_AS_INT, MEAS_MCAL_OFFSET, MEAS_FLAT, Index: trunk/Ohana/src/libdvo/src/dbExtractMeasures.c =================================================================== --- trunk/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 35233) +++ trunk/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 35237) @@ -523,4 +523,19 @@ break; + case MEAS_EXPNAME_AS_INT: + { + Image *image; + image = MatchImageDVO (measure[0].t, measure[0].photcode, measure[0].imageID); + if (image == NULL) break; + // XXX very crude: if this matches oNNNNgNNNNo, then convert to an int + // otherwise, leave as 0 + if ((image->name[0] == 'o') && (image->name[5] == 'g') && (image->name[10] == 'o')) { + int mjd = atoi(&image->name[1]); + int Nexp = atoi(&image->name[6]); + value.Int = mjd * 10000 + Nexp; + } + } + break; + case MEAS_MCAL_OFFSET: /* OK */ value.Flt = measure[0].Mcal; Index: trunk/Ohana/src/libdvo/src/dbFields.c =================================================================== --- trunk/Ohana/src/libdvo/src/dbFields.c (revision 35233) +++ trunk/Ohana/src/libdvo/src/dbFields.c (revision 35237) @@ -231,4 +231,5 @@ if (!strcasecmp (fieldName, "IMAGEID")) ESCAPE (MEAS_IMAGE_ID, MAG_NONE, OPIHI_INT); if (!strcasecmp (fieldName, "EXTERNID")) ESCAPE (MEAS_EXTERN_ID, MAG_NONE, OPIHI_INT); + if (!strcasecmp (fieldName, "EXPNAME")) ESCAPE (MEAS_EXPNAME_AS_INT, MAG_NONE, OPIHI_INT); if (!strcasecmp (fieldName, "PSF_QF")) ESCAPE (MEAS_PSF_QF, MAG_NONE, OPIHI_FLT); if (!strcasecmp (fieldName, "PSF_QF_PERFECT")) ESCAPE (MEAS_PSF_QF_PERFECT, MAG_NONE, OPIHI_FLT); Index: trunk/Ohana/src/libkapa/src/KapaOpen.c =================================================================== --- trunk/Ohana/src/libkapa/src/KapaOpen.c (revision 35233) +++ trunk/Ohana/src/libkapa/src/KapaOpen.c (revision 35237) @@ -8,4 +8,7 @@ static int Nvalid = 0; static int *VALID = NULL; + +char *_parse_nextword (char *string); +int KapaLaunchCommand (char *line); int KapaServerInit (KapaSockAddress *Address) { @@ -225,11 +228,13 @@ if (kapa_name == NULL) { - sprintf (line, "%s &", kapa_exec); + sprintf (line, "%s", kapa_exec); } else { - sprintf (line, "%s -name '%s' &", kapa_exec, kapa_name); - } - if (system (line) == -1) { + sprintf (line, "%s -name '%s'", kapa_exec, kapa_name); + } + + int pid = KapaLaunchCommand (line); + if (!pid) { fprintf (stderr, "failed to launch kapa\n"); - return 0; + return (-1); } @@ -241,4 +246,16 @@ // if (errno != EAGAIN) break; if (errno != ECONNREFUSED) break; + // no connection yet. try again, but first check + // if the kapa job has exited + int waitstatus; + int result = waitpid (pid, &waitstatus, WNOHANG); + if (result == -1) { + fprintf (stderr, "problem checking for kapa\n"); + return (-1); + } + if (result > 0) { + fprintf (stderr, "kapa exited\n"); + return (-1); + } usleep (10000); Ntry ++; @@ -277,7 +294,9 @@ sprintf (temp, "%s -socket %s -name %s &", kapa_exec, socket_name, name); } - if (system (temp) == -1) { + + int pid = KapaLaunchCommand (temp); + if (!pid) { fprintf (stderr, "failed to launch kapa\n"); - return 0; + return (-1); } @@ -291,4 +310,16 @@ if (fd != -1) break; if (errno != EAGAIN) break; + // no connection yet. try again, but first check + // if the kapa job has exited + int waitstatus; + int result = waitpid (pid, &waitstatus, WNOHANG); + if (result == -1) { + fprintf (stderr, "problem checking for kapa\n"); + return (-1); + } + if (result > 0) { + fprintf (stderr, "kapa exited\n"); + return (-1); + } usleep (10000); Ntry ++; @@ -322,2 +353,53 @@ return (sock); } + +int KapaLaunchCommand (char *line) { + + // use fork & exec to launch the command; return the PID for testing exit + int i, done, Nalloc, Nargv; + char **argv, *p, *q; + + i = 0; + Nalloc = 10; + ALLOCATE (argv, char *, Nalloc); + + // split out line into unique words + p = line; + done = FALSE; + while (!done) { + q = _parse_nextword (p); + if (q && *q) { + argv[i] = strncreate (p, q - p); + stripwhite (argv[i]); + p = q; + } else { + argv[i] = strcreate (p); + stripwhite (argv[i]); + done = TRUE; + } + i++; + if (i == Nalloc - 1) { + // need to keep one extra for the NULL pointer + Nalloc += 10; + REALLOCATE (argv, char *, Nalloc); + } + } + Nargv = i; + argv[Nargv] = NULL; + + int pid = fork (); + if (!pid) { /* must be child process */ + execvp (argv[0], argv); + return (0); + } + for (i = 0; i < Nargv; i++) { + if (argv[i]) free (argv[i]); + } + free (argv); + + if (pid == -1) { + return (0); + } + + return (pid); +} Index: trunk/Ohana/src/opihi/cmd.basic/Makefile =================================================================== --- trunk/Ohana/src/opihi/cmd.basic/Makefile (revision 35233) +++ trunk/Ohana/src/opihi/cmd.basic/Makefile (revision 35237) @@ -48,4 +48,5 @@ $(SRC)/run_while.$(ARCH).o \ $(SRC)/scan.$(ARCH).o \ +$(SRC)/scannext.$(ARCH).o \ $(SRC)/shell.$(ARCH).o \ $(SRC)/sprintf.$(ARCH).o \ Index: trunk/Ohana/src/opihi/cmd.basic/init.c =================================================================== --- trunk/Ohana/src/opihi/cmd.basic/init.c (revision 35233) +++ trunk/Ohana/src/opihi/cmd.basic/init.c (revision 35237) @@ -35,4 +35,5 @@ int run_while PROTO((int, char **)); int scan PROTO((int, char **)); +int scannext PROTO((int, char **)); int shell PROTO((int, char **)); int sprintf_opihi PROTO((int, char **)); @@ -86,4 +87,5 @@ {1, "while", run_while, "while loop"}, {1, "scan", scan, "scan line from keyboard or file to variable *"}, + {1, "scannext", scannext, "scan next line from file to variable (file stays open)"}, {1, "!", shell, "system call"}, {1, "exec", shell, "system call"}, Index: trunk/Ohana/src/opihi/cmd.basic/scannext.c =================================================================== --- trunk/Ohana/src/opihi/cmd.basic/scannext.c (revision 35237) +++ trunk/Ohana/src/opihi/cmd.basic/scannext.c (revision 35237) @@ -0,0 +1,52 @@ +# include "basic.h" + +int scannext (int argc, char **argv) { + + int N, status; + char *line; + static char *filename = NULL; + static FILE *f = NULL; + + // clear all sections + if ((N = get_argument (argc, argv, "-close"))) { + remove_argument (N, &argc, argv); + if (f) { + fclose (f); + f = NULL; + } else { + gprint (GP_ERR, "file is not currently open\n"); + } + if (filename) { + free (filename); + filename = NULL; + } + return (TRUE); + } + + if ((argc != 2) && (argc != 3)) { + gprint (GP_ERR, "USAGE: scannext [var]\n"); + return (FALSE); + } + + if (!f || !filename || strcmp(filename, argv[1])) { + if (f) fclose (f); + f = fopen (argv[1], "r"); + if (f == NULL) { + gprint (GP_ERR, "file %s not found\n", argv[1]); + return (FALSE); + } + if (filename) free (filename); + filename = strcreate (argv[1]); + } + + ALLOCATE (line, char, 2048); + status = scan_line_maxlen (f, line, 2048); + if (status == EOF) { + set_str_variable (argv[2], "EOF"); + } else { + set_str_variable (argv[2], line); + } + free (line); + + return (TRUE); +} Index: trunk/Ohana/src/opihi/cmd.data/Makefile =================================================================== --- trunk/Ohana/src/opihi/cmd.data/Makefile (revision 35233) +++ trunk/Ohana/src/opihi/cmd.data/Makefile (revision 35237) @@ -43,4 +43,6 @@ $(SRC)/dbselect.$(ARCH).o \ $(SRC)/dbinsert.$(ARCH).o \ +$(SRC)/dbadmin.$(ARCH).o \ +$(SRC)/dbupdate.$(ARCH).o \ $(SRC)/erase.$(ARCH).o \ $(SRC)/extract.$(ARCH).o \ Index: trunk/Ohana/src/opihi/cmd.data/dbadmin.c =================================================================== --- trunk/Ohana/src/opihi/cmd.data/dbadmin.c (revision 35237) +++ trunk/Ohana/src/opihi/cmd.data/dbadmin.c (revision 35237) @@ -0,0 +1,66 @@ +# include "data.h" +# if (HAVE_MYSQL_H) +# include "mysql.h" + +int dbadmin (int argc, char **argv) { + + int i, Nbytes; + char *query; + + MYSQL *connection = NULL; + + if (argc < 2) { + gprint (GP_ERR, "USAGE: dbadmin [command]\n"); + gprint (GP_ERR, " currently allowed commands: CREATE, DROP, USE\n"); + return FALSE; + } + + connection = db_getConnection (); + if (connection == NULL) { + gprint (GP_ERR, "database not defined; use dbconnect\n"); + return (FALSE); + } + + // check for allowed sql commands + if (!strcasecmp(argv[1], "USE")) goto valid; + if (!strcasecmp(argv[1], "DROP")) goto valid; + if (!strcasecmp(argv[1], "CREATE")) goto valid; + gprint (GP_ERR, "currently allowed dbadmin commands are CREATE, DROP\n"); + return (FALSE); + +valid: + + // generate the query line (concat the argv[i] entries) + Nbytes = 0; + for (i = 1; i < argc; i++) { + Nbytes += strlen(argv[i]) + 1; + } + Nbytes += 10; + + ALLOCATE (query, char, Nbytes); + bzero (query, Nbytes); + for (i = 1; i < argc; i++) { + strcat (query, argv[i]); + strcat (query, " "); + } + + fprintf (stderr, "query: %s\n", query); + + if (mysql_query (connection, query)) { + gprint (GP_ERR, "problem with query\n"); + gprint (GP_ERR, "%s\n", mysql_error (connection)); + free (query); + return (FALSE); + } + free (query); + return (TRUE); +} +# else + +int dbadmin (int argc, char **argv) { + + gprint (GP_ERR, "mysql library is not available\n"); + return FALSE; +} + +# endif Index: trunk/Ohana/src/opihi/cmd.data/dbupdate.c =================================================================== --- trunk/Ohana/src/opihi/cmd.data/dbupdate.c (revision 35237) +++ trunk/Ohana/src/opihi/cmd.data/dbupdate.c (revision 35237) @@ -0,0 +1,57 @@ +# include "data.h" +# if (HAVE_MYSQL_H) +# include "mysql.h" + +int dbupdate (int argc, char **argv) { + + int i, Nbytes; + char *query; + + MYSQL *connection = NULL; + + if (argc < 4) { + gprint (GP_ERR, "USAGE: dbupdate (tablerefs) set field = value,... [where]"); + return FALSE; + } + + connection = db_getConnection (); + if (connection == NULL) { + gprint (GP_ERR, "database not defined; use dbconnect\n"); + return (FALSE); + } + + // generate the query line (concat the argv[i] entries) + Nbytes = 0; + for (i = 1; i < argc; i++) { + Nbytes += strlen(argv[i]) + 1; + } + Nbytes += 10; + + ALLOCATE (query, char, Nbytes); + bzero (query, Nbytes); + strcat (query, "update "); + for (i = 1; i < argc; i++) { + strcat (query, argv[i]); + strcat (query, " "); + } + + fprintf (stderr, "query: %s\n", query); + + if (mysql_query (connection, query)) { + gprint (GP_ERR, "problem with query\n"); + gprint (GP_ERR, "%s\n", mysql_error (connection)); + free (query); + return (FALSE); + } + free (query); + return (TRUE); +} +# else + +int dbupdate (int argc, char **argv) { + + gprint (GP_ERR, "mysql library is not available\n"); + return FALSE; +} + +# endif Index: trunk/Ohana/src/opihi/cmd.data/init.c =================================================================== --- trunk/Ohana/src/opihi/cmd.data/init.c (revision 35233) +++ trunk/Ohana/src/opihi/cmd.data/init.c (revision 35237) @@ -25,4 +25,6 @@ int dbselect PROTO((int, char **)); int dbinsert PROTO((int, char **)); +int dbadmin PROTO((int, char **)); +int dbupdate PROTO((int, char **)); int delete PROTO((int, char **)); int densify PROTO((int, char **)); @@ -172,4 +174,6 @@ {1, "dbselect", dbselect, "extract vectors from mysql database table"}, {1, "dbinsert", dbinsert, "sql insert command"}, + {1, "dbadmin", dbadmin, "sql admin command (create, drop)"}, + {1, "dbupdate", dbupdate, "sql update command"}, {1, "delete", delete, "delete vectors or images"}, {1, "densify", densify, "create an image histogram from a set of vectors"},