Index: trunk/Ohana/src/dvomerge/Makefile =================================================================== --- trunk/Ohana/src/dvomerge/Makefile (revision 39322) +++ trunk/Ohana/src/dvomerge/Makefile (revision 39323) @@ -25,5 +25,5 @@ dvoutils : $(BIN)/dvoutils.$(ARCH) -all: dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client dvoutils +all: dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvorepair_client dvoverify dvoverify_client dvoutils DVOMERGE = \ @@ -125,4 +125,5 @@ DVOREPAIR = \ $(SRC)/dvorepair.$(ARCH).o \ +$(SRC)/dvorepair_args.$(ARCH).o \ $(SRC)/dvorepairFixCPT.$(ARCH).o \ $(SRC)/dvorepairFixWarpIDs.$(ARCH).o \ @@ -137,4 +138,5 @@ $(SRC)/psps_ids.$(ARCH).o \ $(SRC)/LoadImages.$(ARCH).o \ +$(SRC)/DeleteImagesOps.$(ARCH).o \ $(SRC)/ReadDeleteList.$(ARCH).o \ $(SRC)/ReadDeleteListExternID.$(ARCH).o \ @@ -146,6 +148,22 @@ $(SRC)/ImageOps.$(ARCH).o -$(DVOREPAIR) : $(INC)/dvomerge.h +$(DVOREPAIR) : $(INC)/dvorepair.h $(BIN)/dvorepair.$(ARCH) : $(DVOREPAIR) + +DVOREPAIR_CLIENT = \ +$(SRC)/dvorepair_client.$(ARCH).o \ +$(SRC)/dvorepair_args.$(ARCH).o \ +$(SRC)/dvorepairDeleteImagesByExternID.$(ARCH).o \ +$(SRC)/psps_ids.$(ARCH).o \ +$(SRC)/LoadImages.$(ARCH).o \ +$(SRC)/DeleteImagesOps.$(ARCH).o \ +$(SRC)/ReadDeleteListExternID.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ +$(SRC)/myIndex.$(ARCH).o \ +$(SRC)/help.$(ARCH).o + +$(DVOREPAIR_CLIENT) : $(INC)/dvorepair.h +$(BIN)/dvorepair_client.$(ARCH) : $(DVOREPAIR_CLIENT) DVOVERIFY = \ @@ -184,5 +202,5 @@ $(BIN)/dvoutils.$(ARCH) : $(DVOUTILS) -INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client dvoverify dvoutils +INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvorepair_client dvoverify dvoverify_client dvoverify dvoutils # dependancy rules for binary code ######################### Index: trunk/Ohana/src/dvomerge/include/dvorepair.h =================================================================== --- trunk/Ohana/src/dvomerge/include/dvorepair.h (revision 39323) +++ trunk/Ohana/src/dvomerge/include/dvorepair.h (revision 39323) @@ -0,0 +1,111 @@ +# include +# include +# include +# include +# include +# include + +/* linux is happy with this, not solaris */ +# include +# include +# include +# include + +# ifndef MAX_INT +# define MAX_INT 2147483647 +# endif + +typedef enum { + DVOREPAIR_MODE_NONE = 0, + DVOREPAIR_MODE_FixWarpIDs, + DVOREPAIR_MODE_FixStackIDs, + DVOREPAIR_MODE_FixCPT, + DVOREPAIR_MODE_BY_OBJ_ID, + DVOREPAIR_MODE_ImagesVsMeasures, + DVOREPAIR_MODE_DeleteImageList, + DVOREPAIR_MODE_DeleteImagesByExternID, + DVOREPAIR_MODE_DeleteImagesByExternID_v2, + DVOREPAIR_MODE_FixImages, +} dvorepairModes; + +typedef struct { + int minID; + int maxID; + int *index; + int Nindex; + int NINDEX; +} myIndexType; + +typedef struct { + myIndexType *imageIDindex; + int *deleteImage; + int Nimage; +} DeleteImageDataType; + +int PARALLEL; +int PARALLEL_MANUAL; +int PARALLEL_SERIAL; + +int HOST_ID; +char *HOSTDIR; + +int VERBOSE; +char *CATDIR; + +// SkyRegion UserPatch[2]; // used by MODE CAT +// int nUserPatch; + +dvorepairModes MODE; + +int main PROTO((int argc, char **argv)); + +void dvorepair_help PROTO((int argc, char **argv)); +int dvorepair_args PROTO((int *argc, char **argv)); + +void dvorepair_client_help PROTO((int argc, char **argv)); +int dvorepair_client_args PROTO((int *argc, char **argv, SkyRegion *UserPatch)); + +int dvorepairFixCPT PROTO((int argc, char **argv)); +int dvorepairImagesVsMeasures PROTO((int argc, char **argv)); +int dvorepairDeleteImageList PROTO((int argc, char **argv)); +int dvorepairFixImages PROTO((int argc, char **argv)); + +int *ReadDeleteList PROTO((char *filename, int *nindex)); +int RepairTableCPT PROTO((char *cptFilenameSrc, char *cptFilenameTgt, char *cpsFilenameSrc, char *cpsFilenameTgt, Measure *measure, off_t Nmeasure, Image *image, off_t Nimage, char catformat)); + +myIndexType *myIndexInit PROTO((void)); +int myIndexFree PROTO((myIndexType *myIndex)); +int myIndexUpdateLimits PROTO((myIndexType *myIndex, int value)); +int myIndexSetRange PROTO((myIndexType *myIndex)); +int myIndexSetEntry PROTO((myIndexType *myIndex, int value, int entry)); +int myIndexGetEntry PROTO((myIndexType *myIndex, int value)); + +int RepairTableCPT_V1 PROTO((char *cptFilenameSrc, char *cptFilenameTgt, char *cpsFilenameSrc, char *cpsFilenameTgt, Measure *measure, off_t Nmeasure, Image *image, off_t Nimage, myIndexType *imageIDindex, char catformat)); +int *ReadDeleteListExternID PROTO((char *filename, int *nindex)); + +int dvorepair_by_objID PROTO((int argc, char **argv)); +int dvorepairFixWarpIDs PROTO((int argc, char **argv)); +int dvorepairFixStackIDs PROTO((int argc, char **argv)); + + +int dvorepairDeleteImagesByExternID PROTO((int argc, char **argv)); +int dvorepairDeleteImagesByExternID_v2 PROTO((int argc, char **argv)); +int *ReadDeleteListExternID_v2 PROTO((char *filename, int *nindex)); + +int DeleteImagesSave PROTO((char *filename, Image *image, off_t Nimage, int *deleteImage)); +int DeleteImagesLoad PROTO((char *filename, DeleteImageDataType *deleteImageData)); + +int FindDeleteRegion PROTO((SkyRegion *UserPatch, Image *image, off_t Nimage, int *deleteImage)); + +int dvorepairDeleteImagesByExternID_catalogs PROTO((SkyRegion *UserPatch, int nUserPatch, Image *image, off_t Nimage, int *deleteImage, myIndexType *imageIDindex)); +int dvorepairDeleteImagesByExternID_parallel PROTO((SkyRegion *UserPatch, int nUserPatch, Image *image, off_t Nimage, int *deleteImage)); + +int SetSignals PROTO((void)); +void SetProtect PROTO((int mode)); +void TrapSignal PROTO((int sig)); +int Shutdown PROTO((char *format, ...)) OHANA_FORMAT(printf, 1, 2); + +Image *LoadImages PROTO((FITS_DB *db, char *filename, off_t *Nimage)); +uint64_t CreatePSPSDetectionID PROTO((double tobs, int ccdid, int detID)); +uint64_t CreatePSPSObjectID PROTO((double ra, double dec)); + Index: trunk/Ohana/src/dvomerge/src/DeleteImagesOps.c =================================================================== --- trunk/Ohana/src/dvomerge/src/DeleteImagesOps.c (revision 39323) +++ trunk/Ohana/src/dvomerge/src/DeleteImagesOps.c (revision 39323) @@ -0,0 +1,170 @@ +# include "dvorepair.h" + +# define GET_COLUMN(OUT,NAME,TYPE) \ + OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \ + myAssert (!strcmp(type, #TYPE), "wrong column type"); + +// STATUS is value expected for success +# define CHECK_STATUS(STATUS,MSG,...) \ + if (!(STATUS)) { \ + fprintf (stderr, MSG, __VA_ARGS__); \ + return FALSE; \ + } + +// write out the IDmap data for clients to read +int DeleteImagesSave (char *filename, Image *image, off_t Nimage, int *deleteImage) { + + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + gfits_create_table_header (&theader, "BINTABLE", "DELETE_IMAGES"); + + gfits_define_bintable_column (&theader, "J", "IMAGE_ID", "image IDs", NULL, 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "I", "DELETE", "delete image", NULL, 1.0, FT_BZERO_INT16); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + int *imageID; + short *delImage; + + // copy the deleteImage data so gfits_set_bintable_column does not affect it + ALLOCATE (imageID, int, Nimage); + ALLOCATE (delImage, short, Nimage); + + int i; + for (i = 0; i < Nimage; i++) { + imageID[i] = image[i].imageID; + delImage[i] = deleteImage[i]; + } + + gfits_set_bintable_column (&theader, &ftable, "IMAGE_ID", imageID, Nimage); + gfits_set_bintable_column (&theader, &ftable, "DELETE", delImage, Nimage); + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open ID map file for output %s\n", filename); + return FALSE; + } + + int status; + status = gfits_fwrite_header (f, &header); + CHECK_STATUS (status, "ERROR: cannot write header for image subset %s\n", filename); + + status = gfits_fwrite_matrix (f, &matrix); + CHECK_STATUS (status, "ERROR: cannot write matrix for image subset %s\n", filename); + + status = gfits_fwrite_Theader (f, &theader); + CHECK_STATUS (status, "ERROR: cannot write table header for image subset %s\n", filename); + + status = gfits_fwrite_table (f, &ftable); + CHECK_STATUS (status, "ERROR: cannot write table data for image subset %s\n", filename); + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + int fd = fileno (f); + + status = fflush (f); + CHECK_STATUS (!status, "ERROR: cannot flush file image subset %s\n", filename); + + status = fsync (fd); + CHECK_STATUS (!status, "ERROR: cannot flush file image subset %s\n", filename); + + status = fclose (f); + CHECK_STATUS (!status, "ERROR: problem closing image subset file %s\n", filename); + + free (delImage); + free (imageID); + + return TRUE; +} + +int DeleteImagesLoad (char *filename, DeleteImageDataType *deleteImageData) { + + int Ncol; + off_t Nrow; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename); + return FALSE; + } + + /* load in PHU segment (ignore) */ + if (!gfits_fread_header (f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset header\n"); + fclose (f); + return FALSE; + } + if (!gfits_fread_matrix (f, &matrix, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n"); + gfits_free_header (&header); + fclose (f); + return FALSE; + } + + ftable.header = &theader; + + // load data for this header + if (!gfits_load_header (f, &theader)) { + fclose (f); + return FALSE; + } + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + fclose (f); + return FALSE; + } + fclose (f); + + char type[16]; + + int *imageID; + short *delImage; + + GET_COLUMN (imageID, "IMAGE_ID", int); + GET_COLUMN (delImage, "DELETE", short); + deleteImageData->Nimage = Nrow; + fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow); + + // generate an index for imageIDs + deleteImageData->imageIDindex = myIndexInit(); + ALLOCATE (deleteImageData->deleteImage, int, deleteImageData->Nimage); + + // set the min and max ID values for the two strucures: + int i; + for (i = 0; i < deleteImageData->Nimage; i++) { + myIndexUpdateLimits (deleteImageData->imageIDindex, imageID[i]); + deleteImageData->deleteImage[i] = delImage[i]; + } + myIndexSetRange (deleteImageData->imageIDindex); + + // assign the externIDs and imageIDs to their index structures + for (i = 0; i < deleteImageData->Nimage; i++) { + myIndexSetEntry (deleteImageData->imageIDindex, imageID[i], i); + } + + gfits_free_header(&header); + gfits_free_matrix(&matrix); + + gfits_free_header(&theader); + gfits_free_table (&ftable); + + return TRUE; +} + Index: trunk/Ohana/src/dvomerge/src/dvomerge_client.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvomerge_client.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/dvomerge_client.c (revision 39323) @@ -110,6 +110,6 @@ dvomerge_client_args_free (); - ohana_memcheck (TRUE); - ohana_memdump (TRUE); + // ohana_memcheck (TRUE); + // ohana_memdump (TRUE); exit (0); Index: trunk/Ohana/src/dvomerge/src/dvorepair.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepair.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/dvorepair.c (revision 39323) @@ -1,5 +1,48 @@ -# include "dvomerge.h" +# include "dvorepair.h" + +// I have updated this program to use a common arg-parsing function, with operation more similar to other ohana programs +// most of the modes here are not necessarily tested for current databases (parallel, compressed) and are disabled for now int main (int argc, char **argv) { + + SetSignals (); + dvorepair_help(argc, argv); + dvorepair_args(&argc, argv); + + switch (MODE) { + case DVOREPAIR_MODE_FixWarpIDs: + dvorepairFixWarpIDs(argc, argv); + break; + case DVOREPAIR_MODE_FixStackIDs: + dvorepairFixStackIDs(argc, argv); + break; + case DVOREPAIR_MODE_FixCPT: + dvorepairFixCPT(argc, argv); + break; + case DVOREPAIR_MODE_BY_OBJ_ID: + dvorepair_by_objID(argc, argv); + break; + case DVOREPAIR_MODE_ImagesVsMeasures: + dvorepairImagesVsMeasures(argc, argv); + break; + case DVOREPAIR_MODE_DeleteImageList: + dvorepairDeleteImageList(argc, argv); + break; + case DVOREPAIR_MODE_DeleteImagesByExternID: + dvorepairDeleteImagesByExternID(argc, argv); + break; + case DVOREPAIR_MODE_DeleteImagesByExternID_v2: + dvorepairDeleteImagesByExternID_v2(argc, argv); + break; + case DVOREPAIR_MODE_FixImages: + dvorepairFixImages(argc, argv); + break; + default: + fprintf (stderr, "unknown mode\n"); + exit (1); + } + + exit (2); +} // fprintf (stderr, "this program needs to be updated to load old format Measure tables (pre PV1_V5) in which dR,dD are saved, not R,D\n"); @@ -7,19 +50,2 @@ // exit (2); - SetSignals (); - dvorepair_help(argc, argv); - - if (!strcmp(argv[1], "-fix-warp-ids")) dvorepairFixWarpIDs(argc, argv); - if (!strcmp(argv[1], "-fix-stack-ids")) dvorepairFixStackIDs(argc, argv); - - if (!strcmp(argv[1], "-fix-cpt")) dvorepairFixCPT(argc, argv); - if (!strcmp(argv[1], "-fix-cpt-by-objID")) dvorepair_by_objID(argc, argv); - // if (!strcmp(argv[1], "-fix-tables")) dvorepairFixTables(argc, argv); - if (!strcmp(argv[1], "-images-vs-measures")) dvorepairImagesVsMeasures(argc, argv); - if (!strcmp(argv[1], "-delete-image-list")) dvorepairDeleteImageList(argc, argv); - if (!strcmp(argv[1], "-delete-images-by-extern-id")) dvorepairDeleteImagesByExternID(argc, argv); - if (!strcmp(argv[1], "-delete-images-by-extern-id-v2")) dvorepairDeleteImagesByExternID_v2(argc, argv); - if (!strcmp(argv[1], "-fix-images")) dvorepairFixImages(argc, argv); - dvorepair_help(0, NULL); - exit (2); -} Index: trunk/Ohana/src/dvomerge/src/dvorepairDeleteImageList.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairDeleteImageList.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/dvorepairDeleteImageList.c (revision 39323) @@ -19,5 +19,5 @@ off_t i, j, Nmeasure, NmeasureNew, Ndelete, Nvalid, Nimage, Nindex, *imageIdx, index; - int N, seq, Nbytes, NbytesPerRow, Ncheck, nPass, raPass; + int seq, Nbytes, NbytesPerRow, Ncheck, nPass, raPass; double Rthis, Dthis, Rmin, Rmax, Dmin, Dmax, Qthis, Qmin, Qmax, dR, dQ; @@ -48,8 +48,4 @@ DVOCatFormat catformat; - N = get_argument (argc, argv, "-delete-image-list"); - myAssert(N == 1, "programming error: -delete-image-list must be first from main"); - remove_argument (N, &argc, argv); - if (argc != 3) { fprintf (stderr, "USAGE: dvorepair -delete-image-list (catdir) (deleteList)\n"); @@ -58,4 +54,7 @@ exit (2); } + + fprintf (stderr, "is this mode tested?\n"); + exit (2); char *catdir = argv[1]; Index: trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c (revision 39323) @@ -1,3 +1,4 @@ -# include "dvomerge.h" +# include "dvorepair.h" +# define DEBUG 1 // delete images based on a text table of the target IDs @@ -20,11 +21,20 @@ int RepairAverage (Catalog *catalog); +typedef struct { + double Rmin; + double Rmax; + double Qmin; + double Qmax; + double Dmin; + double Dmax; +} DeleteRegion; + + int dvorepairDeleteImagesByExternID (int argc, char **argv) { FITS_DB db; // database handle pointing to input image table - int i, N; + int i; off_t Nimage; - int nPass, raPass; Image *image; @@ -32,33 +42,17 @@ char filename[DVO_MAX_PATH]; - N = get_argument (argc, argv, "-delete-images-by-extern-id"); - myAssert(N == 1, "programming error: -delete-images-by-extern-id must be first argument"); - remove_argument (N, &argc, argv); - - VERBOSE = FALSE; - if ((N = get_argument (argc, argv, "-v"))) { - VERBOSE = TRUE; - remove_argument (N, &argc, argv); - } - - if (argc != 3) { - fprintf (stderr, "USAGE: dvorepair -delete-images-by-extern-id (catdir) (deleteList)\n"); - fprintf (stderr, " catdir : database of interest\n"); - fprintf (stderr, " deleteList : table from dvorepair giving images to be deleted\n"); - exit (2); - } - - char *catdir = argv[1]; + // CATDIR from first arg, not -D CATDIR + CATDIR = argv[1]; char *delList = argv[2]; - sprintf (filename, "%s/Photcodes.dat", catdir); + sprintf (filename, "%s/Photcodes.dat", CATDIR); if (!LoadPhotcodes (filename, NULL, FALSE)) { - fprintf (stderr, "error reading photcodes from %s\n", catdir); + fprintf (stderr, "error reading photcodes from %s\n", CATDIR); exit (1); } // load the image data - ALLOCATE(imageFilename, char, strlen(catdir) + 12); - sprintf (imageFilename, "%s/Images.dat", catdir); + ALLOCATE(imageFilename, char, strlen(CATDIR) + 12); + sprintf (imageFilename, "%s/Images.dat", CATDIR); if ((image = LoadImages (&db, imageFilename, &Nimage)) == NULL) return (FALSE); BuildChipMatch (image, Nimage); // needed for astrometric tranformations @@ -125,50 +119,719 @@ } - // find the RA & DEC range of the images we want to delete - double Rmin = 360.0; - double Rmax = 0.0; - double Dmin = +90.0; - double Dmax = -90.0; - double Qmin = 360.0; - double Qmax = 0.0; + SkyRegion UserPatch[2]; // 1 or 2 UserPatch areas are set by FindDeleteRegion + int nUserPatch = FindDeleteRegion (UserPatch, image, Nimage, deleteImage); + + // delete detections in the external catalog. things I need: + // UserPatch, nUserPatch : definition of the region + // deleteImage : boolean vector of length Nimage (delete or not?) + // imageID : vector of length Nimage, with the image IDs in the same order as deleteImage + + dvorepairDeleteImagesByExternID_catalogs (UserPatch, nUserPatch, image, Nimage, deleteImage, imageIDindex); + + off_t NnewImage; + Image *newImage = DeleteSelectedImages (image, Nimage, deleteImage, imageIDindex, &NnewImage); + + SaveImageTable (newImage, NnewImage, CATDIR, &db); + + myIndexFree (externIDindex); + myIndexFree (imageIDindex); + + free (imageFilename); + free (deleteImage); + free (deleteIDs); + + gfits_db_free (&db); + + ohana_memcheck (TRUE); + + exit (0); +} + +Image *DeleteSelectedImages (Image *image, off_t Nimage, int *deleteImage, myIndexType *imageIDindex, off_t *NnewImage) { + + int i; + + // we need to remove the (WRP) images, but we also want to remove the DIS [PHU] image entries if they have + // had all of their children deleted + + // first, generate an array of the number of children for each parent + + int *Nchildren = NULL; + ALLOCATE (Nchildren, int, Nimage); + memset (Nchildren, 0, Nimage*sizeof(int)); + + // parentID is not necessarily reliable (dvomerge breaks it, for example). rebuild it + // here by using BuildChipMatch (libdvo/src/mosaic_astrom.c): + + BuildChipMatch (image, Nimage); + + // reassign parentID values + int NbadParentIDs = 0; + for (i = 0; i < Nimage; i++) { + if (!image[i].parent) continue; + + if (image[i].parentID != image[i].parent[0].imageID) { + NbadParentIDs ++; + if (NbadParentIDs < 100) { + int n = myIndexGetEntry (imageIDindex, image[i].parentID); + fprintf (stderr, "bad parent ID: %s matchs %s not %s\n", image[i].name, image[i].parent[0].name, image[n].name); + } + image[i].parentID = image[i].parent[0].imageID; + } + } + + // count up the children for each parent + for (i = 0; i < Nimage; i++) { + if (!image[i].parentID) continue; + if (strcmp(&image[i].coords.ctype[4], "-WRP")) { + fprintf (stderr, "warning: child with non WRP coords : %s\n", image[i].name); + } + + int n = myIndexGetEntry (imageIDindex, image[i].parentID); + myAssert (n != -1, "invalid parentID?"); + myAssert (n < Nimage, "invalid parentID??"); + Nchildren[n] ++; + } + + // now subtract the children which need to be deleted for (i = 0; i < Nimage; i++) { if (!deleteImage[i]) continue; + if (!image[i].parentID) continue; // should I warn on this? + + int n = myIndexGetEntry (imageIDindex, image[i].parentID); + myAssert (n != -1, "invalid parentID?"); + myAssert (n < Nimage, "invalid parentID??"); + Nchildren[n] --; + myAssert (Nchildren[n] >= 0, "over deleted???"); + } + + // now check for any parent and delete it + for (i = 0; i < Nimage; i++) { + if (strcmp(&image[i].coords.ctype[4], "-DIS")) continue; + myAssert (!image[i].parentID, "parent ID for DIS image??"); + if (!Nchildren[i]) deleteImage[i] = TRUE; + } + + fprintf (stderr, "deleting the following images:\n"); + + off_t Nnew = 0; + Image *newImage = NULL; + ALLOCATE (newImage, Image, Nimage); + for (i = 0; i < Nimage; i++) { + if (deleteImage[i]) { + fprintf (stderr, "DELETE: %s : %d : %d : %d\n", image[i].name, image[i].imageID, image[i].parentID, image[i].externID); + continue; + } + newImage[Nnew] = image[i]; + Nnew ++; + } + + REALLOCATE (newImage, Image, Nnew); + + *NnewImage = Nnew; + return newImage; +} + +int SaveImageTable (Image *image, off_t Nimage, char *catdir, FITS_DB *oldDB) { + + FITS_DB db; + + char ImageCatSrc[DVO_MAX_PATH]; + char ImageCatTgt[DVO_MAX_PATH]; + + // we are creating a new image table (what about db ID?) + snprintf (ImageCatSrc, DVO_MAX_PATH, "%s/Images.dat", catdir); + snprintf (ImageCatTgt, DVO_MAX_PATH, "%s/Images.dat.broken", catdir); + + // rename the old cpt file: + if (rename (ImageCatSrc, ImageCatTgt)) { + perror ("tried to rename file"); + exit (2); + } + + /* setup image table format and lock */ + gfits_db_init (&db); + db.mode = dvo_catalog_catmode ("SPLIT"); + db.format = dvo_catalog_catformat ("PS1_V5"); + int status = dvo_image_lock (&db, ImageCatSrc, 3600.0, LCK_XCLD); // shorter timeout? + if (!status) { + fprintf (stderr, "ERROR: failure to lock image catalog %s", db.filename); + exit (2); + } + + /* load or create the image table */ + if (db.dbstate != LCK_EMPTY) { + fprintf (stderr, "image table %s exists, exiting", db.filename); + exit (2); + } + dvo_image_create (&db, GetZeroPoint()); + + // I want to keep the old DVO_DBID + char dbID[33]; + int imageIDmax; + if (!gfits_scan (&oldDB->header, "DVO_DBID", "%s", 1, dbID)) { + fprintf (stderr, "source image table is missing DVO_DBID, exiting\n"); + exit (3); + } + if (!gfits_scan (&oldDB->header, "IMAGEID", "%d", 1, &imageIDmax)) { + fprintf (stderr, "max image ID is missing, exiting\n"); + exit (3); + } + gfits_modify (&db.header, "IMAGEID", "%d", 1, imageIDmax); + gfits_modify (&db.header, "DVO_DBID", "%s", 1, dbID); + + // also save the merge history: + int Nmerge; + if (gfits_scan (&oldDB->header, "NMERGE", "%d", 1, &Nmerge)) { + int i; + gfits_modify (&db.header, "NMERGE", "%d", 1, Nmerge); + for (i = 0; i < Nmerge; i++) { + char field[80], value[80]; + snprintf (field, 80, "DM_%05d", i); + if (!gfits_scan (&oldDB->header, field, "%s", 1, value)) { + fprintf (stderr, "missing merge info %s\n", field); + continue; + } + gfits_modify (&db.header, field, "%s", 1, value); + } + } + + /* add the new image and save */ + dvo_image_addrows (&db, image, Nimage); + dvo_image_update (&db, VERBOSE); + dvo_image_unlock (&db); /* unlock? */ + + return TRUE; +} + +int RepairTableCPT_V1(char *cptFilenameSrc, char *cptFilenameTgt, char *cpsFilenameSrc, char *cpsFilenameTgt, Measure *measure, off_t Nmeasure, Image *image, off_t Nimage, myIndexType *imageIDindex, char catformat) { + + off_t *averefMatch; + off_t i, NaveMax, Naverage, NAVERAGE, NaverageOut, Nave, Nout, Nold; + int *found, Nsecfilt; + + Average *average, *averageOut; + + Matrix matrix; + + Header cptHeaderPHU; + Header cptHeaderTBL; + FTable cptFtable; + + cptFtable.header = &cptHeaderTBL; + + NaveMax = 0; + NAVERAGE = 1000; + ALLOCATE (average, Average, NAVERAGE); + memset (average, 0, NAVERAGE*sizeof(Average)); + + ALLOCATE (found, int, NAVERAGE); + memset (found, 0, NAVERAGE*sizeof(int)); + + // examine all measurements and create new objects as needed + // we start with a valid, sorted dvo database, so we use averef to link objects + for (i = 0; i < Nmeasure; i++) { + Nave = measure[i].averef; + + // we only allocate as many as we need + if (Nave >= NAVERAGE) { + Nold = NAVERAGE; + NAVERAGE = MAX(Nave + 1000, NAVERAGE + 1000); + REALLOCATE (average, Average, NAVERAGE); + memset (&average[Nold], 0, (NAVERAGE - Nold)*sizeof(Average)); + + REALLOCATE (found, int, NAVERAGE); + memset (&found[Nold], 0, (NAVERAGE - Nold)*sizeof(int)); + } + + // this measure matches an existing average, just check it is OK and bump Nmeasure + if (found[Nave]) { + average[Nave].Nmeasure ++; + myAssert(average[Nave].objID == measure[i].objID, "objIDs do not match!"); + myAssert(average[Nave].catID == measure[i].catID, "catIDs do not match!"); + continue; + } + + NaveMax = MAX(Nave, NaveMax); + + found[Nave] = TRUE; + + // we are going to leave most of the elements of average unset: they are the result of + // the relastro analysis for this object and can be recreated with a call to relastro + + // need to find image so we can use ccd coordinates to determine RA & DEC + int n = myIndexGetEntry (imageIDindex, measure[i].imageID); + myAssert (n > -1, "impossible!"); + + dvo_average_init (&average[Nave]); + + // I actually have measure->R,D, I could just use those... + XY_to_RD (&average[Nave].R, &average[Nave].D, measure[i].Xccd, measure[i].Yccd, &image[n].coords); + + average[Nave].Nmeasure = 1; + average[Nave].Nmissing = 0; + + // assume the resulting table set is unsorted + average[Nave].measureOffset = -1; + average[Nave].missingOffset = -1; + + average[Nave].objID = measure[i].objID; + average[Nave].catID = measure[i].catID; + average[Nave].extID = CreatePSPSObjectID(average[Nave].R, average[Nave].D); + } + Naverage = NaveMax + 1; + + // we now have an average table, but there will be holes due to deleted measurements + // create a new average table with only existing entries + + ALLOCATE (averageOut, Average, Naverage); + memset (averageOut, 0, Naverage*sizeof(Average)); + + ALLOCATE (averefMatch, off_t, Naverage); + memset (averefMatch, 0, Naverage*sizeof(int)); + + Nave = 0; + for (i = 0; i < Naverage; i++) { + if (!found[i]) continue; + averageOut[Nave] = average[i]; // use a memcpy? + averefMatch[i] = Nave; + Nave ++; + } + NaverageOut = Nave; + + // modify measure.averef to match the new sequence + for (i = 0; i < Nmeasure; i++) { + Nave = measure[i].averef; + Nout = averefMatch[Nave]; + myAssert(Nout < NaverageOut, "output averef is wrong"); + + myAssert(average[Nave].objID == measure[i].objID, "objIDs do not match"); + myAssert(average[Nave].catID == measure[i].catID, "objIDs do not match"); + myAssert(averageOut[Nout].objID == measure[i].objID, "objIDs do not match"); + myAssert(averageOut[Nout].catID == measure[i].catID, "objIDs do not match"); + + measure[i].averef = Nout; + } + + fprintf (stderr, "cpt file : %d obj -> %d obj (%s -> %s)\n", (int) Naverage, (int) NaverageOut, cptFilenameSrc, cptFilenameTgt); + + // open source cpt file + FILE *cptFile = fopen(cptFilenameSrc, "r"); + myAssert(cptFile, "failed to open cpt file"); + + // load the cpt header (use for CATID, RA, DEC range, filenames) + if (!gfits_fread_header (cptFile, &cptHeaderPHU)) { + myAbort("failure to cpt header"); + } + + // update the output header + gfits_modify (&cptHeaderPHU, "NSTARS", OFF_T_FMT, 1, NaverageOut); + gfits_modify (&cptHeaderPHU, "NMEAS", OFF_T_FMT, 1, Nmeasure); + gfits_modify (&cptHeaderPHU, "NMISS", "%d", 1, 0); + gfits_modify_alt (&cptHeaderPHU, "SORTED", "%t", 1, FALSE); + + gfits_scan (&cptHeaderPHU, "NSECFILT", "%d", 1, &Nsecfilt); + + if (0) { + char compressMode[256]; + if (gfits_scan (&cptHeaderTBL, "DVO_CMP", "%s", 1, compressMode)) { + if (strcmp (compressMode, "NONE")) { + myAbort ("fix compression"); + } + } + } + + /* convert internal to external format */ + if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL, TRUE)) { + myAbort("trouble converting format"); + } + fclose(cptFile); + + // rename the old cpt file: + if (rename (cptFilenameSrc, cptFilenameTgt)) { + perror ("tried to rename file"); + exit (2); + } + + // create and write the output file + cptFile = fopen(cptFilenameSrc, "w"); + myAssert(cptFile, "failed to open cpt file"); + + // write PHU header + if (!gfits_fwrite_header (cptFile, &cptHeaderPHU)) { + myAbort("can't write primary header"); + } + + // write the PHU matrix; this is probably a NOP, do I have to keep it in? + gfits_create_matrix (&cptHeaderPHU, &matrix); + if (!gfits_fwrite_matrix (cptFile, &matrix)) { + myAbort("can't write primary matrix"); + } + gfits_free_matrix (&matrix); + + // write the table data + if (!gfits_fwrite_ftable_range (cptFile, &cptFtable, 0, NaverageOut, 0, NaverageOut)) { + myAbort("can't write table data"); + } + fclose(cptFile); + + gfits_free_table (&cptFtable); + gfits_free_header (&cptHeaderPHU); + gfits_free_header (&cptHeaderTBL); + free (average); + free (averageOut); + + free (found); + free (averefMatch); + + { + Header cpsHeaderPHU; + Header cpsHeaderTBL; + FTable cpsFtable; + + SecFilt *secfilt = NULL; + + cpsFtable.header = &cpsHeaderTBL; + + // open source cpt file + FILE *cpsFile = fopen(cpsFilenameSrc, "r"); + myAssert(cpsFile, "failed to open cps file"); + + // load the cps header (use for CATID, RA,DEC range, filenames) + if (!gfits_fread_header (cpsFile, &cpsHeaderPHU)) { + myAbort("failure to cps header"); + } + + int Nrows = Nsecfilt*NaverageOut; + ALLOCATE (secfilt, SecFilt, Nrows); + + for (i = 0; i < Nrows; i++) { + dvo_secfilt_init (&secfilt[i], SECFILT_RESET_ALL); + } + + /* convert internal to external format */ + if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat, TRUE)) { + myAbort("trouble converting format"); + } + fclose(cpsFile); + + // rename the old cpt file: + if (rename (cpsFilenameSrc, cpsFilenameTgt)) { + perror ("tried to rename file"); + exit (2); + } + + // create and write the output file + cpsFile = fopen(cpsFilenameSrc, "w"); + myAssert(cpsFile, "failed to open cps file"); + + // write PHU header + if (!gfits_fwrite_header (cpsFile, &cpsHeaderPHU)) { + myAbort("can't write primary header"); + } + + // write the PHU matrix; this is probably a NOP, do I have to keep it in? + gfits_create_matrix (&cpsHeaderPHU, &matrix); + if (!gfits_fwrite_matrix (cpsFile, &matrix)) { + myAbort("can't write primary matrix"); + } + gfits_free_matrix (&matrix); + + // write the table data + if (!gfits_fwrite_ftable_range (cpsFile, &cpsFtable, 0, Nrows, 0, Nrows)) { + myAbort("can't write table data"); + } + fclose(cpsFile); + + gfits_free_table (&cpsFtable); + gfits_free_header (&cpsHeaderPHU); + gfits_free_header (&cpsHeaderTBL); + free (secfilt); + } + + return (TRUE); +} + +int RepairAverage (Catalog *catalog) { + + off_t i, j, Nave; + + // average is sorted so averef is valid + // secfilt is also sorted so sequence is valid (but this is kind of moot since it will be re-calculated) + // average.measureOffset is NOT valid + // average.lensingOffset is NOT valid + + Average *average = catalog->average; + Measure *measure = catalog->measure; + Lensing *lensing = catalog->lensing; + + // reset the values of average.Nmeasure, average.Nlensing + for (i = 0; i < catalog->Naverage; i++) { + average[i].Nmeasure = 0; + average[i].Nlensing = 0; + } + + // re-calculate average.Nmeasure + for (i = 0; i < catalog->Nmeasure; i++) { + Nave = measure[i].averef; + myAssert (measure[i].objID == average[Nave].objID, "invalid measure:average match"); + average[Nave].Nmeasure ++; + } + + // re-calculate average.Nlensing + for (i = 0; i < catalog->Nlensing; i++) { + Nave = lensing[i].averef; + myAssert (lensing[i].objID == average[Nave].objID, "invalid lensing:average match"); + average[Nave].Nlensing ++; + } + + // create a copy of the average table, keeping only the entries with Nmeasure & Nlensing > 0 + ALLOCATE_PTR (averageNew, Average, catalog->Naverage); + ALLOCATE_PTR (averefNew, int, catalog->Naverage); + + Nave = 0; + for (i = 0; i < catalog->Naverage; i++) { + averefNew[i] = -1; + if (!average[i].Nmeasure && !average[i].Nlensing) continue; + + averageNew[Nave] = average[i]; + averefNew[i] = Nave; + Nave ++; + } + off_t NaverageNew = Nave; + + // update measure.averef values + for (i = 0; i < catalog->Nmeasure; i++) { + Nave = averefNew[measure[i].averef]; + myAssert (Nave >= 0, "oops"); + measure[i].averef = Nave; + } + + // update lensing.averef values + for (i = 0; i < catalog->Nlensing; i++) { + Nave = averefNew[lensing[i].averef]; + myAssert (Nave >= 0, "oops"); + lensing[i].averef = Nave; + } + + // XXX need to update measureOffset and lensingOffset + + // measure[] should be blocked and sequential: + // measure[i+1].averef >= measure[i].averef + + // update average.measureOffset values + Nave = -1; + for (i = 0; i < catalog->Nmeasure; i++) { + if (measure[i].averef == Nave) continue; + Nave = measure[i].averef; + averageNew[Nave].measureOffset = i; + } + + // update average.lensingOffset values + Nave = -1; + for (i = 0; i < catalog->Nlensing; i++) { + if (lensing[i].averef == Nave) continue; + Nave = lensing[i].averef; + averageNew[Nave].lensingOffset = i; + } + + // check the result (measure -> average) + for (i = 0; i < catalog->Nmeasure; i++) { + Nave = measure[i].averef; + myAssert(averageNew[Nave].objID == measure[i].objID, "objIDs do not match"); + myAssert(averageNew[Nave].catID == measure[i].catID, "catIDs do not match"); + } + // check the result (average -> measure) + for (i = 0; i < NaverageNew; i++) { + int m = averageNew[i].measureOffset; + for (j = 0; j < averageNew[i].Nmeasure; j++) { + myAssert(averageNew[i].objID == measure[j+m].objID, "objIDs do not match"); + myAssert(averageNew[i].catID == measure[j+m].catID, "catIDs do not match"); + myAssert(measure[j+m].averef == i, "averef broken"); + } + } + + // check the result (lensing -> average) + for (i = 0; i < catalog->Nlensing; i++) { + Nave = lensing[i].averef; + myAssert(averageNew[Nave].objID == lensing[i].objID, "objIDs do not match"); + myAssert(averageNew[Nave].catID == lensing[i].catID, "catIDs do not match"); + } + // check the result (average -> lensing) + for (i = 0; i < NaverageNew; i++) { + int m = averageNew[i].lensingOffset; + for (j = 0; j < averageNew[i].Nlensing; j++) { + myAssert(averageNew[i].objID == lensing[j+m].objID, "objIDs do not match"); + myAssert(averageNew[i].catID == lensing[j+m].catID, "catIDs do not match"); + myAssert(lensing[j+m].averef == i, "averef broken"); + } + } + + free (catalog->secfilt); + ALLOCATE (catalog->secfilt, SecFilt, NaverageNew*catalog->Nsecfilt); + for (i = 0; i < NaverageNew*catalog->Nsecfilt; i++) { + dvo_secfilt_init (&catalog->secfilt[i], SECFILT_RESET_ALL); + } + + free (averefNew); + free (catalog->average); + catalog->average = averageNew; + catalog->Naverage = NaverageNew; + catalog->Naverage_disk = NaverageNew; + + catalog->Nsecfilt_disk = NaverageNew*catalog->Nsecfilt; + + return (TRUE); +} + +// delete measure entries +int DeleteMeasure (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) { + + int j; + + Measure *measure = catalog->measure; + + // allocate an output array of measures (to replace, if needed) + ALLOCATE_PTR (measureNew, Measure, catalog->Nmeasure); + + int NmeasureNew = 0; + int NmeasureDel = 0; + + // examine all measurements: find ones that need to be deleted + for (j = 0; j < catalog->Nmeasure; j++) { + int imageID = measure[j].imageID; + if (!imageID) continue; + // myAssert(imageID, "measure is missing an image ID"); + // this case is valid if we have REF detections (no associated image) + + int N = myIndexGetEntry(imageIDindex, imageID); + if (N < 0) { + // this detection comes from a non-existant image; delete + NmeasureDel ++; + continue; + } + + // measure matches a bad image; delete + if (deleteImage[N]) { + NmeasureDel ++; + continue; + } + + // keep this measure + measureNew[NmeasureNew] = measure[j]; + NmeasureNew ++; + } + + free (catalog->measure); + catalog->measure = measureNew; + catalog->Nmeasure = NmeasureNew; + catalog->Nmeasure_disk = NmeasureNew; + + *nDelete = NmeasureDel; + + return TRUE; +} + +// delete lensing entries +int DeleteLensing (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) { + + int j; + + Lensing *lensing = catalog->lensing; + + // allocate an output array of measures (to replace, if needed) + ALLOCATE_PTR (lensingNew, Lensing, catalog->Nlensing); + + int NlensingNew = 0; + int NlensingDel = 0; + + // examine all lensing: find ones that need to be deleted + for (j = 0; j < catalog->Nlensing; j++) { + int imageID = lensing[j].imageID; + myAssert(imageID, "lensing is missing an image ID"); + + int N = myIndexGetEntry(imageIDindex, imageID); + if (N < 0) { + // this detection comes from a non-existant image; delete + NlensingDel ++; + continue; + } + + // lensing matches a bad image; delete + if (deleteImage[N]) { + NlensingDel ++; + continue; + } + + // keep this lensing + lensingNew[NlensingNew] = lensing[j]; + NlensingNew ++; + } + + free (catalog->lensing); + catalog->lensing = lensingNew; + catalog->Nlensing = NlensingNew; + catalog->Nlensing_disk = NlensingNew; + + *nDelete = NlensingDel; + + return TRUE; +} + +// return number of SkyRegions which get set +int FindDeleteRegion (SkyRegion *UserPatch, Image *image, off_t Nimage, int *deleteImage) { + + // Rmin,Rmax run from 0 - 360.0; for Rmin < 180.0, Rmin points Qmin,Qmax run from -180.0 - +180.0 + + // find the RA & DEC range of the images we want to delete + double Dmin = +90.0; + double Dmax = -90.0; + double Rmin = +360.0; + double Rmax = 0.0; + double Qmin = +180.0; + double Qmax = -180.0; + + int i; + for (i = 0; i < Nimage; i++) { + if (!deleteImage[i]) continue; double Rthis, Dthis, Qthis; XY_to_RD(&Rthis, &Dthis, 0, 0, &image[i].coords); + Rthis = ohana_normalize_angle_to_midpoint (Rthis, 180.0); // Rthis: 0.0 - 360.0 + Qthis = ohana_normalize_angle_to_midpoint (Rthis, 0.0); // Qthis: -180.0 - 180.0 Rmin = MIN(Rthis, Rmin); Rmax = MAX(Rthis, Rmax); Dmin = MIN(Dthis, Dmin); Dmax = MAX(Dthis, Dmax); - Qthis = (Rthis < 180.0) ? Rthis : Rthis - 360.0; Qmin = MIN(Qthis, Qmin); Qmax = MAX(Qthis, Qmax); - // fprintf (stderr, "image @ %f,%f\n", Rthis, Dthis); - XY_to_RD(&Rthis, &Dthis, image[i].NX, 0, &image[i].coords); + Rthis = ohana_normalize_angle_to_midpoint (Rthis, 180.0); // Rthis: 0.0 - 360.0 + Qthis = ohana_normalize_angle_to_midpoint (Rthis, 0.0); // Qthis: -180.0 - 180.0 Rmin = MIN(Rthis, Rmin); Rmax = MAX(Rthis, Rmax); Dmin = MIN(Dthis, Dmin); Dmax = MAX(Dthis, Dmax); - Qthis = (Rthis < 180.0) ? Rthis : Rthis - 360.0; Qmin = MIN(Qthis, Qmin); Qmax = MAX(Qthis, Qmax); XY_to_RD(&Rthis, &Dthis, 0, image[i].NY, &image[i].coords); + Rthis = ohana_normalize_angle_to_midpoint (Rthis, 180.0); // Rthis: 0.0 - 360.0 + Qthis = ohana_normalize_angle_to_midpoint (Rthis, 0.0); // Qthis: -180.0 - 180.0 Rmin = MIN(Rthis, Rmin); Rmax = MAX(Rthis, Rmax); Dmin = MIN(Dthis, Dmin); Dmax = MAX(Dthis, Dmax); - Qthis = (Rthis < 180.0) ? Rthis : Rthis - 360.0; Qmin = MIN(Qthis, Qmin); Qmax = MAX(Qthis, Qmax); XY_to_RD(&Rthis, &Dthis, image[i].NX, image[i].NY, &image[i].coords); + Rthis = ohana_normalize_angle_to_midpoint (Rthis, 180.0); // Rthis: 0.0 - 360.0 + Qthis = ohana_normalize_angle_to_midpoint (Rthis, 0.0); // Qthis: -180.0 - 180.0 Rmin = MIN(Rthis, Rmin); Rmax = MAX(Rthis, Rmax); Dmin = MIN(Dthis, Dmin); Dmax = MAX(Dthis, Dmax); - Qthis = (Rthis < 180.0) ? Rthis : Rthis - 360.0; Qmin = MIN(Qthis, Qmin); Qmax = MAX(Qthis, Qmax); @@ -178,53 +841,65 @@ double dR = Rmax - Rmin; - // load the sky table for the existing database - SkyTable *insky = SkyTableLoadOptimal (catdir, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE); - myAssert(insky, "can't read SkyTable"); - SkyTableSetFilenames (insky, catdir, "cpt"); - - // XXX apply this...generate the subset matching the user-selected region - SkyRegion UserPatch; - + UserPatch[0].Dmin = Dmin - 0.1; + UserPatch[0].Dmax = Dmax + 0.1; + UserPatch[1].Dmin = Dmin - 0.1; + UserPatch[1].Dmax = Dmax + 0.1; + + int nPass; if (dR < dQ + 0.1) { + UserPatch[0].Rmin = Rmin - 0.1; + UserPatch[0].Rmax = Rmax + 0.1; fprintf (stderr, "R,D range: %f - %f, %f - %f\n", Rmin, Rmax, Dmin, Dmax); nPass = 1; } else { + // Qmax is close to (but above) 0.0 + // Qmin + 360.0 maps to a point close to (but below) 360.0 + UserPatch[0].Rmin = 0.0; + UserPatch[0].Rmax = Qmax + 0.1; + UserPatch[1].Rmin = Qmin + 360.0 - 0.1; + UserPatch[1].Rmax = 360.0; fprintf (stderr, "R,D range: %f - %f, %f - %f\n", Qmin, Qmax, Dmin, Dmax); nPass = 2; } + + return nPass; +} + +int dvorepairDeleteImagesByExternID_catalogs (SkyRegion *UserPatch, int nUserPatch, Image *image, off_t Nimage, int *deleteImage, myIndexType *imageIDindex) { + + if (PARALLEL && !HOST_ID) { + int status = dvorepairDeleteImagesByExternID_parallel (UserPatch, nUserPatch, image, Nimage, deleteImage); + return status; + } + + // load the sky table for the existing database + SkyTable *insky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE); + myAssert(insky, "can't read SkyTable"); + SkyTableSetFilenames (insky, CATDIR, "cpt"); int NmeasureDelTotal = 0; int NlensingDelTotal = 0; - for (raPass = 0; raPass < nPass; raPass++) { - - UserPatch.Dmin = Dmin - 0.1; - UserPatch.Dmax = Dmax + 0.1; - - if (dR < dQ + 0.1) { - UserPatch.Rmin = Rmin - 0.1; - UserPatch.Rmax = Rmax + 0.1; - } else { - if (raPass == 0) { - UserPatch.Rmin = 0.0; - UserPatch.Rmax = Qmax + 0.1; - } else { - UserPatch.Rmin = Qmin + 360.0 - 0.1; - UserPatch.Rmax = 360.0; - } - } - - SkyList *inlist = SkyListByPatch (insky, -1, &UserPatch); + int patch; + for (patch = 0; patch < nUserPatch; patch++) { + SkyList *inlist = SkyListByPatch (insky, -1, &UserPatch[patch]); fprintf (stderr, "%d cpt regions affected\n", (int) inlist->Nregions); - // SkyListPopulatedRange (&Ns, &Ne, inlist, 0); - // depth = inlist[0].regions[Ns][0].depth; - // loop over the populated input regions + int i; for (i = 0; i < inlist[0].Nregions; i++) { if (!inlist[0].regions[i][0].table) continue; - snprintf (filename, DVO_MAX_PATH, "%s/%s.cpt", catdir, inlist[0].regions[i][0].name); + // XXX deal with parallel db filenames + // snprintf (filename, DVO_MAX_PATH, "%s/%s.cpt", CATDIR, inlist[0].regions[i][0].name); + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(inlist[0].regions[i], HOST_ID)) continue; + + // set the parameters which guide catalog open/load/create + char hostfile[DVO_MAX_PATH]; + snprintf (hostfile, DVO_MAX_PATH, "%s/%s.cpt", HOSTDIR, inlist[0].regions[i][0].name); + char *filename = HOST_ID ? hostfile : inlist[0].filename[i]; Catalog catalog; @@ -232,5 +907,5 @@ // set up the basic catalog info dvo_catalog_init (&catalog, TRUE); - catalog.filename = filename; // XXX only allow non-parallel dbs for now + catalog.filename = filename; catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT | DVO_LOAD_LENSING; catalog.Nsecfilt = GetPhotcodeNsecfilt (); @@ -288,633 +963,80 @@ fprintf (stderr, "Deleted %d measure, %d lensing detections\n", NmeasureDelTotal, NlensingDelTotal); - - off_t NnewImage; - Image *newImage = DeleteSelectedImages (image, Nimage, deleteImage, imageIDindex, &NnewImage); - - SaveImageTable (newImage, NnewImage, catdir, &db); - - myIndexFree (externIDindex); - myIndexFree (imageIDindex); - - free (imageFilename); - free (deleteImage); - free (deleteIDs); - - SkyTableFree(insky); - - gfits_db_free (&db); - - ohana_memcheck (TRUE); - - exit (0); -} - -Image *DeleteSelectedImages (Image *image, off_t Nimage, int *deleteImage, myIndexType *imageIDindex, off_t *NnewImage) { - - // we need to remove the (WRP) images, but we also want to remove the DIS [PHU] image entries if they have - // had all of their children deleted - - // first, generate an array of the number of children for each parent - - int *Nchildren = NULL; - ALLOCATE (Nchildren, int, Nimage); - memset (Nchildren, 0, Nimage*sizeof(int)); - - int i; - for (i = 0; i < Nimage; i++) { - if (!image[i].parentID) continue; - if (strcmp(&image[i].coords.ctype[4], "-WRP")) { - fprintf (stderr, "warning: child with non WRP coords : %s\n", image[i].name); - } - - int n = myIndexGetEntry (imageIDindex, image[i].parentID); - myAssert (n != -1, "invalid parentID?"); - myAssert (n < Nimage, "invalid parentID??"); - Nchildren[n] ++; - } - - // now subtract the children which need to be deleted - for (i = 0; i < Nimage; i++) { - if (!deleteImage[i]) continue; - if (!image[i].parentID) continue; // should I warn on this? - - int n = myIndexGetEntry (imageIDindex, image[i].parentID); - myAssert (n != -1, "invalid parentID?"); - myAssert (n < Nimage, "invalid parentID??"); - Nchildren[n] --; - myAssert (Nchildren[n] >= 0, "over deleted???"); - } - - // now check for any parent and delete it - for (i = 0; i < Nimage; i++) { - if (strcmp(&image[i].coords.ctype[4], "-DIS")) continue; - myAssert (!image[i].parentID, "parent ID for DIS image??"); - if (!Nchildren[i]) deleteImage[i] = TRUE; - } - - fprintf (stderr, "deleting the following images:\n"); - - off_t Nnew = 0; - Image *newImage = NULL; - ALLOCATE (newImage, Image, Nimage); - for (i = 0; i < Nimage; i++) { - if (deleteImage[i]) { - fprintf (stderr, "DELETE: %s : %d : %d : %d\n", image[i].name, image[i].imageID, image[i].parentID, image[i].externID); - continue; - } - newImage[Nnew] = image[i]; - Nnew ++; - } - - REALLOCATE (newImage, Image, Nnew); - - *NnewImage = Nnew; - return newImage; -} - -int SaveImageTable (Image *image, off_t Nimage, char *catdir, FITS_DB *oldDB) { - - FITS_DB db; - - char ImageCatSrc[DVO_MAX_PATH]; - char ImageCatTgt[DVO_MAX_PATH]; - - // we are creating a new image table (what about db ID?) - snprintf (ImageCatSrc, DVO_MAX_PATH, "%s/Images.dat", catdir); - snprintf (ImageCatTgt, DVO_MAX_PATH, "%s/Images.dat.broken", catdir); - - // rename the old cpt file: - if (rename (ImageCatSrc, ImageCatTgt)) { - perror ("tried to rename file"); - exit (2); - } - - /* setup image table format and lock */ - gfits_db_init (&db); - db.mode = dvo_catalog_catmode ("SPLIT"); - db.format = dvo_catalog_catformat ("PS1_V5"); - int status = dvo_image_lock (&db, ImageCatSrc, 3600.0, LCK_XCLD); // shorter timeout? - if (!status) { - fprintf (stderr, "ERROR: failure to lock image catalog %s", db.filename); - exit (2); - } - - /* load or create the image table */ - if (db.dbstate != LCK_EMPTY) { - fprintf (stderr, "image table %s exists, exiting", db.filename); - exit (2); - } - dvo_image_create (&db, GetZeroPoint()); - - // I want to keep the old DVO_DBID - char dbID[33]; - int imageIDmax; - if (!gfits_scan (&oldDB->header, "DVO_DBID", "%s", 1, dbID)) { - fprintf (stderr, "source image table is missing DVO_DBID, exiting\n"); - exit (3); - } - if (!gfits_scan (&oldDB->header, "IMAGEID", "%d", 1, &imageIDmax)) { - fprintf (stderr, "max image ID is missing, exiting\n"); - exit (3); - } - gfits_modify (&db.header, "IMAGEID", "%d", 1, imageIDmax); - gfits_modify (&db.header, "DVO_DBID", "%s", 1, dbID); - - // also save the merge history: - int Nmerge; - if (gfits_scan (&oldDB->header, "NMERGE", "%d", 1, &Nmerge)) { - int i; - gfits_modify (&db.header, "NMERGE", "%d", 1, Nmerge); - for (i = 0; i < Nmerge; i++) { - char field[80], value[80]; - snprintf (field, 80, "DM_%05d", i); - if (!gfits_scan (&oldDB->header, field, "%s", 1, value)) { - fprintf (stderr, "missing merge info %s\n", field); - continue; - } - gfits_modify (&db.header, field, "%s", 1, value); - } - } - - /* add the new image and save */ - dvo_image_addrows (&db, image, Nimage); - dvo_image_update (&db, VERBOSE); - dvo_image_unlock (&db); /* unlock? */ - return TRUE; } -int RepairTableCPT_V1(char *cptFilenameSrc, char *cptFilenameTgt, char *cpsFilenameSrc, char *cpsFilenameTgt, Measure *measure, off_t Nmeasure, Image *image, off_t Nimage, myIndexType *imageIDindex, char catformat) { - - off_t *averefMatch; - off_t i, NaveMax, Naverage, NAVERAGE, NaverageOut, Nave, Nout, Nold; - int *found, Nsecfilt; - - Average *average, *averageOut; - - Matrix matrix; - - Header cptHeaderPHU; - Header cptHeaderTBL; - FTable cptFtable; - - cptFtable.header = &cptHeaderTBL; - - NaveMax = 0; - NAVERAGE = 1000; - ALLOCATE (average, Average, NAVERAGE); - memset (average, 0, NAVERAGE*sizeof(Average)); - - ALLOCATE (found, int, NAVERAGE); - memset (found, 0, NAVERAGE*sizeof(int)); - - // examine all measurements and create new objects as needed - // we start with a valid, sorted dvo database, so we use averef to link objects - for (i = 0; i < Nmeasure; i++) { - Nave = measure[i].averef; - - // we only allocate as many as we need - if (Nave >= NAVERAGE) { - Nold = NAVERAGE; - NAVERAGE = MAX(Nave + 1000, NAVERAGE + 1000); - REALLOCATE (average, Average, NAVERAGE); - memset (&average[Nold], 0, (NAVERAGE - Nold)*sizeof(Average)); - - REALLOCATE (found, int, NAVERAGE); - memset (&found[Nold], 0, (NAVERAGE - Nold)*sizeof(int)); - } - - // this measure matches an existing average, just check it is OK and bump Nmeasure - if (found[Nave]) { - average[Nave].Nmeasure ++; - myAssert(average[Nave].objID == measure[i].objID, "objIDs do not match!"); - myAssert(average[Nave].catID == measure[i].catID, "catIDs do not match!"); - continue; - } - - NaveMax = MAX(Nave, NaveMax); - - found[Nave] = TRUE; - - // we are going to leave most of the elements of average unset: they are the result of - // the relastro analysis for this object and can be recreated with a call to relastro - - // need to find image so we can use ccd coordinates to determine RA & DEC - int n = myIndexGetEntry (imageIDindex, measure[i].imageID); - myAssert (n > -1, "impossible!"); - - dvo_average_init (&average[Nave]); - - // I actually have measure->R,D, I could just use those... - XY_to_RD (&average[Nave].R, &average[Nave].D, measure[i].Xccd, measure[i].Yccd, &image[n].coords); - - average[Nave].Nmeasure = 1; - average[Nave].Nmissing = 0; - - // assume the resulting table set is unsorted - average[Nave].measureOffset = -1; - average[Nave].missingOffset = -1; - - average[Nave].objID = measure[i].objID; - average[Nave].catID = measure[i].catID; - average[Nave].extID = CreatePSPSObjectID(average[Nave].R, average[Nave].D); - } - Naverage = NaveMax + 1; - - // we now have an average table, but there will be holes due to deleted measurements - // create a new average table with only existing entries - - ALLOCATE (averageOut, Average, Naverage); - memset (averageOut, 0, Naverage*sizeof(Average)); - - ALLOCATE (averefMatch, off_t, Naverage); - memset (averefMatch, 0, Naverage*sizeof(int)); - - Nave = 0; - for (i = 0; i < Naverage; i++) { - if (!found[i]) continue; - averageOut[Nave] = average[i]; // use a memcpy? - averefMatch[i] = Nave; - Nave ++; - } - NaverageOut = Nave; - - // modify measure.averef to match the new sequence - for (i = 0; i < Nmeasure; i++) { - Nave = measure[i].averef; - Nout = averefMatch[Nave]; - myAssert(Nout < NaverageOut, "output averef is wrong"); - - myAssert(average[Nave].objID == measure[i].objID, "objIDs do not match"); - myAssert(average[Nave].catID == measure[i].catID, "objIDs do not match"); - myAssert(averageOut[Nout].objID == measure[i].objID, "objIDs do not match"); - myAssert(averageOut[Nout].catID == measure[i].catID, "objIDs do not match"); - - measure[i].averef = Nout; - } - - fprintf (stderr, "cpt file : %d obj -> %d obj (%s -> %s)\n", (int) Naverage, (int) NaverageOut, cptFilenameSrc, cptFilenameTgt); - - // open source cpt file - FILE *cptFile = fopen(cptFilenameSrc, "r"); - myAssert(cptFile, "failed to open cpt file"); - - // load the cpt header (use for CATID, RA, DEC range, filenames) - if (!gfits_fread_header (cptFile, &cptHeaderPHU)) { - myAbort("failure to cpt header"); - } - - // update the output header - gfits_modify (&cptHeaderPHU, "NSTARS", OFF_T_FMT, 1, NaverageOut); - gfits_modify (&cptHeaderPHU, "NMEAS", OFF_T_FMT, 1, Nmeasure); - gfits_modify (&cptHeaderPHU, "NMISS", "%d", 1, 0); - gfits_modify_alt (&cptHeaderPHU, "SORTED", "%t", 1, FALSE); - - gfits_scan (&cptHeaderPHU, "NSECFILT", "%d", 1, &Nsecfilt); - - if (0) { - char compressMode[256]; - if (gfits_scan (&cptHeaderTBL, "DVO_CMP", "%s", 1, compressMode)) { - if (strcmp (compressMode, "NONE")) { - myAbort ("fix compression"); +int dvorepairDeleteImagesByExternID_parallel (SkyRegion *UserPatch, int nUserPatch, Image *image, off_t Nimage, int *deleteImage) { + + // ensure that the paths are absolute path names + char *abscatdir = abspath (CATDIR, DVO_MAX_PATH); + + // load the sky table for the existing database + SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE); + myAssert(sky, "can't read SkyTable"); + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + exit (1); + } + + char filename[DVO_MAX_PATH]; + snprintf (filename, DVO_MAX_PATH, "%s/DeleteImages.fits", CATDIR); + if (!DeleteImagesSave (filename, image, Nimage, deleteImage)) { + fprintf (stderr, "ERROR: failure to save delete image info\n"); + exit (2); + } + + int i; + for (i = 0; i < table->Nhosts; i++) { + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + // options / arguments that can affect relastro_client -update-objects: + char *command = NULL; + strextend (&command, "dvorepair_client -delete-images-by-extern-id %s -hostID %d -hostdir %s", abscatdir, table->hosts[i].hostID, table->hosts[i].pathname); + + strextend (&command, "-region %f %f %f %f", UserPatch[0].Rmin, UserPatch[0].Rmax, UserPatch[0].Dmin, UserPatch[0].Dmax); + if (nUserPatch == 2) { + strextend (&command, "-region %f %f %f %f", UserPatch[1].Rmin, UserPatch[1].Rmax, UserPatch[1].Dmin, UserPatch[1].Dmax); + } + + if (VERBOSE) { strextend (&command, "-v"); } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) continue; + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running dvorepair_client\n"); + exit (2); } - } - } - - /* convert internal to external format */ - if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL, TRUE)) { - myAbort("trouble converting format"); - } - fclose(cptFile); - - // rename the old cpt file: - if (rename (cptFilenameSrc, cptFilenameTgt)) { - perror ("tried to rename file"); - exit (2); - } - - // create and write the output file - cptFile = fopen(cptFilenameSrc, "w"); - myAssert(cptFile, "failed to open cpt file"); - - // write PHU header - if (!gfits_fwrite_header (cptFile, &cptHeaderPHU)) { - myAbort("can't write primary header"); - } - - // write the PHU matrix; this is probably a NOP, do I have to keep it in? - gfits_create_matrix (&cptHeaderPHU, &matrix); - if (!gfits_fwrite_matrix (cptFile, &matrix)) { - myAbort("can't write primary matrix"); - } - gfits_free_matrix (&matrix); - - // write the table data - if (!gfits_fwrite_ftable_range (cptFile, &cptFtable, 0, NaverageOut, 0, NaverageOut)) { - myAbort("can't write table data"); - } - fclose(cptFile); - - gfits_free_table (&cptFtable); - gfits_free_header (&cptHeaderPHU); - gfits_free_header (&cptHeaderTBL); - free (average); - free (averageOut); - - free (found); - free (averefMatch); - - { - Header cpsHeaderPHU; - Header cpsHeaderTBL; - FTable cpsFtable; - - SecFilt *secfilt = NULL; - - cpsFtable.header = &cpsHeaderTBL; - - // open source cpt file - FILE *cpsFile = fopen(cpsFilenameSrc, "r"); - myAssert(cpsFile, "failed to open cps file"); - - // load the cps header (use for CATID, RA,DEC range, filenames) - if (!gfits_fread_header (cpsFile, &cpsHeaderPHU)) { - myAbort("failure to cps header"); - } - - int Nrows = Nsecfilt*NaverageOut; - ALLOCATE (secfilt, SecFilt, Nrows); - - for (i = 0; i < Nrows; i++) { - dvo_secfilt_init (&secfilt[i], SECFILT_RESET_ALL); - } - - /* convert internal to external format */ - if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat, TRUE)) { - myAbort("trouble converting format"); - } - fclose(cpsFile); - - // rename the old cpt file: - if (rename (cpsFilenameSrc, cpsFilenameTgt)) { - perror ("tried to rename file"); - exit (2); - } - - // create and write the output file - cpsFile = fopen(cpsFilenameSrc, "w"); - myAssert(cpsFile, "failed to open cps file"); - - // write PHU header - if (!gfits_fwrite_header (cpsFile, &cpsHeaderPHU)) { - myAbort("can't write primary header"); - } - - // write the PHU matrix; this is probably a NOP, do I have to keep it in? - gfits_create_matrix (&cpsHeaderPHU, &matrix); - if (!gfits_fwrite_matrix (cpsFile, &matrix)) { - myAbort("can't write primary matrix"); - } - gfits_free_matrix (&matrix); - - // write the table data - if (!gfits_fwrite_ftable_range (cpsFile, &cpsFtable, 0, Nrows, 0, Nrows)) { - myAbort("can't write table data"); - } - fclose(cpsFile); - - gfits_free_table (&cpsFtable); - gfits_free_header (&cpsHeaderPHU); - gfits_free_header (&cpsHeaderTBL); - free (secfilt); - } - - return (TRUE); -} - -int RepairAverage (Catalog *catalog) { - - off_t i, j, Nave; - - // average is sorted so averef is valid - // secfilt is also sorted so sequence is valid (but this is kind of moot since it will be re-calculated) - // average.measureOffset is NOT valid - // average.lensingOffset is NOT valid - - Average *average = catalog->average; - Measure *measure = catalog->measure; - Lensing *lensing = catalog->lensing; - - // reset the values of average.Nmeasure, average.Nlensing - for (i = 0; i < catalog->Naverage; i++) { - average[i].Nmeasure = 0; - average[i].Nlensing = 0; - } - - // re-calculate average.Nmeasure - for (i = 0; i < catalog->Nmeasure; i++) { - Nave = measure[i].averef; - myAssert (measure[i].objID == average[Nave].objID, "invalid measure:average match"); - average[Nave].Nmeasure ++; - } - - // re-calculate average.Nlensing - for (i = 0; i < catalog->Nlensing; i++) { - Nave = lensing[i].averef; - myAssert (lensing[i].objID == average[Nave].objID, "invalid lensing:average match"); - average[Nave].Nlensing ++; - } - - // create a copy of the average table, keeping only the entries with Nmeasure & Nlensing > 0 - ALLOCATE_PTR (averageNew, Average, catalog->Naverage); - ALLOCATE_PTR (averefNew, int, catalog->Naverage); - - Nave = 0; - for (i = 0; i < catalog->Naverage; i++) { - averefNew[i] = -1; - if (!average[i].Nmeasure && !average[i].Nlensing) continue; - - averageNew[Nave] = average[i]; - averefNew[i] = Nave; - Nave ++; - } - off_t NaverageNew = Nave; - - // update measure.averef values - for (i = 0; i < catalog->Nmeasure; i++) { - Nave = averefNew[measure[i].averef]; - myAssert (Nave >= 0, "oops"); - measure[i].averef = Nave; - } - - // update lensing.averef values - for (i = 0; i < catalog->Nlensing; i++) { - Nave = averefNew[lensing[i].averef]; - myAssert (Nave >= 0, "oops"); - lensing[i].averef = Nave; - } - - // XXX need to update measureOffset and lensingOffset - - // measure[] should be blocked and sequential: - // measure[i+1].averef >= measure[i].averef - - // update average.measureOffset values - Nave = -1; - for (i = 0; i < catalog->Nmeasure; i++) { - if (measure[i].averef == Nave) continue; - Nave = measure[i].averef; - averageNew[Nave].measureOffset = i; - } - - // update average.lensingOffset values - Nave = -1; - for (i = 0; i < catalog->Nlensing; i++) { - if (lensing[i].averef == Nave) continue; - Nave = lensing[i].averef; - averageNew[Nave].lensingOffset = i; - } - - // check the result (measure -> average) - for (i = 0; i < catalog->Nmeasure; i++) { - Nave = measure[i].averef; - myAssert(averageNew[Nave].objID == measure[i].objID, "objIDs do not match"); - myAssert(averageNew[Nave].catID == measure[i].catID, "catIDs do not match"); - } - // check the result (average -> measure) - for (i = 0; i < NaverageNew; i++) { - int m = averageNew[i].measureOffset; - for (j = 0; j < averageNew[i].Nmeasure; j++) { - myAssert(averageNew[i].objID == measure[j+m].objID, "objIDs do not match"); - myAssert(averageNew[i].catID == measure[j+m].catID, "catIDs do not match"); - myAssert(measure[j+m].averef == i, "averef broken"); - } - } - - // check the result (lensing -> average) - for (i = 0; i < catalog->Nlensing; i++) { - Nave = lensing[i].averef; - myAssert(averageNew[Nave].objID == lensing[i].objID, "objIDs do not match"); - myAssert(averageNew[Nave].catID == lensing[i].catID, "catIDs do not match"); - } - // check the result (average -> lensing) - for (i = 0; i < NaverageNew; i++) { - int m = averageNew[i].lensingOffset; - for (j = 0; j < averageNew[i].Nlensing; j++) { - myAssert(averageNew[i].objID == lensing[j+m].objID, "objIDs do not match"); - myAssert(averageNew[i].catID == lensing[j+m].catID, "catIDs do not match"); - myAssert(lensing[j+m].averef == i, "averef broken"); - } - } - - free (catalog->secfilt); - ALLOCATE (catalog->secfilt, SecFilt, NaverageNew*catalog->Nsecfilt); - for (i = 0; i < NaverageNew*catalog->Nsecfilt; i++) { - dvo_secfilt_init (&catalog->secfilt[i], SECFILT_RESET_ALL); - } - - free (averefNew); - free (catalog->average); - catalog->average = averageNew; - catalog->Naverage = NaverageNew; - catalog->Naverage_disk = NaverageNew; - - catalog->Nsecfilt_disk = NaverageNew*catalog->Nsecfilt; - - return (TRUE); -} - -// delete measure entries -int DeleteMeasure (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) { - - int j; - - Measure *measure = catalog->measure; - - // allocate an output array of measures (to replace, if needed) - ALLOCATE_PTR (measureNew, Measure, catalog->Nmeasure); - - int NmeasureNew = 0; - int NmeasureDel = 0; - - // examine all measurements: find ones that need to be deleted - for (j = 0; j < catalog->Nmeasure; j++) { - int imageID = measure[j].imageID; - myAssert(imageID, "measure is missing an image ID"); - // XXX this case is valid if we have REF detections (no associated image) - - int N = myIndexGetEntry(imageIDindex, imageID); - if (N < 0) { - // this detection comes from a non-existant image; delete - NmeasureDel ++; - continue; - } - - // measure matches a bad image; delete - if (deleteImage[N]) { - NmeasureDel ++; - continue; - } - - // keep this measure - measureNew[NmeasureNew] = measure[j]; - NmeasureNew ++; - } - - free (catalog->measure); - catalog->measure = measureNew; - catalog->Nmeasure = NmeasureNew; - catalog->Nmeasure_disk = NmeasureNew; - - *nDelete = NmeasureDel; + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + free (command); + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the dvorepair_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + } return TRUE; -} - -// delete lensing entries -int DeleteLensing (Catalog *catalog, myIndexType *imageIDindex, int *deleteImage, int *nDelete) { - - int j; - - Lensing *lensing = catalog->lensing; - - // allocate an output array of measures (to replace, if needed) - ALLOCATE_PTR (lensingNew, Lensing, catalog->Nlensing); - - int NlensingNew = 0; - int NlensingDel = 0; - - // examine all lensingments: find ones that need to be deleted - for (j = 0; j < catalog->Nlensing; j++) { - int imageID = lensing[j].imageID; - myAssert(imageID, "lensing is missing an image ID"); - // XXX this case is valid if we have REF detections (no associated image) - - int N = myIndexGetEntry(imageIDindex, imageID); - if (N < 0) { - // this detection comes from a non-existant image; delete - NlensingDel ++; - continue; - } - - // lensing matches a bad image; delete - if (deleteImage[N]) { - NlensingDel ++; - continue; - } - - // keep this lensing - lensingNew[NlensingNew] = lensing[j]; - NlensingNew ++; - } - - free (catalog->lensing); - catalog->lensing = lensingNew; - catalog->Nlensing = NlensingNew; - catalog->Nlensing_disk = NlensingNew; - - *nDelete = NlensingDel; - - return TRUE; -} - +} Index: trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID_v2.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID_v2.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID_v2.c (revision 39323) @@ -1,3 +1,3 @@ -# include "dvomerge.h" +# include "dvorepair.h" // delete images based on a text table of the target IDs @@ -24,5 +24,5 @@ FITS_DB db; // database handle pointing to input image table - int i, N; + int i; off_t Nimage; @@ -30,14 +30,4 @@ char *imageFilename = NULL; char filename[DVO_MAX_PATH]; - - N = get_argument (argc, argv, "-delete-images-by-extern-id-v2"); - myAssert(N == 1, "programming error: -delete-images-by-extern-id must be first argument"); - remove_argument (N, &argc, argv); - - VERBOSE = FALSE; - if ((N = get_argument (argc, argv, "-v"))) { - VERBOSE = TRUE; - remove_argument (N, &argc, argv); - } if (argc != 3) { Index: trunk/Ohana/src/dvomerge/src/dvorepairFixCPT.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairFixCPT.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/dvorepairFixCPT.c (revision 39323) @@ -12,5 +12,5 @@ int dvorepairFixCPT (int argc, char **argv) { - int i, j, N, Nfiles, NFILES; + int i, j, Nfiles, NFILES; off_t Nimage; FITS_DB db; // database handle pointing to input image table @@ -19,12 +19,11 @@ char **files; - N = get_argument (argc, argv, "-fix-cpt"); - myAssert(N == 1, "programming error: -fix-cpt must be first from main"); - remove_argument (N, &argc, argv); - if (argc != 3) { fprintf (stderr, "USAGE: dvorepair -fix-cpt (images) (filelist)\n"); exit (2); } + + fprintf (stderr, "is this mode tested?\n"); + exit (2); imageFilename = argv[1]; Index: trunk/Ohana/src/dvomerge/src/dvorepairFixImages.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairFixImages.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/dvorepairFixImages.c (revision 39323) @@ -21,14 +21,8 @@ off_t i, Nimage, Nout, Nindex, *imageIdx, index; - int N; - Image *image, *imageOut; char *imageFilenameOld = NULL; char *imageFilenameNew = NULL; - - N = get_argument (argc, argv, "-fix-images"); - myAssert(N == 1, "programming error: -fix-images must be first from main"); - remove_argument (N, &argc, argv); if (argc != 3) { @@ -38,4 +32,7 @@ exit (2); } + + fprintf (stderr, "is this mode tested?\n"); + exit (2); char *catdir = argv[1]; Index: trunk/Ohana/src/dvomerge/src/dvorepairFixStackIDs.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairFixStackIDs.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/dvorepairFixStackIDs.c (revision 39323) @@ -11,8 +11,4 @@ Image *image; - int N = get_argument (argc, argv, "-fix-stack-ids"); - myAssert(N == 1, "programming error: -fix-stack-ids must be first from main"); - remove_argument (N, &argc, argv); - if (argc != 2) { fprintf (stderr, "USAGE: dvorepair -fix-stack-ids (catdir.list)\n"); @@ -23,4 +19,7 @@ exit (2); } + + fprintf (stderr, "is this mode tested?\n"); + exit (2); char *catdir_list = argv[1]; Index: trunk/Ohana/src/dvomerge/src/dvorepairFixWarpIDs.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairFixWarpIDs.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/dvorepairFixWarpIDs.c (revision 39323) @@ -17,13 +17,8 @@ off_t Nimage; - int N; - Image *image; - N = get_argument (argc, argv, "-fix-warp-ids"); - myAssert(N == 1, "programming error: -fix-images must be first from main"); - remove_argument (N, &argc, argv); - if (argc != 3) { + // NOTE: mode has been stripped fprintf (stderr, "USAGE: dvorepair -fix-warp-ids (catdir.list) (warp.ids)\n"); fprintf (stderr, " catdir.list : list of databases of interest\n"); @@ -31,4 +26,7 @@ exit (2); } + + fprintf (stderr, "is this mode tested?\n"); + exit (2); char *catdir_list = argv[1]; Index: trunk/Ohana/src/dvomerge/src/dvorepairImagesVsMeasures.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepairImagesVsMeasures.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/dvorepairImagesVsMeasures.c (revision 39323) @@ -13,5 +13,5 @@ off_t i, j, Nmeasure, Nvalid, Nimage, Nindex, *imageIdx, index, imageSeq; - int N, Nbytes, NbytesPerRow, Nbad, Ncheck, Ntol; + int Nbytes, NbytesPerRow, Nbad, Ncheck, Ntol; int *detCounts; @@ -34,8 +34,4 @@ DVOCatFormat catformat; - N = get_argument (argc, argv, "-images-vs-measures"); - myAssert(N == 1, "programming error: -images-vs-measures must be first from main"); - remove_argument (N, &argc, argv); - if (argc != 3) { fprintf (stderr, "USAGE: dvorepair -images-vs-measures (catdir) (Ntol)\n"); @@ -44,4 +40,7 @@ exit (2); } + + fprintf (stderr, "is this mode tested?\n"); + exit (2); catdir = argv[1]; Index: trunk/Ohana/src/dvomerge/src/dvorepair_args.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepair_args.c (revision 39323) +++ trunk/Ohana/src/dvomerge/src/dvorepair_args.c (revision 39323) @@ -0,0 +1,179 @@ +# include "dvorepair.h" + +/*** check for command line options ***/ +int dvorepair_args (int *argc, char **argv) { + + int N; + + HOST_ID = 0; + HOSTDIR = NULL; + MODE = DVOREPAIR_MODE_NONE; + + if ((N = get_argument (*argc, argv, "-fix-warp-ids" ))) { MODE = DVOREPAIR_MODE_FixWarpIDs ; remove_argument (N, argc, argv); } + if ((N = get_argument (*argc, argv, "-fix-stack-ids" ))) { MODE = DVOREPAIR_MODE_FixStackIDs ; remove_argument (N, argc, argv); } + if ((N = get_argument (*argc, argv, "-fix-cpt" ))) { MODE = DVOREPAIR_MODE_FixCPT ; remove_argument (N, argc, argv); } + if ((N = get_argument (*argc, argv, "-fix-cpt-by-objID" ))) { MODE = DVOREPAIR_MODE_BY_OBJ_ID ; remove_argument (N, argc, argv); } + if ((N = get_argument (*argc, argv, "-images-vs-measures" ))) { MODE = DVOREPAIR_MODE_ImagesVsMeasures ; remove_argument (N, argc, argv); } + if ((N = get_argument (*argc, argv, "-delete-image-list" ))) { MODE = DVOREPAIR_MODE_DeleteImageList ; remove_argument (N, argc, argv); } + if ((N = get_argument (*argc, argv, "-delete-images-by-extern-id" ))) { MODE = DVOREPAIR_MODE_DeleteImagesByExternID ; remove_argument (N, argc, argv); } + if ((N = get_argument (*argc, argv, "-delete-images-by-extern-id-v2"))) { MODE = DVOREPAIR_MODE_DeleteImagesByExternID_v2 ; remove_argument (N, argc, argv); } + if ((N = get_argument (*argc, argv, "-fix-images" ))) { MODE = DVOREPAIR_MODE_FixImages ; remove_argument (N, argc, argv); } + + if (MODE == DVOREPAIR_MODE_NONE) dvorepair_help(0, NULL); + + VERBOSE = FALSE; + if ((N = get_argument (*argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, argc, argv); + } + + PARALLEL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel"))) { + PARALLEL = TRUE; + remove_argument (N, argc, argv); + } + + // this is a test mode : rather than launching the remote jobs and waiting for completion, + // relphot will simply list the remote command and wait for the user to signal completion + PARALLEL_MANUAL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel-manual"))) { + PARALLEL = TRUE; // -parallel-manual implies -parallel + PARALLEL_MANUAL = TRUE; + remove_argument (N, argc, argv); + } + // this is a test mode : rather than launching the relphot_client jobs remotely, they are + // run in serial via 'system' + PARALLEL_SERIAL = FALSE; + if ((N = get_argument (*argc, argv, "-parallel-serial"))) { + if (PARALLEL_MANUAL) { + fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n"); + exit (1); + } + PARALLEL = TRUE; // -parallel-serial implies -parallel + PARALLEL_SERIAL = TRUE; + remove_argument (N, argc, argv); + } + + dvorepair_help (*argc, argv); + return TRUE; +} + +/*** check for command line options ***/ +int dvorepair_client_args (int *argc, char **argv, SkyRegion *UserPatch) { + + int N; + + MODE = DVOREPAIR_MODE_NONE; + + if ((N = get_argument (*argc, argv, "-delete-images-by-extern-id" ))) { MODE = DVOREPAIR_MODE_DeleteImagesByExternID ; remove_argument (N, argc, argv); } + + if (MODE == DVOREPAIR_MODE_NONE) dvorepair_client_help (0, NULL); + + // by definition, the client is not parallel + PARALLEL = FALSE; + PARALLEL_MANUAL = FALSE; + PARALLEL_SERIAL = FALSE; + + HOST_ID = 0; + if ((N = get_argument (*argc, argv, "-hostID"))) { + remove_argument (N, argc, argv); + HOST_ID = atoi (argv[N]); + remove_argument (N, argc, argv); + } + if (!HOST_ID) dvorepair_client_help(0, NULL); + + HOSTDIR = NULL; + if ((N = get_argument (*argc, argv, "-hostdir"))) { + remove_argument (N, argc, argv); + HOSTDIR = strcreate (argv[N]); + remove_argument (N, argc, argv); + } + if (!HOSTDIR) dvorepair_client_help(0, NULL); + + VERBOSE = FALSE; + if ((N = get_argument (*argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, argc, argv); + } + + /*** provide additional data ***/ + /* restrict to a portion of the sky? */ + int i; + int nUserPatch = 0; + for (i = 0; i < 2; i++) { + UserPatch[i].Rmin = 0; + UserPatch[i].Rmax= 360; + UserPatch[i].Dmin = -90; + UserPatch[i].Dmax = +90; + if ((N = get_argument (*argc, argv, "-region"))) { + remove_argument (N, argc, argv); + UserPatch[i].Rmin = atof (argv[N]); + remove_argument (N, argc, argv); + UserPatch[i].Rmax = atof (argv[N]); + remove_argument (N, argc, argv); + UserPatch[i].Dmin = atof (argv[N]); + remove_argument (N, argc, argv); + UserPatch[i].Dmax = atof (argv[N]); + remove_argument (N, argc, argv); + nUserPatch ++; + } + } + + dvorepair_client_help (*argc, argv); + return nUserPatch; +} + +void dvorepair_help (int argc, char **argv) { + + /* check for help request */ + if (!argv) goto show_help; + if (get_argument (argc, argv, "-help")) goto show_help; + if (get_argument (argc, argv, "-h")) goto show_help; + if (argc < 2) goto show_help; + return; + +show_help: + + fprintf (stderr, "USAGE\n"); + fprintf (stderr, " dvorepair -fix-warp-ids (catdir.list) (idfile) - regenerate images.dat warp_skyfile_ids (EXTERN_ID) values\n"); + fprintf (stderr, " dvorepair -fix-stack-ids (catdir.list) - regenerate images.dat stackID (EXTERN_ID) values\n"); + fprintf (stderr, " dvorepair -fix-cpt (images) (rootlist) - regenerate cpt & cps files from the cpm files\n"); + fprintf (stderr, " dvorepair -images-vs-measures (catdir) (Ntol) - find images with too many missing detections\n"); + fprintf (stderr, " dvorepair -delete-image-list (catdir) (deleteList) - delete a set of images based on image IDs (output from -images-vs-measures)\n"); + fprintf (stderr, " dvorepair -delete-images-by-extern-id (catdir) (deleteList) - delete a set of images based on externIDs\n"); + fprintf (stderr, " ** deletes the detections associated with the image\n"); + fprintf (stderr, " ** deleteList has 9 words per line, 2nd word is the externID\n"); + fprintf (stderr, " dvorepair -delete-images-by-extern-id-v2 (catdir) (deleteList) - delete a set of images based on extern image IDs\n"); + fprintf (stderr, " ** does NOT delete the detections associated with the image\n"); + fprintf (stderr, " ** deleteList has 3 words per line, 2nd word is the externID\n"); + fprintf (stderr, " dvorepair -fix-images (catdir) (deleteList) - delete a set of images based on image IDs (output from -images-vs-measures)\n"); + fprintf (stderr, "\n"); + + fprintf (stderr, " optional flags:\n"); + fprintf (stderr, " -help : this list\n"); + fprintf (stderr, " -h : this list\n\n"); + exit (2); +} + +void dvorepair_client_help (int argc, char **argv) { + + /* check for help request */ + if (!argv) goto show_help; + if (get_argument (argc, argv, "-help")) goto show_help; + if (get_argument (argc, argv, "-h")) goto show_help; + if (argc < 2) goto show_help; + return; + +show_help: + + fprintf (stderr, "USAGE\n"); + fprintf (stderr, " dvorepair_client -delete-images-by-extern-id (catdir) (deleteList) - delete a set of images based on externIDs\n"); + fprintf (stderr, " ** deletes the detections associated with the image\n"); + fprintf (stderr, " ** deleteList has 9 words per line, 2nd word is the externID\n"); + fprintf (stderr, "\n"); + + fprintf (stderr, " optional flags:\n"); + fprintf (stderr, " -help : this list\n"); + fprintf (stderr, " -h : this list\n\n"); + exit (2); +} Index: trunk/Ohana/src/dvomerge/src/dvorepair_by_objID.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepair_by_objID.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/dvorepair_by_objID.c (revision 39323) @@ -2,11 +2,4 @@ int dvorepair_by_objID (int argc, char **argv) { - - int N = get_argument (argc, argv, "-fix-cpt-by-objID"); - if (N != 1) { - fprintf (stderr, "-fix-cpt must be first from main\n"); - exit (2); - } - remove_argument (N, &argc, argv); if (argc != 2) { @@ -14,4 +7,7 @@ exit (2); } + + fprintf (stderr, "is this mode tested?\n"); + exit (2); char *cptFile = argv[1]; Index: trunk/Ohana/src/dvomerge/src/dvorepair_client.c =================================================================== --- trunk/Ohana/src/dvomerge/src/dvorepair_client.c (revision 39323) +++ trunk/Ohana/src/dvomerge/src/dvorepair_client.c (revision 39323) @@ -0,0 +1,39 @@ +# include "dvorepair.h" + +int main (int argc, char **argv) { + + // check various options + SetSignals (); + + SkyRegion UserPatch[2]; // 1 or 2 UserPatch areas are set by FindDeleteRegion + int nUserPatch = dvorepair_client_args (&argc, argv, UserPatch); + + if (argc != 2) dvorepair_client_args (0, NULL, NULL); + CATDIR = argv[1]; + + DeleteImageDataType deleteImageData; + + char filename[DVO_MAX_PATH]; + + sprintf (filename, "%s/Photcodes.dat", CATDIR); + if (!LoadPhotcodes (filename, NULL, FALSE)) { + fprintf (stderr, "error reading photcodes from %s\n", CATDIR); + exit (1); + } + + snprintf (filename, DVO_MAX_PATH, "%s/DeleteImages.fits", CATDIR); + if (!DeleteImagesLoad (filename, &deleteImageData)) { + fprintf (stderr, "ERROR: failure to save delete image info\n"); + exit (2); + } + + int status = dvorepairDeleteImagesByExternID_catalogs (UserPatch, nUserPatch, NULL, deleteImageData.Nimage, deleteImageData.deleteImage, deleteImageData.imageIDindex); + + if (!status) { + fprintf (stderr, "ERROR: failure on hostID %d\n", HOST_ID); + exit (1); + } + fprintf (stderr, "SUCCESS on hostID %d\n", HOST_ID); + exit (0); +} + Index: trunk/Ohana/src/dvomerge/src/help.c =================================================================== --- trunk/Ohana/src/dvomerge/src/help.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/help.c (revision 39323) @@ -159,28 +159,2 @@ } -void dvorepair_help (int argc, char **argv) { - - /* check for help request */ - if (!argv) goto show_help; - if (get_argument (argc, argv, "-help")) goto show_help; - if (get_argument (argc, argv, "-h")) goto show_help; - if (argc < 2) goto show_help; - return; - -show_help: - - fprintf (stderr, "USAGE\n"); - fprintf (stderr, " dvorepair -fix-warp-ids (catdir.list) (idfile) - regenerate images.dat warp_skyfile_ids (EXTERN_ID) values\n"); - fprintf (stderr, " dvorepair -fix-stack-ids (catdir.list) - regenerate images.dat stackID (EXTERN_ID) values\n"); - fprintf (stderr, " dvorepair -fix-cpt (images) (rootlist) - regenerate cpt & cps files from the cpm files\n"); - fprintf (stderr, " dvorepair -images-vs-measures (catdir) (Ntol) - find images with too many missing detections\n"); - fprintf (stderr, " dvorepair -delete-image-list (catdir) (deleteList) - delete a set of images based on image IDs (output from -images-vs-measures)\n"); - fprintf (stderr, " dvorepair -fix-images (catdir) (deleteList) - delete a set of images based on image IDs (output from -images-vs-measures)\n"); - fprintf (stderr, "\n"); - - fprintf (stderr, " optional flags:\n"); - fprintf (stderr, " -help : this list\n"); - fprintf (stderr, " -h : this list\n\n"); - exit (2); -} - Index: trunk/Ohana/src/dvomerge/src/myIndex.c =================================================================== --- trunk/Ohana/src/dvomerge/src/myIndex.c (revision 39322) +++ trunk/Ohana/src/dvomerge/src/myIndex.c (revision 39323) @@ -55,5 +55,9 @@ myAssert (n >= 0, "impossible!"); - myAssert(myIndex->index[n] == -1, "stomping on an earlier index"); + + if (myIndex->index[n] != -1) { + fprintf (stderr, "skipping duplicate ID value: %d\n", value); + return -1; + } myIndex->index[n] = entry;