Index: trunk/Ohana/src/uniphot/Makefile =================================================================== --- trunk/Ohana/src/uniphot/Makefile (revision 35763) +++ trunk/Ohana/src/uniphot/Makefile (revision 35764) @@ -1,3 +1,3 @@ -default: uniphot setphot setphot_client setfwhm +default: uniphot setphot setphot_client setfwhm setposangle setposangle_client setastrom setastrom_client fiximids fiximids_client fixstkids fixstkids_client ckids ckids_client help: @echo "make options: uniphot setphot setfwhm default help install default (uniphot setphot setfwhm)" @@ -25,6 +25,12 @@ setastrom: $(BIN)/setastrom.$(ARCH) setastrom_client: $(BIN)/setastrom_client.$(ARCH) - -install: $(DESTBIN)/uniphot $(DESTBIN)/setfwhm $(DESTBIN)/setphot $(DESTBIN)/setphot_client $(DESTBIN)/setposangle $(DESTBIN)/setposangle_client $(DESTBIN)/setastrom $(DESTBIN)/setastrom_client +fiximids: $(BIN)/fiximids.$(ARCH) +fiximids_client: $(BIN)/fiximids_client.$(ARCH) +fixstkids: $(BIN)/fixstkids.$(ARCH) +fixstkids_client: $(BIN)/fixstkids_client.$(ARCH) +ckids: $(BIN)/ckids.$(ARCH) +ckids_client: $(BIN)/ckids_client.$(ARCH) + +install: $(DESTBIN)/uniphot $(DESTBIN)/setfwhm $(DESTBIN)/setphot $(DESTBIN)/setphot_client $(DESTBIN)/setposangle $(DESTBIN)/setposangle_client $(DESTBIN)/setastrom $(DESTBIN)/setastrom_client $(DESTBIN)/fiximids $(DESTBIN)/fiximids_client $(DESTBIN)/fixstkids $(DESTBIN)/fixstkids_client $(DESTBIN)/ckids $(DESTBIN)/ckids_client UNIPHOT = \ @@ -137,2 +143,73 @@ $(SETFWHM): $(INC)/uniphot.h $(BIN)/setfwhm.$(ARCH): $(SETFWHM) + +FIXIMIDS = \ +$(SRC)/fiximids.$(ARCH).o \ +$(SRC)/initialize_fiximids.$(ARCH).o \ +$(SRC)/load_images_fiximids.$(ARCH).o \ +$(SRC)/ImageSubsetFixImageIDs.$(ARCH).o \ +$(SRC)/update_dvo_fiximids.$(ARCH).o \ +$(SRC)/update_catalog_fiximids.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o + +$(FIXIMIDS): $(INC)/fiximids.h +$(BIN)/fiximids.$(ARCH): $(FIXIMIDS) + +FIXIMIDS_CLIENT = \ +$(SRC)/fiximids_client.$(ARCH).o \ +$(SRC)/initialize_fiximids.$(ARCH).o \ +$(SRC)/load_images_fiximids.$(ARCH).o \ +$(SRC)/ImageSubsetFixImageIDs.$(ARCH).o \ +$(SRC)/update_dvo_fiximids.$(ARCH).o \ +$(SRC)/update_catalog_fiximids.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o + +$(FIXIMIDS_CLIENT): $(INC)/fiximids.h +$(BIN)/fiximids_client.$(ARCH): $(FIXIMIDS_CLIENT) + +FIXSTKIDS = \ +$(SRC)/fixstkids.$(ARCH).o \ +$(SRC)/initialize_fixstkids.$(ARCH).o \ +$(SRC)/load_images_fixstkids.$(ARCH).o \ +$(SRC)/ImageSubsetFixStackIDs.$(ARCH).o \ +$(SRC)/update_dvo_fixstkids.$(ARCH).o \ +$(SRC)/update_catalog_fixstkids.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o + +$(FIXSTKIDS): $(INC)/fixstkids.h +$(BIN)/fixstkids.$(ARCH): $(FIXSTKIDS) + +FIXSTKIDS_CLIENT = \ +$(SRC)/fixstkids_client.$(ARCH).o \ +$(SRC)/initialize_fixstkids.$(ARCH).o \ +$(SRC)/load_images_fixstkids.$(ARCH).o \ +$(SRC)/ImageSubsetFixStackIDs.$(ARCH).o \ +$(SRC)/update_dvo_fixstkids.$(ARCH).o \ +$(SRC)/update_catalog_fixstkids.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o + +$(FIXSTKIDS_CLIENT): $(INC)/fixstkids.h +$(BIN)/fixstkids_client.$(ARCH): $(FIXSTKIDS_CLIENT) + +CKIDS = \ +$(SRC)/ckids.$(ARCH).o \ +$(SRC)/initialize_ckids.$(ARCH).o \ +$(SRC)/update_dvo_ckids.$(ARCH).o \ +$(SRC)/update_catalog_ckids.$(ARCH).o + +$(CKIDS): $(INC)/ckids.h +$(BIN)/ckids.$(ARCH): $(CKIDS) + +CKIDS_CLIENT = \ +$(SRC)/ckids_client.$(ARCH).o \ +$(SRC)/initialize_ckids.$(ARCH).o \ +$(SRC)/update_dvo_ckids.$(ARCH).o \ +$(SRC)/update_catalog_ckids.$(ARCH).o + +$(CKIDS_CLIENT): $(INC)/ckids.h +$(BIN)/ckids_client.$(ARCH): $(CKIDS_CLIENT) + Index: trunk/Ohana/src/uniphot/include/ckids.h =================================================================== --- trunk/Ohana/src/uniphot/include/ckids.h (revision 35764) +++ trunk/Ohana/src/uniphot/include/ckids.h (revision 35764) @@ -0,0 +1,35 @@ +# include +# include +# include + +/* global variables set in parameter file */ +# define DVO_MAX_PATH 1024 +char ImageCat[DVO_MAX_PATH]; +char *CATDIR; +int HOST_ID; +char *HOSTDIR; +char *SINGLE_CPT; +int VERBOSE; +int UPDATE; +int PARALLEL; +int PARALLEL_MANUAL; +int PARALLEL_SERIAL; + +SkyRegion UserPatch; + +/***** prototypes ****/ +int main PROTO((int argc, char **argv)); +void ConfigInit PROTO((int *argc, char **argv)); + +void usage_ckids PROTO(()); +void initialize_ckids PROTO((int argc, char **argv)); +int args_ckids PROTO((int argc, char **argv)); + +void usage_ckids_client PROTO(()); +void initialize_ckids_client PROTO((int argc, char **argv)); +int args_ckids_client PROTO((int argc, char **argv)); + +int update_dvo_ckids PROTO(()); +int update_dvo_ckids_parallel PROTO((SkyList *sky)); + +void update_catalog_ckids PROTO((Catalog *catalog, FILE *foutput)); Index: trunk/Ohana/src/uniphot/include/fiximids.h =================================================================== --- trunk/Ohana/src/uniphot/include/fiximids.h (revision 35764) +++ trunk/Ohana/src/uniphot/include/fiximids.h (revision 35764) @@ -0,0 +1,73 @@ +# include +# include +# include + +// to determine the image ID, we need time (tzero + trate) and photcode +typedef struct { + unsigned int imageID; + unsigned short NX; // image width + unsigned short NY; // image height + unsigned int nstar; // number of stars on the image + e_time tzero; // readout time (row 0) + unsigned char trate; // scan rate (100 usec/pixel) + short photcode; +} ImageSubset; + +/* global variables set in parameter file */ +# define DVO_MAX_PATH 1024 +char ImageCat[DVO_MAX_PATH]; +char *CATDIR; +int HOST_ID; +char *HOSTDIR; +char *IMAGES; +char *SINGLE_CPT; +int VERBOSE; +int VERBOSE_IMSTATS; +int SUMMARY_IMSTATS; +char *IMSTATS_FILE; // location of partial imstats file for remote client +int UPDATE; +int PARALLEL; +int PARALLEL_MANUAL; +int PARALLEL_SERIAL; + +SkyRegion UserPatch; + +/***** prototypes ****/ +int main PROTO((int argc, char **argv)); +void ConfigInit PROTO((int *argc, char **argv)); + +void usage_fiximids PROTO(()); +void initialize_fiximids PROTO((int argc, char **argv)); +int args_fiximids PROTO((int argc, char **argv)); + +void usage_fiximids_client PROTO(()); +void initialize_fiximids_client PROTO((int argc, char **argv)); +int args_fiximids_client PROTO((int argc, char **argv)); + +ImageSubset *ImageSubsetLoad PROTO((char *filename, off_t *nimage)); +int ImageSubsetSave PROTO((char *filename, ImageSubset *image, off_t Nimage)); +ImageSubset *ImagesToSubset PROTO((Image *image, off_t N)); + +void set_db PROTO((FITS_DB *in)); +int Shutdown PROTO((char *format, ...) OHANA_FORMAT(printf, 1, 2) ); +void TrapSignal PROTO((int sig)); +void SetProtect PROTO((int mode)); +int SetSignals PROTO((void)); + +Image *load_images_fiximids PROTO((FITS_DB *db, off_t *Nimage)); +int update_dvo_fiximids PROTO((ImageSubset *image, off_t Nimage)); +int update_dvo_fiximids_parallel PROTO((SkyTable *sky, ImageSubset *image, off_t Nimage)); + +void update_catalog_fiximids PROTO((Catalog *catalog)); + +void sort_image_times PROTO((e_time *S, e_time *E, short *C, unsigned int *I, unsigned int *Q, off_t N)); +int FindImageID PROTO((off_t *ID, off_t *Seq, e_time time, short photcode)); +void initImageIndex PROTO((ImageSubset *image, off_t Nimage_init)); + +void BumpValidImage PROTO((int Seq)); +void BumpInvalidImage PROTO((int Seq)); +void CompareImageCounts PROTO((ImageSubset *image, off_t Nimage_comp)); +void SummaryImageStats PROTO((ImageSubset *image, off_t Nimage_comp)); + +int ImageValidSave PROTO((char *filename)); +int ImageValidLoad PROTO((char *filename)); Index: trunk/Ohana/src/uniphot/include/fixstkids.h =================================================================== --- trunk/Ohana/src/uniphot/include/fixstkids.h (revision 35764) +++ trunk/Ohana/src/uniphot/include/fixstkids.h (revision 35764) @@ -0,0 +1,73 @@ +# include +# include +# include + +// to determine the image ID, we need time (tzero + trate) and photcode +typedef struct { + unsigned int imageID; + unsigned int externID; + unsigned int nstar; // number of stars on the image + short photcode; +} ImageSubset; + +/* global variables set in parameter file */ +# define DVO_MAX_PATH 1024 +char ImageCat[DVO_MAX_PATH]; +char *CATDIR; +int HOST_ID; +char *HOSTDIR; +char *IMAGES; +char *SINGLE_CPT; +int VERBOSE; +int VERBOSE_IMSTATS; +int SUMMARY_IMSTATS; +char *IMSTATS_FILE; // location of partial imstats file for remote client +int UPDATE; +int PARALLEL; +int PARALLEL_MANUAL; +int PARALLEL_SERIAL; + +SkyRegion UserPatch; + +/***** prototypes ****/ +int main PROTO((int argc, char **argv)); +void ConfigInit PROTO((int *argc, char **argv)); + +void usage_fixstkids PROTO(()); +void initialize_fixstkids PROTO((int argc, char **argv)); +int args_fixstkids PROTO((int argc, char **argv)); + +void usage_fixstkids_client PROTO(()); +void initialize_fixstkids_client PROTO((int argc, char **argv)); +int args_fixstkids_client PROTO((int argc, char **argv)); + +ImageSubset *ImageSubsetLoad PROTO((char *filename, off_t *nimage)); +int ImageSubsetSave PROTO((char *filename, ImageSubset *image, off_t Nimage)); +ImageSubset *ImagesToSubset PROTO((Image *image, off_t N)); + +void set_db PROTO((FITS_DB *in)); +int Shutdown PROTO((char *format, ...) OHANA_FORMAT(printf, 1, 2) ); +void TrapSignal PROTO((int sig)); +void SetProtect PROTO((int mode)); +int SetSignals PROTO((void)); + +Image *load_images_fixstkids PROTO((FITS_DB *db, off_t *Nimage)); +int update_dvo_fixstkids PROTO((ImageSubset *image, off_t Nimage)); +int update_dvo_fixstkids_parallel PROTO((SkyList *sky, ImageSubset *image, off_t Nimage)); + +void update_catalog_fixstkids PROTO((Catalog *catalog)); + +void sort_image_externID PROTO((unsigned int *I, unsigned int *E, short *C, unsigned int *Q, off_t N)); +void sort_image_times PROTO((e_time *S, e_time *E, short *C, unsigned int *I, unsigned int *Q, off_t N)); +int FindImageID PROTO((off_t *ID, off_t *Seq, short *photcode, off_t extID)); +void initImageIndex PROTO((ImageSubset *image, off_t Nimage_init)); + +void BumpValidImage PROTO((int Seq)); +void BumpInvalidImage PROTO((int Seq)); +void CompareImageCounts PROTO((ImageSubset *image, off_t Nimage_comp)); +void SummaryImageStats PROTO((ImageSubset *image, off_t Nimage_comp)); + +int ImageValidSave PROTO((char *filename)); +int ImageValidLoad PROTO((char *filename)); + +int UnpackPSPSStackDetectionID PROTO((int *sourceID, int *imageID, int *detID, uint64_t pspsStackID)); Index: trunk/Ohana/src/uniphot/src/ImageSubsetFixImageIDs.c =================================================================== --- trunk/Ohana/src/uniphot/src/ImageSubsetFixImageIDs.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/ImageSubsetFixImageIDs.c (revision 35764) @@ -0,0 +1,203 @@ +# include "fiximids.h" +# define FT_BZERO_INT16 1.0*0x8000 +# define FT_BZERO_INT32 1.0*0x80000000 + +# define GET_COLUMN(OUT,NAME,TYPE) \ + TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \ + myAssert (!strcmp(type, #TYPE), "wrong column type"); + +ImageSubset *ImageSubsetLoad(char *filename, off_t *nimage) { + + int Ncol; + off_t i; + off_t Nrow; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + *nimage = 0; + ImageSubset *image = NULL; + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename); + return NULL; + } + + /* load in PHU segment (ignore) */ + if (!gfits_fread_header (f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset header\n"); + fclose (f); + return NULL; + } + if (!gfits_fread_matrix (f, &matrix, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n"); + gfits_free_header (&header); + fclose (f); + return NULL; + } + + ftable.header = &theader; + + // load data for this header + if (!gfits_load_header (f, &theader)) { + fclose (f); + return NULL; + } + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + fclose (f); + return NULL; + } + fclose (f); + + char type[16]; + + GET_COLUMN (imageID, "IMAGE_ID", int); + GET_COLUMN (NX, "NX", short); + GET_COLUMN (NY, "NY", short); + GET_COLUMN (nstar, "nstar", int); + GET_COLUMN (tzero, "TZERO", int); + GET_COLUMN (trate, "TRATE", short); + GET_COLUMN (photcode, "PHOTCODE" , short); + + ALLOCATE (image, ImageSubset, Nrow); + for (i = 0; i < Nrow; i++) { + image[i].imageID = imageID[i]; + image[i].NX = NX[i]; + image[i].NY = NY[i]; + image[i].nstar = nstar[i]; + image[i].tzero = tzero[i]; + image[i].trate = trate[i]; + image[i].photcode = photcode[i]; + } + fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow); + + free (imageID); + free (NX); + free (NY); + free (nstar); + free (tzero); + free (trate); + free (photcode); + + *nimage = Nrow; + return image; +} + +int ImageSubsetSave(char *filename, ImageSubset *image, off_t Nimage) { + + off_t i; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + gfits_create_table_header (&theader, "BINTABLE", "IMAGE_SUBSET"); + + // an unsigned int needs to have bzero of 0x8000 + gfits_define_bintable_column (&theader, "J", "IMAGE_ID", "image ID", "tmp", 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "I", "NX", "tmp", "tmp", 1.0, 0.0); + gfits_define_bintable_column (&theader, "I", "NY", "tmp", "tmp", 1.0, 0.0); + gfits_define_bintable_column (&theader, "J", "NSTAR", "tmp", "tmp", 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "TZERO", "tmp", "tmp", 1.0, 0.0); + gfits_define_bintable_column (&theader, "I", "TRATE", "tmp", "tmp", 1.0, 0.0); + gfits_define_bintable_column (&theader, "I", "PHOTCODE", "tmp", "tmp", 1.0, 0.0); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + unsigned int *imageID, *nstar; + short *NX, *NY; + e_time *tzero; + short *trate; + short *photcode; + + // create intermediate storage arrays + ALLOCATE (imageID, unsigned int, Nimage); + ALLOCATE (NX, short, Nimage); + ALLOCATE (NY, short, Nimage); + ALLOCATE (nstar, unsigned int, Nimage); + ALLOCATE (tzero, e_time, Nimage); + ALLOCATE (trate, short, Nimage); + ALLOCATE (photcode, short, Nimage); + + // assign the storage arrays + for (i = 0; i < Nimage; i++) { + imageID[i] = image[i].imageID ; + NX[i] = image[i].NX ; + NY[i] = image[i].NY ; + nstar[i] = image[i].nstar ; + tzero[i] = image[i].tzero ; + trate[i] = image[i].trate ; + photcode[i] = image[i].photcode; + } + + // add the columns to the output array + gfits_set_bintable_column (&theader, &ftable, "IMAGE_ID", imageID, Nimage); + gfits_set_bintable_column (&theader, &ftable, "NX", NX, Nimage); + gfits_set_bintable_column (&theader, &ftable, "NY", NY, Nimage); + gfits_set_bintable_column (&theader, &ftable, "NSTAR", nstar, Nimage); + gfits_set_bintable_column (&theader, &ftable, "TZERO", tzero, Nimage); + gfits_set_bintable_column (&theader, &ftable, "TRATE", trate, Nimage); + gfits_set_bintable_column (&theader, &ftable, "PHOTCODE", photcode, Nimage); + + free (imageID ); + free (NX ); + free (NY ); + free (nstar ); + free (tzero ); + free (trate ); + free (photcode); + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file for output %s\n", filename); + return FALSE; + } + + gfits_fwrite_header (f, &header); + gfits_fwrite_matrix (f, &matrix); + gfits_fwrite_Theader (f, &theader); + gfits_fwrite_table (f, &ftable); + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + fclose (f); + fflush (f); + + return TRUE; +} + +ImageSubset *ImagesToSubset (Image *image, off_t N) { + + off_t i; + + // we have been given an ImageSubset array, containing a reduced set of image fields + // create full a Image array and save the needed values + ImageSubset *subset = NULL; + ALLOCATE (subset, ImageSubset, N); + + for (i = 0; i < N; i++) { + subset[i].imageID = image[i].imageID ; + subset[i].NX = image[i].NX ; + subset[i].NY = image[i].NY ; + subset[i].nstar = image[i].nstar ; + subset[i].tzero = image[i].tzero ; + subset[i].trate = image[i].trate ; + subset[i].photcode = image[i].photcode ; + } + return subset; +} + + Index: trunk/Ohana/src/uniphot/src/ImageSubsetFixStackIDs.c =================================================================== --- trunk/Ohana/src/uniphot/src/ImageSubsetFixStackIDs.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/ImageSubsetFixStackIDs.c (revision 35764) @@ -0,0 +1,173 @@ +# include "fixstkids.h" +# define FT_BZERO_INT16 1.0*0x8000 +# define FT_BZERO_INT32 1.0*0x80000000 + +# define GET_COLUMN(OUT,NAME,TYPE) \ + TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \ + myAssert (!strcmp(type, #TYPE), "wrong column type"); + +ImageSubset *ImageSubsetLoad(char *filename, off_t *nimage) { + + int Ncol; + off_t i; + off_t Nrow; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + *nimage = 0; + ImageSubset *image = NULL; + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename); + return NULL; + } + + /* load in PHU segment (ignore) */ + if (!gfits_fread_header (f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset header\n"); + fclose (f); + return NULL; + } + if (!gfits_fread_matrix (f, &matrix, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n"); + gfits_free_header (&header); + fclose (f); + return NULL; + } + + ftable.header = &theader; + + // load data for this header + if (!gfits_load_header (f, &theader)) { + fclose (f); + return NULL; + } + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + fclose (f); + return NULL; + } + fclose (f); + + char type[16]; + + GET_COLUMN (imageID, "IMAGE_ID", int); + GET_COLUMN (externID, "EXTERN_ID", int); + GET_COLUMN (nstar, "NSTAR", int); + GET_COLUMN (photcode, "PHOTCODE", short); + + ALLOCATE (image, ImageSubset, Nrow); + for (i = 0; i < Nrow; i++) { + image[i].imageID = imageID[i]; + image[i].externID = externID[i]; + image[i].nstar = nstar[i]; + image[i].photcode = photcode[i]; + } + fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow); + + free (imageID); + free (externID); + free (nstar); + free (photcode); + + *nimage = Nrow; + return image; +} + +int ImageSubsetSave(char *filename, ImageSubset *image, off_t Nimage) { + + off_t i; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + gfits_create_table_header (&theader, "BINTABLE", "IMAGE_SUBSET"); + + // an unsigned int needs to have bzero of 0x8000 + gfits_define_bintable_column (&theader, "J", "IMAGE_ID", "image ID", "tmp", 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "EXTERN_ID", "extern ID", "tmp", 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "NSTAR", "tmp", "tmp", 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "I", "PHOTCODE", "tmp", "tmp", 1.0, 0.0); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + unsigned int *imageID, *externID, *nstar; + short *photcode; + + // create intermediate storage arrays + ALLOCATE (imageID, unsigned int, Nimage); + ALLOCATE (externID, unsigned int, Nimage); + ALLOCATE (nstar, unsigned int, Nimage); + ALLOCATE (photcode, short, Nimage); + + // assign the storage arrays + for (i = 0; i < Nimage; i++) { + imageID[i] = image[i].imageID ; + externID[i] = image[i].externID; + nstar[i] = image[i].nstar ; + photcode[i] = image[i].photcode; + } + + // add the columns to the output array + gfits_set_bintable_column (&theader, &ftable, "IMAGE_ID", imageID, Nimage); + gfits_set_bintable_column (&theader, &ftable, "EXTERN_ID", externID, Nimage); + gfits_set_bintable_column (&theader, &ftable, "NSTAR", nstar, Nimage); + gfits_set_bintable_column (&theader, &ftable, "PHOTCODE", photcode, Nimage); + + free (imageID ); + free (externID); + free (nstar ); + free (photcode); + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file for output %s\n", filename); + return FALSE; + } + + gfits_fwrite_header (f, &header); + gfits_fwrite_matrix (f, &matrix); + gfits_fwrite_Theader (f, &theader); + gfits_fwrite_table (f, &ftable); + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + fclose (f); + fflush (f); + + return TRUE; +} + +ImageSubset *ImagesToSubset (Image *image, off_t N) { + + off_t i; + + // we have been given an ImageSubset array, containing a reduced set of image fields + // create full a Image array and save the needed values + ImageSubset *subset = NULL; + ALLOCATE (subset, ImageSubset, N); + + for (i = 0; i < N; i++) { + subset[i].imageID = image[i].imageID ; + subset[i].externID = image[i].externID ; + subset[i].nstar = image[i].nstar ; + subset[i].photcode = image[i].photcode ; + } + return subset; +} + + Index: trunk/Ohana/src/uniphot/src/ckids.c =================================================================== --- trunk/Ohana/src/uniphot/src/ckids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/ckids.c (revision 35764) @@ -0,0 +1,20 @@ +# include "ckids.h" + +int main (int argc, char **argv) { + + int status; + + /* get configuration info, args, lockfile */ + initialize_ckids (argc, argv); + + status = update_dvo_ckids (); + + if (!status) exit (1); + exit (0); +} + + +/* ckids : check for duplicate IDs for measurements in the db + ** load catalogs + ** count duplicate detections + */ Index: trunk/Ohana/src/uniphot/src/ckids_client.c =================================================================== --- trunk/Ohana/src/uniphot/src/ckids_client.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/ckids_client.c (revision 35764) @@ -0,0 +1,19 @@ +# include "ckids.h" + +// ckids_client is run on a remote host and is responsible for updating the catalogs +// owned by that host. + +// ckids_client -hostname (hostname) -catdir (catdir) -hostdir (hostdir) +// load the SkyTable and HostTable from (catdir) [contains the host responsibilities] +// loop over tables from SkyTable with my host ID +// the calling program tells the client which host they are (or host ID?); we do not actually have to run this on the real host + +int main (int argc, char **argv) { + + // get configuration info, args, lockfile (set CATDIR, HOST_ID, HOSTDIR, IMAGES) + initialize_ckids_client (argc, argv); + + update_dvo_ckids (); + + exit (0); +} Index: trunk/Ohana/src/uniphot/src/fiximids.c =================================================================== --- trunk/Ohana/src/uniphot/src/fiximids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/fiximids.c (revision 35764) @@ -0,0 +1,44 @@ +# include "fiximids.h" + +int main (int argc, char **argv) { + + off_t Nimage; + int status; + FITS_DB db; + + /* get configuration info, args, lockfile */ + initialize_fiximids (argc, argv); + + set_db (&db); + status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT)); + if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + if (db.dbstate == LCK_EMPTY) Shutdown ("ERROR: No images in catalog %s (1)", db.filename); + + // load images (images are not modified, but we need the astrometry transformations + Image *image = load_images_fiximids (&db, &Nimage); + dvo_image_unlock (&db); + + // the imageSubset here is a reduced set of fields, not a reduced set of images + ImageSubset *subset = ImagesToSubset (image, Nimage); + + status = update_dvo_fiximids (subset, Nimage); + + if (VERBOSE_IMSTATS) { + CompareImageCounts (subset, Nimage); + } + + if (SUMMARY_IMSTATS) { + SummaryImageStats (subset, Nimage); + } + + if (!status) exit (1); + exit (0); +} + + +/* fiximids : set the posangle & pltscale for measurements in the db + ** load images (save subset with imageID, time, photcode, seq) + ** load catalogs + ** update detection + ** count detections / image ID? + */ Index: trunk/Ohana/src/uniphot/src/fiximids_client.c =================================================================== --- trunk/Ohana/src/uniphot/src/fiximids_client.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/fiximids_client.c (revision 35764) @@ -0,0 +1,24 @@ +# include "fiximids.h" + +// fiximids_client is run on a remote host and is responsible for updating the catalogs +// owned by that host. + +// fiximids_client -hostname (hostname) -catdir (catdir) -hostdir (hostdir) -images (images) +// load the SkyTable and HostTable from (catdir) [contains the host responsibilities] +// load the image data from (images) [contains the zp infomation to apply] +// loop over tables from SkyTable with my host ID +// the calling program tells the client which host they are (or host ID?); we do not actually have to run this on the real host + +int main (int argc, char **argv) { + + // get configuration info, args, lockfile (set CATDIR, HOST_ID, HOSTDIR, IMAGES) + initialize_fiximids_client (argc, argv); + + // load the image subset table from the specified location + off_t Nimage; + ImageSubset *subset = ImageSubsetLoad (IMAGES, &Nimage); + + update_dvo_fiximids (subset, Nimage); + + exit (0); +} Index: trunk/Ohana/src/uniphot/src/fixstkids.c =================================================================== --- trunk/Ohana/src/uniphot/src/fixstkids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/fixstkids.c (revision 35764) @@ -0,0 +1,44 @@ +# include "fixstkids.h" + +int main (int argc, char **argv) { + + off_t Nimage; + int status; + FITS_DB db; + + /* get configuration info, args, lockfile */ + initialize_fixstkids (argc, argv); + + set_db (&db); + status = dvo_image_lock (&db, ImageCat, 60.0, (UPDATE ? LCK_XCLD : LCK_SOFT)); + if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + if (db.dbstate == LCK_EMPTY) Shutdown ("ERROR: No images in catalog %s (1)", db.filename); + + // load images (images are not modified, but we need the astrometry transformations + Image *image = load_images_fixstkids (&db, &Nimage); + dvo_image_unlock (&db); + + // the imageSubset here is a reduced set of fields, not a reduced set of images + ImageSubset *subset = ImagesToSubset (image, Nimage); + + status = update_dvo_fixstkids (subset, Nimage); + + if (VERBOSE_IMSTATS) { + CompareImageCounts (subset, Nimage); + } + + if (SUMMARY_IMSTATS) { + SummaryImageStats (subset, Nimage); + } + + if (!status) exit (1); + exit (0); +} + + +/* fixstkids : set the posangle & pltscale for measurements in the db + ** load images (save subset with imageID, time, photcode, seq) + ** load catalogs + ** update detection + ** count detections / image ID? + */ Index: trunk/Ohana/src/uniphot/src/fixstkids_client.c =================================================================== --- trunk/Ohana/src/uniphot/src/fixstkids_client.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/fixstkids_client.c (revision 35764) @@ -0,0 +1,24 @@ +# include "fixstkids.h" + +// fixstkids_client is run on a remote host and is responsible for updating the catalogs +// owned by that host. + +// fixstkids_client -hostname (hostname) -catdir (catdir) -hostdir (hostdir) -images (images) +// load the SkyTable and HostTable from (catdir) [contains the host responsibilities] +// load the image data from (images) [contains the zp infomation to apply] +// loop over tables from SkyTable with my host ID +// the calling program tells the client which host they are (or host ID?); we do not actually have to run this on the real host + +int main (int argc, char **argv) { + + // get configuration info, args, lockfile (set CATDIR, HOST_ID, HOSTDIR, IMAGES) + initialize_fixstkids_client (argc, argv); + + // load the image subset table from the specified location + off_t Nimage; + ImageSubset *subset = ImageSubsetLoad (IMAGES, &Nimage); + + update_dvo_fixstkids (subset, Nimage); + + exit (0); +} Index: trunk/Ohana/src/uniphot/src/initialize_ckids.c =================================================================== --- trunk/Ohana/src/uniphot/src/initialize_ckids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/initialize_ckids.c (revision 35764) @@ -0,0 +1,266 @@ +# include "ckids.h" + +void usage_ckids () { + fprintf (stderr, "USAGE: ckids [options]\n"); + fprintf (stderr, " options:\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -cpt n0000/0000.00.cpt : only repair this cpt file \n"); + fprintf (stderr, " -region Rmin Rmax Dmin Dmax : only repair this region\n"); + fprintf (stderr, " -update : actually write results to detections tables\n"); + fprintf (stderr, " -parallel : run in parallel mode\n\n"); + + fprintf (stderr, " -h : this help list\n"); + fprintf (stderr, " -help : this help list\n"); + fprintf (stderr, " --h : this help list\n"); + fprintf (stderr, " --help : this help list\n"); + + fprintf (stderr, " Note that the dvo db can be specified by -D CATDIR (directory)\n"); + exit (2); +} + +void initialize_ckids (int argc, char **argv) { + + int N; + struct stat statbuffer; + char CatdirPhotcodeFile[256]; + + if ((N = get_argument (argc, argv, "-h"))) usage_ckids(); + if ((N = get_argument (argc, argv, "-help"))) usage_ckids(); + if ((N = get_argument (argc, argv, "--h"))) usage_ckids(); + if ((N = get_argument (argc, argv, "--help"))) usage_ckids(); + + /* are these set correctly? */ + ConfigInit (&argc, argv); + args_ckids (argc, argv); + + if (stat (CATDIR, &statbuffer)) { + fprintf (stderr, "error accessing dvo database directory '%s'\n", CATDIR); + exit (1); + } + + // load the photcode table + sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); + if (!LoadPhotcodes (CatdirPhotcodeFile, NULL, FALSE)) { + fprintf (stderr, "error loading photcode table %s\n", CatdirPhotcodeFile); + exit (1); + } +} + +int args_ckids (int argc, char **argv) { + + int N; + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + SINGLE_CPT = NULL; + if ((N = get_argument (argc, argv, "-cpt"))) { + remove_argument (N, &argc, argv); + SINGLE_CPT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // region of interest + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + PARALLEL = FALSE; + if ((N = get_argument (argc, argv, "-parallel"))) { + PARALLEL = TRUE; + remove_argument (N, &argc, argv); + } + // this is a test mode : rather than launching the remote jobs and waiting for completion, + // ckids will simply list the remote command and wait for the user to signal completion + PARALLEL_MANUAL = FALSE; + if ((N = get_argument (argc, argv, "-parallel-manual"))) { + PARALLEL = TRUE; // -parallel-manual implies -parallel + PARALLEL_MANUAL = TRUE; + remove_argument (N, &argc, argv); + } + // this is a test mode : rather than launching the ckids_client jobs remotely, they are + // run in serial via 'system' + PARALLEL_SERIAL = FALSE; + if ((N = get_argument (argc, argv, "-parallel-serial"))) { + if (PARALLEL_MANUAL) { + fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n"); + exit (1); + } + PARALLEL = TRUE; // -parallel-serial implies -parallel + PARALLEL_SERIAL = TRUE; + remove_argument (N, &argc, argv); + } + + UPDATE = FALSE; + // XXX do not allow update until we have added a modification concept + // if ((N = get_argument (argc, argv, "-update"))) { + // remove_argument (N, &argc, argv); + // UPDATE = TRUE; + // } + + if (argc != 1) usage_ckids(); + + return (TRUE); +} + +void usage_ckids_client () { + fprintf (stderr, "USAGE: ckids_client -hostID (hostID) -catdir (catdir) -hostdir (hostdir) [options]\n"); + fprintf (stderr, " options:\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -update : actually write results to detections tables\n"); + exit (2); +} + +void initialize_ckids_client (int argc, char **argv) { + + int N; + struct stat statbuffer; + char CatdirPhotcodeFile[256]; + + if ((N = get_argument (argc, argv, "-h"))) usage_ckids_client(); + if ((N = get_argument (argc, argv, "-help"))) usage_ckids_client(); + if ((N = get_argument (argc, argv, "--h"))) usage_ckids_client(); + if ((N = get_argument (argc, argv, "--help"))) usage_ckids_client(); + + SetZeroPoint (25.0); // XXX is this needed? + args_ckids_client (argc, argv); + + if (stat (CATDIR, &statbuffer)) { + fprintf (stderr, "error accessing dvo database directory '%s'\n", CATDIR); + exit (1); + } + + // load the photcode table : XXX needed? + sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); + if (!LoadPhotcodes (CatdirPhotcodeFile, NULL, FALSE)) { + fprintf (stderr, "error loading photcode table %s\n", CatdirPhotcodeFile); + exit (1); + } +} + +int args_ckids_client (int argc, char **argv) { + + int N; + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + UPDATE = FALSE; + // if ((N = get_argument (argc, argv, "-update"))) { + // remove_argument (N, &argc, argv); + // UPDATE = TRUE; + // } + + SINGLE_CPT = NULL; + if ((N = get_argument (argc, argv, "-cpt"))) { + remove_argument (N, &argc, argv); + SINGLE_CPT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // region of interest + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + HOST_ID = 0; + if ((N = get_argument (argc, argv, "-hostID"))) { + remove_argument (N, &argc, argv); + HOST_ID = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if (!HOST_ID) usage_ckids_client(); + + HOSTDIR = NULL; + if ((N = get_argument (argc, argv, "-hostdir"))) { + remove_argument (N, &argc, argv); + HOSTDIR = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!HOSTDIR) usage_ckids_client(); + + if ((N = get_argument (argc, argv, "-catdir"))) { + remove_argument (N, &argc, argv); + CATDIR = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!CATDIR) usage_ckids_client(); + + if (argc != 1) usage_ckids_client(); + return (TRUE); +} + +void GetConfig (char *config, char *field, char *format, int N, void *ptr); + +void ConfigInit (int *argc, char **argv) { + + double ZERO_POINT; + char *config, *file; + + /*** load configuration info ***/ + file = SelectConfigFile (argc, argv, "ptolemy"); + config = LoadConfigFile (file); + if (config == (char *) NULL) { + fprintf (stderr, "ERROR: can't find configuration file %s\n", file); + if (file != (char *) NULL) free (file); + exit (0); + } + if (VERBOSE) fprintf (stderr, "loaded config file: %s\n", file); + + // force CATDIR to be absolute (so parallel mode will work) + char *tmpcatdir = NULL; + ALLOCATE (tmpcatdir, char, DVO_MAX_PATH); + GetConfig (config, "CATDIR", "%s", 0, tmpcatdir); + CATDIR = abspath (tmpcatdir, DVO_MAX_PATH); + free (tmpcatdir); + + sprintf (ImageCat, "%s/Images.dat", CATDIR); + + ScanConfig (config, "ZERO_PT", "%lf", 0, &ZERO_POINT); + SetZeroPoint (ZERO_POINT); + + free (config); + free (file); +} + +void GetConfig (char *config, char *field, char *format, int N, void *ptr) { + + char *status; + + status = ScanConfig (config, field, format, N, ptr); + if (status == NULL) { + fprintf (stderr, "error in config, cannot find %s\n", field); + exit (1); + } + return; +} Index: trunk/Ohana/src/uniphot/src/initialize_fiximids.c =================================================================== --- trunk/Ohana/src/uniphot/src/initialize_fiximids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/initialize_fiximids.c (revision 35764) @@ -0,0 +1,298 @@ +# include "fiximids.h" + +void usage_fiximids () { + fprintf (stderr, "USAGE: fiximids [options]\n"); + fprintf (stderr, " options:\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -cpt n0000/0000.00.cpt : only repair this cpt file \n"); + fprintf (stderr, " -region Rmin Rmax Dmin Dmax : only repair this region\n"); + fprintf (stderr, " -update : actually write results to detections tables\n"); + fprintf (stderr, " -parallel : run in parallel mode\n\n"); + + fprintf (stderr, " -h : this help list\n"); + fprintf (stderr, " -help : this help list\n"); + fprintf (stderr, " --h : this help list\n"); + fprintf (stderr, " --help : this help list\n"); + + fprintf (stderr, " Note that the dvo db can be specified by -D CATDIR (directory)\n"); + exit (2); +} + +void initialize_fiximids (int argc, char **argv) { + + int N; + struct stat statbuffer; + char CatdirPhotcodeFile[256]; + + if ((N = get_argument (argc, argv, "-h"))) usage_fiximids(); + if ((N = get_argument (argc, argv, "-help"))) usage_fiximids(); + if ((N = get_argument (argc, argv, "--h"))) usage_fiximids(); + if ((N = get_argument (argc, argv, "--help"))) usage_fiximids(); + + /* are these set correctly? */ + ConfigInit (&argc, argv); + args_fiximids (argc, argv); + + if (stat (CATDIR, &statbuffer)) { + fprintf (stderr, "error accessing dvo database directory '%s'\n", CATDIR); + exit (1); + } + + // load the photcode table + sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); + if (!LoadPhotcodes (CatdirPhotcodeFile, NULL, FALSE)) { + fprintf (stderr, "error loading photcode table %s\n", CatdirPhotcodeFile); + exit (1); + } +} + +int args_fiximids (int argc, char **argv) { + + int N; + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + SUMMARY_IMSTATS = FALSE; + if ((N = get_argument (argc, argv, "-imstats"))) { + SUMMARY_IMSTATS = TRUE; + remove_argument (N, &argc, argv); + } + VERBOSE_IMSTATS = FALSE; + if ((N = get_argument (argc, argv, "-vimstats"))) { + SUMMARY_IMSTATS = TRUE; + VERBOSE_IMSTATS = TRUE; + remove_argument (N, &argc, argv); + } + + // XXX I want to avoid running this on all data all the time + // some options: gpc1 only (hardwired), require photcodes (fairly tedious), + // exclude some photcodes? + // XXX for now, hardwire gpc1 photcodes + + SINGLE_CPT = NULL; + if ((N = get_argument (argc, argv, "-cpt"))) { + remove_argument (N, &argc, argv); + SINGLE_CPT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // region of interest + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + PARALLEL = FALSE; + if ((N = get_argument (argc, argv, "-parallel"))) { + PARALLEL = TRUE; + remove_argument (N, &argc, argv); + } + // this is a test mode : rather than launching the remote jobs and waiting for completion, + // fiximids will simply list the remote command and wait for the user to signal completion + PARALLEL_MANUAL = FALSE; + if ((N = get_argument (argc, argv, "-parallel-manual"))) { + PARALLEL = TRUE; // -parallel-manual implies -parallel + PARALLEL_MANUAL = TRUE; + remove_argument (N, &argc, argv); + } + // this is a test mode : rather than launching the fiximids_client jobs remotely, they are + // run in serial via 'system' + PARALLEL_SERIAL = FALSE; + if ((N = get_argument (argc, argv, "-parallel-serial"))) { + if (PARALLEL_MANUAL) { + fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n"); + exit (1); + } + PARALLEL = TRUE; // -parallel-serial implies -parallel + PARALLEL_SERIAL = TRUE; + remove_argument (N, &argc, argv); + } + + UPDATE = FALSE; + if ((N = get_argument (argc, argv, "-update"))) { + remove_argument (N, &argc, argv); + UPDATE = TRUE; + } + + if (argc != 1) usage_fiximids(); + + return (TRUE); +} + +void usage_fiximids_client () { + fprintf (stderr, "USAGE: fiximids_client -hostID (hostID) -catdir (catdir) -hostdir (hostdir) -images (images) [options]\n"); + fprintf (stderr, " options:\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -update : actually write results to detections tables\n"); + exit (2); +} + +void initialize_fiximids_client (int argc, char **argv) { + + int N; + struct stat statbuffer; + char CatdirPhotcodeFile[256]; + + if ((N = get_argument (argc, argv, "-h"))) usage_fiximids_client(); + if ((N = get_argument (argc, argv, "-help"))) usage_fiximids_client(); + if ((N = get_argument (argc, argv, "--h"))) usage_fiximids_client(); + if ((N = get_argument (argc, argv, "--help"))) usage_fiximids_client(); + + SetZeroPoint (25.0); // XXX is this needed? + args_fiximids_client (argc, argv); + + if (stat (CATDIR, &statbuffer)) { + fprintf (stderr, "error accessing dvo database directory '%s'\n", CATDIR); + exit (1); + } + + // load the photcode table : XXX needed? + sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); + if (!LoadPhotcodes (CatdirPhotcodeFile, NULL, FALSE)) { + fprintf (stderr, "error loading photcode table %s\n", CatdirPhotcodeFile); + exit (1); + } +} + +int args_fiximids_client (int argc, char **argv) { + + int N; + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + VERBOSE_IMSTATS = FALSE; + IMSTATS_FILE = NULL; + if ((N = get_argument (argc, argv, "-imstats"))) { + remove_argument (N, &argc, argv); + IMSTATS_FILE = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + UPDATE = FALSE; + if ((N = get_argument (argc, argv, "-update"))) { + remove_argument (N, &argc, argv); + UPDATE = TRUE; + } + + SINGLE_CPT = NULL; + if ((N = get_argument (argc, argv, "-cpt"))) { + remove_argument (N, &argc, argv); + SINGLE_CPT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // region of interest + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + HOST_ID = 0; + if ((N = get_argument (argc, argv, "-hostID"))) { + remove_argument (N, &argc, argv); + HOST_ID = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if (!HOST_ID) usage_fiximids_client(); + + HOSTDIR = NULL; + if ((N = get_argument (argc, argv, "-hostdir"))) { + remove_argument (N, &argc, argv); + HOSTDIR = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!HOSTDIR) usage_fiximids_client(); + + if ((N = get_argument (argc, argv, "-catdir"))) { + remove_argument (N, &argc, argv); + CATDIR = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!CATDIR) usage_fiximids_client(); + + IMAGES = NULL; + if ((N = get_argument (argc, argv, "-images"))) { + remove_argument (N, &argc, argv); + IMAGES = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!IMAGES) usage_fiximids_client(); + + if (argc != 1) usage_fiximids_client(); + return (TRUE); +} + +void GetConfig (char *config, char *field, char *format, int N, void *ptr); + +void ConfigInit (int *argc, char **argv) { + + double ZERO_POINT; + char *config, *file; + + /*** load configuration info ***/ + file = SelectConfigFile (argc, argv, "ptolemy"); + config = LoadConfigFile (file); + if (config == (char *) NULL) { + fprintf (stderr, "ERROR: can't find configuration file %s\n", file); + if (file != (char *) NULL) free (file); + exit (0); + } + if (VERBOSE) fprintf (stderr, "loaded config file: %s\n", file); + + // force CATDIR to be absolute (so parallel mode will work) + char *tmpcatdir = NULL; + ALLOCATE (tmpcatdir, char, DVO_MAX_PATH); + GetConfig (config, "CATDIR", "%s", 0, tmpcatdir); + CATDIR = abspath (tmpcatdir, DVO_MAX_PATH); + free (tmpcatdir); + + sprintf (ImageCat, "%s/Images.dat", CATDIR); + + ScanConfig (config, "ZERO_PT", "%lf", 0, &ZERO_POINT); + SetZeroPoint (ZERO_POINT); + + free (config); + free (file); +} + +void GetConfig (char *config, char *field, char *format, int N, void *ptr) { + + char *status; + + status = ScanConfig (config, field, format, N, ptr); + if (status == NULL) { + fprintf (stderr, "error in config, cannot find %s\n", field); + exit (1); + } + return; +} Index: trunk/Ohana/src/uniphot/src/initialize_fixstkids.c =================================================================== --- trunk/Ohana/src/uniphot/src/initialize_fixstkids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/initialize_fixstkids.c (revision 35764) @@ -0,0 +1,293 @@ +# include "fixstkids.h" + +void usage_fixstkids () { + fprintf (stderr, "USAGE: fixstkids [options]\n"); + fprintf (stderr, " options:\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -cpt n0000/0000.00.cpt : only repair this cpt file \n"); + fprintf (stderr, " -region Rmin Rmax Dmin Dmax : only repair this region\n"); + fprintf (stderr, " -update : actually write results to detections tables\n"); + fprintf (stderr, " -parallel : run in parallel mode\n\n"); + + fprintf (stderr, " -h : this help list\n"); + fprintf (stderr, " -help : this help list\n"); + fprintf (stderr, " --h : this help list\n"); + fprintf (stderr, " --help : this help list\n"); + + fprintf (stderr, " Note that the dvo db can be specified by -D CATDIR (directory)\n"); + exit (2); +} + +void initialize_fixstkids (int argc, char **argv) { + + int N; + struct stat statbuffer; + char CatdirPhotcodeFile[256]; + + if ((N = get_argument (argc, argv, "-h"))) usage_fixstkids(); + if ((N = get_argument (argc, argv, "-help"))) usage_fixstkids(); + if ((N = get_argument (argc, argv, "--h"))) usage_fixstkids(); + if ((N = get_argument (argc, argv, "--help"))) usage_fixstkids(); + + /* are these set correctly? */ + ConfigInit (&argc, argv); + args_fixstkids (argc, argv); + + if (stat (CATDIR, &statbuffer)) { + fprintf (stderr, "error accessing dvo database directory '%s'\n", CATDIR); + exit (1); + } + + // load the photcode table + sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); + if (!LoadPhotcodes (CatdirPhotcodeFile, NULL, FALSE)) { + fprintf (stderr, "error loading photcode table %s\n", CatdirPhotcodeFile); + exit (1); + } +} + +int args_fixstkids (int argc, char **argv) { + + int N; + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + SUMMARY_IMSTATS = FALSE; + if ((N = get_argument (argc, argv, "-imstats"))) { + SUMMARY_IMSTATS = TRUE; + remove_argument (N, &argc, argv); + } + VERBOSE_IMSTATS = FALSE; + if ((N = get_argument (argc, argv, "-vimstats"))) { + SUMMARY_IMSTATS = TRUE; + VERBOSE_IMSTATS = TRUE; + remove_argument (N, &argc, argv); + } + + SINGLE_CPT = NULL; + if ((N = get_argument (argc, argv, "-cpt"))) { + remove_argument (N, &argc, argv); + SINGLE_CPT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // region of interest + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + PARALLEL = FALSE; + if ((N = get_argument (argc, argv, "-parallel"))) { + PARALLEL = TRUE; + remove_argument (N, &argc, argv); + } + // this is a test mode : rather than launching the remote jobs and waiting for completion, + // fixstkids will simply list the remote command and wait for the user to signal completion + PARALLEL_MANUAL = FALSE; + if ((N = get_argument (argc, argv, "-parallel-manual"))) { + PARALLEL = TRUE; // -parallel-manual implies -parallel + PARALLEL_MANUAL = TRUE; + remove_argument (N, &argc, argv); + } + // this is a test mode : rather than launching the fixstkids_client jobs remotely, they are + // run in serial via 'system' + PARALLEL_SERIAL = FALSE; + if ((N = get_argument (argc, argv, "-parallel-serial"))) { + if (PARALLEL_MANUAL) { + fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n"); + exit (1); + } + PARALLEL = TRUE; // -parallel-serial implies -parallel + PARALLEL_SERIAL = TRUE; + remove_argument (N, &argc, argv); + } + + UPDATE = FALSE; + if ((N = get_argument (argc, argv, "-update"))) { + remove_argument (N, &argc, argv); + UPDATE = TRUE; + } + + if (argc != 1) usage_fixstkids(); + + return (TRUE); +} + +void usage_fixstkids_client () { + fprintf (stderr, "USAGE: fixstkids_client -hostID (hostID) -catdir (catdir) -hostdir (hostdir) -images (images) [options]\n"); + fprintf (stderr, " options:\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -update : actually write results to detections tables\n"); + exit (2); +} + +void initialize_fixstkids_client (int argc, char **argv) { + + int N; + struct stat statbuffer; + char CatdirPhotcodeFile[256]; + + if ((N = get_argument (argc, argv, "-h"))) usage_fixstkids_client(); + if ((N = get_argument (argc, argv, "-help"))) usage_fixstkids_client(); + if ((N = get_argument (argc, argv, "--h"))) usage_fixstkids_client(); + if ((N = get_argument (argc, argv, "--help"))) usage_fixstkids_client(); + + SetZeroPoint (25.0); // XXX is this needed? + args_fixstkids_client (argc, argv); + + if (stat (CATDIR, &statbuffer)) { + fprintf (stderr, "error accessing dvo database directory '%s'\n", CATDIR); + exit (1); + } + + // load the photcode table : XXX needed? + sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); + if (!LoadPhotcodes (CatdirPhotcodeFile, NULL, FALSE)) { + fprintf (stderr, "error loading photcode table %s\n", CatdirPhotcodeFile); + exit (1); + } +} + +int args_fixstkids_client (int argc, char **argv) { + + int N; + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + VERBOSE_IMSTATS = FALSE; + IMSTATS_FILE = NULL; + if ((N = get_argument (argc, argv, "-imstats"))) { + remove_argument (N, &argc, argv); + IMSTATS_FILE = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + UPDATE = FALSE; + if ((N = get_argument (argc, argv, "-update"))) { + remove_argument (N, &argc, argv); + UPDATE = TRUE; + } + + SINGLE_CPT = NULL; + if ((N = get_argument (argc, argv, "-cpt"))) { + remove_argument (N, &argc, argv); + SINGLE_CPT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // region of interest + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + } + + HOST_ID = 0; + if ((N = get_argument (argc, argv, "-hostID"))) { + remove_argument (N, &argc, argv); + HOST_ID = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if (!HOST_ID) usage_fixstkids_client(); + + HOSTDIR = NULL; + if ((N = get_argument (argc, argv, "-hostdir"))) { + remove_argument (N, &argc, argv); + HOSTDIR = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!HOSTDIR) usage_fixstkids_client(); + + if ((N = get_argument (argc, argv, "-catdir"))) { + remove_argument (N, &argc, argv); + CATDIR = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!CATDIR) usage_fixstkids_client(); + + IMAGES = NULL; + if ((N = get_argument (argc, argv, "-images"))) { + remove_argument (N, &argc, argv); + IMAGES = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!IMAGES) usage_fixstkids_client(); + + if (argc != 1) usage_fixstkids_client(); + return (TRUE); +} + +void GetConfig (char *config, char *field, char *format, int N, void *ptr); + +void ConfigInit (int *argc, char **argv) { + + double ZERO_POINT; + char *config, *file; + + /*** load configuration info ***/ + file = SelectConfigFile (argc, argv, "ptolemy"); + config = LoadConfigFile (file); + if (config == (char *) NULL) { + fprintf (stderr, "ERROR: can't find configuration file %s\n", file); + if (file != (char *) NULL) free (file); + exit (0); + } + if (VERBOSE) fprintf (stderr, "loaded config file: %s\n", file); + + // force CATDIR to be absolute (so parallel mode will work) + char *tmpcatdir = NULL; + ALLOCATE (tmpcatdir, char, DVO_MAX_PATH); + GetConfig (config, "CATDIR", "%s", 0, tmpcatdir); + CATDIR = abspath (tmpcatdir, DVO_MAX_PATH); + free (tmpcatdir); + + sprintf (ImageCat, "%s/Images.dat", CATDIR); + + ScanConfig (config, "ZERO_PT", "%lf", 0, &ZERO_POINT); + SetZeroPoint (ZERO_POINT); + + free (config); + free (file); +} + +void GetConfig (char *config, char *field, char *format, int N, void *ptr) { + + char *status; + + status = ScanConfig (config, field, format, N, ptr); + if (status == NULL) { + fprintf (stderr, "error in config, cannot find %s\n", field); + exit (1); + } + return; +} Index: trunk/Ohana/src/uniphot/src/load_images_fiximids.c =================================================================== --- trunk/Ohana/src/uniphot/src/load_images_fiximids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/load_images_fiximids.c (revision 35764) @@ -0,0 +1,309 @@ +# include "fiximids.h" +# define FT_BZERO_INT32 1.0*0x80000000 + +off_t Nimage = 0; +e_time *startImage = NULL; +e_time *stopImage = NULL; +short *photcodeImage = NULL; +unsigned int *imageID = NULL; +unsigned int *imageSeq = NULL; +int *Nimage_valid = NULL; +int *Nimage_invalid = NULL; + +Image *load_images_fiximids (FITS_DB *db, off_t *Nimage_load) { + + Image *image; + + if (VERBOSE) fprintf (stderr, "finding images\n"); + + /* read entire db table */ + if (!dvo_image_load (db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db[0].filename); + + /* use a vtable to keep the images to be calibrated */ + image = gfits_table_get_Image (&db[0].ftable, Nimage_load, &db[0].swapped); + if (!image) { + fprintf (stderr, "ERROR: failed to read images\n"); + exit (2); + } + + fprintf (stderr, "loaded "OFF_T_FMT" images\n", *Nimage_load); + + return (image); +} + +// find mosaic frames (unique time periods & photcode name matches mosaic) +void initImageIndex (ImageSubset *image, off_t Nimage_init) { + + off_t i; + + Nimage = Nimage_init; + ALLOCATE (startImage, e_time, Nimage); + ALLOCATE (stopImage, e_time, Nimage); + ALLOCATE (photcodeImage, short, Nimage); + ALLOCATE (imageID, unsigned int, Nimage); + ALLOCATE (imageSeq, unsigned int, Nimage); + + ALLOCATE (Nimage_valid, int, Nimage); + ALLOCATE (Nimage_invalid, int, Nimage); + + memset (Nimage_valid, 0, Nimage*sizeof(int)); + memset (Nimage_invalid, 0, Nimage*sizeof(int)); + + // save the image data in the local static arrays + for (i = 0; i < Nimage; i++) { + + /* set image time range (small boundary buffer) */ + e_time start = image[i].tzero - MAX(0.01*image[i].trate*image[i].NY, 1); + e_time stop = image[i].tzero + MAX(1.01*image[i].trate*image[i].NY, 1); + + startImage[i] = start; + stopImage[i] = stop; + photcodeImage[i] = image[i].photcode; + imageID[i] = image[i].imageID; + imageSeq[i] = i; + } + + // sort the index, start, and stop by the start times: + sort_image_times (startImage, stopImage, photcodeImage, imageID, imageSeq, Nimage); + + return; +} + +// I need an index to go from image to associated mosaic (if existant) +// This code is copied from relastro/src/MosaicOps.c + +// find the image by time. this search makes the following assumptions: +// 1) there is only a single EXPOSURE from a single camera with the given exposure time +// 2) there may be multiple IMAGES from that exposure, distinguished by photcode +// 3) the image data has been sorted by start time +// 4) once we have (time > start[i]) we are past the images of interest +int FindImageID (off_t *ID, off_t *Seq, e_time time, short photcode) { + + off_t Nlo, Nhi, N, Ni; + + *ID = 0; + *Seq = -1; + + // Find the last image ending BEFORE 'time' (time > start[i]). + // Use bisection to find the overlapping images. + Nlo = 0; Nhi = Nimage; + while (Nhi - Nlo > 10) { + N = 0.5*(Nlo + Nhi); + if (stopImage[N] < time) { + Nlo = MAX(N, 0); + } else { + Nhi = MIN(N + 1, Nimage); + } + } + + // check for the matched images starting from Nlo + for (N = Nlo; N < Nimage; N++) { + if (time > stopImage[N]) continue; + if (time < startImage[N]) return FALSE; + // a possible image match (start <= time <= stop) + for (Ni = N; (Ni < Nimage) && (time <= stopImage[Ni]); Ni++) { + if (photcodeImage[Ni] == photcode) { + *ID = imageID[Ni]; + *Seq = imageSeq[Ni]; + return TRUE; + } + } + return FALSE; + } + return FALSE; +} + +// increment the count of valid detections for this image +void BumpValidImage (int Seq) { + myAssert (Seq >= 0, "impossible seq"); + myAssert (Seq < Nimage, "impossible seq"); + Nimage_valid[Seq] ++; +} + +// increment the count of invalid detections for this image +void BumpInvalidImage (int Seq) { + myAssert (Seq >= 0, "impossible seq"); + myAssert (Seq < Nimage, "impossible seq"); + Nimage_invalid[Seq] ++; +} + +void CompareImageCounts (ImageSubset *image, off_t Nimage_comp) { + + off_t i; + + for (i = 0; i < Nimage_comp; i++) { + // XXX can I limit the images to those in range of the user region? + + if (Nimage_valid[i] + Nimage_invalid[i] == 0) continue; // XXX for now, skip (optionally?) images with no matched detections + if (image[i].photcode == 0) continue; // skip images with 0 photcode (eg, PHU) + + char *date = ohana_sec_to_date (image[i].tzero); + char *code = GetPhotcodeNamebyCode (image[i].photcode); + fprintf (stderr, "image %s %s : %d %d %d : %f\n", date, code, image[i].nstar, Nimage_valid[i], Nimage_invalid[i], + (Nimage_valid[i] + Nimage_invalid[i]) / (float) image[i].nstar); + } +} + +void SummaryImageStats (ImageSubset *image, off_t Nimage_comp) { + + off_t i; + + // let's report the following: + // Nimages with corrected image IDs + // Nimages with Nsum / Nstar < 0.95 + // Nimages with Nsum / Nstar < 0.99 + + int Nfixed = 0; + int Nbad_1 = 0; + int Nbad_2 = 0; + + for (i = 0; i < Nimage_comp; i++) { + if (Nimage_valid[i] + Nimage_invalid[i] == 0) continue; // XXX for now, skip (optionally?) images with no matched detections + if (image[i].photcode == 0) continue; // skip images with 0 photcode (eg, PHU) + + if (Nimage_invalid[i] > 0) Nfixed ++; + + float found_ratio = (Nimage_valid[i] + Nimage_invalid[i]) / (float) image[i].nstar; + + if (found_ratio < 0.99) Nbad_1 ++; + if (found_ratio < 0.95) Nbad_2 ++; + } + + fprintf (stderr, "Nfixed: %d, Ndrop (>1%%) %d, Ndrop (>5%%) %d\n", Nfixed, Nbad_1, Nbad_2); +} + +// sort two times vectors and an index by first time vector +void sort_image_times (e_time *S, e_time *E, short *C, unsigned int *I, unsigned int *Q, off_t N) { + +# define SWAPFUNC(A,B){ e_time tmp_t; unsigned int tmp_u; short tmp_c; \ + tmp_t = S[A]; S[A] = S[B]; S[B] = tmp_t; \ + tmp_t = E[A]; E[A] = E[B]; E[B] = tmp_t; \ + tmp_c = C[A]; C[A] = C[B]; C[B] = tmp_c; \ + tmp_u = I[A]; I[A] = I[B]; I[B] = tmp_u; \ + tmp_u = Q[A]; Q[A] = Q[B]; Q[B] = tmp_u; \ +} +# define COMPARE(A,B)(S[A] < S[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + +# define GET_COLUMN(OUT,NAME,TYPE) \ + TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \ + myAssert (!strcmp(type, #TYPE), "wrong column type"); + +int ImageValidLoad(char *filename) { + + int Ncol; + off_t i; + off_t Nrow; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename); + return FALSE; + } + + /* load in PHU segment (ignore) */ + if (!gfits_fread_header (f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset header\n"); + fclose (f); + return FALSE; + } + if (!gfits_fread_matrix (f, &matrix, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n"); + gfits_free_header (&header); + fclose (f); + return FALSE; + } + + ftable.header = &theader; + + // load data for this header + if (!gfits_load_header (f, &theader)) { + if (VERBOSE) fprintf (stderr, "can't read table header\n"); + fclose (f); + return FALSE; + } + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + if (VERBOSE) fprintf (stderr, "can't read table data\n"); + fclose (f); + return FALSE; + } + fclose (f); + + char type[16]; + + GET_COLUMN (Nvalid, "NVALID", int); + GET_COLUMN (Ninvalid, "NINVALID", int); + + myAssert (Nrow == Nimage, "wrong size imstats result file\n"); + + // merge the new arrays into the single array? + for (i = 0; i < Nrow; i++) { + Nimage_valid[i] += Nvalid[i]; + Nimage_invalid[i] += Ninvalid[i]; + } + fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow); + + free (Nvalid); + free (Ninvalid); + + return TRUE; +} + +int ImageValidSave(char *filename) { + + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + gfits_create_table_header (&theader, "BINTABLE", "IMAGE_VALID"); + + // create the table layout + gfits_define_bintable_column (&theader, "J", "NVALID", "tmp", "tmp", 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "NINVALID", "tmp", "tmp", 1.0, FT_BZERO_INT32); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + // add the columns to the output array + gfits_set_bintable_column (&theader, &ftable, "NVALID", Nimage_valid, Nimage); + gfits_set_bintable_column (&theader, &ftable, "NINVALID", Nimage_invalid, Nimage); + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file for output %s\n", filename); + return FALSE; + } + + gfits_fwrite_header (f, &header); + gfits_fwrite_matrix (f, &matrix); + gfits_fwrite_Theader (f, &theader); + gfits_fwrite_table (f, &ftable); + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + fclose (f); + fflush (f); + + return TRUE; +} Index: trunk/Ohana/src/uniphot/src/load_images_fixstkids.c =================================================================== --- trunk/Ohana/src/uniphot/src/load_images_fixstkids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/load_images_fixstkids.c (revision 35764) @@ -0,0 +1,297 @@ +# include "fixstkids.h" +# define FT_BZERO_INT32 1.0*0x80000000 + +off_t Nimage = 0; +short *photcodeImage = NULL; +unsigned int *imageID = NULL; +unsigned int *externID = NULL; +unsigned int *imageSeq = NULL; +int *Nimage_valid = NULL; +int *Nimage_invalid = NULL; + +Image *load_images_fixstkids (FITS_DB *db, off_t *Nimage_load) { + + Image *image; + + if (VERBOSE) fprintf (stderr, "finding images\n"); + + /* read entire db table */ + if (!dvo_image_load (db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db[0].filename); + + /* use a vtable to keep the images to be calibrated */ + image = gfits_table_get_Image (&db[0].ftable, Nimage_load, &db[0].swapped); + if (!image) { + fprintf (stderr, "ERROR: failed to read images\n"); + exit (2); + } + + fprintf (stderr, "loaded "OFF_T_FMT" images\n", *Nimage_load); + + return (image); +} + +// find mosaic frames (unique time periods & photcode name matches mosaic) +void initImageIndex (ImageSubset *image, off_t Nimage_init) { + + off_t i; + + Nimage = Nimage_init; + ALLOCATE (photcodeImage, short, Nimage); + ALLOCATE (imageID, unsigned int, Nimage); + ALLOCATE (externID, unsigned int, Nimage); + ALLOCATE (imageSeq, unsigned int, Nimage); + + ALLOCATE (Nimage_valid, int, Nimage); + ALLOCATE (Nimage_invalid, int, Nimage); + + memset (Nimage_valid, 0, Nimage*sizeof(int)); + memset (Nimage_invalid, 0, Nimage*sizeof(int)); + + // save the image data in the local static arrays + for (i = 0; i < Nimage; i++) { + + photcodeImage[i] = image[i].photcode; + imageID[i] = image[i].imageID; + externID[i] = image[i].externID; + imageSeq[i] = i; + } + + // sort the index, start, and stop by the start times: + sort_image_externID (imageID, externID, photcodeImage, imageSeq, Nimage); + + return; +} + +int FindImageID (off_t *ID, off_t *Seq, short *photcode, off_t extID) { + + off_t Nlo, Nhi, N, Ni; + + *ID = 0; + *Seq = -1; + *photcode = 0; + + // Find the last image with externID < extID + // Use bisection to find the overlapping images. + Nlo = 0; Nhi = Nimage; + while (Nhi - Nlo > 10) { + N = 0.5*(Nlo + Nhi); + if (externID[N] < extID) { + Nlo = MAX(N, 0); + } else { + Nhi = MIN(N + 1, Nimage); + } + } + + // check for the matched images starting from Nlo + for (N = Nlo; N < Nimage; N++) { + if (extID > externID[N]) continue; + if (extID < externID[N]) return FALSE; + + // try to find other matches? + *photcode = photcodeImage[N]; + *ID = imageID[N]; + *Seq = imageSeq[N]; + + for (Ni = N + 1; Ni < Nimage; Ni++) { + if (extID < externID[Ni]) break; + if (externID[Ni] == extID) { + fprintf (stderr, "WARNING: found duplicate extern IDs : ext ID %d, image ID 1: %d (%d), image ID 2: %d (%d)\n", externID[N], imageID[N], photcodeImage[N], imageID[Ni], photcodeImage[Ni]); + } + } + return TRUE; + } + return FALSE; +} + +// increment the count of valid detections for this image +void BumpValidImage (int Seq) { + myAssert (Seq >= 0, "impossible seq"); + myAssert (Seq < Nimage, "impossible seq"); + Nimage_valid[Seq] ++; +} + +// increment the count of invalid detections for this image +void BumpInvalidImage (int Seq) { + myAssert (Seq >= 0, "impossible seq"); + myAssert (Seq < Nimage, "impossible seq"); + Nimage_invalid[Seq] ++; +} + +void CompareImageCounts (ImageSubset *image, off_t Nimage_comp) { + + off_t i; + + for (i = 0; i < Nimage_comp; i++) { + // XXX can I limit the images to those in range of the user region? + + if (Nimage_valid[i] + Nimage_invalid[i] == 0) continue; // XXX for now, skip (optionally?) images with no matched detections + if (image[i].photcode == 0) continue; // skip images with 0 photcode (eg, PHU) + + char *code = GetPhotcodeNamebyCode (image[i].photcode); + fprintf (stderr, "image %s extID %d imageID %d : %d %d %d : %f\n", code, image[i].imageID, image[i].externID, image[i].nstar, Nimage_valid[i], Nimage_invalid[i], + (Nimage_valid[i] + Nimage_invalid[i]) / (float) image[i].nstar); + } +} + +void SummaryImageStats (ImageSubset *image, off_t Nimage_comp) { + + off_t i; + + // let's report the following: + // Nimages with corrected image IDs + // Nimages with Nsum / Nstar < 0.95 + // Nimages with Nsum / Nstar < 0.99 + + int Nfixed = 0; + int Nbad_1 = 0; + int Nbad_2 = 0; + + for (i = 0; i < Nimage_comp; i++) { + if (Nimage_valid[i] + Nimage_invalid[i] == 0) continue; // XXX for now, skip (optionally?) images with no matched detections + if (image[i].photcode == 0) continue; // skip images with 0 photcode (eg, PHU) + + if (Nimage_invalid[i] > 0) Nfixed ++; + + float found_ratio = (Nimage_valid[i] + Nimage_invalid[i]) / (float) image[i].nstar; + + if (found_ratio < 0.99) Nbad_1 ++; + if (found_ratio < 0.95) Nbad_2 ++; + } + + fprintf (stderr, "Nfixed: %d, Ndrop (>1%%) %d, Ndrop (>5%%) %d\n", Nfixed, Nbad_1, Nbad_2); +} + +// sort two times vectors and an index by first time vector +void sort_image_externID (unsigned int *I, unsigned int *E, short *C, unsigned int *Q, off_t N) { + +# define SWAPFUNC(A,B){ unsigned int tmp_u; short tmp_c; \ + tmp_u = I[A]; I[A] = I[B]; I[B] = tmp_u; \ + tmp_u = E[A]; E[A] = E[B]; E[B] = tmp_u; \ + tmp_c = C[A]; C[A] = C[B]; C[B] = tmp_c; \ + tmp_u = Q[A]; Q[A] = Q[B]; Q[B] = tmp_u; \ +} +# define COMPARE(A,B)(E[A] < E[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + +# define GET_COLUMN(OUT,NAME,TYPE) \ + TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \ + myAssert (!strcmp(type, #TYPE), "wrong column type"); + +int ImageValidLoad(char *filename) { + + int Ncol; + off_t i; + off_t Nrow; + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + FILE *f = fopen (filename, "r"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename); + return FALSE; + } + + /* load in PHU segment (ignore) */ + if (!gfits_fread_header (f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset header\n"); + fclose (f); + return FALSE; + } + if (!gfits_fread_matrix (f, &matrix, &header)) { + if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n"); + gfits_free_header (&header); + fclose (f); + return FALSE; + } + + ftable.header = &theader; + + // load data for this header + if (!gfits_load_header (f, &theader)) { + if (VERBOSE) fprintf (stderr, "can't read table header\n"); + fclose (f); + return FALSE; + } + + // read the fits table bytes + if (!gfits_fread_ftable_data (f, &ftable, FALSE)) { + if (VERBOSE) fprintf (stderr, "can't read table data\n"); + fclose (f); + return FALSE; + } + fclose (f); + + char type[16]; + + GET_COLUMN (Nvalid, "NVALID", int); + GET_COLUMN (Ninvalid, "NINVALID", int); + + myAssert (Nrow == Nimage, "wrong size imstats result file\n"); + + // merge the new arrays into the single array? + for (i = 0; i < Nrow; i++) { + Nimage_valid[i] += Nvalid[i]; + Nimage_invalid[i] += Ninvalid[i]; + } + fprintf (stderr, "loaded data for %lld images\n", (long long) Nrow); + + free (Nvalid); + free (Ninvalid); + + return TRUE; +} + +int ImageValidSave(char *filename) { + + Header header; + Header theader; + Matrix matrix; + FTable ftable; + + gfits_init_header (&header); + header.extend = TRUE; + gfits_create_header (&header); + gfits_create_matrix (&header, &matrix); + + gfits_create_table_header (&theader, "BINTABLE", "IMAGE_VALID"); + + // create the table layout + gfits_define_bintable_column (&theader, "J", "NVALID", "tmp", "tmp", 1.0, FT_BZERO_INT32); + gfits_define_bintable_column (&theader, "J", "NINVALID", "tmp", "tmp", 1.0, FT_BZERO_INT32); + + // generate the output array that carries the data + gfits_create_table (&theader, &ftable); + + // add the columns to the output array + gfits_set_bintable_column (&theader, &ftable, "NVALID", Nimage_valid, Nimage); + gfits_set_bintable_column (&theader, &ftable, "NINVALID", Nimage_invalid, Nimage); + + FILE *f = fopen (filename, "w"); + if (!f) { + fprintf (stderr, "ERROR: cannot open image subset file for output %s\n", filename); + return FALSE; + } + + gfits_fwrite_header (f, &header); + gfits_fwrite_matrix (f, &matrix); + gfits_fwrite_Theader (f, &theader); + gfits_fwrite_table (f, &ftable); + + gfits_free_header (&header); + gfits_free_matrix (&matrix); + gfits_free_header (&theader); + gfits_free_table (&ftable); + + fclose (f); + fflush (f); + + return TRUE; +} Index: trunk/Ohana/src/uniphot/src/match_zpts_to_images.c =================================================================== --- trunk/Ohana/src/uniphot/src/match_zpts_to_images.c (revision 35763) +++ trunk/Ohana/src/uniphot/src/match_zpts_to_images.c (revision 35764) @@ -62,4 +62,17 @@ Ni = index[i]; + + // UBERCAL zero points are only applied to gpc1 exposures (not stacks) + if (UBERCAL) { + if (image[Ni].photcode < 10000) { + i++; + continue; + } + if (image[Ni].photcode > 10600) { + i++; + continue; + } + } + dT = image[Ni].tzero - zpts[Nz].time; Index: trunk/Ohana/src/uniphot/src/update_catalog_ckids.c =================================================================== --- trunk/Ohana/src/uniphot/src/update_catalog_ckids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/update_catalog_ckids.c (revision 35764) @@ -0,0 +1,84 @@ +# include "ckids.h" +void sort_fullIDs (uint64_t *I, int *S, off_t N); + +void update_catalog_ckids (Catalog *catalog, FILE *foutput) { + + // I am looking for duplicate detections based on the image ID + detection ID values + // I assume that all duplicate detections land in the same catalog, so I don't have to do a global search + + // 1) make an array (int64) of the full ID (imageID & detID) [I could do this in a 32bit value by finding the max values, but why bother?] + // 2) sort the array + // 3) identify unique values (count number of duplciates) + + // what do I want in the end? + // a) unique[i] == 0, 1, count[j] : (i = 0, Nmeasure) [so I can tell which ones to drop / flag] + + off_t i; + uint64_t *fullID; + int *seq, *duplicates; + + if (catalog[0].Nmeasure == 0) return; + + Measure *measure = catalog[0].measure; + Average *average = catalog[0].average; + + ALLOCATE (fullID, uint64_t, catalog[0].Nmeasure); + ALLOCATE (seq, int, catalog[0].Nmeasure); + ALLOCATE (duplicates, int, catalog[0].Nmeasure); + + memset (duplicates, 0, sizeof(int) * catalog[0].Nmeasure); + + // make an array of the full IDs + for (i = 0; i < catalog[0].Nmeasure; i++) { + + // is an OR faster or a PLUS? + fullID[i] = ((uint64_t) (measure[i].imageID) << 32) | (measure[i].detID); + seq[i] = i; + } + + // sort the array and sequence number + sort_fullIDs (fullID, seq, catalog[0].Nmeasure); + + uint64_t current = fullID[0]; // current unique value + int count = 0; // number of duplicates for the current unique value + duplicates[0] = count; // duplicate sequence + + for (i = 1; i < catalog[0].Nmeasure; i++) { + if (fullID[i] == current) { + count ++; + } else { + count = 0; + current = fullID[i]; + } + duplicates[i] = count; + } + + for (i = 0; i < catalog[0].Nmeasure; i++) { + if (!duplicates[i]) continue; + off_t j = seq[i]; + off_t N = measure[j].averef; + fprintf (foutput, "0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[j].imageID, measure[j].detID, average[N].R, average[N].D, measure[j].photcode); + } + free (fullID); + free (seq); + free (duplicates); + + return; +} + +// sort fullId vector and an index vector +void sort_fullIDs (uint64_t *I, int *S, off_t N) { + +# define SWAPFUNC(A,B){ uint64_t tmp_i; int tmp_s; \ + tmp_i = I[A]; I[A] = I[B]; I[B] = tmp_i; \ + tmp_s = S[A]; S[A] = S[B]; S[B] = tmp_s; \ +} +# define COMPARE(A,B)(I[A] < I[B]) + + OHANA_SORT (N, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + +} + Index: trunk/Ohana/src/uniphot/src/update_catalog_fiximids.c =================================================================== --- trunk/Ohana/src/uniphot/src/update_catalog_fiximids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/update_catalog_fiximids.c (revision 35764) @@ -0,0 +1,45 @@ +# include "fiximids.h" + +void update_catalog_fiximids (Catalog *catalog) { + + off_t i; + + off_t found = 0; + off_t Nvalid = 0; + off_t Ninvalid = 0; + + for (i = 0; i < catalog[0].Nmeasure; i++) { + + // only do PS1 / GPC1 photcodes? + // XXX if (catalog[0].measure[i].photcode < 10000) continue; + // XXX if (catalog[0].measure[i].photcode > 10600) continue; + + // we have a measure with a given photcode and time: + short photcode = catalog[0].measure[i].photcode; + e_time time = catalog[0].measure[i].t; + + // skip detections with no valid imageID (eg, ref photcode) + + off_t imageID, Seq; + if (!FindImageID (&imageID, &Seq, time, photcode)) continue; + if (imageID <= 0) continue; + + if (imageID == catalog[0].measure[i].imageID) { + // existing image ID is valid + Nvalid ++; + BumpValidImage (Seq); + } else { + // existing image ID is NOT valid + Ninvalid ++; + catalog[0].measure[i].imageID = imageID; + BumpInvalidImage (Seq); + } + + found ++; + } + + if (found) { + fprintf (stderr, "found "OFF_T_FMT" matches: "OFF_T_FMT" valid, "OFF_T_FMT" invalid\n", found, Nvalid, Ninvalid); + } +} + Index: trunk/Ohana/src/uniphot/src/update_catalog_fixstkids.c =================================================================== --- trunk/Ohana/src/uniphot/src/update_catalog_fixstkids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/update_catalog_fixstkids.c (revision 35764) @@ -0,0 +1,81 @@ +# include "fixstkids.h" + +void update_catalog_fixstkids (Catalog *catalog) { + + off_t i; + + off_t found = 0; + off_t Nvalid = 0; + off_t Ninvalid = 0; + + int sourceID, externID, detID; + int Nmissed = 0; + + for (i = 0; i < catalog[0].Nmeasure; i++) { + + // only do PS1 stack photcodes + if (catalog[0].measure[i].photcode < 11000) continue; + if (catalog[0].measure[i].photcode > 11500) continue; + + // reverse engineer the detID, imageID, sourceID from the psps det ID (extID) + UnpackPSPSStackDetectionID (&sourceID, &externID, &detID, catalog[0].measure[i].extID); + myAssert (sourceID == 35, "problem with source ID %d (%0x08 %0x08)", sourceID, catalog[0].measure[i].objID, catalog[0].measure[i].catID); + myAssert (detID == catalog[0].measure[i].detID, "problem with det ID %d (%0x08 %0x08)", detID, catalog[0].measure[i].objID, catalog[0].measure[i].catID); + + // we now have the externID of the given detection -- find the real image in the image table + off_t imageID, Seq; + short Photcode; + if (!FindImageID (&imageID, &Seq, &Photcode, externID)) { + Nmissed ++; + continue; + } + + // if only one matched (or none), the we got the right ID the first time around + if (imageID <= 0) { + fprintf (stderr, "?"); + } + + // validate the photcode and time? + myAssert (catalog[0].measure[i].photcode == Photcode, "bad photcode match? objID, catID: (%0x08 %0x08), imageID, externID: "OFF_T_FMT", %d", catalog[0].measure[i].objID, catalog[0].measure[i].catID, imageID, externID) + + if (imageID == catalog[0].measure[i].imageID) { + // existing image ID is valid + Nvalid ++; + BumpValidImage (Seq); + } else { + // existing image ID is NOT valid + Ninvalid ++; + catalog[0].measure[i].imageID = imageID; + BumpInvalidImage (Seq); + } + + found ++; + } + + if (found) { + fprintf (stderr, "found "OFF_T_FMT" matches: "OFF_T_FMT" valid, "OFF_T_FMT" invalid\n", found, Nvalid, Ninvalid); + } + if (Nmissed) { + fprintf (stderr, "missed %d stack entries\n", Nmissed); + } +} + +int UnpackPSPSStackDetectionID(int *sourceID, int *imageID, int *detID, uint64_t pspsStackID) +{ + // sourceID : ID of database + table that tracked the image (< 0x7f = 127) + // imageID : external ID of the image which provided the detections (< 0x1000.0000 ~ 2.7e8) + // detID : detection sequence in image (< 0x1000.0000 ~ 2.7e8) + + // 0x0000.0000.0000.0000 + + // bits 0 - 27 : 0x0000.0000.0fff.ffff + // bits 28 - 55 : 0x00ff.ffff.f000.0000 + // bits 56 - 63 : 0xff00.0000.0000.0000 + + *sourceID = (pspsStackID & (uint64_t) 0xff00000000000000) >> 56; + *imageID = (pspsStackID & (uint64_t) 0x00fffffff0000000) >> 28; + *detID = (pspsStackID & (uint64_t) 0x000000000fffffff); + + return TRUE; +} + Index: trunk/Ohana/src/uniphot/src/update_dvo_ckids.c =================================================================== --- trunk/Ohana/src/uniphot/src/update_dvo_ckids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/update_dvo_ckids.c (revision 35764) @@ -0,0 +1,157 @@ +# include "ckids.h" + +int update_dvo_ckids () { + + SkyTable *sky = NULL; + SkyList *skylist = NULL; + Catalog catalog; + off_t i; + + // load the current sky table (layout of all SkyRegions) + sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, -1, VERBOSE); + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // determine the populated SkyRegions overlapping the requested area (default depth) + if (SINGLE_CPT) { + skylist = SkyRegionByCPT (sky, SINGLE_CPT); + } else { + skylist = SkyListByPatch (sky, -1, &UserPatch); + } + myAssert (skylist, "ooops!"); + + if (PARALLEL && !HOST_ID) { + update_dvo_ckids_parallel (skylist); + return TRUE; + } + + char filename[256]; + snprintf (filename, 256, "dvo.ckimids.%02d.dat", HOST_ID); + FILE *foutput = fopen (filename, "w"); + if (!foutput) { + fprintf (stderr, "failed to open output file %s\n", filename); + exit (3); + } + + // update measurements for each populated catalog + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue; + + char localFilename[1024]; + snprintf (localFilename, 1024, "%s/%s.cpt", HOSTDIR, skylist->regions[i]->name); + + // set up the basic catalog info + catalog.filename = HOST_ID ? localFilename : skylist[0].filename[i]; + catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs + catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs + catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naves_disk) { + if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + update_catalog_ckids (&catalog, foutput); + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename); + + dvo_catalog_save (&catalog, VERBOSE); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + } + fclose (foutput); + + return (TRUE); +} + +# define DEBUG 1 + +int update_dvo_ckids_parallel (SkyList *sky) { + + // now launch the ckids_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + + int i, j; + for (i = 0; i < table->Nhosts; i++) { + + if (sky->Nregions < table->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + char command[1024]; + snprintf (command, 1024, "ckids_client -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", + table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, + UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + char tmpline[1024]; + if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } + if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } + if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) continue; + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running ckids_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + } + + // wait for the remote jobs to be completed + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the ckids_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + if (!status) return FALSE; + } + + return (TRUE); +} + Index: trunk/Ohana/src/uniphot/src/update_dvo_fiximids.c =================================================================== --- trunk/Ohana/src/uniphot/src/update_dvo_fiximids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/update_dvo_fiximids.c (revision 35764) @@ -0,0 +1,174 @@ +# include "fiximids.h" + +int update_dvo_fiximids (ImageSubset *image, off_t Nimage) { + + SkyTable *sky = NULL; + SkyList *skylist = NULL; + Catalog catalog; + off_t i; + + // load the current sky table (layout of all SkyRegions) + sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, -1, VERBOSE); + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // generate index tables + initImageIndex (image, Nimage); + + if (PARALLEL && !HOST_ID) { + update_dvo_fiximids_parallel (sky, image, Nimage); + return TRUE; + } + + // determine the populated SkyRegions overlapping the requested area (default depth) + if (SINGLE_CPT) { + skylist = SkyRegionByCPT (sky, SINGLE_CPT); + } else { + skylist = SkyListByPatch (sky, -1, &UserPatch); + } + myAssert (skylist, "ooops!"); + + // update measurements for each populated catalog + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue; + + char localFilename[1024]; + snprintf (localFilename, 1024, "%s/%s.cpt", HOSTDIR, skylist->regions[i]->name); + + // set up the basic catalog info + catalog.filename = HOST_ID ? localFilename : skylist[0].filename[i]; + catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs + catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs + catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naves_disk) { + if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + update_catalog_fiximids (&catalog); + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename); + + dvo_catalog_save (&catalog, VERBOSE); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + } + + // for the remote clients, we need to dump out the results table (Nvalid, Ninvalid) + if (HOST_ID && IMSTATS_FILE) { + ImageValidSave (IMSTATS_FILE); + } + + return (TRUE); +} + +# define DEBUG 1 + +int update_dvo_fiximids_parallel (SkyTable *sky, ImageSubset *image, off_t Nimage) { + + char uniquer[12]; + int TIME = time(NULL); + int PID = getpid(); + snprintf (uniquer, 12, "%05d.%05d", PID, TIME % 100000); + + // write out the subset table of image information + char imageFile[512]; + snprintf (imageFile, 512, "%s/fiximids.images.%s.dat", CATDIR, uniquer); + + if (!ImageSubsetSave (imageFile, image, Nimage)) { + fprintf (stderr, "failed to write image subset\n"); + exit (1); + } + + // now launch the fiximids_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + + int i; + for (i = 0; i < table->Nhosts; i++) { + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + ALLOCATE (table->hosts[i].results, char, 1024); + snprintf (table->hosts[i].results, 1024, "%s/fiximids.results.%s.dat", table->hosts[i].pathname, uniquer); + + char command[1024]; + snprintf (command, 1024, "fiximids_client -hostID %d -catdir %s -hostdir %s -images %s -region %f %f %f %f", + table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, imageFile, + UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + int saveImstats = VERBOSE_IMSTATS || SUMMARY_IMSTATS; + + char tmpline[1024]; + if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } + if (saveImstats) { snprintf (tmpline, 1024, "%s -imstats %s", command, table->hosts[i].results); strcpy (command, tmpline); } + if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } + if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) continue; + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running fiximids_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + } + + // wait for the remote jobs to be completed + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the fiximids_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + if (!status) return FALSE; + } + + if (VERBOSE_IMSTATS || SUMMARY_IMSTATS) { + for (i = 0; i < table->Nhosts; i++) { + while (!ImageValidLoad (table->hosts[i].results)) { + // failed to get the data from this host. This can happen for various reasons. Give the user a chance to try again... + fprintf (stderr, "failed to read data from %s, stopping operations until this can be fixed\n", table->hosts[i].hostname); + fprintf (stderr, "you may run the command manually and send this process the CONT signal\n"); + int pid = getpid(); + kill (pid, SIGSTOP); + fprintf (stderr, "retrying %s\n", table->hosts[i].results); + } + } + } + + return (TRUE); +} + Index: trunk/Ohana/src/uniphot/src/update_dvo_fixstkids.c =================================================================== --- trunk/Ohana/src/uniphot/src/update_dvo_fixstkids.c (revision 35764) +++ trunk/Ohana/src/uniphot/src/update_dvo_fixstkids.c (revision 35764) @@ -0,0 +1,190 @@ +# include "fixstkids.h" + +int update_dvo_fixstkids (ImageSubset *image, off_t Nimage) { + + SkyTable *sky = NULL; + SkyList *skylist = NULL; + Catalog catalog; + off_t i; + + // load the current sky table (layout of all SkyRegions) + sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, -1, VERBOSE); + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // generate index tables + initImageIndex (image, Nimage); + + // determine the populated SkyRegions overlapping the requested area (default depth) + if (SINGLE_CPT) { + skylist = SkyRegionByCPT (sky, SINGLE_CPT); + } else { + skylist = SkyListByPatch (sky, -1, &UserPatch); + } + myAssert (skylist, "ooops!"); + + if (PARALLEL && !HOST_ID) { + update_dvo_fixstkids_parallel (skylist, image, Nimage); + return TRUE; + } + + // update measurements for each populated catalog + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue; + + char localFilename[1024]; + snprintf (localFilename, 1024, "%s/%s.cpt", HOSTDIR, skylist->regions[i]->name); + + // set up the basic catalog info + catalog.filename = HOST_ID ? localFilename : skylist[0].filename[i]; + catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs + catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs + catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naves_disk) { + if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + update_catalog_fixstkids (&catalog); + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + if (VERBOSE) fprintf (stderr, "saving catalog %s\n", catalog.filename); + + dvo_catalog_save (&catalog, VERBOSE); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + } + + // for the remote clients, we need to dump out the results table (Nvalid, Ninvalid) + if (HOST_ID && IMSTATS_FILE) { + ImageValidSave (IMSTATS_FILE); + } + + return (TRUE); +} + +# define DEBUG 1 + +int update_dvo_fixstkids_parallel (SkyList *sky, ImageSubset *image, off_t Nimage) { + + char uniquer[12]; + int TIME = time(NULL); + int PID = getpid(); + snprintf (uniquer, 12, "%05d.%05d", PID, TIME % 100000); + + // write out the subset table of image information + char imageFile[512]; + snprintf (imageFile, 512, "%s/fixstkids.images.%s.dat", CATDIR, uniquer); + + if (!ImageSubsetSave (imageFile, image, Nimage)) { + fprintf (stderr, "failed to write image subset\n"); + exit (1); + } + + // now launch the fixstkids_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + + int i, j; + for (i = 0; i < table->Nhosts; i++) { + + if (sky->Nregions < table->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + ALLOCATE (table->hosts[i].results, char, 1024); + snprintf (table->hosts[i].results, 1024, "%s/fixstkids.results.%s.dat", table->hosts[i].pathname, uniquer); + + char command[1024]; + snprintf (command, 1024, "fixstkids_client -hostID %d -catdir %s -hostdir %s -images %s -region %f %f %f %f", + table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, imageFile, + UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + int saveImstats = VERBOSE_IMSTATS || SUMMARY_IMSTATS; + + char tmpline[1024]; + if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } + if (saveImstats) { snprintf (tmpline, 1024, "%s -imstats %s", command, table->hosts[i].results); strcpy (command, tmpline); } + if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } + if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) continue; + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running fixstkids_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + } + + // wait for the remote jobs to be completed + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the fixstkids_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + if (!status) return FALSE; + } + + if (VERBOSE_IMSTATS || SUMMARY_IMSTATS) { + for (i = 0; i < table->Nhosts; i++) { + if (!table->hosts[i].results) continue; + while (!ImageValidLoad (table->hosts[i].results)) { + // failed to get the data from this host. This can happen for various reasons. Give the user a chance to try again... + fprintf (stderr, "failed to read data from %s, stopping operations until this can be fixed\n", table->hosts[i].hostname); + fprintf (stderr, "you may run the command manually and send this process the CONT signal\n"); + int pid = getpid(); + kill (pid, SIGSTOP); + fprintf (stderr, "retrying %s\n", table->hosts[i].results); + } + } + } + + return (TRUE); +} +