Index: branches/czw_branch/20160809/Ohana/src/delstar/Makefile =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/Makefile (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/Makefile (revision 39719) @@ -31,4 +31,5 @@ $(SRC)/delete_duplicate_images.$(ARCH).o \ $(SRC)/delete_duplicate_measures.$(ARCH).o \ +$(SRC)/delete_measures_by_match.$(ARCH).o \ $(SRC)/delete_fix_LAP.$(ARCH).o \ $(SRC)/delete_fix_LAP_edges.$(ARCH).o \ @@ -56,4 +57,5 @@ $(SRC)/delete_duplicate_images.$(ARCH).o \ $(SRC)/delete_duplicate_measures.$(ARCH).o \ +$(SRC)/delete_measures_by_match.$(ARCH).o \ $(SRC)/delete_fix_LAP.$(ARCH).o \ $(SRC)/delete_fix_LAP_edges.$(ARCH).o \ Index: branches/czw_branch/20160809/Ohana/src/delstar/include/delstar.h =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/include/delstar.h (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/include/delstar.h (revision 39719) @@ -93,4 +93,5 @@ int SAVE_DUPLICATES; +int SAVE_DELETES; int SKIP_IMAGES; char *BACKUP_EXTNAME; @@ -104,5 +105,5 @@ int MODE; -enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES, MODE_DUP_MEASURES, MODE_FIX_LAP, MODE_FIX_LAP_STATS, MODE_FIX_LAP_EDGES, MODE_FIX_LAP_EDGES_DELETE}; +enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES, MODE_DUP_MEASURES, MODE_DELETE_MEASURES_BY_MATCH, MODE_FIX_LAP, MODE_FIX_LAP_STATS, MODE_FIX_LAP_EDGES, MODE_FIX_LAP_EDGES_DELETE}; char DateKeyword[64], DateMode[64], UTKeyword[64], MJDKeyword[64], JDKeyword[64]; @@ -185,4 +186,8 @@ int ImageIDSave(char *filename, IndexArray *imageID); +int delete_measures_by_match (); +int delete_measures_by_match_parallel (SkyList *sky); +DeleteMeasureResult delete_measures_by_match_catalog (Catalog *catalog); + int delete_duplicate_measures (); int delete_duplicate_measures_parallel (SkyList *sky); Index: branches/czw_branch/20160809/Ohana/src/delstar/src/args.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/src/args.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/src/args.c (revision 39719) @@ -12,4 +12,5 @@ fprintf (stderr, " delstar -dup-images : delete duplicate images (by externID)\n\n"); fprintf (stderr, " delstar -dup-measures : delete duplicate measures (by imageID + detID)\n\n"); + fprintf (stderr, " delstar -delete-measures-by-match : delete duplicate measures by imageID, photcode, time constratins\n\n"); fprintf (stderr, " optional flags:\n"); fprintf (stderr, " -v : verbose mode\n"); @@ -194,4 +195,10 @@ SKIP_IMAGES = TRUE; // we do not need to load the images for -dup-measures } + if ((N = get_argument (argc, argv, "-delete-measures-by-match"))) { + if (MODE != MODE_NONE) usage(); + MODE = MODE_DELETE_MEASURES_BY_MATCH; + remove_argument (N, &argc, argv); + SKIP_IMAGES = TRUE; // we do not need to load the images for -dup-measures + } DELETE_MIN_DET_ID = 0; @@ -222,4 +229,5 @@ remove_argument (N, &argc, argv); } + BACKUP_EXTNAME = NULL; if ((N = get_argument (argc, argv, "-backup-extname"))) { @@ -229,4 +237,10 @@ } if (!BACKUP_EXTNAME) BACKUP_EXTNAME = strcreate (".bck"); + + SAVE_DELETES = FALSE; + if ((N = get_argument (argc, argv, "-save-deletes"))) { + SAVE_DELETES = TRUE; + remove_argument (N, &argc, argv); + } if ((N = get_argument (argc, argv, "-fix-LAP"))) { Index: branches/czw_branch/20160809/Ohana/src/delstar/src/delete_duplicate_measures.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/src/delete_duplicate_measures.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/src/delete_duplicate_measures.c (revision 39719) @@ -373,5 +373,5 @@ } -# if (0) +# if (1) FILE *fsave = NULL; if (SAVE_DUPLICATES) { @@ -408,7 +408,7 @@ off_t N = measure[j].averef; if (VERBOSE) fprintf (stderr, "0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[j].imageID, measure[j].detID, average[N].R, average[N].D, measure[j].photcode); -// if (fsave) { -// fprintf (fsave, "0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[j].imageID, measure[j].detID, average[N].R, average[N].D, measure[j].photcode); -// } + if (fsave) { + fprintf (fsave, "0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[j].imageID, measure[j].detID, average[N].R, average[N].D, measure[j].photcode); + } if (isGPC1chip(measure[j].photcode)) { result.NdelChip ++; @@ -426,5 +426,5 @@ } - // if (fsave) fclose (fsave); + if (fsave) fclose (fsave); // set up the measure sequence lists Index: branches/czw_branch/20160809/Ohana/src/delstar/src/delstar.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/src/delstar.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/src/delstar.c (revision 39719) @@ -27,4 +27,11 @@ case MODE_DUP_MEASURES: if (!delete_duplicate_measures ()) exit (1); + delstar_args_free (); + ohana_memcheck (TRUE); + ohana_memdump (TRUE); + exit (0); + break; + case MODE_DELETE_MEASURES_BY_MATCH: + if (!delete_measures_by_match ()) exit (1); delstar_args_free (); ohana_memcheck (TRUE); Index: branches/czw_branch/20160809/Ohana/src/delstar/src/delstar_client.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/src/delstar_client.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/src/delstar_client.c (revision 39719) @@ -31,4 +31,11 @@ if (!delete_duplicate_measures ()) exit (1); delstar_client_args_free (); + ohana_memcheck (TRUE); + ohana_memdump (TRUE); + exit (0); + break; + case MODE_DELETE_MEASURES_BY_MATCH: + if (!delete_measures_by_match ()) exit (1); + delstar_args_free (); ohana_memcheck (TRUE); ohana_memdump (TRUE); Index: branches/czw_branch/20160809/Ohana/src/kapa2/src/Graphs.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/kapa2/src/Graphs.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/kapa2/src/Graphs.c (revision 39719) @@ -26,51 +26,10 @@ graph[0].axis[i].fMinor = 5.0; } - graph[0].data.ticktextPad = NAN; - graph[0].data.labelPadXm = NAN; - graph[0].data.labelPadXp = NAN; - graph[0].data.labelPadYm = NAN; - graph[0].data.labelPadYp = NAN; - - graph[0].data.padXm = NAN; - graph[0].data.padXp = NAN; - graph[0].data.padYm = NAN; - graph[0].data.padYp = NAN; - - graph[0].data.fLabelRangeXm = 1.0; - graph[0].data.fLabelRangeXp = 1.0; - graph[0].data.fLabelRangeYm = 1.0; - graph[0].data.fLabelRangeYp = 1.0; - - graph[0].data.fMinorXm = 5.0; - graph[0].data.fMinorXp = 5.0; - graph[0].data.fMinorYm = 5.0; - graph[0].data.fMinorYp = 5.0; + KapaInitGraph (&graph[0].data); for (i = 0; i < 8; i++) { strcpy (graph[0].label[i].text, ""); } - - graph[0].data.xmin = 0.0; - graph[0].data.xmax = 1.0; - graph[0].data.ymin = 0.0; - graph[0].data.ymax = 1.0; - - graph[0].data.style = 2; // points - graph[0].data.ptype = 2; // + for points - graph[0].data.ltype = 0; // solid line - graph[0].data.etype = 0; // no error bars - graph[0].data.ebar = 0; // no cross bar - graph[0].data.color = 0; // black - graph[0].data.lweight = 0.5; // line weight of 0.5 - graph[0].data.size = 1.0; // point size of 1.0 - - graph[0].data.flipeast = TRUE; // +East = -X by default - graph[0].data.flipnorth = FALSE; // +North = +Y by default - - InitCoords (&graph[0].data.coords, "DEC--LIN"); - strcpy (graph[0].data.axis, "2222"); - strcpy (graph[0].data.ticks, "2222"); - strcpy (graph[0].data.labels, "2222"); graph[0].Nobjects = 0; Index: branches/czw_branch/20160809/Ohana/src/libdvo/include/dvo.h =================================================================== --- branches/czw_branch/20160809/Ohana/src/libdvo/include/dvo.h (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/libdvo/include/dvo.h (revision 39719) @@ -921,4 +921,6 @@ float PhotFluxCatErr (Measure *measure, dvoMagClassType class); float PhotFluxAveErr (PhotCode *code, Average *average, SecFilt *secfilt, dvoMagClassType class, dvoMagSourceType source); +float PhotFluxSysErr (Measure *measure, Average *average, SecFilt *secfilt, dvoMagClassType class); +float PhotFluxRelErr (Measure *measure, Average *average, SecFilt *secfilt, dvoMagClassType class); float PhotXm (PhotCode *code, Average *average, SecFilt *secfilt); Index: branches/czw_branch/20160809/Ohana/src/libdvo/src/dbExtractMeasures.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 39719) @@ -228,11 +228,17 @@ break; case MAG_LEVEL_SYS: + value.Flt = PhotFluxSysErr (measure, average, secfilt, field->magClass); + break; case MAG_LEVEL_REL: + value.Flt = PhotFluxRelErr (measure, average, secfilt, field->magClass); + break; case MAG_LEVEL_CAL: - // value.Flt = PhotFluxErr (measure, field->magClass); + // XXX not defined break; case MAG_LEVEL_AVE: + value.Flt = PhotFluxAveErr (equiv, average, secfilt, field->magClass, field->magSource); + break; case MAG_LEVEL_REF: - // value.Flt = PhotFluxAveErr (equiv, average, secfilt, field->magClass, field->magSource); + // XXX not defined break; case MAG_LEVEL_NONE: Index: branches/czw_branch/20160809/Ohana/src/libdvo/src/dvo_photcode_ops.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/libdvo/src/dvo_photcode_ops.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/libdvo/src/dvo_photcode_ops.c (revision 39719) @@ -1507,4 +1507,131 @@ } return (dFcat); +} + +float PhotFluxSysErr (Measure *measure, Average *average, SecFilt *secfilt, dvoMagClassType class) { + + int Np = photcodes[0].hashcode[measure[0].photcode]; + if (Np == -1) return (NAN); + PhotCode *code = &photcodes[0].code[Np]; + + // measure.M has the static ZERO_POINT (25.0) applied, but not measure.Flux + float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C; + float Moff = Mcal - ZERO_POINT + 8.9; + float Foff = 3630.8 * MagToFlux(Mcal); + + // use dFlux if we can, but use dMag if we must: + // dFlux = Flux * dMag + + float dFcat = NAN; + switch (class) { + case MAG_CLASS_PSF: + if (isnan (measure[0].dFluxPSF)) { + float Finst = MagToFlux(measure[0].M + Moff); + dFcat = measure[0].dM * Finst; + } else { + dFcat = measure[0].dFluxPSF * Foff; + } + break; + case MAG_CLASS_KRON: + if (isnan (measure[0].dFluxKron)) { + float Finst = MagToFlux(measure[0].Mkron + Moff); + dFcat = measure[0].dMkron * Finst; + } else { + dFcat = measure[0].dFluxKron * Foff; + } + break; + case MAG_CLASS_APER: + if (isnan (measure[0].dFluxAp)) { + float Finst = MagToFlux(measure[0].Map + Moff); + dFcat = measure[0].dMap * Finst; + } else { + dFcat = measure[0].dFluxAp * Foff; + } + break; + default: + break; + } + + /* color correction */ + float mc = PhotColorForCode (average, secfilt, NULL, code); + if (isnan(mc)) return (dFcat); + mc -= SCALE*code[0].dX; + + double Mc = mc; + float Mcol = 0; + int i = 0; + for (i = 0; i < code[0].Nc; i++) { + Mcol += code[0].X[i]*Mc; + Mc *= mc; + } + float Fcol = MagToFlux (Mcol); + + float dFsys = dFcat * Fcol; + + return (dFsys); +} + +float PhotFluxRelErr (Measure *measure, Average *average, SecFilt *secfilt, dvoMagClassType class) { + + int Np = photcodes[0].hashcode[measure[0].photcode]; + if (Np == -1) return (NAN); + PhotCode *code = &photcodes[0].code[Np]; + + // measure.M has the static ZERO_POINT (25.0) applied, but not measure.Flux + float Mflat = isfinite(measure[0].Mflat) ? measure[0].Mflat : 0.0; + float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - measure[0].Mcal - Mflat; + float Moff = Mcal - ZERO_POINT + 8.9; + float Foff = 3630.8 * MagToFlux(Mcal); + + // use dFlux if we can, but use dMag if we must: + // dFlux = Flux * dMag + + float dFcat = NAN; + switch (class) { + case MAG_CLASS_PSF: + if (isnan (measure[0].dFluxPSF)) { + float Finst = MagToFlux(measure[0].M + Moff); + dFcat = measure[0].dM * Finst; + } else { + dFcat = measure[0].dFluxPSF * Foff; + } + break; + case MAG_CLASS_KRON: + if (isnan (measure[0].dFluxKron)) { + float Finst = MagToFlux(measure[0].Mkron + Moff); + dFcat = measure[0].dMkron * Finst; + } else { + dFcat = measure[0].dFluxKron * Foff; + } + break; + case MAG_CLASS_APER: + if (isnan (measure[0].dFluxAp)) { + float Finst = MagToFlux(measure[0].Map + Moff); + dFcat = measure[0].dMap * Finst; + } else { + dFcat = measure[0].dFluxAp * Foff; + } + break; + default: + break; + } + + /* color correction */ + float mc = PhotColorForCode (average, secfilt, NULL, code); + if (isnan(mc)) return (dFcat); + mc -= SCALE*code[0].dX; + + double Mc = mc; + float Mcol = 0; + int i = 0; + for (i = 0; i < code[0].Nc; i++) { + Mcol += code[0].X[i]*Mc; + Mc *= mc; + } + float Fcol = MagToFlux (Mcol); + + float dFrel = dFcat * Fcol; + + return (dFrel); } Index: branches/czw_branch/20160809/Ohana/src/libkapa/src/KapaWindow.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/libkapa/src/KapaWindow.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/libkapa/src/KapaWindow.c (revision 39719) @@ -93,5 +93,6 @@ graphdata[0].xmin = graphdata[0].ymin = 0.0; graphdata[0].xmax = graphdata[0].ymax = 1.0; - graphdata[0].style = graphdata[0].ptype = 0; + + graphdata[0].style = graphdata[0].ptype = 2; graphdata[0].ltype = graphdata[0].color = 0; graphdata[0].etype = graphdata[0].ebar = 0; @@ -117,4 +118,14 @@ graphdata[0].padYm = NAN; graphdata[0].padYp = NAN; + + graphdata[0].fLabelRangeXm = 1.0; + graphdata[0].fLabelRangeXp = 1.0; + graphdata[0].fLabelRangeYm = 1.0; + graphdata[0].fLabelRangeYp = 1.0; + + graphdata[0].fMinorXm = 5.0; + graphdata[0].fMinorXp = 5.0; + graphdata[0].fMinorYm = 5.0; + graphdata[0].fMinorYp = 5.0; return (TRUE); Index: branches/czw_branch/20160809/Ohana/src/relastro/Makefile =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/Makefile (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/Makefile (revision 39719) @@ -26,4 +26,6 @@ RELASTRO = \ $(SRC)/CheckMeasureToImage.$(ARCH).o \ +$(SRC)/RepairObjectIDs.$(ARCH).o \ +$(SRC)/RepairStacks.$(ARCH).o \ $(SRC)/RepairStackMeasures.$(ARCH).o \ $(SRC)/StackImageMaps.$(ARCH).o \ @@ -114,4 +116,6 @@ RELASTRO_CLIENT = \ $(SRC)/CheckMeasureToImage.$(ARCH).o \ +$(SRC)/RepairObjectIDs.$(ARCH).o \ +$(SRC)/RepairStacks.$(ARCH).o \ $(SRC)/RepairStackMeasures.$(ARCH).o \ $(SRC)/StackImageMaps.$(ARCH).o \ Index: branches/czw_branch/20160809/Ohana/src/relastro/include/relastro.h =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/include/relastro.h (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/include/relastro.h (revision 39719) @@ -31,5 +31,5 @@ typedef enum {FIT_NONE, FIT_AVERAGE, FIT_PM_ONLY, FIT_PAR_ONLY, FIT_PM_AND_PAR} FitMode; -typedef enum {OP_NONE, OP_IMAGES, OP_HIGH_SPEED, OP_MERGE_SOURCE, OP_UPDATE_OBJECTS, OP_UPDATE_OFFSETS, OP_LOAD_OBJECTS, OP_HPM, OP_PARALLEL_REGIONS, OP_PARALLEL_IMAGES, OP_REPAIR_WARPS} RelastroOp; +typedef enum {OP_NONE, OP_IMAGES, OP_HIGH_SPEED, OP_MERGE_SOURCE, OP_UPDATE_OBJECTS, OP_UPDATE_OFFSETS, OP_LOAD_OBJECTS, OP_HPM, OP_PARALLEL_REGIONS, OP_PARALLEL_IMAGES, OP_REPAIR_STACKS, OP_REPAIR_WARPS, OP_REPAIR_OBJECT_ID} RelastroOp; typedef enum {TARGET_NONE, TARGET_SIMPLE, TARGET_CHIPS, TARGET_MOSAICS} FitTarget; @@ -864,4 +864,8 @@ uint64_t CreatePSPSDetectionID(double tobs, int ccdid, int detID); +int RepairStacks (SkyList *skylist, int hostID, char *hostpath); + +int RepairObjectIDs (SkyList *skylist, int hostID, char *hostpath); + void sort_by_ra (double *R, double *D, int *I, int *S, int N); void FreeStackGroups (void); Index: branches/czw_branch/20160809/Ohana/src/relastro/src/RepairObjectIDs.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/RepairObjectIDs.c (revision 39719) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/RepairObjectIDs.c (revision 39719) @@ -0,0 +1,177 @@ +# include "relastro.h" + +int RepairObjectIDs_parallel (SkyList *sky); +int RepairObjectIDs_catalog (Catalog *catalog); + +// some average entries are missing the PSPS object ID values. set them correctly here +int RepairObjectIDs (SkyList *skylist, int hostID, char *hostpath) { + + int i; + Catalog catalog; + + if (PARALLEL && !hostID) { + RepairObjectIDs_parallel (skylist); + return TRUE; + } + + // load data from each region file, only use bright stars + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue; + + // define the catalog file name + char hostfile[1024]; + snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name); + + dvo_catalog_init (&catalog, TRUE); + catalog.filename = hostID ? hostfile : skylist[0].filename[i]; + + // set up the basic catalog info + catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE2, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naverage_disk) { + if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + RepairObjectIDs_catalog (&catalog); + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + struct timeval now; + gettimeofday (&now, (void *) NULL); + char *moddate = ohana_sec_to_date (now.tv_sec); + gfits_modify (&catalog.header, "REPAIR", "%s", 1, moddate); + free (moddate); + + // write the updated detections to disk + save_catalogs (&catalog, 1); + } + return (TRUE); +} + +int RepairObjectIDs_parallel_table (HostTable *table, SkyList *sky); + +int RepairObjectIDs_parallel (SkyList *sky) { + + // launch the setphot_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + exit (1); + } + + RepairObjectIDs_parallel_table (table, sky); + + return TRUE; +} + +// CATDIR is supplied globally +# define DEBUG 1 +int RepairObjectIDs_parallel_table (HostTable *table, SkyList *sky) { + + // launch the relastro_client jobs to the parallel hosts + + int i, j; + for (i = 0; i < table->Nhosts; i++) { + + if (sky->Nregions < table->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + char *command = NULL; + strextend (&command, "relastro_client -repair-object-id"); + strextend (&command, "-hostID %d", table->hosts[i].hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-hostdir %s", table->hosts[i].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (UPDATE) strextend (&command, "-update"); + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) { + free (command); + continue; + } + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running relastro_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + free (command); + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the relastro_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + } + + FreeHostTable (table); + return TRUE; +} + +int RepairObjectIDs_catalog (Catalog *catalog) { + + myAssert (!catalog->Naverage || catalog->average, "programming error"); + + int onePercent = catalog->Naverage / 100; + + for (off_t j = 0; j < catalog->Naverage; j++) { + if (j % onePercent == 0) fprintf (stderr, "."); + + catalog->average[j].extID = CreatePSPSObjectID (catalog->average[j].R, catalog->average[j].D); + } + + return TRUE; +} + Index: branches/czw_branch/20160809/Ohana/src/relastro/src/RepairStacks.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/RepairStacks.c (revision 39719) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/RepairStacks.c (revision 39719) @@ -0,0 +1,279 @@ +# include "relastro.h" + +int RepairStacks_parallel (SkyList *sky); + +int RepairStacks (SkyList *skylist, int hostID, char *hostpath) { + + int i; + Catalog catalog; + + if (PARALLEL && !hostID) { + RepairStacks_parallel (skylist); + return TRUE; + } + + MakeStackIndex (); + + // load data from each region file, only use bright stars + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue; + + // define the catalog file name + char hostfile[1024]; + snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name); + + dvo_catalog_init (&catalog, TRUE); + catalog.filename = hostID ? hostfile : skylist[0].filename[i]; + + // set up the basic catalog info + catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE2, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naverage_disk) { + if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + // set the values in MeasureTiny needed by UpdateObjects + populate_tiny_values(&catalog, DVO_TV_MEASURE); + + // match measurements with images + initImageBins (&catalog, 1, FALSE); + findImages (&catalog, 1, FALSE); + + RepairStackMeasures (&catalog); + + free_tiny_values(&catalog); + + freeImageBins (1); + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + struct timeval now; + gettimeofday (&now, (void *) NULL); + char *moddate = ohana_sec_to_date (now.tv_sec); + gfits_modify (&catalog.header, "RELASTRO", "%s", 1, moddate); + free (moddate); + + // write the updated detections to disk + save_catalogs (&catalog, 1); + } + printNcatTotal(); + + FreeStackGroups (); + + return (TRUE); +} + +int RepairStacks_parallel_group (HostTableGroup *group, SkyList *sky); +int RepairStacks_parallel_table (HostTable *table, SkyList *sky); + +// CATDIR is supplied globally +# define DEBUG 1 +int RepairStacks_parallel (SkyList *sky) { + + // launch the setphot_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + exit (1); + } + +# if (1) + + RepairStacks_parallel_table (table, sky); + +# else + + int Ngroups; + HostTableGroup *groups = HostTableGroupsUniqueMachines (table, &Ngroups); + // split the table into Ngroups, each with a unique set of machines (this avoids overloading the remote host machines) + + int i; + for (i = 0; i < Ngroups; i++) { + // update only a group of unique machines at a time + RepairStacks_parallel_group (&groups[i], sky); + } +# endif + + return TRUE; +} + +// CATDIR is supplied globally +# define DEBUG 1 +int RepairStacks_parallel_group (HostTableGroup *group, SkyList *sky) { + + // launch the relastro_client jobs to the parallel hosts + + int i, j; + for (i = 0; i < group->Nhosts; i++) { + + if (sky->Nregions < group->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], group->hosts[i][0].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", group->hosts[i][0].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (group->hosts[i][0].pathname, DVO_MAX_PATH); + free (group->hosts[i][0].pathname); + group->hosts[i][0].pathname = tmppath; + + char *command = NULL; + strextend (&command, "relastro_client -repair-stacks"); + strextend (&command, "-hostID %d", group->hosts[i][0].hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-hostdir %s", group->hosts[i][0].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (UPDATE) strextend (&command, "-update"); + + if (USE_BASIC_CHECK) strextend (&command, "-basic-image-search"); + if (USE_ALL_IMAGES) strextend (&command, "-use-all-images"); + if (CHECK_MEASURE_TO_IMAGE) strextend (&command, "-check-measures"); + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) { + free (command); + continue; + } + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running relastro_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", group->hosts[i][0].hostname, command, group->hosts[i][0].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", group->hosts[i][0].hostname, errorInfo); + exit (1); + } + group->hosts[i][0].pid = pid; // save for future reference + } + free (command); + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the relastro_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableGroupWaitJobsGetIO (group, __FILE__, __LINE__, VERBOSE); + } + + return TRUE; +} + +// CATDIR is supplied globally +# define DEBUG 1 +int RepairStacks_parallel_table (HostTable *table, SkyList *sky) { + + // launch the relastro_client jobs to the parallel hosts + + int i, j; + for (i = 0; i < table->Nhosts; i++) { + + if (sky->Nregions < table->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + char *command = NULL; + strextend (&command, "relastro_client -repair-stacks"); + strextend (&command, "-hostID %d", table->hosts[i].hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-hostdir %s", table->hosts[i].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (UPDATE) strextend (&command, "-update"); + + if (USE_BASIC_CHECK) strextend (&command, "-basic-image-search"); + if (USE_ALL_IMAGES) strextend (&command, "-use-all-images"); + if (CHECK_MEASURE_TO_IMAGE) strextend (&command, "-check-measures"); + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) { + free (command); + continue; + } + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running relastro_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + free (command); + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the relastro_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + } + + FreeHostTable (table); + return TRUE; +} Index: branches/czw_branch/20160809/Ohana/src/relastro/src/StackImageMaps.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/StackImageMaps.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/StackImageMaps.c (revision 39719) @@ -87,5 +87,5 @@ // we have the stack centers; find all stacks within 0.3 degrees of this point Rstk = ohana_normalize_angle (Rstk); - double dD = 0.3; + double dD = 0.4; double dR = dD / cos(RAD_DEG*Dstk); double Rmin = Rstk - dR; @@ -93,4 +93,10 @@ double Dmin = Dstk - dD; double Dmax = Dstk + dD; + + // for the north pole, just test the whole circle + if (Dstk > 88) { + Rmin = 0.0; + Rmax = 360.0; + } double dPosMin = NAN; Index: branches/czw_branch/20160809/Ohana/src/relastro/src/UpdateObjectOffsets.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 39719) @@ -194,5 +194,5 @@ if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); - if (REPAIR_STACKS) strextend (&command, "-repair-stacks"); + if (REPAIR_STACKS) strextend (&command, "-repair-stacks-on-update"); if (CHECK_MEASURE_TO_IMAGE) strextend (&command, "-check-measures"); @@ -324,7 +324,8 @@ if (ExcludeBogus) strextend (&command, "-exclude-bogus %f", ExcludeBogusRadius); + if (USE_ALL_IMAGES) strextend (&command, "-use-all-images"); if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); - if (REPAIR_STACKS) strextend (&command, "-repair-stacks"); + if (REPAIR_STACKS) strextend (&command, "-repair-stacks-on-update"); if (CHECK_MEASURE_TO_IMAGE) strextend (&command, "-check-measures"); Index: branches/czw_branch/20160809/Ohana/src/relastro/src/args.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/args.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/args.c (revision 39719) @@ -53,4 +53,13 @@ } + if ((N = get_argument (argc, argv, "-repair-stacks"))) { + remove_argument (N, &argc, argv); + RELASTRO_OP = OP_REPAIR_STACKS; + } + if ((N = get_argument (argc, argv, "-repair-object-id"))) { + remove_argument (N, &argc, argv); + RELASTRO_OP = OP_REPAIR_OBJECT_ID; + } + if ((N = get_argument (argc, argv, "-repair-warps"))) { remove_argument (N, &argc, argv); @@ -59,5 +68,5 @@ REPAIR_STACKS = FALSE; - if ((N = get_argument (argc, argv, "-repair-stacks"))) { + if ((N = get_argument (argc, argv, "-repair-stacks-on-update"))) { remove_argument (N, &argc, argv); REPAIR_STACKS = TRUE; @@ -691,4 +700,14 @@ } + if ((N = get_argument (argc, argv, "-repair-stacks"))) { + remove_argument (N, &argc, argv); + RELASTRO_OP = OP_REPAIR_STACKS; + } + + if ((N = get_argument (argc, argv, "-repair-object-id"))) { + remove_argument (N, &argc, argv); + RELASTRO_OP = OP_REPAIR_OBJECT_ID; + } + if ((N = get_argument (argc, argv, "-repair-warps"))) { remove_argument (N, &argc, argv); @@ -697,5 +716,5 @@ REPAIR_STACKS = FALSE; - if ((N = get_argument (argc, argv, "-repair-stacks"))) { + if ((N = get_argument (argc, argv, "-repair-stacks-on-update"))) { remove_argument (N, &argc, argv); REPAIR_STACKS = TRUE; Index: branches/czw_branch/20160809/Ohana/src/relastro/src/relastro.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/relastro.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/relastro.c (revision 39719) @@ -104,4 +104,44 @@ } + case OP_REPAIR_STACKS: { + FITS_DB db; + if (!PARALLEL) { + + set_db (&db); + gfits_db_init (&db); + + /* lock and load the image db table */ + int status = dvo_image_lock (&db, ImageCat, 60.0, LCK_SOFT); + if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + if (db.dbstate == LCK_EMPTY) Shutdown ("ERROR: No images in catalog %s (1)", db.filename); + if (!dvo_image_load (&db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db.filename); + // the raw FITS data is freed by dvo_image_load, leaving only the Image structure data + + /* load regions and images based on specified sky patch (default depth) */ + load_images (&db, skylist, FALSE, USE_ALL_IMAGES); + } + + // iterate over catalogs to make detection coordinates consistant + RepairStacks (skylist, 0, NULL); + + if (!PARALLEL) { + freeImages (db.ftable.buffer); + gfits_db_free (&db); + freeMosaics (); + } + + relastro_free (sky, skylist); + exit (0); + } + + case OP_REPAIR_OBJECT_ID: { + + // iterate over catalogs to make detection coordinates consistant + RepairObjectIDs (skylist, 0, NULL); + + relastro_free (sky, skylist); + exit (0); + } + case OP_HIGH_SPEED: /* high-speed is a 2pt cross-correlation process for linking moving objects (high PM) */ Index: branches/czw_branch/20160809/Ohana/src/relastro/src/relastro_client.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/relastro_client.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/relastro_client.c (revision 39719) @@ -104,4 +104,40 @@ } + case OP_REPAIR_OBJECT_ID: { + + // iterate over catalogs to make detection coordinates consistant + RepairObjectIDs (skylist, HOST_ID, HOSTDIR); + + relastro_free (sky, skylist); + exit (0); + } + + // XXX loading the images is fairly costly -- see if we can do an image subset? + case OP_REPAIR_STACKS: { + FITS_DB db; + + set_db (&db); + gfits_db_init (&db); + + /* lock and load the image db table */ + int status = dvo_image_lock (&db, ImageCat, 60.0, LCK_SOFT); + if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + if (db.dbstate == LCK_EMPTY) Shutdown ("ERROR: No images in catalog %s (1)", db.filename); + if (!dvo_image_load (&db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db.filename); + // the raw FITS data is freed by dvo_image_load, leaving only the Image structure data + + /* load regions and images based on specified sky patch (default depth) */ + load_images (&db, skylist, FALSE, USE_ALL_IMAGES); + + RepairStacks (skylist, HOST_ID, HOSTDIR); + + freeImages (db.ftable.buffer); + gfits_db_free (&db); + freeMosaics (); + + relastro_client_free (sky, skylist); + break; + } + // XXX loading the images is fairly costly -- see if we can do an image subset? case OP_REPAIR_WARPS: {