Index: branches/eam_branches/ipp-20140904/Ohana/src/addstar/include/WISE.h =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/addstar/include/WISE.h (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/addstar/include/WISE.h (revision 37803) @@ -21,9 +21,10 @@ short WISE_W1, WISE_W2, WISE_W3, WISE_W4; -int ALLSKY; +enum {MODE_NONE, MODE_ALLWISE, MODE_ALLSKY, MODE_PRELIM}; +int MODE; AddstarClientOptions args_loadwise (int argc, char **argv, AddstarClientOptions options); -int loadwise_rawdata (SkyList *skytable, char *filename, AddstarClientOptions options, int ALLSKY); +int loadwise_rawdata (SkyList *skytable, char *filename, AddstarClientOptions options); int getWISE_setup (); Index: branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/MatchHeaders.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/MatchHeaders.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/MatchHeaders.c (revision 37803) @@ -64,4 +64,5 @@ if (!strcmp (exttype, "PS1_SV2")) goto keep; if (!strcmp (exttype, "PS1_SV3")) goto keep; + if (!strcmp (exttype, "PS1_SV4")) goto keep; if (!strcmp (exttype, "PS1_DV1")) goto keep; Index: branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/ReadStarsFITS.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/ReadStarsFITS.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/ReadStarsFITS.c (revision 37803) @@ -14,8 +14,10 @@ Stars *Convert_PS1_V5 PROTO((FTable *table, unsigned int *nstars)); Stars *Convert_PS1_V5_Lensing PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_V5_Lensing_Alt PROTO((FTable *table, unsigned int *nstars)); Stars *Convert_PS1_SV1 PROTO((FTable *table, unsigned int *nstars)); Stars *Convert_PS1_SV1_Alt PROTO((FTable *table, unsigned int *nstars)); Stars *Convert_PS1_SV2 PROTO((FTable *table, unsigned int *nstars)); Stars *Convert_PS1_SV3 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_SV4 PROTO((FTable *table, unsigned int *nstars)); Stars *Convert_PS1_DV3 PROTO((FTable *table, unsigned int *nstars)); Stars *Convert_PS1_DV4 PROTO((FTable *table, unsigned int *nstars)); @@ -78,8 +80,17 @@ } if (!strcmp (type, "PS1_V5")) { - if (table.header[0].Naxis[0] == 312) { - stars = Convert_PS1_V5_Lensing (&table, &Nstars); - } else { - stars = Convert_PS1_V5 (&table, &Nstars); + switch (table.header[0].Naxis[0]) { + case 232: + stars = Convert_PS1_V5 (&table, &Nstars); + break; + case 312: + stars = Convert_PS1_V5_Lensing_Alt (&table, &Nstars); + break; + case 320: + stars = Convert_PS1_V5_Lensing (&table, &Nstars); + break; + default: + fprintf (stderr, "invalid PS1_V5 table size %d\n", (int) table.header[0].Naxis[0]); + return NULL; } } @@ -92,4 +103,7 @@ if (!strcmp (type, "PS1_SV3")) { stars = Convert_PS1_SV3 (&table, &Nstars); + } + if (!strcmp (type, "PS1_SV4")) { + stars = Convert_PS1_SV4 (&table, &Nstars); } if (!strcmp (type, "PS1_DV3")) { @@ -971,4 +985,129 @@ } +Stars *Convert_PS1_V5_Lensing_Alt (FTable *table, unsigned int *nstars) { + + off_t Nstars; + unsigned int i; + double ZeroPt; + Stars *stars; + CMF_PS1_V5_Lensing_Alt *ps1data; + + ps1data = gfits_table_get_CMF_PS1_V5_Lensing_Alt (table, &Nstars, NULL); + if (!ps1data) { + fprintf (stderr, "skipping inconsistent entry\n"); + return (NULL); + } + ZeroPt = GetZeroPoint(); + + ALLOCATE (stars, Stars, Nstars); + for (i = 0; i < Nstars; i++) { + InitStar (&stars[i]); + stars[i].measure.Xccd = ps1data[i].X; + stars[i].measure.Yccd = ps1data[i].Y; + stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); + + stars[i].measure.posangle = ToShortDegrees(ps1data[i].posangle); + stars[i].measure.pltscale = ps1data[i].pltscale; + + if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) { + stars[i].measure.M = NAN; + } else { + stars[i].measure.M = ps1data[i].M + ZeroPt; + } + stars[i].measure.dM = ps1data[i].dM; + stars[i].measure.dMcal = ps1data[i].dMcal; + stars[i].measure.Map = ps1data[i].Map + ZeroPt; + stars[i].measure.dMap = (ps1data[i].apFlux > 0.0) ? fabs(ps1data[i].apFluxErr / ps1data[i].apFlux) : NAN; + + stars[i].measure.Mkron = (ps1data[i].kronFlux > 0.0) ? -2.5*log10(ps1data[i].kronFlux) + ZeroPt : NAN; + stars[i].measure.dMkron = (ps1data[i].kronFlux > 0.0) ? ps1data[i].kronFluxErr / ps1data[i].kronFlux : NAN; + + // these fluxes are converted from counts to counts/sec in FilterStars.c + stars[i].measure.FluxPSF = GetFluxFromFluxOrMag (ps1data[i].Flux, ps1data[i].M); + stars[i].measure.dFluxPSF = GetFluxErrFromFluxOrMag (ps1data[i].dFlux, stars[i].measure.FluxPSF, ps1data[i].dM); + stars[i].measure.FluxKron = ps1data[i].kronFlux; + stars[i].measure.dFluxKron = ps1data[i].kronFluxErr; + stars[i].measure.FluxAp = GetFluxFromFluxOrMag (ps1data[i].apFlux, ps1data[i].Map); + stars[i].measure.dFluxAp = GetFluxErrFromFluxOrMag (ps1data[i].apFluxErr, stars[i].measure.FluxAp, stars[i].measure.dMap); + + stars[i].measure.Sky = ps1data[i].sky; + stars[i].measure.dSky = ps1data[i].dSky; + + stars[i].measure.psfChisq = ps1data[i].psfChisq; + stars[i].measure.psfQF = ps1data[i].psfQF; + stars[i].measure.psfQFperf = ps1data[i].psfQFperf; + + stars[i].measure.psfNdof = ps1data[i].psfNdof; + stars[i].measure.psfNpix = ps1data[i].psfNpix; + stars[i].measure.extNsigma = ps1data[i].extNsigma; + + stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); + stars[i].measure.theta = ToShortDegrees(ps1data[i].df); + + stars[i].measure.Mxx = ToShortPixels(ps1data[i].Mxx); + stars[i].measure.Mxy = ToShortPixels(ps1data[i].Mxy); + stars[i].measure.Myy = ToShortPixels(ps1data[i].Myy); + + stars[i].measure.photFlags = ps1data[i].flags; + stars[i].measure.photFlags2 = ps1data[i].flags2; + + // this is may optionally be replaced by the internal sequence (see FilterStars.c) + stars[i].measure.detID = ps1data[i].detID; + + ALLOCATE (stars[i].lensing, Lensing, 1); + dvo_lensing_init (stars[i].lensing); + + stars[i].lensing->X11_sm_obj = ps1data[i].X11_sm_obj; + stars[i].lensing->X12_sm_obj = ps1data[i].X12_sm_obj; + stars[i].lensing->X22_sm_obj = ps1data[i].X22_sm_obj; + stars[i].lensing->E1_sm_obj = ps1data[i].E1_sm_obj; + stars[i].lensing->E2_sm_obj = ps1data[i].E2_sm_obj; + + stars[i].lensing->X11_sh_obj = ps1data[i].X11_sh_obj; + stars[i].lensing->X12_sh_obj = ps1data[i].X12_sh_obj; + stars[i].lensing->X22_sh_obj = ps1data[i].X22_sh_obj; + stars[i].lensing->E1_sh_obj = ps1data[i].E1_sh_obj; + stars[i].lensing->E2_sh_obj = ps1data[i].E2_sh_obj; + + stars[i].lensing->X11_sm_psf = ps1data[i].X11_sm_psf; + stars[i].lensing->X12_sm_psf = ps1data[i].X12_sm_psf; + stars[i].lensing->X22_sm_psf = ps1data[i].X22_sm_psf; + stars[i].lensing->E1_sm_psf = ps1data[i].E1_sm_psf; + stars[i].lensing->E2_sm_psf = ps1data[i].E2_sm_psf; + + stars[i].lensing->X11_sh_psf = ps1data[i].X11_sh_psf; + stars[i].lensing->X12_sh_psf = ps1data[i].X12_sh_psf; + stars[i].lensing->X22_sh_psf = ps1data[i].X22_sh_psf; + stars[i].lensing->E1_sh_psf = ps1data[i].E1_sh_psf; + stars[i].lensing->E2_sh_psf = ps1data[i].E2_sh_psf; + + // stars[i].lensing->F_ApR5 = ps1data[i].F_ApR5; + // stars[i].lensing->dF_ApR5 = ps1data[i].dF_ApR5; + // stars[i].lensing->sF_ApR5 = ps1data[i].sF_ApR5; + // stars[i].lensing->fF_ApR5 = ps1data[i].fF_ApR5; + // + // stars[i].lensing->F_ApR6 = ps1data[i].F_ApR6; + // stars[i].lensing->dF_ApR6 = ps1data[i].dF_ApR6; + // stars[i].lensing->sF_ApR6 = ps1data[i].sF_ApR6; + // stars[i].lensing->fF_ApR6 = ps1data[i].fF_ApR6; + + // this is may optionally be replaced by the internal sequence (see FilterStars.c) + stars[i].lensing->detID = ps1data[i].detID; + + // the Average fields and the following Measure fields are set in FilterStars after + // the image metadata is in hand: dR, dD, Mcal, dt, airmass, az, t, imageID, extID. + + // averef is set in find_matches + + // dbFlags is zero on ingest. + + // the following fields are currently not being set anywhere: t_msec + } + *nstars = Nstars; + return (stars); +} + Stars *Convert_PS1_SV1 (FTable *table, unsigned int *nstars) { @@ -1304,5 +1443,5 @@ } -Stars *Convert_PS1_DV3 (FTable *table, unsigned int *nstars) { +Stars *Convert_PS1_SV4 (FTable *table, unsigned int *nstars) { off_t Nstars; @@ -1310,7 +1449,7 @@ double ZeroPt; Stars *stars; - CMF_PS1_DV3 *ps1data; - - ps1data = gfits_table_get_CMF_PS1_DV3 (table, &Nstars, NULL); + CMF_PS1_SV4 *ps1data; + + ps1data = gfits_table_get_CMF_PS1_SV4 (table, &Nstars, NULL); if (!ps1data) { fprintf (stderr, "skipping inconsistent entry\n"); @@ -1342,5 +1481,5 @@ stars[i].measure.Mkron = (ps1data[i].kronFlux > 0.0) ? -2.5*log10(ps1data[i].kronFlux) + ZeroPt : NAN; stars[i].measure.dMkron = (ps1data[i].kronFlux > 0.0) ? ps1data[i].kronFluxErr / ps1data[i].kronFlux : NAN; - + // these fluxes are converted from counts to counts/sec in FilterStars.c stars[i].measure.FluxPSF = GetFluxFromFluxOrMag (ps1data[i].Flux, ps1data[i].M); @@ -1357,4 +1496,5 @@ stars[i].measure.psfQF = ps1data[i].psfQF; stars[i].measure.psfQFperf = ps1data[i].psfQFperf; + stars[i].measure.psfNdof = ps1data[i].psfNdof; stars[i].measure.psfNpix = ps1data[i].psfNpix; @@ -1376,9 +1516,6 @@ // the Average fields and the following Measure fields are set in FilterStars after - // the image metadata is in hand: dR, dD, Mcal, dt, airmass, az, t, imageID, extID. - - // averef is set in find_matches - - // dbFlags is zero on ingest. + // the image metadata is in hand: dR, dD, Mcal, dt, airmass, az, t, imageID, extID, + // averef is set in find_matches, dbFlags is zero on ingest. // the following fields are currently not being set anywhere: t_msec @@ -1388,5 +1525,5 @@ } -Stars *Convert_PS1_DV4 (FTable *table, unsigned int *nstars) { +Stars *Convert_PS1_DV3 (FTable *table, unsigned int *nstars) { off_t Nstars; @@ -1394,7 +1531,7 @@ double ZeroPt; Stars *stars; - CMF_PS1_DV4 *ps1data; - - ps1data = gfits_table_get_CMF_PS1_DV4 (table, &Nstars, NULL); + CMF_PS1_DV3 *ps1data; + + ps1data = gfits_table_get_CMF_PS1_DV3 (table, &Nstars, NULL); if (!ps1data) { fprintf (stderr, "skipping inconsistent entry\n"); @@ -1402,6 +1539,4 @@ } ZeroPt = GetZeroPoint(); - - fprintf (stderr, "WARNING: Convert_PS1_DV4 not yet updated to match real format\n"); ALLOCATE (stars, Stars, Nstars); @@ -1474,4 +1609,90 @@ } - - +Stars *Convert_PS1_DV4 (FTable *table, unsigned int *nstars) { + + off_t Nstars; + unsigned int i; + double ZeroPt; + Stars *stars; + CMF_PS1_DV4 *ps1data; + + ps1data = gfits_table_get_CMF_PS1_DV4 (table, &Nstars, NULL); + if (!ps1data) { + fprintf (stderr, "skipping inconsistent entry\n"); + return (NULL); + } + ZeroPt = GetZeroPoint(); + + fprintf (stderr, "WARNING: Convert_PS1_DV4 not yet updated to match real format\n"); + + ALLOCATE (stars, Stars, Nstars); + for (i = 0; i < Nstars; i++) { + InitStar (&stars[i]); + stars[i].measure.Xccd = ps1data[i].X; + stars[i].measure.Yccd = ps1data[i].Y; + stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); + + stars[i].measure.posangle = ToShortDegrees(ps1data[i].posangle); + stars[i].measure.pltscale = ps1data[i].pltscale; + + if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) { + stars[i].measure.M = NAN; + } else { + stars[i].measure.M = ps1data[i].M + ZeroPt; + } + stars[i].measure.dM = ps1data[i].dM; + stars[i].measure.dMcal = ps1data[i].dMcal; + stars[i].measure.Map = ps1data[i].Map + ZeroPt; + stars[i].measure.dMap = (ps1data[i].apFlux > 0.0) ? fabs(ps1data[i].apFluxErr / ps1data[i].apFlux) : NAN; + + stars[i].measure.Mkron = (ps1data[i].kronFlux > 0.0) ? -2.5*log10(ps1data[i].kronFlux) + ZeroPt : NAN; + stars[i].measure.dMkron = (ps1data[i].kronFlux > 0.0) ? ps1data[i].kronFluxErr / ps1data[i].kronFlux : NAN; + + // these fluxes are converted from counts to counts/sec in FilterStars.c + stars[i].measure.FluxPSF = GetFluxFromFluxOrMag (ps1data[i].Flux, ps1data[i].M); + stars[i].measure.dFluxPSF = GetFluxErrFromFluxOrMag (ps1data[i].dFlux, stars[i].measure.FluxPSF, ps1data[i].dM); + stars[i].measure.FluxKron = ps1data[i].kronFlux; + stars[i].measure.dFluxKron = ps1data[i].kronFluxErr; + stars[i].measure.FluxAp = GetFluxFromFluxOrMag (ps1data[i].apFlux, ps1data[i].Map); + stars[i].measure.dFluxAp = GetFluxErrFromFluxOrMag (ps1data[i].apFluxErr, stars[i].measure.FluxAp, stars[i].measure.dMap); + + stars[i].measure.Sky = ps1data[i].sky; + stars[i].measure.dSky = ps1data[i].dSky; + + stars[i].measure.psfChisq = ps1data[i].psfChisq; + stars[i].measure.psfQF = ps1data[i].psfQF; + stars[i].measure.psfQFperf = ps1data[i].psfQFperf; + stars[i].measure.psfNdof = ps1data[i].psfNdof; + stars[i].measure.psfNpix = ps1data[i].psfNpix; + stars[i].measure.extNsigma = ps1data[i].extNsigma; + + stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); + stars[i].measure.theta = ToShortDegrees(ps1data[i].df); + + stars[i].measure.Mxx = ToShortPixels(ps1data[i].Mxx); + stars[i].measure.Mxy = ToShortPixels(ps1data[i].Mxy); + stars[i].measure.Myy = ToShortPixels(ps1data[i].Myy); + + stars[i].measure.photFlags = ps1data[i].flags; + stars[i].measure.photFlags2 = ps1data[i].flags2; + + // this is may optionally be replaced by the internal sequence (see FilterStars.c) + stars[i].measure.detID = ps1data[i].detID; + + // the Average fields and the following Measure fields are set in FilterStars after + // the image metadata is in hand: dR, dD, Mcal, dt, airmass, az, t, imageID, extID. + + // averef is set in find_matches + + // dbFlags is zero on ingest. + + // the following fields are currently not being set anywhere: t_msec + } + *nstars = Nstars; + return (stars); +} + + + Index: branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/args_loadwise.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/args_loadwise.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/args_loadwise.c (revision 37803) @@ -53,8 +53,19 @@ } +enum {MODE_NONE, MODE_ALLWISE, MODE_ALLSKY, MODE_PRELIM}; +int MODE; + /* load the prelim data dump */ - ALLSKY = TRUE; - if ((N = get_argument (argc, argv, "-prelim"))) { - ALLSKY = FALSE; + MODE = MODE_ALLWISE; + if ((N = get_argument (argc, argv, "-mode"))) { + MODE = MODE_NONE; + remove_argument (N, &argc, argv); + if (!strcasecmp(argv[N], "allwise")) MODE = MODE_ALLWISE; + if (!strcasecmp(argv[N], "allsky")) MODE = MODE_ALLSKY; + if (!strcasecmp(argv[N], "prelim")) MODE = MODE_PRELIM; + if (MODE == MODE_NONE) { + fprintf (stderr, "invalid mode: %s\n", argv[N]); + exit (2); + } remove_argument (N, &argc, argv); } Index: branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadsupercos_plates.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadsupercos_plates.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadsupercos_plates.c (revision 37803) @@ -92,5 +92,5 @@ image[Nimage].Radius = NAN; - snprintf (image[Nimage].name, 121, "SCOS.%02d.%06d", survey_id, plate_id); + snprintf (image[Nimage].name, DVO_IMAGE_NAME_LEN, "SCOS.%02d.%06d", survey_id, plate_id); image[Nimage].imageID = Nimage + 1; image[Nimage].externID = plate_id; Index: branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadwise.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadwise.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadwise.c (revision 37803) @@ -40,5 +40,5 @@ for (i = 1; i < argc; i++) { fprintf (stderr, "loading %s\n", argv[i]); - loadwise_rawdata (skylist, argv[i], options, ALLSKY); + loadwise_rawdata (skylist, argv[i], options); } exit (0); Index: branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadwise_rawdata.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadwise_rawdata.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadwise_rawdata.c (revision 37803) @@ -10,5 +10,5 @@ # define DEBUG 0 -int loadwise_rawdata (SkyList *skytable, char *filename, AddstarClientOptions options, int ALLSKY) { +int loadwise_rawdata (SkyList *skytable, char *filename, AddstarClientOptions options) { int i, j, verbose; @@ -159,8 +159,15 @@ stars[Nstars+3][0].average.D = tstars[j].D; - if (ALLSKY) { - loadwise_star_allsky (&stars[Nstars], &buffer[offset], Nbyte - offset); - } else { - loadwise_star_prelim (&stars[Nstars], &buffer[offset], Nbyte - offset); + switch (MODE) { + case MODE_PRELIM: + loadwise_star_prelim (&stars[Nstars], &buffer[offset], Nbyte - offset); + break; + case MODE_ALLSKY: + loadwise_star_allsky (&stars[Nstars], &buffer[offset], Nbyte - offset); + break; + case MODE_ALLWISE: + default: + // loadwise_star_allwise (&stars[Nstars], &buffer[offset], Nbyte - offset); + break; } Index: branches/eam_branches/ipp-20140904/Ohana/src/dvomerge/src/args.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/dvomerge/src/args.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/dvomerge/src/args.c (revision 37803) @@ -59,8 +59,8 @@ UPDATE_CATFORMAT = NULL; - if ((N = get_argument (argc, argv, "-update-catformat"))) { - remove_argument (N, &argc, argv); + if ((N = get_argument (*argc, argv, "-update-catformat"))) { + remove_argument (N, argc, argv); UPDATE_CATFORMAT = strcreate (argv[N]); - remove_argument (N, &argc, argv); + remove_argument (N, argc, argv); } Index: branches/eam_branches/ipp-20140904/Ohana/src/dvomerge/src/build_links.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/dvomerge/src/build_links.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/dvomerge/src/build_links.c (revision 37803) @@ -241,5 +241,8 @@ for (i = 0; i < Naverage; i++) { if (!average[i].Nlensing) continue; + off_t m = average[i].lensingOffset; + myAssert (lensing[m].averef == i, "not sorted"); for (j = 0; j < average[i].Nlensing - 1; j++, N++) { + myAssert (lensing[m+j+1].averef == i, "not sorted"); next_lens[N] = N + 1; if (N >= Nlensing) { @@ -329,5 +332,6 @@ } - if (m == -1) { + // if Nlensing is 0, m may have been mis-set; add to the end + if ((average[0].Nlensing == 0) || (m == -1)) { average[0].lensingOffset = Nlensing; } else { @@ -355,4 +359,5 @@ /* fix order of Lensing (memory intensive, but fast) */ + np = -1; N = 0; ALLOCATE (tmplensing, Lensing, Nlensing); Index: branches/eam_branches/ipp-20140904/Ohana/src/dvomerge/src/merge_catalogs_new.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/dvomerge/src/merge_catalogs_new.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/dvomerge/src/merge_catalogs_new.c (revision 37803) @@ -15,4 +15,5 @@ Naverage = output[0].Naverage; Nmeasure = output[0].Nmeasure; + Nlensing = output[0].Nlensing; NsecfiltOut = output[0].Nsecfilt; @@ -21,8 +22,10 @@ NAVERAGE = Naverage + 100; NMEASURE = Nmeasure + 1000; + NLENSING = Nlensing + 1000; REALLOCATE (output[0].average, Average, NAVERAGE); REALLOCATE (output[0].secfilt, SecFilt, NAVERAGE*NsecfiltOut); REALLOCATE (output[0].measure, Measure, NMEASURE); + REALLOCATE (output[0].lensing, Lensing, NLENSING); // add all of these entries to the existing catalog Index: branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/Makefile =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/Makefile (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/Makefile (revision 37803) @@ -30,4 +30,6 @@ $(SRC)/insert_objects_dvopsps.$(ARCH).o \ $(SRC)/insert_objects_dvopsps_catalog.$(ARCH).o \ +$(SRC)/insert_FWobjects_dvopsps.$(ARCH).o \ +$(SRC)/insert_FWobjects_dvopsps_catalog.$(ARCH).o \ $(SRC)/insert_detections_dvopsps.$(ARCH).o \ $(SRC)/insert_detections_dvopsps_catalog.$(ARCH).o @@ -46,4 +48,6 @@ $(SRC)/insert_objects_dvopsps.$(ARCH).o \ $(SRC)/insert_objects_dvopsps_catalog.$(ARCH).o \ +$(SRC)/insert_FWobjects_dvopsps.$(ARCH).o \ +$(SRC)/insert_FWobjects_dvopsps_catalog.$(ARCH).o \ $(SRC)/insert_detections_dvopsps.$(ARCH).o \ $(SRC)/insert_detections_dvopsps_catalog.$(ARCH).o Index: branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/include/dvopsps.h =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/include/dvopsps.h (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/include/dvopsps.h (revision 37803) @@ -102,8 +102,16 @@ int insert_objects_mysql_create_tables PROTO((char *basename, MYSQL *mysql)); - int insert_objects_mysql_commit PROTO((IOBuffer *ave_buffer, IOBuffer *sec_buffer, MYSQL *mysql)); int insert_objects_mysql_value PROTO((IOBuffer *ave_buffer, IOBuffer *sec_buffer, Average *average, SecFilt *secfilt, int Nsecfilt)); int insert_objects_mysql_init PROTO((IOBuffer *ave_buffer, IOBuffer *sec_buffer, char *basename)); + +int insert_FWobjects_dvopsps PROTO(()); +int insert_FWobjects_dvopsps_parallel PROTO((SkyList *sky)); +int insert_FWobjects_dvopsps_catalog PROTO((Catalog *catalog, char *basename, MYSQL *mysql)); + +int insert_FWobjects_mysql_create_tables PROTO((char *basename, MYSQL *mysql)); +int insert_FWobjects_mysql_init PROTO((IOBuffer *ave_buffer, IOBuffer *sec_buffer, IOBuffer *cpy_buffer, char *basename)); +int insert_FWobjects_mysql_value PROTO((IOBuffer *ave_buffer, IOBuffer *sec_buffer, IOBuffer *cpy_buffer, Average *average, SecFilt *secfilt, int Nsecfilt, Lensobj *lensobj, int Nlensobj)); +int insert_FWobjects_mysql_commit PROTO((IOBuffer *ave_buffer, IOBuffer *sec_buffer, IOBuffer *cpy_buffer, MYSQL *mysql)); int insert_skytable PROTO(()); Index: branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/dvopsps.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/dvopsps.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/dvopsps.c (revision 37803) @@ -14,4 +14,7 @@ if (!strcasecmp (argv[1], "objects")) { status = insert_objects_dvopsps (); + } + if (!strcasecmp (argv[1], "forced_warp_objects")) { + status = insert_FWobjects_dvopsps (); } if (!strcasecmp (argv[1], "skytable")) { Index: branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/dvopsps_client.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/dvopsps_client.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/dvopsps_client.c (revision 37803) @@ -22,4 +22,7 @@ status = insert_objects_dvopsps (); } + if (!strcasecmp (argv[1], "forced_warp_objects")) { + status = insert_FWobjects_dvopsps (); + } if (status == -1) { fprintf (stderr, "invalid mode, should be : detections, skytable\n"); Index: branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/initialize_dvopsps.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/initialize_dvopsps.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/initialize_dvopsps.c (revision 37803) @@ -21,5 +21,5 @@ fprintf (stderr, "\n"); - fprintf (stderr, " (mode) is one of detections, objects, skytable\n"); + fprintf (stderr, " (mode) is one of detections, objects, forced_warp_objects, skytable\n"); fprintf (stderr, "\n"); Index: branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps.c (revision 37803) +++ branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps.c (revision 37803) @@ -0,0 +1,175 @@ +# include "dvopsps.h" +# define USE_MYSQL 1 + +// determine the relevant catalogs, launch parallel clients if desired +int insert_FWobjects_dvopsps () { + + SkyTable *sky = NULL; + SkyList *skylist = NULL; + Catalog catalog; + off_t i; + + // load the current sky table (layout of all SkyRegions) + sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, -1, VERBOSE); + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // determine the populated SkyRegions overlapping the requested area (default depth) + if (SINGLE_CPT) { + skylist = SkyRegionByCPT (sky, SINGLE_CPT); + } else { + skylist = SkyListByPatch (sky, -1, &UserPatch); + } + myAssert (skylist, "ooops!"); + + // pass the skylist to the parallel version, and only launch on relevant hosts + if (PARALLEL && !HOST_ID) { + int status = insert_FWobjects_dvopsps_parallel (skylist); + return status; + } + +# if (USE_MYSQL) + // NOTE: mysql connection happens here since each dvopsps_client makes its own connection + MYSQL mysqlBase; + MYSQL *mysqlReal = mysql_dvopsps_connect (&mysqlBase); + if (!mysqlReal) { + fprintf (stderr, "failed to connect to mysql\n"); + exit (1); + } +# else + MYSQL *mysqlReal = NULL; +# endif + + // select measurements for each populated catalog + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue; + + char localFilename[1024]; + snprintf (localFilename, 1024, "%s/%s.cpt", HOSTDIR, skylist->regions[i]->name); + + // set up the basic catalog info + catalog.filename = HOST_ID ? localFilename : skylist[0].filename[i]; + catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs + catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs + catalog.catflags = LOAD_AVES | LOAD_SECF | LOAD_LENSOBJ; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "r")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naves_disk) { + if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + // NOTE: this is where the real action happens + insert_FWobjects_dvopsps_catalog (&catalog, skylist->regions[i]->name, mysqlReal); + + // NOTE : unlike setastrom or relphot, this program is read-only wrt dvo + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + } + + if (0) { + char query[256]; + MYSQL_RES *result; + + sprintf (query, "commit;"); + if (mysql_query (mysqlReal, query)) { + fprintf (stderr, "failed to turn off autocommit\n"); + fprintf (stderr, "%s\n", mysql_error (mysqlReal)); + return FALSE; + } + result = mysql_store_result (mysqlReal); + mysql_free_result (result); + } + + return (TRUE); +} + +# define DEBUG 1 + +int insert_FWobjects_dvopsps_parallel (SkyList *sky) { + + // launch the dvopsps_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + + // if sky->Nregions < 0.5*Nhosts, check if the host used before launching... + + int i, j; + for (i = 0; i < table->Nhosts; i++) { + + if (sky->Nregions < table->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + char command[1024]; + snprintf (command, 1024, "dvopsps_client forced_warp_objects -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", + table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, + UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + char tmpline[1024]; + snprintf (tmpline, 1024, "%s -dbhost %s", command, DATABASE_HOST); strcpy (command, tmpline); + snprintf (tmpline, 1024, "%s -dbuser %s", command, DATABASE_USER); strcpy (command, tmpline); + snprintf (tmpline, 1024, "%s -dbpass %s", command, DATABASE_PASS); strcpy (command, tmpline); + snprintf (tmpline, 1024, "%s -dbname %s", command, DATABASE_NAME); strcpy (command, tmpline); + + if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } + if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) continue; + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running dvopsps_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + } + + // wait for the remote jobs to be completed + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the dvopsps_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + if (!status) return FALSE; + } + + return (TRUE); +} Index: branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps_catalog.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps_catalog.c (revision 37803) +++ branches/eam_branches/ipp-20140904/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps_catalog.c (revision 37803) @@ -0,0 +1,489 @@ +# include "dvopsps.h" +# define DEBUG 0 + +int insert_FWobjects_dvopsps_catalog (Catalog *catalog, char *basename, MYSQL *mysql) { + + off_t i; + int missingID = 0; + + IOBuffer ave_buffer, sec_buffer, cpy_buffer; + ave_buffer.Nalloc = 0; + sec_buffer.Nalloc = 0; + cpy_buffer.Nalloc = 0; + + Average *average = catalog->average; + SecFilt *secfilt = catalog->secfilt; + Lensobj *lensobj = catalog->lensobj; + int Nsecfilt = catalog->Nsecfilt; + + off_t found = 0; + + char *cleanname = strcreate (basename); + for (i = 0; i < strlen(cleanname); i++) { + if (cleanname[i] == '.') cleanname[i] = '_'; + if (cleanname[i] == '/') cleanname[i] = '_'; + } + insert_FWobjects_mysql_create_tables (cleanname, mysql); + + INITTIME; + insert_FWobjects_mysql_init (&ave_buffer, &sec_buffer, &cpy_buffer, cleanname); + int Ninsert = 0; + + // NOTE for testing, just do a few objects + if (!mysql) { + catalog[0].Naverage = 5; + } + for (i = 0; i < catalog[0].Naverage; i++) { + + if (average[i].extID == 0) { + missingID ++; + } + + off_t m = average[i].lensobjOffset; + off_t Nlensobj = average[i].Nlensobj; + + // XXX check return status + insert_FWobjects_mysql_value (&ave_buffer, &sec_buffer, &cpy_buffer, + &average[i], &secfilt[i*Nsecfilt], Nsecfilt, &lensobj[m], Nlensobj); + + // fprintf (stderr, "%f : %d %d %d\n", average[i].ChiSqPM, isinf(average[i].ChiSqPM), isnan(average[i].ChiSqPM), isfinite(average[i].ChiSqPM)); + + // Average is a bigger table, but Nsecfilt*secfilt might be bigger, so check both + int fullBuffer = FALSE; + fullBuffer = fullBuffer || (ave_buffer.Nbuffer > MAX_BUFFER); + fullBuffer = fullBuffer || (sec_buffer.Nbuffer > MAX_BUFFER); + fullBuffer = fullBuffer || (cpy_buffer.Nbuffer > MAX_BUFFER); + + if (fullBuffer) { + insert_FWobjects_mysql_commit (&ave_buffer, &sec_buffer, &cpy_buffer, mysql); + if (DEBUG) fprintf (stderr, "inserted %d rows\n", Ninsert); + Ninsert = 0; + if (0) { + FlushIOBuffer (&ave_buffer); + FlushIOBuffer (&sec_buffer); + FlushIOBuffer (&cpy_buffer); + } else { + ave_buffer.Nbuffer = 0; + bzero (ave_buffer.buffer, ave_buffer.Nalloc); + sec_buffer.Nbuffer = 0; + bzero (sec_buffer.buffer, sec_buffer.Nalloc); + cpy_buffer.Nbuffer = 0; + bzero (cpy_buffer.buffer, cpy_buffer.Nalloc); + } + insert_FWobjects_mysql_init (&ave_buffer, &sec_buffer, &cpy_buffer, cleanname); + } + Ninsert ++; + found ++; + } + insert_FWobjects_mysql_commit (&ave_buffer, &sec_buffer, &cpy_buffer, mysql); + if (VERBOSE) fprintf (stderr, "inserted "OFF_T_FMT" average, "OFF_T_FMT" secfilt\n", + catalog[0].Naverage, Nsecfilt*catalog[0].Naverage); + FreeIOBuffer (&ave_buffer); + FreeIOBuffer (&sec_buffer); + FreeIOBuffer (&cpy_buffer); + free (cleanname); + + MARKTIME("-- inserted "OFF_T_FMT" objects in %f sec, skipped %d without IDs\n", found, dtime, missingID); + return (TRUE); +} + +int insert_FWobjects_mysql_create_tables (char *basename, MYSQL *mysql) { + + int status; + IOBuffer buffer; + InitIOBuffer (&buffer, 1024); + // MYSQL_RES *result; + + PrintIOBuffer (&buffer, "DROP TABLE %s_cpt\n", basename); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to drop table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + } + buffer.Nbuffer = 0; + bzero (buffer.buffer, buffer.Nalloc); + + // Only send the necessary fields (eg, do not sent parallax and pm) + PrintIOBuffer (&buffer, "CREATE TABLE %s_cpt (" + "EXT_ID BIGINT, " + "OBJ_ID INT, " + "CAT_ID INT " + ")\n", basename); + + if (DEBUG) fprintf (stderr, "%s\n", buffer.buffer); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to create table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer); + } + buffer.Nbuffer = 0; + bzero (buffer.buffer, buffer.Nalloc); + + PrintIOBuffer (&buffer, "DROP TABLE %s_cps\n", basename); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to drop table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + } + buffer.Nbuffer = 0; + bzero (buffer.buffer, buffer.Nalloc); + + PrintIOBuffer (&buffer, "CREATE TABLE %s_cps (" + "NWARP SMALLINT, " + "NUSED_WRP SMALLINT, " + "NUSED_KRON_WRP SMALLINT, " + "NUSED_AP_WRP SMALLINT, " + + "FLUX_PSF_WRP FLOAT, " + "FLUX_PSF_WRP_ERR FLOAT, " + "FLUX_PSF_WRP_STDEV FLOAT, " + + "FLUX_KRON_WRP FLOAT, " + "FLUX_KRON_WRP_ERR FLOAT, " + "FLUX_KRON_WRP_STDEV FLOAT, " + + "FLUX_AP_WRP FLOAT, " + "FLUX_AP_WRP_ERR FLOAT, " + "FLUX_AP_WRP_STDEV FLOAT, " + + "FLAGS INT " + ")\n", basename); + + if (DEBUG) fprintf (stderr, "%s\n", buffer.buffer); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to create table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer); + } + + PrintIOBuffer (&buffer, "DROP TABLE %s_cpy\n", basename); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to drop table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + } + buffer.Nbuffer = 0; + bzero (buffer.buffer, buffer.Nalloc); + + PrintIOBuffer (&buffer, "CREATE TABLE %s_cpy (" + "OBJ_ID INT, " + "CAT_ID INT, " + "PHOTCODE SMALLINT, " + + "FLUX_AP_R5, FLOAT, " + "FLUX_ERR_AP_R5, FLOAT, " + "FLUX_STD_AP_R5, FLOAT, " + "FLUX_FIL_AP_R5, FLOAT, " + "FLUX_AP_R6, FLOAT, " + "FLUX_ERR_AP_R6, FLOAT, " + "FLUX_STD_AP_R6, FLOAT, " + "FLUX_FIL_AP_R6, FLOAT, " + + "X11_SM_OBJ, FLOAT, " + "X12_SM_OBJ, FLOAT, " + "X22_SM_OBJ, FLOAT, " + "E1_SM_OBJ, FLOAT, " + "E2_SM_OBJ, FLOAT, " + "X11_SH_OBJ, FLOAT, " + "X12_SH_OBJ, FLOAT, " + "X22_SH_OBJ, FLOAT, " + "E1_SH_OBJ, FLOAT, " + "E2_SH_OBJ, FLOAT, " + + "X11_SM_PSF, FLOAT, " + "X12_SM_PSF, FLOAT, " + "X22_SM_PSF, FLOAT, " + "E1_SM_PSF, FLOAT, " + "E2_SM_PSF, FLOAT, " + "X11_SH_PSF, FLOAT, " + "X12_SH_PSF, FLOAT, " + "X22_SH_PSF, FLOAT, " + "E1_SH_PSF, FLOAT, " + "E2_SH_PSF, FLOAT, " + + "GAMMA, FLOAT, " + "E1, FLOAT, " + "E2, FLOAT, " + + "NMEAS INT " + ")\n", basename); + + if (DEBUG) fprintf (stderr, "%s\n", buffer.buffer); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to create table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer); + } + + FreeIOBuffer (&buffer); + return TRUE; +} + +int insert_FWobjects_mysql_init (IOBuffer *ave_buffer, IOBuffer *sec_buffer, IOBuffer *cpy_buffer, char *basename) { + + if (ave_buffer->Nalloc == 0) { + InitIOBuffer (ave_buffer, 1024); + } + PrintIOBuffer (ave_buffer, "INSERT INTO %s_cpt (" + "EXT_ID, " + "OBJ_ID, " + "CAT_ID " + ") VALUES \n", basename); + + if (sec_buffer->Nalloc == 0) { + InitIOBuffer (sec_buffer, 1024); + } + PrintIOBuffer (sec_buffer, "INSERT INTO %s_cps (" + "NWARP, " + "NUSED_WRP, " + "NUSED_KRON_WRP, " + "NUSED_AP_WRP, " + + "FLUX_PSF_WRP, " + "FLUX_PSF_WRP_ERR, " + "FLUX_PSF_WRP_STDEV, " + + "FLUX_KRON_WRP, " + "FLUX_KRON_WRP_ERR, " + "FLUX_KRON_WRP_STDEV, " + + "FLUX_AP_WRP, " + "FLUX_AP_WRP_ERR, " + "FLUX_AP_WRP_STDEV, " + + "FLAGS" + ") VALUES \n", basename); + + if (cpy_buffer->Nalloc == 0) { + InitIOBuffer (cpy_buffer, 1024); + } + PrintIOBuffer (cpy_buffer, "INSERT INTO %s_cpy (" + "OBJ_ID, " + "CAT_ID, " + "PHOTCODE, " + + "FLUX_AP_R5, " + "FLUX_ERR_AP_R5, " + "FLUX_STD_AP_R5, " + "FLUX_FIL_AP_R5, " + "FLUX_AP_R6, " + "FLUX_ERR_AP_R6, " + "FLUX_STD_AP_R6, " + "FLUX_FIL_AP_R6, " + + "X11_SM_OBJ, " + "X12_SM_OBJ, " + "X22_SM_OBJ, " + "E1_SM_OBJ, " + "E2_SM_OBJ, " + "X11_SH_OBJ, " + "X12_SH_OBJ, " + "X22_SH_OBJ, " + "E1_SH_OBJ, " + "E2_SH_OBJ, " + + "X11_SM_PSF, " + "X12_SM_PSF, " + "X22_SM_PSF, " + "E1_SM_PSF, " + "E2_SM_PSF, " + "X11_SH_PSF, " + "X12_SH_PSF, " + "X22_SH_PSF, " + "E1_SH_PSF, " + "E2_SH_PSF, " + + "GAMMA, " + "E1, " + "E2, " + "NMEAS" + ") VALUES \n", basename); + + return TRUE; +} + +# define PRINT_FLOAT(BUFFER,FIELD,FORMAT) \ + if (isinf(FIELD) || isnan(FIELD)) PrintIOBuffer (BUFFER, "NULL, "); \ + else PrintIOBuffer (BUFFER, FORMAT, FIELD); + +int insert_FWobjects_mysql_value (IOBuffer *ave_buffer, IOBuffer *sec_buffer, IOBuffer *cpy_buffer, Average *average, SecFilt *secfilt, int Nsecfilt, Lensobj *lensobj, int Nlensobj) { + + int i; + + PrintIOBuffer (ave_buffer, " ("); + PrintIOBuffer (ave_buffer, "%lu, ", average->extID); + PrintIOBuffer (ave_buffer, "%u, ", average->objID); + PrintIOBuffer (ave_buffer, "%u, ", average->catID); + PrintIOBuffer (ave_buffer, "),\n"); + + // XXX what rules for keeping or NAN-ing various mags? + for (i = 0; i < Nsecfilt; i++) { + float meanPSFFlux = -999.0; + float meanPSFFluxErr = -999.0; + float meanPSFFluxStd = -999.0; + if (isfinite(secfilt->dFpsfWrp) && isfinite(secfilt->FpsfWrp)) { + meanPSFFlux = secfilt-> FpsfWrp; + meanPSFFluxErr = secfilt->dFpsfWrp; + meanPSFFluxStd = secfilt->sFpsfWrp; + } + + float meanKronFlux = -999.0; + float meanKronFluxErr = -999.0; + float meanKronFluxStd = -999.0; + if (isfinite(secfilt->dFkronWrp) && isfinite(secfilt->FkronWrp)) { + meanKronFlux = secfilt-> FkronWrp; + meanKronFluxErr = secfilt->dFkronWrp; + meanKronFluxStd = secfilt->sFkronWrp; + } + + float meanApFlux = -999.0; + float meanApFluxErr = -999.0; + float meanApFluxStd = -999.0; + if (isfinite(secfilt->dFapWrp) && isfinite(secfilt->FapWrp)) { + meanApFlux = secfilt-> FapWrp; + meanApFluxErr = secfilt->dFapWrp; + meanApFluxStd = secfilt->sFapWrp; + } + + PrintIOBuffer (sec_buffer, " ("); + PrintIOBuffer (sec_buffer, "%hd, ", secfilt->Nwarp); + PrintIOBuffer (sec_buffer, "%hd, ", secfilt->NusedWrp); + PrintIOBuffer (sec_buffer, "%hd, ", secfilt->NusedKronWrp); + PrintIOBuffer (sec_buffer, "%hd, ", secfilt->NusedApWrp); + + // use %e? + PRINT_FLOAT(sec_buffer, meanPSFFlux, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanPSFFluxErr, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanPSFFluxStd, "%.6f, "); // uflux precision + + PRINT_FLOAT(sec_buffer, meanKronFlux, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanKronFluxErr, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanKronFluxStd, "%.6f, "); // uflux precision + + PRINT_FLOAT(sec_buffer, meanApFlux, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanApFluxErr, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanApFluxStd, "%.6f, "); // uflux precision + + PrintIOBuffer (sec_buffer, "%u ", secfilt->flags); + PrintIOBuffer (sec_buffer, "),\n"); + secfilt ++; + } + + for (i = 0; i < Nlensobj; i++) { + PrintIOBuffer (cpy_buffer, " ("); + PrintIOBuffer (cpy_buffer, "%u, ", lensobj->objID); + PrintIOBuffer (cpy_buffer, "%u, ", lensobj->catID); + PrintIOBuffer (cpy_buffer, "%hd, ", lensobj->photcode); + + PRINT_FLOAT(cpy_buffer, lensobj-> F_ApR5, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->dF_ApR5, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->sF_ApR5, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->fF_ApR5, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj-> F_ApR6, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->dF_ApR6, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->sF_ApR6, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->fF_ApR6, "%.8f"); + + PRINT_FLOAT(cpy_buffer, lensobj->X11_sm_obj, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->X12_sm_obj, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->X22_sm_obj, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->E1_sm_obj, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->E2_sm_obj, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->X11_sh_obj, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->X12_sh_obj, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->X22_sh_obj, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->E1_sh_obj, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->E2_sh_obj, "%.8f"); + + PRINT_FLOAT(cpy_buffer, lensobj->X11_sm_psf, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->X12_sm_psf, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->X22_sm_psf, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->E1_sm_psf, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->E2_sm_psf, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->X11_sh_psf, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->X12_sh_psf, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->X22_sh_psf, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->E1_sh_psf, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->E2_sh_psf, "%.8f"); + + PRINT_FLOAT(cpy_buffer, lensobj->gamma, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->E1, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj->E2, "%.8f"); + + PrintIOBuffer (cpy_buffer, "%u ", lensobj->Nmeas); // NOTE: PRINT_FLOAT always adds a trailing comma -- need to use a different method on the last entry + PrintIOBuffer (cpy_buffer, "),\n"); + lensobj ++; + } + + return TRUE; +} + +int insert_FWobjects_mysql_commit (IOBuffer *ave_buffer, IOBuffer *sec_buffer, IOBuffer *cpy_buffer, MYSQL *mysql) { + + MYSQL_RES *result; + + // check that the last two chars are ,\n and replace with ;\n + if (!strcmp(&ave_buffer->buffer[ave_buffer->Nbuffer-2], ",\n")) { + ave_buffer->buffer[ave_buffer->Nbuffer-2] = ';'; + } else { + fprintf (stderr, "invalid sql?\n"); + return FALSE; + } + + // check that the last two chars are ,\n and replace with ;\n + if (!strcmp(&sec_buffer->buffer[sec_buffer->Nbuffer-2], ",\n")) { + sec_buffer->buffer[sec_buffer->Nbuffer-2] = ';'; + } else { + fprintf (stderr, "invalid sql?\n"); + return FALSE; + } + + // check that the last two chars are ,\n and replace with ;\n + if (!strcmp(&cpy_buffer->buffer[cpy_buffer->Nbuffer-2], ",\n")) { + cpy_buffer->buffer[cpy_buffer->Nbuffer-2] = ';'; + } else { + fprintf (stderr, "invalid sql?\n"); + return FALSE; + } + + // XXX check return status + if (mysql) { + int status; + if (DEBUG) fprintf (stderr, "%s\n", ave_buffer->buffer); + status = mysql_query(mysql, ave_buffer->buffer); + if (status) { + fprintf (stderr, "error with insert:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", ave_buffer->Nbuffer); + } + result = mysql_store_result (mysql); + if (result) mysql_free_result (result); + + if (DEBUG) fprintf (stderr, "%s\n", sec_buffer->buffer); + status = mysql_query(mysql, sec_buffer->buffer); + if (status) { + fprintf (stderr, "error with insert:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", sec_buffer->Nbuffer); + } + result = mysql_store_result (mysql); + if (result) mysql_free_result (result); + + if (DEBUG) fprintf (stderr, "%s\n", cpy_buffer->buffer); + status = mysql_query(mysql, cpy_buffer->buffer); + if (status) { + fprintf (stderr, "error with insert:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", cpy_buffer->Nbuffer); + } + result = mysql_store_result (mysql); + if (result) mysql_free_result (result); + } else { + fprintf (stderr, "%s\n", sec_buffer->buffer); + } + + return TRUE; +} Index: branches/eam_branches/ipp-20140904/Ohana/src/kapa2/src/InterpretKeys.c =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/kapa2/src/InterpretKeys.c (revision 37802) +++ branches/eam_branches/ipp-20140904/Ohana/src/kapa2/src/InterpretKeys.c (revision 37803) @@ -50,10 +50,11 @@ Z = -1; - if (graph) { + int haveGraph = graph && graph->haveGraph; + if (haveGraph) { X = (event[0].x - graph[0].axis[0].fx)*(graph[0].axis[0].max - graph[0].axis[0].min)/graph[0].axis[0].dfx + graph[0].axis[0].min; Y = (event[0].y - graph[0].axis[1].fy)*(graph[0].axis[1].max - graph[0].axis[1].min)/graph[0].axis[1].dfy + graph[0].axis[1].min; XY_to_RD (&R, &D, X, Y, &graph[0].data.coords); } - if (image && !graph) { + if (image && !haveGraph) { if (event[0].x < image[0].picture.x) goto skip_cursor; if (event[0].y < image[0].picture.y) goto skip_cursor; Index: branches/eam_branches/ipp-20140904/Ohana/src/libautocode/def/cmf-ps1-sv4.d =================================================================== --- branches/eam_branches/ipp-20140904/Ohana/src/libautocode/def/cmf-ps1-sv4.d (revision 37803) +++ branches/eam_branches/ipp-20140904/Ohana/src/libautocode/def/cmf-ps1-sv4.d (revision 37803) @@ -0,0 +1,72 @@ +# name of structure type +STRUCT CMF_PS1_SV4 +EXTNAME CMF_PS1_SV4 +TYPE BINTABLE +SIZE 224 + +# elements of data structure / FITS table +FIELD detID, IPP_IDET, unsigned int, detection ID +FIELD X, X_PSF, float, x coord, pixels +FIELD Y, Y_PSF, float, y coord, pixels +FIELD dX, X_PSF_SIG, float, x coord error, pixels +FIELD dY, Y_PSF_SIG, float, y coord error, pixels +FIELD posangle, POSANGLE, float, Posangle at source, degrees +FIELD pltscale, PLTSCALE, float, Plate Scale at source, arcsec/pixel +FIELD M, PSF_INST_MAG, float, inst mags, mags +FIELD dM, PSF_INST_MAG_SIG, float, inst mag error, mags +FIELD Flux, PSF_INST_FLUX, float, psf flux, counts +FIELD dFlux, PSF_INST_FLUX_SIG, float, psf flux error, counts +FIELD Map, AP_MAG, float, standard aperture mag, mags +FIELD MapRaw, AP_MAG_RAW, float, raw aperture mag, mags +FIELD apRadius, AP_MAG_RADIUS, float, radius used for fit, pixels +FIELD apFlux, AP_FLUX, float, aperture flux, counts +FIELD apFluxErr, AP_FLUX_SIG, float, error on ap flux, counts +FIELD apNpix, AP_NPIX, int, pixels used for aper +FIELD Mcalib, CAL_PSF_MAG, float, calibrated psf mag, mags +FIELD dMcal, CAL_PSF_MAG_SIG, float, zero point scatter, mags + +# this field is in the wrong order (is between DEC and sky in cmf, but breaks byte boundary). +# I need to move these bytes around on read (fix PS1_V5?) +FIELD Mpeak, PEAK_FLUX_AS_MAG, float, peak flux as a mag, mags + +# NOTE: RA & DEC (both double) need to be on an 8-byte boundary... +FIELD RA, RA_PSF, double, PSF RA coord, degrees +FIELD DEC, DEC_PSF, double, PSF DEC coord, degrees + +FIELD sky, SKY, float, sky flux, cnts/sec +FIELD dSky, SKY_SIG, float, sky flux error, cnts/sec +FIELD psfChisq, PSF_CHISQ, float, psf fit chisq + +FIELD crNsigma, CR_NSIGMA, float, Nsigma deviations from PSF to CF +FIELD extNsigma, EXT_NSIGMA, float, Nsigma deviations from PSF to EXT +FIELD fx, PSF_MAJOR, float, psf fit major axis, pixels +FIELD fy, PSF_MINOR, float, psf fit minor axis, pixels +FIELD df, PSF_THETA, float, ellipse angle, degrees +FIELD k, PSF_CORE, float, extra PSF parameter, unitless +FIELD fwhmMaj, PSF_FWHM_MAJ, float, true fwhm of psf, pixels +FIELD fwhmMin, PSF_FWHM_MIN, float, true fwhm (minor), pixels +FIELD psfQF, PSF_QF, float, quality factor +FIELD psfQFperf, PSF_QF_PERFECT, float, quality factor perfect +FIELD psfNdof, PSF_NDOF, int, psf degrees of freedom +FIELD psfNpix, PSF_NPIX, int, psf number of pixels + +FIELD Mxx, MOMENTS_XX, float, second moment X, pixels^2 +FIELD Mxy, MOMENTS_XY, float, second moment Y, pixels^2 +FIELD Myy, MOMENTS_YY, float, second moment XY, pixels^2 +FIELD M3c, MOMENTS_M3C, float, third moment cos(t), pixels^3 +FIELD M3s, MOMENTS_M3S, float, third moment sin(t), pixels^3 +FIELD M4c, MOMENTS_M4C, float, fourth moment cos(t), pixels^4 +FIELD M4s, MOMENTS_M4S, float, fourth moment sin(t), pixels^4 +FIELD Mr1, MOMENTS_R1, float, first radial moment, pixels +FIELD Mrh, MOMENTS_RH, float, half radial moment, pixels^1/2 +FIELD kronFlux, KRON_FLUX, float, kron flux, counts +FIELD kronFluxErr, KRON_FLUX_ERR, float, kron flux error, counts +FIELD kronInner, KRON_FLUX_INNER, float, kron flux 1