Index: /branches/eam_branches/ipp-20150112/Ohana/Makefile.in =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/Makefile.in (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/Makefile.in (revision 38059) @@ -30,4 +30,5 @@ addstar \ delstar \ +dvolens \ dvosplit \ dvomerge \ Index: /branches/eam_branches/ipp-20150112/Ohana/configure.tcsh =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/configure.tcsh (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/configure.tcsh (revision 38059) @@ -230,5 +230,5 @@ set syslibpath = "/lib /usr/lib /usr/X11R6/lib /usr/local/lib" -set xtrlibpath = `checkpaths.pl lib` +set xtrlibpath = `./checkpaths.pl lib` set syslibpath = "$syslibpath $xtrlibpath" @@ -249,5 +249,5 @@ set sysincpath = "/usr/include /usr/local/include /usr/X11R6/include" -set xtrincpath = `checkpaths.pl include` +set xtrincpath = `./checkpaths.pl include` set sysincpath = "$sysincpath $xtrincpath" Index: /branches/eam_branches/ipp-20150112/Ohana/src/addstar/src/MatchHeaders.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/addstar/src/MatchHeaders.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/addstar/src/MatchHeaders.c (revision 38059) @@ -70,4 +70,5 @@ if (!strcmp (exttype, "PS1_DV3")) goto keep; if (!strcmp (exttype, "PS1_DV4")) goto keep; + if (!strcmp (exttype, "PS1_DV5")) goto keep; continue; Index: /branches/eam_branches/ipp-20150112/Ohana/src/addstar/src/ReadStarsFITS.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/addstar/src/ReadStarsFITS.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/addstar/src/ReadStarsFITS.c (revision 38059) @@ -4,22 +4,24 @@ // (excluding SDSS data and reference database info, such as 2MASS) // NOTE: these must also be listed in MatchHeaders.c (line ~ 62) -Stars *Convert_SMPDATA PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_DEV_0 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_DEV_1 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_V1 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_V1_Alt PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_V2 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_V3 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_V4 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_V5 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_V5_Lensing PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_V5_Lensing_Alt PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_SV1 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_SV1_Alt PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_SV2 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_SV3 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_SV4 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_DV3 PROTO((FTable *table, unsigned int *nstars)); -Stars *Convert_PS1_DV4 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_SMPDATA PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_DEV_0 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_DEV_1 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_V1 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_V1_Alt PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_V2 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_V3 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_V4 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_V5 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_V5_R0_Lensing PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_V5_R1_Lensing PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_V5_R2_Lensing PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_SV1 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_SV1_Alt PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_SV2 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_SV3 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_SV4 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_DV3 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_DV4 PROTO((FTable *table, unsigned int *nstars)); +Stars *Convert_PS1_DV5 PROTO((FTable *table, unsigned int *nstars)); // given a file with the pointer at the start of the table block and the @@ -85,8 +87,11 @@ break; case 312: - stars = Convert_PS1_V5_Lensing_Alt (&table, &Nstars); + stars = Convert_PS1_V5_R0_Lensing (&table, &Nstars); break; case 320: - stars = Convert_PS1_V5_Lensing (&table, &Nstars); + stars = Convert_PS1_V5_R1_Lensing (&table, &Nstars); + break; + case 328: + stars = Convert_PS1_V5_R2_Lensing (&table, &Nstars); break; default: @@ -112,4 +117,7 @@ if (!strcmp (type, "PS1_DV4")) { stars = Convert_PS1_DV4 (&table, &Nstars); + } + if (!strcmp (type, "PS1_DV5")) { + stars = Convert_PS1_DV5 (&table, &Nstars); } if (stars == NULL) { @@ -860,5 +868,5 @@ } -Stars *Convert_PS1_V5_Lensing (FTable *table, unsigned int *nstars) { +Stars *Convert_PS1_V5_R0_Lensing (FTable *table, unsigned int *nstars) { off_t Nstars; @@ -866,7 +874,7 @@ double ZeroPt; Stars *stars; - CMF_PS1_V5_Lensing *ps1data; - - ps1data = gfits_table_get_CMF_PS1_V5_Lensing (table, &Nstars, NULL); + CMF_PS1_V5_R0_Lensing *ps1data; + + ps1data = gfits_table_get_CMF_PS1_V5_R0_Lensing (table, &Nstars, NULL); if (!ps1data) { fprintf (stderr, "skipping inconsistent entry\n"); @@ -985,5 +993,5 @@ } -Stars *Convert_PS1_V5_Lensing_Alt (FTable *table, unsigned int *nstars) { +Stars *Convert_PS1_V5_R1_Lensing (FTable *table, unsigned int *nstars) { off_t Nstars; @@ -991,7 +999,7 @@ double ZeroPt; Stars *stars; - CMF_PS1_V5_Lensing_Alt *ps1data; - - ps1data = gfits_table_get_CMF_PS1_V5_Lensing_Alt (table, &Nstars, NULL); + CMF_PS1_V5_R1_Lensing *ps1data; + + ps1data = gfits_table_get_CMF_PS1_V5_R1_Lensing (table, &Nstars, NULL); if (!ps1data) { fprintf (stderr, "skipping inconsistent entry\n"); @@ -1110,5 +1118,5 @@ } -Stars *Convert_PS1_SV1 (FTable *table, unsigned int *nstars) { +Stars *Convert_PS1_V5_R2_Lensing (FTable *table, unsigned int *nstars) { off_t Nstars; @@ -1116,12 +1124,7 @@ double ZeroPt; Stars *stars; - CMF_PS1_SV1 *ps1data; - - if (table[0].header[0].Naxis[0] == 196) { - stars = Convert_PS1_SV1_Alt (table, nstars); - return (stars); - } - - ps1data = gfits_table_get_CMF_PS1_SV1 (table, &Nstars, NULL); + CMF_PS1_V5_R2_Lensing *ps1data; + + ps1data = gfits_table_get_CMF_PS1_V5_R2_Lensing (table, &Nstars, NULL); if (!ps1data) { fprintf (stderr, "skipping inconsistent entry\n"); @@ -1149,4 +1152,137 @@ stars[i].measure.dMcal = ps1data[i].dMcal; stars[i].measure.Map = ps1data[i].Map + ZeroPt; + stars[i].measure.dMap = (ps1data[i].apFlux > 0.0) ? fabs(ps1data[i].apFluxErr / ps1data[i].apFlux) : NAN; + + stars[i].measure.Mkron = (ps1data[i].kronFlux > 0.0) ? -2.5*log10(ps1data[i].kronFlux) + ZeroPt : NAN; + stars[i].measure.dMkron = (ps1data[i].kronFlux > 0.0) ? ps1data[i].kronFluxErr / ps1data[i].kronFlux : NAN; + + // these fluxes are converted from counts to counts/sec in FilterStars.c + stars[i].measure.FluxPSF = GetFluxFromFluxOrMag (ps1data[i].Flux, ps1data[i].M); + stars[i].measure.dFluxPSF = GetFluxErrFromFluxOrMag (ps1data[i].dFlux, stars[i].measure.FluxPSF, ps1data[i].dM); + stars[i].measure.FluxKron = ps1data[i].kronFlux; + stars[i].measure.dFluxKron = ps1data[i].kronFluxErr; + stars[i].measure.FluxAp = GetFluxFromFluxOrMag (ps1data[i].apFlux, ps1data[i].Map); + stars[i].measure.dFluxAp = GetFluxErrFromFluxOrMag (ps1data[i].apFluxErr, stars[i].measure.FluxAp, stars[i].measure.dMap); + + stars[i].measure.Sky = ps1data[i].sky; + stars[i].measure.dSky = ps1data[i].dSky; + + stars[i].measure.psfChisq = ps1data[i].psfChisq; + stars[i].measure.psfQF = ps1data[i].psfQF; + stars[i].measure.psfQFperf = ps1data[i].psfQFperf; + + stars[i].measure.psfNdof = ps1data[i].psfNdof; + stars[i].measure.psfNpix = ps1data[i].psfNpix; + stars[i].measure.extNsigma = ps1data[i].extNsigma; + + stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); + stars[i].measure.theta = ToShortDegrees(ps1data[i].df); + + stars[i].measure.Mxx = ToShortPixels(ps1data[i].Mxx); + stars[i].measure.Mxy = ToShortPixels(ps1data[i].Mxy); + stars[i].measure.Myy = ToShortPixels(ps1data[i].Myy); + + stars[i].measure.photFlags = ps1data[i].flags; + stars[i].measure.photFlags2 = ps1data[i].flags2; + + // this is may optionally be replaced by the internal sequence (see FilterStars.c) + stars[i].measure.detID = ps1data[i].detID; + + ALLOCATE (stars[i].lensing, Lensing, 1); + dvo_lensing_init (stars[i].lensing); + + stars[i].lensing->X11_sm_obj = ps1data[i].X11_sm_obj; + stars[i].lensing->X12_sm_obj = ps1data[i].X12_sm_obj; + stars[i].lensing->X22_sm_obj = ps1data[i].X22_sm_obj; + stars[i].lensing->E1_sm_obj = ps1data[i].E1_sm_obj; + stars[i].lensing->E2_sm_obj = ps1data[i].E2_sm_obj; + + stars[i].lensing->X11_sh_obj = ps1data[i].X11_sh_obj; + stars[i].lensing->X12_sh_obj = ps1data[i].X12_sh_obj; + stars[i].lensing->X22_sh_obj = ps1data[i].X22_sh_obj; + stars[i].lensing->E1_sh_obj = ps1data[i].E1_sh_obj; + stars[i].lensing->E2_sh_obj = ps1data[i].E2_sh_obj; + + stars[i].lensing->X11_sm_psf = ps1data[i].X11_sm_psf; + stars[i].lensing->X12_sm_psf = ps1data[i].X12_sm_psf; + stars[i].lensing->X22_sm_psf = ps1data[i].X22_sm_psf; + stars[i].lensing->E1_sm_psf = ps1data[i].E1_sm_psf; + stars[i].lensing->E2_sm_psf = ps1data[i].E2_sm_psf; + + stars[i].lensing->X11_sh_psf = ps1data[i].X11_sh_psf; + stars[i].lensing->X12_sh_psf = ps1data[i].X12_sh_psf; + stars[i].lensing->X22_sh_psf = ps1data[i].X22_sh_psf; + stars[i].lensing->E1_sh_psf = ps1data[i].E1_sh_psf; + stars[i].lensing->E2_sh_psf = ps1data[i].E2_sh_psf; + + stars[i].lensing->E1_psf = ps1data[i].E1_psf; + stars[i].lensing->E2_psf = ps1data[i].E2_psf; + + // stars[i].lensing->F_ApR5 = ps1data[i].F_ApR5; + // stars[i].lensing->dF_ApR5 = ps1data[i].dF_ApR5; + // stars[i].lensing->sF_ApR5 = ps1data[i].sF_ApR5; + // stars[i].lensing->fF_ApR5 = ps1data[i].fF_ApR5; + // + // stars[i].lensing->F_ApR6 = ps1data[i].F_ApR6; + // stars[i].lensing->dF_ApR6 = ps1data[i].dF_ApR6; + // stars[i].lensing->sF_ApR6 = ps1data[i].sF_ApR6; + // stars[i].lensing->fF_ApR6 = ps1data[i].fF_ApR6; + + // this is may optionally be replaced by the internal sequence (see FilterStars.c) + stars[i].lensing->detID = ps1data[i].detID; + + // the Average fields and the following Measure fields are set in FilterStars after + // the image metadata is in hand: dR, dD, Mcal, dt, airmass, az, t, imageID, extID. + + // averef is set in find_matches + + // dbFlags is zero on ingest. + + // the following fields are currently not being set anywhere: t_msec + } + *nstars = Nstars; + return (stars); +} + +Stars *Convert_PS1_SV1 (FTable *table, unsigned int *nstars) { + + off_t Nstars; + unsigned int i; + double ZeroPt; + Stars *stars; + CMF_PS1_SV1 *ps1data; + + if (table[0].header[0].Naxis[0] == 196) { + stars = Convert_PS1_SV1_Alt (table, nstars); + return (stars); + } + + ps1data = gfits_table_get_CMF_PS1_SV1 (table, &Nstars, NULL); + if (!ps1data) { + fprintf (stderr, "skipping inconsistent entry\n"); + return (NULL); + } + ZeroPt = GetZeroPoint(); + + ALLOCATE (stars, Stars, Nstars); + for (i = 0; i < Nstars; i++) { + InitStar (&stars[i]); + stars[i].measure.Xccd = ps1data[i].X; + stars[i].measure.Yccd = ps1data[i].Y; + stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); + + stars[i].measure.posangle = ToShortDegrees(ps1data[i].posangle); + stars[i].measure.pltscale = ps1data[i].pltscale; + + if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) { + stars[i].measure.M = NAN; + } else { + stars[i].measure.M = ps1data[i].M + ZeroPt; + } + stars[i].measure.dM = ps1data[i].dM; + stars[i].measure.dMcal = ps1data[i].dMcal; + stars[i].measure.Map = ps1data[i].Map + ZeroPt; stars[i].measure.dMap = ps1data[i].dM; // a proxy measure @@ -1695,4 +1831,88 @@ } - - +Stars *Convert_PS1_DV5 (FTable *table, unsigned int *nstars) { + + off_t Nstars; + unsigned int i; + double ZeroPt; + Stars *stars; + CMF_PS1_DV5 *ps1data; + + ps1data = gfits_table_get_CMF_PS1_DV5 (table, &Nstars, NULL); + if (!ps1data) { + fprintf (stderr, "skipping inconsistent entry\n"); + return (NULL); + } + ZeroPt = GetZeroPoint(); + + ALLOCATE (stars, Stars, Nstars); + for (i = 0; i < Nstars; i++) { + InitStar (&stars[i]); + stars[i].measure.Xccd = ps1data[i].X; + stars[i].measure.Yccd = ps1data[i].Y; + stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); + + stars[i].measure.posangle = ToShortDegrees(ps1data[i].posangle); + stars[i].measure.pltscale = ps1data[i].pltscale; + + if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) { + stars[i].measure.M = NAN; + } else { + stars[i].measure.M = ps1data[i].M + ZeroPt; + } + stars[i].measure.dM = ps1data[i].dM; + stars[i].measure.dMcal = ps1data[i].dMcal; + stars[i].measure.Map = ps1data[i].Map + ZeroPt; + stars[i].measure.dMap = (ps1data[i].apFlux > 0.0) ? fabs(ps1data[i].apFluxErr / ps1data[i].apFlux) : NAN; + + stars[i].measure.Mkron = (ps1data[i].kronFlux > 0.0) ? -2.5*log10(ps1data[i].kronFlux) + ZeroPt : NAN; + stars[i].measure.dMkron = (ps1data[i].kronFlux > 0.0) ? ps1data[i].kronFluxErr / ps1data[i].kronFlux : NAN; + + // these fluxes are converted from counts to counts/sec in FilterStars.c + stars[i].measure.FluxPSF = GetFluxFromFluxOrMag (ps1data[i].Flux, ps1data[i].M); + stars[i].measure.dFluxPSF = GetFluxErrFromFluxOrMag (ps1data[i].dFlux, stars[i].measure.FluxPSF, ps1data[i].dM); + stars[i].measure.FluxKron = ps1data[i].kronFlux; + stars[i].measure.dFluxKron = ps1data[i].kronFluxErr; + stars[i].measure.FluxAp = GetFluxFromFluxOrMag (ps1data[i].apFlux, ps1data[i].Map); + stars[i].measure.dFluxAp = GetFluxErrFromFluxOrMag (ps1data[i].apFluxErr, stars[i].measure.FluxAp, stars[i].measure.dMap); + + stars[i].measure.Sky = ps1data[i].sky; + stars[i].measure.dSky = ps1data[i].dSky; + + stars[i].measure.psfChisq = ps1data[i].psfChisq; + stars[i].measure.psfQF = ps1data[i].psfQF; + stars[i].measure.psfQFperf = ps1data[i].psfQFperf; + stars[i].measure.psfNdof = ps1data[i].psfNdof; + stars[i].measure.psfNpix = ps1data[i].psfNpix; + stars[i].measure.extNsigma = ps1data[i].extNsigma; + + stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); + stars[i].measure.theta = ToShortDegrees(ps1data[i].df); + + stars[i].measure.Mxx = ToShortPixels(ps1data[i].Mxx); + stars[i].measure.Mxy = ToShortPixels(ps1data[i].Mxy); + stars[i].measure.Myy = ToShortPixels(ps1data[i].Myy); + + stars[i].measure.photFlags = ps1data[i].flags; + stars[i].measure.photFlags2 = ps1data[i].flags2; + + // this is may optionally be replaced by the internal sequence (see FilterStars.c) + stars[i].measure.detID = ps1data[i].detID; + + // the Average fields and the following Measure fields are set in FilterStars after + // the image metadata is in hand: dR, dD, Mcal, dt, airmass, az, t, imageID, extID. + + // averef is set in find_matches + + // dbFlags is zero on ingest. + + // the following fields are currently not being set anywhere: t_msec + } + *nstars = Nstars; + return (stars); +} + + + Index: /branches/eam_branches/ipp-20150112/Ohana/src/addstar/src/mkcmf.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/addstar/src/mkcmf.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/addstar/src/mkcmf.c (revision 38059) @@ -1003,10 +1003,10 @@ int i; - CMF_PS1_V5_Lensing *stars; + CMF_PS1_V5_R2_Lensing *stars; float flux, fSN; // XXX add gaussian-distributed noise based on counts // this needs to make different output 'stars' entries depending on the desired type - ALLOCATE (stars, CMF_PS1_V5_Lensing, Nstars); + ALLOCATE (stars, CMF_PS1_V5_R2_Lensing, Nstars); for (i = 0; i < Nstars; i++) { @@ -1129,5 +1129,5 @@ } - gfits_table_set_CMF_PS1_V5_Lensing (ftable, stars, Nstars); + gfits_table_set_CMF_PS1_V5_R2_Lensing (ftable, stars, Nstars); gfits_modify (ftable->header, "EXTTYPE", "%s", 1, "PS1_V5"); } Index: /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/Makefile =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/Makefile (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/Makefile (revision 38059) @@ -32,4 +32,6 @@ $(SRC)/insert_FWobjects_dvopsps.$(ARCH).o \ $(SRC)/insert_FWobjects_dvopsps_catalog.$(ARCH).o \ +$(SRC)/insert_diffobj_dvopsps.$(ARCH).o \ +$(SRC)/insert_diffobj_dvopsps_catalog.$(ARCH).o \ $(SRC)/insert_detections_dvopsps.$(ARCH).o \ $(SRC)/insert_detections_dvopsps_catalog.$(ARCH).o @@ -50,4 +52,6 @@ $(SRC)/insert_FWobjects_dvopsps.$(ARCH).o \ $(SRC)/insert_FWobjects_dvopsps_catalog.$(ARCH).o \ +$(SRC)/insert_diffobj_dvopsps.$(ARCH).o \ +$(SRC)/insert_diffobj_dvopsps_catalog.$(ARCH).o \ $(SRC)/insert_detections_dvopsps.$(ARCH).o \ $(SRC)/insert_detections_dvopsps_catalog.$(ARCH).o Index: /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/include/dvopsps.h =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/include/dvopsps.h (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/include/dvopsps.h (revision 38059) @@ -121,4 +121,13 @@ int insert_skytable_mysql_init PROTO((IOBuffer *buffer)); +int insert_diffobj_dvopsps PROTO(()); +int insert_diffobj_dvopsps_parallel PROTO((SkyList *sky)); +int insert_diffobj_dvopsps_catalog PROTO((Catalog *catalog, char *basename, MYSQL *mysql)); + +int insert_diffobj_mysql_create_tables PROTO((char *basename, MYSQL *mysql)); +int insert_diffobj_mysql_init PROTO((IOBuffer *ave_buffer, IOBuffer *sec_buffer, char *basename)); +int insert_diffobj_mysql_value PROTO((IOBuffer *ave_buffer, IOBuffer *sec_buffer, Average *average, SecFilt *secfilt, int Nsecfilt)); +int insert_diffobj_mysql_commit PROTO((IOBuffer *ave_buffer, IOBuffer *sec_buffer, MYSQL *mysql)); + Detections *DetectionsLoad PROTO((char *filename, int *Ndetections)); int DetectionsSave PROTO((char *filename, Detections *detections, int Ndetections)); Index: /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/dvopsps.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/dvopsps.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/dvopsps.c (revision 38059) @@ -14,4 +14,7 @@ if (!strcasecmp (argv[1], "objects")) { status = insert_objects_dvopsps (); + } + if (!strcasecmp (argv[1], "diffobj")) { + status = insert_diffobj_dvopsps (); } if (!strcasecmp (argv[1], "forced_warp_objects")) { Index: /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/dvopsps_client.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/dvopsps_client.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/dvopsps_client.c (revision 38059) @@ -22,4 +22,7 @@ status = insert_objects_dvopsps (); } + if (!strcasecmp (argv[1], "diffobj")) { + status = insert_diffobj_dvopsps (); + } if (!strcasecmp (argv[1], "forced_warp_objects")) { status = insert_FWobjects_dvopsps (); Index: /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps_catalog.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps_catalog.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/insert_FWobjects_dvopsps_catalog.c (revision 38059) @@ -6,4 +6,5 @@ off_t i; int missingID = 0; + int noLensobj = 0; IOBuffer ave_buffer, sec_buffer, cpy_buffer; @@ -42,4 +43,10 @@ off_t m = average[i].lensobjOffset; off_t Nlensobj = average[i].Nlensobj; + + // skip objects with no lensing object + if (!Nlensobj) { + noLensobj ++; + continue; + } // XXX check return status @@ -76,7 +83,9 @@ found ++; } - insert_FWobjects_mysql_commit (&ave_buffer, &sec_buffer, &cpy_buffer, mysql); - if (VERBOSE) fprintf (stderr, "inserted "OFF_T_FMT" average, "OFF_T_FMT" secfilt\n", - catalog[0].Naverage, Nsecfilt*catalog[0].Naverage); + if (Ninsert) { + insert_FWobjects_mysql_commit (&ave_buffer, &sec_buffer, &cpy_buffer, mysql); + } + if (VERBOSE) fprintf (stderr, "inserted "OFF_T_FMT" average, "OFF_T_FMT" secfilt, "OFF_T_FMT" lensobj\n", + catalog[0].Naverage, Nsecfilt*catalog[0].Naverage, catalog[0].Nlensobj); FreeIOBuffer (&ave_buffer); FreeIOBuffer (&sec_buffer); @@ -84,5 +93,5 @@ free (cleanname); - MARKTIME("-- inserted "OFF_T_FMT" objects in %f sec, skipped %d without IDs\n", found, dtime, missingID); + MARKTIME("-- inserted "OFF_T_FMT" objects in %f sec, skipped %d without IDs, %d with no lensobj\n", found, dtime, missingID, noLensobj); return (TRUE); } @@ -158,4 +167,6 @@ fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer); } + buffer.Nbuffer = 0; + bzero (buffer.buffer, buffer.Nalloc); PrintIOBuffer (&buffer, "DROP TABLE %s_cpy\n", basename); @@ -173,38 +184,38 @@ "PHOTCODE SMALLINT, " - "FLUX_AP_R5, FLOAT, " - "FLUX_ERR_AP_R5, FLOAT, " - "FLUX_STD_AP_R5, FLOAT, " - "FLUX_FIL_AP_R5, FLOAT, " - "FLUX_AP_R6, FLOAT, " - "FLUX_ERR_AP_R6, FLOAT, " - "FLUX_STD_AP_R6, FLOAT, " - "FLUX_FIL_AP_R6, FLOAT, " - - "X11_SM_OBJ, FLOAT, " - "X12_SM_OBJ, FLOAT, " - "X22_SM_OBJ, FLOAT, " - "E1_SM_OBJ, FLOAT, " - "E2_SM_OBJ, FLOAT, " - "X11_SH_OBJ, FLOAT, " - "X12_SH_OBJ, FLOAT, " - "X22_SH_OBJ, FLOAT, " - "E1_SH_OBJ, FLOAT, " - "E2_SH_OBJ, FLOAT, " - - "X11_SM_PSF, FLOAT, " - "X12_SM_PSF, FLOAT, " - "X22_SM_PSF, FLOAT, " - "E1_SM_PSF, FLOAT, " - "E2_SM_PSF, FLOAT, " - "X11_SH_PSF, FLOAT, " - "X12_SH_PSF, FLOAT, " - "X22_SH_PSF, FLOAT, " - "E1_SH_PSF, FLOAT, " - "E2_SH_PSF, FLOAT, " - - "GAMMA, FLOAT, " - "E1, FLOAT, " - "E2, FLOAT, " + "FLUX_AP_R5 FLOAT, " + "FLUX_ERR_AP_R5 FLOAT, " + "FLUX_STD_AP_R5 FLOAT, " + "FLUX_FIL_AP_R5 FLOAT, " + "FLUX_AP_R6 FLOAT, " + "FLUX_ERR_AP_R6 FLOAT, " + "FLUX_STD_AP_R6 FLOAT, " + "FLUX_FIL_AP_R6 FLOAT, " + + "X11_SM_OBJ FLOAT, " + "X12_SM_OBJ FLOAT, " + "X22_SM_OBJ FLOAT, " + "E1_SM_OBJ FLOAT, " + "E2_SM_OBJ FLOAT, " + "X11_SH_OBJ FLOAT, " + "X12_SH_OBJ FLOAT, " + "X22_SH_OBJ FLOAT, " + "E1_SH_OBJ FLOAT, " + "E2_SH_OBJ FLOAT, " + + "X11_SM_PSF FLOAT, " + "X12_SM_PSF FLOAT, " + "X22_SM_PSF FLOAT, " + "E1_SM_PSF FLOAT, " + "E2_SM_PSF FLOAT, " + "X11_SH_PSF FLOAT, " + "X12_SH_PSF FLOAT, " + "X22_SH_PSF FLOAT, " + "E1_SH_PSF FLOAT, " + "E2_SH_PSF FLOAT, " + + "GAMMA FLOAT, " + "E1 FLOAT, " + "E2 FLOAT, " "NMEAS INT " @@ -317,5 +328,5 @@ PrintIOBuffer (ave_buffer, "%lu, ", average->extID); PrintIOBuffer (ave_buffer, "%u, ", average->objID); - PrintIOBuffer (ave_buffer, "%u, ", average->catID); + PrintIOBuffer (ave_buffer, "%u ", average->catID); PrintIOBuffer (ave_buffer, "),\n"); @@ -379,38 +390,38 @@ PrintIOBuffer (cpy_buffer, "%hd, ", lensobj->photcode); - PRINT_FLOAT(cpy_buffer, lensobj-> F_ApR5, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->dF_ApR5, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->sF_ApR5, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->fF_ApR5, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj-> F_ApR6, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->dF_ApR6, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->sF_ApR6, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->fF_ApR6, "%.8f"); - - PRINT_FLOAT(cpy_buffer, lensobj->X11_sm_obj, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->X12_sm_obj, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->X22_sm_obj, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->E1_sm_obj, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->E2_sm_obj, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->X11_sh_obj, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->X12_sh_obj, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->X22_sh_obj, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->E1_sh_obj, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->E2_sh_obj, "%.8f"); - - PRINT_FLOAT(cpy_buffer, lensobj->X11_sm_psf, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->X12_sm_psf, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->X22_sm_psf, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->E1_sm_psf, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->E2_sm_psf, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->X11_sh_psf, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->X12_sh_psf, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->X22_sh_psf, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->E1_sh_psf, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->E2_sh_psf, "%.8f"); - - PRINT_FLOAT(cpy_buffer, lensobj->gamma, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->E1, "%.8f"); - PRINT_FLOAT(cpy_buffer, lensobj->E2, "%.8f"); + PRINT_FLOAT(cpy_buffer, lensobj-> F_ApR5, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->dF_ApR5, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->sF_ApR5, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->fF_ApR5, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj-> F_ApR6, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->dF_ApR6, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->sF_ApR6, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->fF_ApR6, "%.8f, "); + + PRINT_FLOAT(cpy_buffer, lensobj->X11_sm_obj, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->X12_sm_obj, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->X22_sm_obj, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->E1_sm_obj, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->E2_sm_obj, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->X11_sh_obj, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->X12_sh_obj, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->X22_sh_obj, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->E1_sh_obj, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->E2_sh_obj, "%.8f, "); + + PRINT_FLOAT(cpy_buffer, lensobj->X11_sm_psf, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->X12_sm_psf, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->X22_sm_psf, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->E1_sm_psf, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->E2_sm_psf, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->X11_sh_psf, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->X12_sh_psf, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->X22_sh_psf, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->E1_sh_psf, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->E2_sh_psf, "%.8f, "); + + PRINT_FLOAT(cpy_buffer, lensobj->gamma, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->E1, "%.8f, "); + PRINT_FLOAT(cpy_buffer, lensobj->E2, "%.8f, "); PrintIOBuffer (cpy_buffer, "%u ", lensobj->Nmeas); // NOTE: PRINT_FLOAT always adds a trailing comma -- need to use a different method on the last entry Index: /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/insert_diffobj_dvopsps.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/insert_diffobj_dvopsps.c (revision 38059) +++ /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/insert_diffobj_dvopsps.c (revision 38059) @@ -0,0 +1,175 @@ +# include "dvopsps.h" +# define USE_MYSQL 1 + +// determine the relevant catalogs, launch parallel clients if desired +int insert_diffobj_dvopsps () { + + SkyTable *sky = NULL; + SkyList *skylist = NULL; + Catalog catalog; + off_t i; + + // load the current sky table (layout of all SkyRegions) + sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, -1, VERBOSE); + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // determine the populated SkyRegions overlapping the requested area (default depth) + if (SINGLE_CPT) { + skylist = SkyRegionByCPT (sky, SINGLE_CPT); + } else { + skylist = SkyListByPatch (sky, -1, &UserPatch); + } + myAssert (skylist, "ooops!"); + + // pass the skylist to the parallel version, and only launch on relevant hosts + if (PARALLEL && !HOST_ID) { + int status = insert_diffobj_dvopsps_parallel (skylist); + return status; + } + +# if (USE_MYSQL) + // NOTE: mysql connection happens here since each dvopsps_client makes its own connection + MYSQL mysqlBase; + MYSQL *mysqlReal = mysql_dvopsps_connect (&mysqlBase); + if (!mysqlReal) { + fprintf (stderr, "failed to connect to mysql\n"); + exit (1); + } +# else + MYSQL *mysqlReal = NULL; +# endif + + // select measurements for each populated catalog + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue; + + char localFilename[1024]; + snprintf (localFilename, 1024, "%s/%s.cpt", HOSTDIR, skylist->regions[i]->name); + + // set up the basic catalog info + catalog.filename = HOST_ID ? localFilename : skylist[0].filename[i]; + catalog.catformat = DVO_FORMAT_UNDEF; // not needed since we skip empty catalogs + catalog.catmode = DVO_MODE_UNDEF; // not needed since we skip empty catalogs + catalog.catflags = LOAD_AVES | LOAD_SECF; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "r")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naves_disk) { + if (VERBOSE) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + // NOTE: this is where the real action happens + insert_diffobj_dvopsps_catalog (&catalog, skylist->regions[i]->name, mysqlReal); + + // NOTE : unlike setastrom or relphot, this program is read-only wrt dvo + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + } + + if (0) { + char query[256]; + MYSQL_RES *result; + + sprintf (query, "commit;"); + if (mysql_query (mysqlReal, query)) { + fprintf (stderr, "failed to turn off autocommit\n"); + fprintf (stderr, "%s\n", mysql_error (mysqlReal)); + return FALSE; + } + result = mysql_store_result (mysqlReal); + mysql_free_result (result); + } + + return (TRUE); +} + +# define DEBUG 1 + +int insert_diffobj_dvopsps_parallel (SkyList *sky) { + + // launch the dvopsps_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + + // if sky->Nregions < 0.5*Nhosts, check if the host used before launching... + + int i, j; + for (i = 0; i < table->Nhosts; i++) { + + if (sky->Nregions < table->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + char command[1024]; + snprintf (command, 1024, "dvopsps_client diffobj -hostID %d -catdir %s -hostdir %s -region %f %f %f %f", + table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, + UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + char tmpline[1024]; + snprintf (tmpline, 1024, "%s -dbhost %s", command, DATABASE_HOST); strcpy (command, tmpline); + snprintf (tmpline, 1024, "%s -dbuser %s", command, DATABASE_USER); strcpy (command, tmpline); + snprintf (tmpline, 1024, "%s -dbpass %s", command, DATABASE_PASS); strcpy (command, tmpline); + snprintf (tmpline, 1024, "%s -dbname %s", command, DATABASE_NAME); strcpy (command, tmpline); + + if (VERBOSE) { snprintf (tmpline, 1024, "%s -v", command); strcpy (command, tmpline); } + if (SINGLE_CPT) { snprintf (tmpline, 1024, "%s -cpt %s", command, SINGLE_CPT); strcpy (command, tmpline); } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) continue; + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running dvopsps_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + } + + // wait for the remote jobs to be completed + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the dvopsps_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + if (!status) return FALSE; + } + + return (TRUE); +} Index: /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/insert_diffobj_dvopsps_catalog.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/insert_diffobj_dvopsps_catalog.c (revision 38059) +++ /branches/eam_branches/ipp-20150112/Ohana/src/dvopsps/src/insert_diffobj_dvopsps_catalog.c (revision 38059) @@ -0,0 +1,327 @@ +# include "dvopsps.h" +# define DEBUG 0 + +int insert_diffobj_dvopsps_catalog (Catalog *catalog, char *basename, MYSQL *mysql) { + + off_t i; + int missingID = 0; + + IOBuffer ave_buffer, sec_buffer; + ave_buffer.Nalloc = 0; + sec_buffer.Nalloc = 0; + + Average *average = catalog->average; + SecFilt *secfilt = catalog->secfilt; + int Nsecfilt = catalog->Nsecfilt; + + off_t found = 0; + + char *cleanname = strcreate (basename); + for (i = 0; i < strlen(cleanname); i++) { + if (cleanname[i] == '.') cleanname[i] = '_'; + if (cleanname[i] == '/') cleanname[i] = '_'; + } + insert_diffobj_mysql_create_tables (cleanname, mysql); + + INITTIME; + insert_diffobj_mysql_init (&ave_buffer, &sec_buffer, cleanname); + int Ninsert = 0; + + // NOTE for testing, just do a few objects + if (!mysql) { + catalog[0].Naverage = 5; + } + for (i = 0; i < catalog[0].Naverage; i++) { + + if (average[i].extID == 0) { + missingID ++; + } + + // XXX check return status + insert_diffobj_mysql_value (&ave_buffer, &sec_buffer, &average[i], &secfilt[i*Nsecfilt], Nsecfilt); + + // fprintf (stderr, "%f : %d %d %d\n", average[i].ChiSqPM, isinf(average[i].ChiSqPM), isnan(average[i].ChiSqPM), isfinite(average[i].ChiSqPM)); + + // Average is a bigger table, but Nsecfilt*secfilt might be bigger, so check both + if ((ave_buffer.Nbuffer > MAX_BUFFER) || (sec_buffer.Nbuffer > MAX_BUFFER)) { + insert_diffobj_mysql_commit (&ave_buffer, &sec_buffer, mysql); + if (DEBUG) fprintf (stderr, "inserted %d rows\n", Ninsert); + Ninsert = 0; + if (0) { + FlushIOBuffer (&ave_buffer); + FlushIOBuffer (&sec_buffer); + } else { + ave_buffer.Nbuffer = 0; + bzero (ave_buffer.buffer, ave_buffer.Nalloc); + sec_buffer.Nbuffer = 0; + bzero (sec_buffer.buffer, sec_buffer.Nalloc); + } + insert_diffobj_mysql_init (&ave_buffer, &sec_buffer, cleanname); + } + Ninsert ++; + found ++; + } + insert_diffobj_mysql_commit (&ave_buffer, &sec_buffer, mysql); + if (VERBOSE) fprintf (stderr, "inserted "OFF_T_FMT" average, "OFF_T_FMT" secfilt\n", catalog[0].Naverage, Nsecfilt*catalog[0].Naverage); + FreeIOBuffer (&ave_buffer); + FreeIOBuffer (&sec_buffer); + free (cleanname); + + MARKTIME("-- inserted "OFF_T_FMT" diffobj in %f sec, skipped %d without IDs\n", found, dtime, missingID); + return (TRUE); +} + +int insert_diffobj_mysql_create_tables (char *basename, MYSQL *mysql) { + + int status; + IOBuffer buffer; + InitIOBuffer (&buffer, 1024); + // MYSQL_RES *result; + + PrintIOBuffer (&buffer, "DROP TABLE %s_cpt\n", basename); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to drop table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + } + buffer.Nbuffer = 0; + bzero (buffer.buffer, buffer.Nalloc); + + // Only send the necessary fields (eg, do not sent parallax and pm) + PrintIOBuffer (&buffer, "CREATE TABLE %s_cpt (" + "EXT_ID BIGINT, " + "OBJ_ID INT, " + "CAT_ID INT, " + "RA_MEAN DOUBLE, " + "DEC_MEAN DOUBLE, " + "RA_ERR FLOAT, " + "DEC_ERR FLOAT, " + "FLAGS INT " + ")\n", basename); + + if (DEBUG) fprintf (stderr, "%s\n", buffer.buffer); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to create table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer); + } + buffer.Nbuffer = 0; + bzero (buffer.buffer, buffer.Nalloc); + + PrintIOBuffer (&buffer, "DROP TABLE %s_cps\n", basename); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to drop table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + } + buffer.Nbuffer = 0; + bzero (buffer.buffer, buffer.Nalloc); + + PrintIOBuffer (&buffer, "CREATE TABLE %s_cps (" + "NWARP SMALLINT, " + "NUSED_WRP SMALLINT, " + "NUSED_KRON_WRP SMALLINT, " + "NUSED_AP_WRP SMALLINT, " + "PSF_QF_PERF_MAX FLOAT, " + + "FLUX_PSF_WRP FLOAT, " + "FLUX_PSF_WRP_ERR FLOAT, " + "FLUX_PSF_WRP_STDEV FLOAT, " + + "FLUX_KRON_WRP FLOAT, " + "FLUX_KRON_WRP_ERR FLOAT, " + "FLUX_KRON_WRP_STDEV FLOAT, " + + "FLUX_AP_WRP FLOAT, " + "FLUX_AP_WRP_ERR FLOAT, " + "FLUX_AP_WRP_STDEV FLOAT, " + + "FLAGS INT " + ")\n", basename); + + if (DEBUG) fprintf (stderr, "%s\n", buffer.buffer); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to create table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer); + } + // result = mysql_store_result (mysql); + // mysql_free_result (result); + + FreeIOBuffer (&buffer); + return TRUE; +} + +int insert_diffobj_mysql_init (IOBuffer *ave_buffer, IOBuffer *sec_buffer, char *basename) { + + if (ave_buffer->Nalloc == 0) { + InitIOBuffer (ave_buffer, 1024); + } + PrintIOBuffer (ave_buffer, "INSERT INTO %s_cpt (" + "EXT_ID, " + "OBJ_ID, " + "CAT_ID, " + "RA_MEAN, " + "DEC_MEAN, " + "RA_ERR, " + "DEC_ERR, " + "FLAGS " + ") VALUES \n", basename); + + if (sec_buffer->Nalloc == 0) { + InitIOBuffer (sec_buffer, 1024); + } + PrintIOBuffer (sec_buffer, "INSERT INTO %s_cps (" + "NWARP, " + "NUSED_WRP, " + "NUSED_KRON_WRP, " + "NUSED_AP_WRP, " + "PSF_QF_PERF_MAX, " + + "FLUX_PSF_WRP, " + "FLUX_PSF_WRP_ERR, " + "FLUX_PSF_WRP_STDEV, " + + "FLUX_KRON_WRP, " + "FLUX_KRON_WRP_ERR, " + "FLUX_KRON_WRP_STDEV, " + + "FLUX_AP_WRP, " + "FLUX_AP_WRP_ERR, " + "FLUX_AP_WRP_STDEV, " + + "FLAGS" + ") VALUES \n", basename); + + return TRUE; +} + +# define PRINT_FLOAT(BUFFER,FIELD,FORMAT) \ + if (isinf(FIELD) || isnan(FIELD)) PrintIOBuffer (BUFFER, "NULL, "); \ + else PrintIOBuffer (BUFFER, FORMAT, FIELD); + +int insert_diffobj_mysql_value (IOBuffer *ave_buffer, IOBuffer *sec_buffer, Average *average, SecFilt *secfilt, int Nsecfilt) { + + int i; + +// XXX this bit could/should be autocoded... + PrintIOBuffer (ave_buffer, " ("); + PrintIOBuffer (ave_buffer, OFF_T_FMT", ", average->extID); + PrintIOBuffer (ave_buffer, "%u, ", average->objID); + PrintIOBuffer (ave_buffer, "%u, ", average->catID); + + PRINT_FLOAT(ave_buffer, average->R, "%.8f, "); // 0.036 mas precision + PRINT_FLOAT(ave_buffer, average->D, "%.8f, "); // 0.036 mas precision + PRINT_FLOAT(ave_buffer, average->dR, "%.5f, "); // 0.010 mas precision + PRINT_FLOAT(ave_buffer, average->dD, "%.5f, "); // 0.010 mas precision + + PrintIOBuffer (ave_buffer, "%u ", average->flags); + PrintIOBuffer (ave_buffer, "),\n"); + + // XXX what rules for keeping or NAN-ing various mags? + for (i = 0; i < Nsecfilt; i++) { + float meanPSFFlux = -999.0; + float meanPSFFluxErr = -999.0; + float meanPSFFluxStd = -999.0; + if (isfinite(secfilt->dFpsfWrp) && isfinite(secfilt->FpsfWrp)) { + meanPSFFlux = secfilt-> FpsfWrp; + meanPSFFluxErr = secfilt->dFpsfWrp; + meanPSFFluxStd = secfilt->sFpsfWrp; + } + + float meanKronFlux = -999.0; + float meanKronFluxErr = -999.0; + float meanKronFluxStd = -999.0; + if (isfinite(secfilt->dFkronWrp) && isfinite(secfilt->FkronWrp)) { + meanKronFlux = secfilt-> FkronWrp; + meanKronFluxErr = secfilt->dFkronWrp; + meanKronFluxStd = secfilt->sFkronWrp; + } + + float meanApFlux = -999.0; + float meanApFluxErr = -999.0; + float meanApFluxStd = -999.0; + if (isfinite(secfilt->dFapWrp) && isfinite(secfilt->FapWrp)) { + meanApFlux = secfilt-> FapWrp; + meanApFluxErr = secfilt->dFapWrp; + meanApFluxStd = secfilt->sFapWrp; + } + + PrintIOBuffer (sec_buffer, " ("); + PrintIOBuffer (sec_buffer, "%hd, ", secfilt->Nwarp); + PrintIOBuffer (sec_buffer, "%hd, ", secfilt->NusedWrp); + PrintIOBuffer (sec_buffer, "%hd, ", secfilt->NusedKronWrp); + PrintIOBuffer (sec_buffer, "%hd, ", secfilt->NusedApWrp); + PRINT_FLOAT(sec_buffer, secfilt->psfQfPerfMax, "%.6f,"); + + // use %e? + PRINT_FLOAT(sec_buffer, meanPSFFlux, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanPSFFluxErr, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanPSFFluxStd, "%.6f, "); // uflux precision + + PRINT_FLOAT(sec_buffer, meanKronFlux, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanKronFluxErr, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanKronFluxStd, "%.6f, "); // uflux precision + + PRINT_FLOAT(sec_buffer, meanApFlux, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanApFluxErr, "%.6f, "); // uflux precision + PRINT_FLOAT(sec_buffer, meanApFluxStd, "%.6f, "); // uflux precision + + PrintIOBuffer (sec_buffer, "%u ", secfilt->flags); + PrintIOBuffer (sec_buffer, "),\n"); + secfilt ++; + } + + return TRUE; +} + +int insert_diffobj_mysql_commit (IOBuffer *ave_buffer, IOBuffer *sec_buffer, MYSQL *mysql) { + + MYSQL_RES *result; + + // check that the last two chars are ,\n and replace with ;\n + if (!strcmp(&ave_buffer->buffer[ave_buffer->Nbuffer-2], ",\n")) { + ave_buffer->buffer[ave_buffer->Nbuffer-2] = ';'; + } else { + fprintf (stderr, "invalid sql?\n"); + return FALSE; + } + + // check that the last two chars are ,\n and replace with ;\n + if (!strcmp(&sec_buffer->buffer[sec_buffer->Nbuffer-2], ",\n")) { + sec_buffer->buffer[sec_buffer->Nbuffer-2] = ';'; + } else { + fprintf (stderr, "invalid sql?\n"); + return FALSE; + } + + // XXX check return status + if (mysql) { + int status; + if (DEBUG) fprintf (stderr, "%s\n", ave_buffer->buffer); + status = mysql_query(mysql, ave_buffer->buffer); + if (status) { + fprintf (stderr, "error with insert:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", ave_buffer->Nbuffer); + } + result = mysql_store_result (mysql); + if (result) mysql_free_result (result); + + if (DEBUG) fprintf (stderr, "%s\n", sec_buffer->buffer); + status = mysql_query(mysql, sec_buffer->buffer); + if (status) { + fprintf (stderr, "error with insert:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", sec_buffer->Nbuffer); + } + result = mysql_store_result (mysql); + if (result) mysql_free_result (result); + } else { + fprintf (stderr, "%s\n", sec_buffer->buffer); + } + + return TRUE; +} Index: /branches/eam_branches/ipp-20150112/Ohana/src/fakeastro/Makefile =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/fakeastro/Makefile (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/fakeastro/Makefile (revision 38059) @@ -20,5 +20,5 @@ fakeastro_client: $(BIN)/fakeastro_client.$(ARCH) -install: $(DESTBIN)/fakeastro $(DESTBIN)/fakeastro_client $(DESTBIN)/testparallax +install: $(DESTBIN)/fakeastro $(DESTBIN)/fakeastro_client FAKEASTRO = \ Index: /branches/eam_branches/ipp-20150112/Ohana/src/libautocode/Makefile.Targets =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/libautocode/Makefile.Targets (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/libautocode/Makefile.Targets (revision 38059) @@ -45,6 +45,7 @@ $(ASRC)/measure-ps1-ref.$(ARCH).o \ $(ASRC)/lensing.$(ARCH).o \ -$(ASRC)/lensing-ps1-v5.$(ARCH).o \ -$(ASRC)/lensing-ps1-v5-alt.$(ARCH).o \ +$(ASRC)/lensing-ps1-v5-r0.$(ARCH).o \ +$(ASRC)/lensing-ps1-v5-r1.$(ARCH).o \ +$(ASRC)/lensing-ps1-v5-r2.$(ARCH).o \ $(ASRC)/lensobj.$(ARCH).o \ $(ASRC)/lensobj-ps1-v5.$(ARCH).o \ @@ -91,4 +92,5 @@ $(ASRC)/cmf-ps1-sv2.$(ARCH).o \ $(ASRC)/cmf-ps1-dv4.$(ARCH).o \ +$(ASRC)/cmf-ps1-dv5.$(ARCH).o \ $(ASRC)/cmf-smpdata.$(ARCH).o \ $(ASRC)/getstar-ps1-dev-0.$(ARCH).o \ @@ -151,6 +153,7 @@ $(AINC)/measure-ps1-ref.h \ $(AINC)/lensing.h \ -$(AINC)/lensing-ps1-v5.h \ -$(AINC)/lensing-ps1-v5-alt.h \ +$(AINC)/lensing-ps1-v5-r0.h \ +$(AINC)/lensing-ps1-v5-r1.h \ +$(AINC)/lensing-ps1-v5-r2.h \ $(AINC)/lensobj.h \ $(AINC)/lensobj-ps1-v5.h \ @@ -197,4 +200,5 @@ $(AINC)/cmf-ps1-sv2.h \ $(AINC)/cmf-ps1-dv4.h \ +$(AINC)/cmf-ps1-dv5.h \ $(AINC)/cmf-smpdata.h \ $(AINC)/getstar-ps1-dev-0.h \ Index: /branches/eam_branches/ipp-20150112/Ohana/src/libautocode/def/cmf-ps1-dv5.d =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/libautocode/def/cmf-ps1-dv5.d (revision 38059) +++ /branches/eam_branches/ipp-20150112/Ohana/src/libautocode/def/cmf-ps1-dv5.d (revision 38059) @@ -0,0 +1,138 @@ +# name of structure type +STRUCT CMF_PS1_DV5 +EXTNAME CMF_PS1_DV5 +TYPE BINTABLE +SIZE 248 + +# elements of data structure / FITS table +FIELD detID, IPP_IDET, unsigned int, detection ID +FIELD X, X_PSF, float, x coord, pixels +FIELD Y, Y_PSF, float, y coord, pixels +FIELD dX, X_PSF_SIG, float, x coord error, pixels +FIELD dY, Y_PSF_SIG, float, y coord error, pixels +FIELD posangle, POSANGLE, float, Posangle at source, degrees +FIELD pltscale, PLTSCALE, float, Plate Scale at source, arcsec/pixel +FIELD M, PSF_INST_MAG, float, inst mags, mags +FIELD dM, PSF_INST_MAG_SIG, float, inst mag error, mags +FIELD Flux, PSF_INST_FLUX, float, psf flux, counts +FIELD dFlux, PSF_INST_FLUX_SIG, float, psf flux error, counts +FIELD Map, AP_MAG, float, standard aperture mag, mags +FIELD MapRaw, AP_MAG_RAW, float, raw aperture mag, mags +FIELD apRadius, AP_MAG_RADIUS, float, radius used for fit, pixels +FIELD apFlux, AP_FLUX, float, ap flux +FIELD apFluxErr, AP_FLUX_SIG, float, ap flux err +FIELD apNpix, AP_NPIX, int, number of pixels in aperture +FIELD Mpeak, PEAK_FLUX_AS_MAG, float, peak flux as a mag, mags +FIELD Mcalib, CAL_PSF_MAG, float, calibrated psf mag, mags +FIELD dMcal, CAL_PSF_MAG_SIG, float, zero point scatter, mags + +# NOTE: RA & DEC (both double) need to be on an 8-byte boundary... +FIELD RA, RA_PSF, double, PSF RA coord, degrees +FIELD DEC, DEC_PSF, double, PSF DEC coord, degrees + +FIELD sky, SKY, float, sky flux, cnts/sec +FIELD dSky, SKY_SIGMA, float, sky flux error, cnts/sec +FIELD psfChisq, PSF_CHISQ, float, psf fit chisq +FIELD crNsigma, CR_NSIGMA, float, Nsigma deviations from PSF to CF +FIELD extNsigma, EXT_NSIGMA, float, Nsigma deviations from PSF to EXT +FIELD fx, PSF_MAJOR, float, psf fit major axis, pixels +FIELD fy, PSF_MINOR, float, psf fit minor axis, pixels +FIELD df, PSF_THETA, float, ellipse angle, degrees +FIELD psfCore, PSF_CORE, float, radial slope term +FIELD psfFwhmMajor, PSF_FWHM_MAJ, float, true fwhm in major axis direction +FIELD psfFwhmMinor, PSF_FWHM_MIN, float, true fwhm in minor axis direction +FIELD psfQF, PSF_QF, float, quality factor +FIELD psfQFperf, PSF_QF_PERFECT, float, quality factor perfect +FIELD psfNdof, PSF_NDOF, int, psf degrees of freedom +FIELD psfNpix, PSF_NPIX, int, psf number of pixels +FIELD Mxx, MOMENTS_XX, float, second moment X, pixels^2 +FIELD Mxy, MOMENTS_XY, float, second moment Y, pixels^2 +FIELD Myy, MOMENTS_YY, float, second moment XY, pixels^2 +FIELD Mr1, MOMENTS_R1, float, first radial moment, pixels +FIELD Mrh, MOMENTS_RH, float, half radial moment, pixels^1/2 +FIELD kronFlux, KRON_FLUX, float, kron flux, counts +FIELD kronFluxErr, KRON_FLUX_ERR, float, kron flux error, counts +FIELD kronInner, KRON_FLUX_INNER, float, kron flux 1E1_sh_psf = NAN; lensing->E2_sh_psf = NAN; + + lensing->E1_psf = NAN; + lensing->E2_psf = NAN; lensing->F_ApR5 = NAN; Index: /branches/eam_branches/ipp-20150112/Ohana/src/libdvo/src/dvo_convert.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/libdvo/src/dvo_convert.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/libdvo/src/dvo_convert.c (revision 38059) @@ -432,11 +432,23 @@ if (!strcmp (extname, "DVO_LENSING_PS1_V5") && (ftable[0].header[0].Naxis[0] == 128)) { - Lensing_PS1_V5_ALT *tmpLensing; - tmpLensing = gfits_table_get_Lensing_PS1_V5_ALT (ftable, Nlensing, NULL); + Lensing_PS1_V5_R0 *tmpLensing; + tmpLensing = gfits_table_get_Lensing_PS1_V5_R0 (ftable, Nlensing, NULL); if (!tmpLensing) { fprintf (stderr, "ERROR: failed to read lensings\n"); exit (2); } - lensing = Lensing_PS1_V5_ALT_ToInternal (tmpLensing, *Nlensing); + lensing = Lensing_PS1_V5_R0_ToInternal (tmpLensing, *Nlensing); + free (tmpLensing); + *format = DVO_FORMAT_PS1_V5; + return (lensing); } + + if (!strcmp (extname, "DVO_LENSING_PS1_V5") && (ftable[0].header[0].Naxis[0] == 136)) { + Lensing_PS1_V5_R1 *tmpLensing; + tmpLensing = gfits_table_get_Lensing_PS1_V5_R1 (ftable, Nlensing, NULL); + if (!tmpLensing) { + fprintf (stderr, "ERROR: failed to read lensings\n"); + exit (2); + } + lensing = Lensing_PS1_V5_R1_ToInternal (tmpLensing, *Nlensing); free (tmpLensing); *format = DVO_FORMAT_PS1_V5; @@ -484,5 +496,5 @@ SKIPPING_FORMAT ("DVO_LENSING_PS1_V3", PS1_V3, PS1_V3); SKIPPING_FORMAT ("DVO_LENSING_PS1_V4", PS1_V4, PS1_V4); - CONVERT_FORMAT ("DVO_LENSING_PS1_V5", PS1_V5, PS1_V5); + CONVERT_FORMAT ("DVO_LENSING_PS1_V5", PS1_V5, PS1_V5_R2); # undef CONVERT_FORMAT @@ -521,5 +533,5 @@ // FORMAT_CASE (PS1_V3, PS1_V3); // FORMAT_CASE (PS1_V4, PS1_V4); - FORMAT_CASE (PS1_V5, PS1_V5); + FORMAT_CASE (PS1_V5, PS1_V5_R2); # undef FORMAT_CASE Index: /branches/eam_branches/ipp-20150112/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c (revision 38059) @@ -448,5 +448,5 @@ } -Lensing *Lensing_PS1_V5_ToInternal (Lensing_PS1_V5 *in, off_t Nvalues) { +Lensing *Lensing_PS1_V5_R2_ToInternal (Lensing_PS1_V5_R2 *in, off_t Nvalues) { off_t i; @@ -481,4 +481,7 @@ out[i].E1_sh_psf = in[i].E1_sh_psf; out[i].E2_sh_psf = in[i].E2_sh_psf; + + out[i].E1_psf = in[i].E1_psf; + out[i].E2_psf = in[i].E2_psf; out[i].F_ApR5 = in[i].F_ApR5; @@ -495,5 +498,4 @@ out[i].objID = in[i].objID; out[i].catID = in[i].catID; - out[i].averef = in[i].averef; @@ -503,8 +505,8 @@ } -Lensing_PS1_V5 *LensingInternalTo_PS1_V5 (Lensing *in, off_t Nvalues) { - - off_t i; - Lensing_PS1_V5 *out; +Lensing_PS1_V5_R2 *LensingInternalTo_PS1_V5_R2 (Lensing *in, off_t Nvalues) { + + off_t i; + Lensing_PS1_V5_R2 *out; ALLOCATE_ZERO (out, Lensing_PS1_V5, Nvalues); @@ -535,4 +537,7 @@ out[i].E1_sh_psf = in[i].E1_sh_psf; out[i].E2_sh_psf = in[i].E2_sh_psf; + + out[i].E1_psf = in[i].E1_psf; + out[i].E2_psf = in[i].E2_psf; out[i].F_ApR5 = in[i].F_ApR5; @@ -549,6 +554,6 @@ out[i].objID = in[i].objID; out[i].catID = in[i].catID; - out[i].averef = in[i].averef; + out[i].imageID = in[i].imageID; } @@ -1268,6 +1273,5 @@ } - -Lensing *Lensing_PS1_V5_ALT_ToInternal (Lensing_PS1_V5_ALT *in, off_t Nvalues) { +Lensing *Lensing_PS1_V5_R0_ToInternal (Lensing_PS1_V5_R0 *in, off_t Nvalues) { off_t i; @@ -1316,6 +1320,59 @@ out[i].objID = in[i].objID; out[i].catID = in[i].catID; - out[i].averef = in[i].averef; + } + return (out); +} + +Lensing *Lensing_PS1_V5_R1_ToInternal (Lensing_PS1_V5_R1 *in, off_t Nvalues) { + + off_t i; + Lensing *out; + + ALLOCATE_ZERO (out, Lensing, Nvalues); + + for (i = 0; i < Nvalues; i++) { + dvo_lensing_init (&out[i]); + + out[i].X11_sm_obj = in[i].X11_sm_obj; + out[i].X12_sm_obj = in[i].X12_sm_obj; + out[i].X22_sm_obj = in[i].X22_sm_obj; + out[i].E1_sm_obj = in[i].E1_sm_obj; + out[i].E2_sm_obj = in[i].E2_sm_obj; + + out[i].X11_sh_obj = in[i].X11_sh_obj; + out[i].X12_sh_obj = in[i].X12_sh_obj; + out[i].X22_sh_obj = in[i].X22_sh_obj; + out[i].E1_sh_obj = in[i].E1_sh_obj; + out[i].E2_sh_obj = in[i].E2_sh_obj; + + out[i].X11_sm_psf = in[i].X11_sm_psf; + out[i].X12_sm_psf = in[i].X12_sm_psf; + out[i].X22_sm_psf = in[i].X22_sm_psf; + out[i].E1_sm_psf = in[i].E1_sm_psf; + out[i].E2_sm_psf = in[i].E2_sm_psf; + + out[i].X11_sh_psf = in[i].X11_sh_psf; + out[i].X12_sh_psf = in[i].X12_sh_psf; + out[i].X22_sh_psf = in[i].X22_sh_psf; + out[i].E1_sh_psf = in[i].E1_sh_psf; + out[i].E2_sh_psf = in[i].E2_sh_psf; + + out[i].F_ApR5 = in[i].F_ApR5; + out[i].dF_ApR5 = in[i].dF_ApR5; + out[i].sF_ApR5 = in[i].sF_ApR5; + out[i].fF_ApR5 = in[i].fF_ApR5; + + out[i].F_ApR6 = in[i].F_ApR6; + out[i].dF_ApR6 = in[i].dF_ApR6; + out[i].sF_ApR6 = in[i].sF_ApR6; + out[i].fF_ApR6 = in[i].fF_ApR6; + + out[i].detID = in[i].detID; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].averef = in[i].averef; + + out[i].imageID = in[i].imageID; } return (out); Index: /branches/eam_branches/ipp-20150112/Ohana/src/opihi/lib.shell/expand_vectors.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/opihi/lib.shell/expand_vectors.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/opihi/lib.shell/expand_vectors.c (revision 38059) @@ -1,3 +1,7 @@ # include "opihi.h" + +// converts a[], a[4], a[2][3] to their numeric values +// add a?[] and a?[][] as equivalent to $?a +// a[][?] char *expand_vectors (char *line) { @@ -5,5 +9,5 @@ char *newline, *tmpline, strValue[128], *val; char *L, *N, *p, *q, *p2, *w; - int n, I, J, size, showLength, NLINE, Noff, isBuffer, inRange; + int n, I, J, size, showLength, showXsize, showYsize, checkType, NLINE, Noff, isBuffer, inRange; float *ptr; double f1; @@ -41,6 +45,10 @@ val = NULL; showLength = FALSE; + showXsize = FALSE; + showYsize = FALSE; + if (n == 0) { showLength = TRUE; + showXsize = TRUE; } else { tmpline = strncreate (p+1, n); @@ -60,8 +68,8 @@ p2 = strchr (q + 1, ']'); if (p2 != NULL) { - if (showLength) { - gprint (GP_ERR, "unsupported : name[][..]\n"); - goto asVector; - } + // if (showLength) { + // gprint (GP_ERR, "unsupported : name[][..]\n"); + // goto asVector; + // } isBuffer = TRUE; @@ -71,14 +79,16 @@ val = NULL; if (n == 0) { - gprint (GP_ERR, "unsupported : name[..][]\n"); - isBuffer = FALSE; - goto asVector; - } - tmpline = strncreate (q+2, n); - val = dvomath (1, &tmpline, &size, 0); - free (tmpline); - if (val == NULL) { - isBuffer = FALSE; - goto asVector; /* not a valid vector subscript */ + showYsize = TRUE; + // gprint (GP_ERR, "unsupported : name[..][]\n"); + // isBuffer = FALSE; + // goto asVector; + } else { + tmpline = strncreate (q+2, n); + val = dvomath (1, &tmpline, &size, 0); + free (tmpline); + if (val == NULL) { + isBuffer = FALSE; + goto asVector; /* not a valid vector subscript */ + } } J = 0; @@ -93,4 +103,9 @@ asVector: /* find vector/buffer name */ + checkType = FALSE; + if (*(p-1) == '?') { + checkType = TRUE; + p--; + } for (w = p - 1; (w >= line) && !OHANA_WHITESPACE(*w) && (isalnum(*w) || (*w == ':') || (*w == '_')); w--); w ++; @@ -99,6 +114,22 @@ if (isBuffer) { - if ((buf = SelectBuffer (tmpline, OLDBUFFER, TRUE)) == NULL) goto dumpline; - + buf = SelectBuffer (tmpline, OLDBUFFER, !checkType); + if (checkType) { + f1 = (buf != NULL); + goto skipBuffer; + } + if (showXsize && showYsize) { + gprint (GP_ERR, "ambiguous size option buf[][]\n"); + goto dumpline; + } + if (buf == NULL) goto dumpline; + if (showXsize) { + f1 = buf[0].header.Naxis[0]; + goto skipBuffer; + } + if (showYsize) { + f1 = buf[0].header.Naxis[1]; + goto skipBuffer; + } /* find buffer element */ inRange = TRUE; @@ -116,19 +147,25 @@ f1 = ptr[I + J*buf[0].header.Naxis[0]]; } else { - if ((vec = SelectVector (tmpline, OLDVECTOR, TRUE)) == NULL) goto dumpline; - - /* find vector element */ - if (showLength) { - f1 = vec[0].Nelements; + vec = SelectVector (tmpline, OLDVECTOR, !checkType); + if (checkType) { + f1 = (vec != NULL); } else { - if ((I >= vec[0].Nelements) || (I < -1*vec[0].Nelements)) { - gprint (GP_ERR, "vector subscript out of range\n"); - goto escape; + if (vec == NULL) goto dumpline; + + /* find vector element */ + if (showLength) { + f1 = vec[0].Nelements; + } else { + if ((I >= vec[0].Nelements) || (I < -1*vec[0].Nelements)) { + gprint (GP_ERR, "vector subscript out of range\n"); + goto escape; + } + if (I < 0) I += vec[0].Nelements; + f1 = (vec[0].type == OPIHI_FLT) ? vec[0].elements.Flt[I] : vec[0].elements.Int[I]; } - if (I < 0) I += vec[0].Nelements; - f1 = (vec[0].type == OPIHI_FLT) ? vec[0].elements.Flt[I] : vec[0].elements.Int[I]; - } - } - + } + } + + skipBuffer: free (tmpline); if ((int)f1 == f1) Index: /branches/eam_branches/ipp-20150112/Ohana/src/opihi/pantasks/LocalJob.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/opihi/pantasks/LocalJob.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/opihi/pantasks/LocalJob.c (revision 38059) @@ -1,3 +1,4 @@ # include "pantasks.h" +# include /* local jobs are forked in the background Index: /branches/eam_branches/ipp-20150112/Ohana/src/opihi/pclient/job.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/opihi/pclient/job.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/opihi/pclient/job.c (revision 38059) @@ -1,3 +1,4 @@ # include "pclient.h" +# include int job (int argc, char **argv) { Index: /branches/eam_branches/ipp-20150112/Ohana/src/relphot/include/relphot.h =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/relphot/include/relphot.h (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/relphot/include/relphot.h (revision 38059) @@ -214,4 +214,5 @@ int SET_MREL_VERSION; +int IS_DIFF_DB; double MAG_LIM; Index: /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/args.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/args.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/args.c (revision 38059) @@ -64,4 +64,10 @@ } + IS_DIFF_DB = FALSE; + if ((N = get_argument (argc, argv, "-is-diff-db"))) { + remove_argument (N, &argc, argv); + IS_DIFF_DB = TRUE; + } + USE_FULL_OVERLAP = TRUE; if ((N = get_argument (argc, argv, "-sloppy-image-overlap"))) { @@ -517,4 +523,10 @@ } + IS_DIFF_DB = FALSE; + if ((N = get_argument (argc, argv, "-is-diff-db"))) { + remove_argument (N, &argc, argv); + IS_DIFF_DB = TRUE; + } + SET_MREL_VERSION = 1; if ((N = get_argument (argc, argv, "-set-mrel-version"))) { Index: /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/extra.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/extra.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/extra.c (revision 38059) @@ -43,4 +43,15 @@ int isGPC1warp (int photcode) { + // diff warps get stack-like photcodes (kind of lame) + if (IS_DIFF_DB) { + if (photcode == 11000) return TRUE; // g-band + if (photcode == 11100) return TRUE; // r-band + if (photcode == 11200) return TRUE; // i-band + if (photcode == 11300) return TRUE; // z-band + if (photcode == 11400) return TRUE; // y-band + if (photcode == 11500) return TRUE; // w-band + return FALSE; + } + if (photcode == 12000) return TRUE; // g-band if (photcode == 12100) return TRUE; // r-band Index: /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/reload_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/reload_catalogs.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/reload_catalogs.c (revision 38059) @@ -226,4 +226,5 @@ if (RESET_ZEROPTS) { snprintf (tmpline, 1024, "%s -reset-zpts", command); strcpy (command, tmpline); } if (UPDATE) { snprintf (tmpline, 1024, "%s -update", command); strcpy (command, tmpline); } + if (IS_DIFF_DB) { snprintf (tmpline, 1024, "%s -is-diff-db", command); strcpy (command, tmpline); } if (!KEEP_UBERCAL) { snprintf (tmpline, 1024, "%s -reset-ubercal", command); strcpy (command, tmpline); } if (UPDATE_CATFORMAT) { snprintf (tmpline, 1024, "%s -update-catformat %s", command, UPDATE_CATFORMAT); strcpy (command, tmpline); } Index: /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/setMrelCatalog.c =================================================================== --- /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/setMrelCatalog.c (revision 38058) +++ /branches/eam_branches/ipp-20150112/Ohana/src/relphot/src/setMrelCatalog.c (revision 38059) @@ -62,5 +62,5 @@ // only apply Stack operation on setMrelFinal in first pass - if (isSetMrelFinal && (pass == 0)) { + if (isSetMrelFinal && (pass == 0) && !IS_DIFF_DB) { setMrelAverageStack (m, Nc, flatcorr, results, average, secfilt, &catalog[Nc].measure[m], catalog[Nc].Nmeasure); setGlobalObjStats (&catalog[Nc].average[j], &catalog[Nc].measure[m]); @@ -703,4 +703,10 @@ } + // for a diff db, we just want primary, if available or not + int priTessID, priProjID, priSkycellID; + if (IS_DIFF_DB) { + get_tess_ids(&priTessID, &priProjID, &priSkycellID, average[0].R, average[0].D); + } + int Ns; for (Ns = 0; Ns < Nphotcodes; Ns++) { @@ -718,6 +724,11 @@ int stkTessID, stkProjID, stkSkycellID; - myAssert (secfilt[Nsec].stackBestOff <= Nmeasure, "stackBestOff out of range"); - if (!FindImageSkycellID (secfilt[Nsec].stackBestOff, cat, &stkTessID, &stkProjID, &stkSkycellID)) continue; + if (!IS_DIFF_DB) { + myAssert (secfilt[Nsec].stackBestOff <= Nmeasure, "stackBestOff out of range"); + if (!FindImageSkycellID (secfilt[Nsec].stackBestOff, cat, &stkTessID, &stkProjID, &stkSkycellID)) continue; + } + + float psfQfMax = 0.0; + float psfQfPerfMax = 0.0; off_t meas = measureOffset; @@ -744,6 +755,10 @@ measure[k].dbFlags &= ~ID_MEAS_WARP_USED; - if (!MatchImageSkycellID (meas, cat, stkTessID, stkProjID, stkSkycellID)) continue; - + if (IS_DIFF_DB) { + if (!MatchImageSkycellID (meas, cat, priTessID, priProjID, priSkycellID)) continue; + } else { + if (!MatchImageSkycellID (meas, cat, stkTessID, stkProjID, stkSkycellID)) continue; + } + if (measure[k].dbFlags & MEAS_BAD) SKIP_THIS_MEAS(Nbad); @@ -761,4 +776,10 @@ float Fcal = MagToFlux(-Mcal); + // these are only used below if IS_DIFF_DB + if (IS_DIFF_DB) { + if (measure[k].psfQF > psfQfMax) psfQfMax = measure[k].psfQF; + if (measure[k].psfQFperf > psfQfPerfMax) psfQfPerfMax = measure[k].psfQFperf; + } + // in the calculations below, // ...list gives the error per measurement, wlist gives the weight @@ -811,9 +832,22 @@ } } - if (Npsf < 1) continue; + if (Npsf < 1) { + secfilt[Nsec].Nwarp = Nwarp; + secfilt[Nsec].NwarpGood = NwarpGood; + secfilt[Nsec].NusedWrp = Npsf; + secfilt[Nsec].NusedApWrp = Nap; + secfilt[Nsec].NusedKronWrp = Nkron; + continue; + } found[Nsec] = TRUE; liststats (Fpsflist, dpsflist, wpsflist, Npsf, psfstats); + + if (IS_DIFF_DB) { + // for non DIFF_DB, these are set in setMrelAverageExposure + secfilt[Nsec].psfQfMax = psfQfMax; + secfilt[Nsec].psfQfPerfMax = psfQfPerfMax; + } secfilt[Nsec].FpsfWrp = psfstats->mean;