Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/batch.py (revision 37133) @@ -230,4 +230,14 @@ ''' def safeDictionaryAccess(self, header, key): + + if key in header: return header[key] + else: + self.logger.errorPair("Missing header field", key) + return "NULL" + + ''' + Returns the string keyword value from this header or else "NULL" + ''' + def getKeyValue(self, header, key): if key in header: return header[key] Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/detectionbatch.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/detectionbatch.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/detectionbatch.py (revision 37133) @@ -125,4 +125,8 @@ sqlLine = sqlUtility("INSERT INTO FrameMeta (") + # sqlLine.group("p1Recip", ' '); + # sqlLine.group("p2Recip", ' '); + # sqlLine.group("p3Recip", ' '); + sqlLine.group("frameID", str(self.expID)); sqlLine.group("frameName", self.expName); @@ -130,41 +134,38 @@ sqlLine.group("cameraConfigID", "1"); sqlLine.group("telescopeID", "1"); - sqlLine.group("analysisVer", str(self.analysisVer)); - sqlLine.group("p1Recip", " "); - sqlLine.group("p2Recip", " "); - sqlLine.group("p3Recip", " "); - sqlLine.group("photoScat", "self.safeDictionaryAccess(self.header, 'ZPT_ERR')"); - sqlLine.group("expStart", "self.safeDictionaryAccess(self.header, 'MJD-OBS')"); - sqlLine.group("expTime", "self.safeDictionaryAccess(self.header, 'EXPREQ')"); - sqlLine.group("airmass", "self.safeDictionaryAccess(self.header, 'AIRMASS')"); - sqlLine.group("raBore", "self.safeDictionaryAccess(self.header, 'RA')"); - sqlLine.group("decBore", "self.safeDictionaryAccess(self.header, 'DEC')"); - sqlLine.group("ctype1", self.safeDictionaryAccess(self.header, 'CTYPE1')); - sqlLine.group("ctype2", self.safeDictionaryAccess(self.header, 'CTYPE2')); - sqlLine.group("crval1", "self.safeDictionaryAccess(self.header, 'CRVAL1')"); - sqlLine.group("crval2", "self.safeDictionaryAccess(self.header, 'CRVAL2')"); - sqlLine.group("crpix1", "self.safeDictionaryAccess(self.header, 'CRPIX1')"); - sqlLine.group("crpix2", "self.safeDictionaryAccess(self.header, 'CRPIX2')"); - sqlLine.group("cdelt1", "self.safeDictionaryAccess(self.header, 'CDELT1')"); - sqlLine.group("cdelt2", "self.safeDictionaryAccess(self.header, 'CDELT2')"); - sqlLine.group("pc001001", "self.safeDictionaryAccess(self.header, 'PC001001')"); - sqlLine.group("pc001002", "self.safeDictionaryAccess(self.header, 'PC001002')"); - sqlLine.group("pc002001", "self.safeDictionaryAccess(self.header, 'PC002001')"); - sqlLine.group("pc002002", "self.safeDictionaryAccess(self.header, 'PC002002')"); - sqlLine.group("polyOrder", "self.safeDictionaryAccess(self.header, 'NPLYTERM')"); - sqlLine.group("pca1x3y0", "self.safeDictionaryAccess(self.header, 'PCA1X3Y0')"); - sqlLine.group("pca1x2y1", "self.safeDictionaryAccess(self.header, 'PCA1X2Y1')"); - sqlLine.group("pca1x1y2", "self.safeDictionaryAccess(self.header, 'PCA1X1Y2')"); - sqlLine.group("pca1x0y3", "self.safeDictionaryAccess(self.header, 'PCA1X0Y3')"); - sqlLine.group("pca1x2y0", "self.safeDictionaryAccess(self.header, 'PCA1X2Y0')"); - sqlLine.group("pca1x1y1", "self.safeDictionaryAccess(self.header, 'PCA1X1Y1')"); - sqlLine.group("pca1x0y2", "self.safeDictionaryAccess(self.header, 'PCA1X0Y2')"); - sqlLine.group("pca2x3y0", "self.safeDictionaryAccess(self.header, 'PCA2X3Y0')"); - sqlLine.group("pca2x2y1", "self.safeDictionaryAccess(self.header, 'PCA2X2Y1')"); - sqlLine.group("pca2x1y2", "self.safeDictionaryAccess(self.header, 'PCA2X1Y2')"); - sqlLine.group("pca2x0y3", "self.safeDictionaryAccess(self.header, 'PCA2X0Y3')"); - sqlLine.group("pca2x2y0", "self.safeDictionaryAccess(self.header, 'PCA2X2Y0')"); - sqlLine.group("pca2x1y1", "self.safeDictionaryAccess(self.header, 'PCA2X1Y1')"); - sqlLine.group("pca2x0y2", "self.safeDictionaryAccess(self.header, 'PCA2X0Y2')"); + sqlLine.group("analysisVer", self.analysisVer); + sqlLine.group("photoScat", self.getKeyValue(self.header, 'ZPT_ERR')); + sqlLine.group("expStart", self.getKeyValue(self.header, 'MJD-OBS')); + sqlLine.group("expTime", self.getKeyValue(self.header, 'EXPREQ')); + sqlLine.group("airmass", self.getKeyValue(self.header, 'AIRMASS')); + sqlLine.group("raBore", self.getKeyValue(self.header, 'RA')); + sqlLine.group("decBore", self.getKeyValue(self.header, 'DEC')); + sqlLine.group("ctype1", self.getKeyValue(self.header, 'CTYPE1')); + sqlLine.group("ctype2", self.getKeyValue(self.header, 'CTYPE2')); + sqlLine.group("crval1", self.getKeyValue(self.header, 'CRVAL1')); + sqlLine.group("crval2", self.getKeyValue(self.header, 'CRVAL2')); + sqlLine.group("crpix1", self.getKeyValue(self.header, 'CRPIX1')); + sqlLine.group("crpix2", self.getKeyValue(self.header, 'CRPIX2')); + sqlLine.group("cdelt1", self.getKeyValue(self.header, 'CDELT1')); + sqlLine.group("cdelt2", self.getKeyValue(self.header, 'CDELT2')); + sqlLine.group("pc001001", self.getKeyValue(self.header, 'PC001001')); + sqlLine.group("pc001002", self.getKeyValue(self.header, 'PC001002')); + sqlLine.group("pc002001", self.getKeyValue(self.header, 'PC002001')); + sqlLine.group("pc002002", self.getKeyValue(self.header, 'PC002002')); + sqlLine.group("polyOrder", self.getKeyValue(self.header, 'NPLYTERM')); + sqlLine.group("pca1x3y0", self.getKeyValue(self.header, 'PCA1X3Y0')); + sqlLine.group("pca1x2y1", self.getKeyValue(self.header, 'PCA1X2Y1')); + sqlLine.group("pca1x1y2", self.getKeyValue(self.header, 'PCA1X1Y2')); + sqlLine.group("pca1x0y3", self.getKeyValue(self.header, 'PCA1X0Y3')); + sqlLine.group("pca1x2y0", self.getKeyValue(self.header, 'PCA1X2Y0')); + sqlLine.group("pca1x1y1", self.getKeyValue(self.header, 'PCA1X1Y1')); + sqlLine.group("pca1x0y2", self.getKeyValue(self.header, 'PCA1X0Y2')); + sqlLine.group("pca2x3y0", self.getKeyValue(self.header, 'PCA2X3Y0')); + sqlLine.group("pca2x2y1", self.getKeyValue(self.header, 'PCA2X2Y1')); + sqlLine.group("pca2x1y2", self.getKeyValue(self.header, 'PCA2X1Y2')); + sqlLine.group("pca2x0y3", self.getKeyValue(self.header, 'PCA2X0Y3')); + sqlLine.group("pca2x2y0", self.getKeyValue(self.header, 'PCA2X2Y0')); + sqlLine.group("pca2x1y1", self.getKeyValue(self.header, 'PCA2X1Y1')); + sqlLine.group("pca2x0y2", self.getKeyValue(self.header, 'PCA2X0Y2')); sql = sqlLine.make(") VALUES ( ", ")") @@ -193,5 +194,10 @@ else: ccdID = 0 - psfFwhm = 0.5*(self.safeDictionaryAccess(header, 'FWHM_MAJ') + self.safeDictionaryAccess(header, 'FWHM_MIN')) + psfFwhmMajor = self.getKeyValue(header, 'FWHM_MAJ') + psfFwhmMinor = self.getKeyValue(header, 'FWHM_MIN') + if ((psfFwhmMajor == "NULL") or (psfFwhmMajor == "NULL")): + psfFwhm = "NULL" + else: + psfFwhm = 0.5*(psfFwhmMajor + psfFwhmMinor) # insert image metadata into table @@ -202,18 +208,18 @@ sqlLine.group("bias", str(self.bias)) sqlLine.group("biasScat", str(self.biasScat)) - sqlLine.group("sky", "self.safeDictionaryAccess(header, 'MSKY_MN')") - sqlLine.group("skyScat", "self.safeDictionaryAccess(header, 'MSKY_SIG')") - sqlLine.group("magSat", "self.safeDictionaryAccess(header, 'FSATUR')") - sqlLine.group("completMag", "self.safeDictionaryAccess(header, 'FLIMIT')") - sqlLine.group("astroScat", "self.safeDictionaryAccess(header, 'CERROR')") - sqlLine.group("photoScat", "self.safeDictionaryAccess(self.header, 'ZPT_OBS')") - sqlLine.group("numAstroRef", "self.safeDictionaryAccess(header, 'NASTRO')") - sqlLine.group("numPhotoRef", "self.safeDictionaryAccess(header, 'NASTRO')") - sqlLine.group("nx", "self.safeDictionaryAccess(header, 'CNAXIS1')") - sqlLine.group("ny", "self.safeDictionaryAccess(header, 'CNAXIS2')") + sqlLine.group("sky", self.getKeyValue(header, 'MSKY_MN')) + sqlLine.group("skyScat", self.getKeyValue(header, 'MSKY_SIG')) + sqlLine.group("magSat", self.getKeyValue(header, 'FSATUR')) + sqlLine.group("completMag", self.getKeyValue(header, 'FLIMIT')) + sqlLine.group("astroScat", self.getKeyValue(header, 'CERROR')) + sqlLine.group("photoScat", self.getKeyValue(self.header, 'ZPT_OBS')) + sqlLine.group("numAstroRef", self.getKeyValue(header, 'NASTRO')) + sqlLine.group("numPhotoRef", self.getKeyValue(header, 'NASTRO')) + sqlLine.group("nx", self.getKeyValue(header, 'CNAXIS1')) + sqlLine.group("ny", self.getKeyValue(header, 'CNAXIS2')) sqlLine.group("psfFwhm", str(psfFwhm)) - sqlLine.group("psfWidMajor", "self.safeDictionaryAccess(header, 'FWHM_MAJ')") - sqlLine.group("psfWidMinor", "self.safeDictionaryAccess(header, 'FWHM_MIN')") - sqlLine.group("psfTheta", "self.safeDictionaryAccess(header, 'ANGLE')") + sqlLine.group("psfWidMajor", str(psfFwhmMajor)) + sqlLine.group("psfWidMinor", str(psfFwhmMinor)) + sqlLine.group("psfTheta", self.getKeyValue(header, 'ANGLE')) sqlLine.group("momentXX", "-999") sqlLine.group("momentXY", "-999") @@ -225,44 +231,44 @@ sqlLine.group("momentR1", "-999") sqlLine.group("momentRH", "-999") - sqlLine.group("apResid", "self.safeDictionaryAccess(header, 'APMIFIT')") - sqlLine.group("dapResid", "self.safeDictionaryAccess(header, 'DAPMIFIT')") - sqlLine.group("detectorID", self.safeDictionaryAccess(header, 'DETECTOR')) + sqlLine.group("apResid", self.getKeyValue(header, 'APMIFIT')) + sqlLine.group("dapResid", self.getKeyValue(header, 'DAPMIFIT')) + sqlLine.group("detectorID", self.getKeyValue(header, 'DETECTOR')) sqlLine.group("qaFlags", str(self.scratchDb.getDvoImageFlags(header['IMAGEID']))) - sqlLine.group("detrend1", self.safeDictionaryAccess(header, 'DETREND.MASK')) - sqlLine.group("detrend2", self.safeDictionaryAccess(header, 'DETREND.DARK')) - sqlLine.group("detrend3", self.safeDictionaryAccess(header, 'DETREND.FLAT')) - sqlLine.group("detrend4", " ") - sqlLine.group("detrend5", " ") - sqlLine.group("detrend6", " ") - sqlLine.group("detrend7", " ") - sqlLine.group("detrend8", " ") - sqlLine.group("photoZero", "self.safeDictionaryAccess(self.header, 'ZPT_OBS')") - sqlLine.group("ctype1", "self.safeDictionaryAccess(header, 'CTYPE1')") - sqlLine.group("ctype2", "self.safeDictionaryAccess(header, 'CTYPE2')") - sqlLine.group("crval1", "self.safeDictionaryAccess(header, 'CRVAL1')") - sqlLine.group("crval2", "self.safeDictionaryAccess(header, 'CRVAL2')") - sqlLine.group("crpix1", "self.safeDictionaryAccess(header, 'CRPIX1')") - sqlLine.group("crpix2", "self.safeDictionaryAccess(header, 'CRPIX2')") - sqlLine.group("cdelt1", "self.safeDictionaryAccess(header, 'CDELT1')") - sqlLine.group("cdelt2", "self.safeDictionaryAccess(header, 'CDELT2')") - sqlLine.group("pc001001", "self.safeDictionaryAccess(header, 'PC001001')") - sqlLine.group("pc001002", "self.safeDictionaryAccess(header, 'PC001002')") - sqlLine.group("pc002001", "self.safeDictionaryAccess(header, 'PC002001')") - sqlLine.group("pc002002", "self.safeDictionaryAccess(header, 'PC002002')") - sqlLine.group("polyOrder", "self.safeDictionaryAccess(header, 'NPLYTERM')") - sqlLine.group("pca1x3y0", "self.safeDictionaryAccess(header, 'PCA1X3Y0')") - sqlLine.group("pca1x2y1", "self.safeDictionaryAccess(header, 'PCA1X2Y1')") - sqlLine.group("pca1x1y2", "self.safeDictionaryAccess(header, 'PCA1X1Y2')") - sqlLine.group("pca1x0y3", "self.safeDictionaryAccess(header, 'PCA1X0Y3')") - sqlLine.group("pca1x2y0", "self.safeDictionaryAccess(header, 'PCA1X2Y0')") - sqlLine.group("pca1x1y1", "self.safeDictionaryAccess(header, 'PCA1X1Y1')") - sqlLine.group("pca1x0y2", "self.safeDictionaryAccess(header, 'PCA1X0Y2')") - sqlLine.group("pca2x3y0", "self.safeDictionaryAccess(header, 'PCA2X3Y0')") - sqlLine.group("pca2x2y1", "self.safeDictionaryAccess(header, 'PCA2X2Y1')") - sqlLine.group("pca2x1y2", "self.safeDictionaryAccess(header, 'PCA2X1Y2')") - sqlLine.group("pca2x0y3", "self.safeDictionaryAccess(header, 'PCA2X0Y3')") - sqlLine.group("pca2x2y0", "self.safeDictionaryAccess(header, 'PCA2X2Y0')") - sqlLine.group("pca2x1y1", "self.safeDictionaryAccess(header, 'PCA2X1Y1')") - sqlLine.group("pca2x0y2", "self.safeDictionaryAccess(header, 'PCA2X0Y2')") + sqlLine.group("detrend1", self.getKeyValue(header, 'DETREND.MASK')) + sqlLine.group("detrend2", self.getKeyValue(header, 'DETREND.DARK')) + sqlLine.group("detrend3", self.getKeyValue(header, 'DETREND.FLAT')) + # sqlLine.group("detrend4", " ") + # sqlLine.group("detrend5", " ") + # sqlLine.group("detrend6", " ") + # sqlLine.group("detrend7", " ") + # sqlLine.group("detrend8", " ") + sqlLine.group("photoZero", self.getKeyValue(self.header, 'ZPT_OBS')) + sqlLine.group("ctype1", self.getKeyValue(header, 'CTYPE1')) + sqlLine.group("ctype2", self.getKeyValue(header, 'CTYPE2')) + sqlLine.group("crval1", self.getKeyValue(header, 'CRVAL1')) + sqlLine.group("crval2", self.getKeyValue(header, 'CRVAL2')) + sqlLine.group("crpix1", self.getKeyValue(header, 'CRPIX1')) + sqlLine.group("crpix2", self.getKeyValue(header, 'CRPIX2')) + sqlLine.group("cdelt1", self.getKeyValue(header, 'CDELT1')) + sqlLine.group("cdelt2", self.getKeyValue(header, 'CDELT2')) + sqlLine.group("pc001001", self.getKeyValue(header, 'PC001001')) + sqlLine.group("pc001002", self.getKeyValue(header, 'PC001002')) + sqlLine.group("pc002001", self.getKeyValue(header, 'PC002001')) + sqlLine.group("pc002002", self.getKeyValue(header, 'PC002002')) + sqlLine.group("polyOrder", self.getKeyValue(header, 'NPLYTERM')) + sqlLine.group("pca1x3y0", self.getKeyValue(header, 'PCA1X3Y0')) + sqlLine.group("pca1x2y1", self.getKeyValue(header, 'PCA1X2Y1')) + sqlLine.group("pca1x1y2", self.getKeyValue(header, 'PCA1X1Y2')) + sqlLine.group("pca1x0y3", self.getKeyValue(header, 'PCA1X0Y3')) + sqlLine.group("pca1x2y0", self.getKeyValue(header, 'PCA1X2Y0')) + sqlLine.group("pca1x1y1", self.getKeyValue(header, 'PCA1X1Y1')) + sqlLine.group("pca1x0y2", self.getKeyValue(header, 'PCA1X0Y2')) + sqlLine.group("pca2x3y0", self.getKeyValue(header, 'PCA2X3Y0')) + sqlLine.group("pca2x2y1", self.getKeyValue(header, 'PCA2X2Y1')) + sqlLine.group("pca2x1y2", self.getKeyValue(header, 'PCA2X1Y2')) + sqlLine.group("pca2x0y3", self.getKeyValue(header, 'PCA2X0Y3')) + sqlLine.group("pca2x2y0", self.getKeyValue(header, 'PCA2X2Y0')) + sqlLine.group("pca2x1y1", self.getKeyValue(header, 'PCA2X1Y1')) + sqlLine.group("pca2x0y2", self.getKeyValue(header, 'PCA2X0Y2')) sql = sqlLine.make(") VALUES ( ", ")") @@ -340,5 +346,6 @@ sqlLine.group("apFlux", "AP_FLUX / " + extTimeString) sqlLine.group("apFluxErr", "AP_FLUX_SIG / " + extTimeString) - sqlLine.group("apFluxNpix", "AP_NPIX") + sqlLine.group("apFillF", "AP_NPIX / (3.1416 * AP_MAG_RADIUS * AP_MAG_RADIUS)") + sqlLine.group("apRadius", "AP_MAG_RADIUS") sqlLine.group("kronFlux", "KRON_FLUX / " + extTimeString) sqlLine.group("kronFluxErr", "KRON_FLUX_ERR / " + extTimeString) @@ -350,6 +357,5 @@ sqlLine.group("dataRelease", str(self.skychunk.dataRelease)) - sqlLine.make(") SELECT ", " FROM " + ippTableName) - + sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) print "sql: " + sql @@ -450,9 +456,9 @@ # check we have valid sourceID/imageID pair from the header - if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": return False - if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": return False + if self.getKeyValue(header, 'SOURCEID') == "NULL": return False + if self.getKeyValue(header, 'IMAGEID') == "NULL": return False - # store sourceID/imageID combo in Db so DVO can look up later - self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID']) + # store sourceID/imageID combo in Db so DVO can look up later (XXX deprecated?) + # self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID']) # store these for later @@ -501,6 +507,6 @@ self.alterPspsTablesChip("Chip", "Chip.hdr", 0, 0) - # now run DVO code to get all IDs - if not self.getIDsFromDVO(): return False + # now run DVO code to get all IDs ( XXX deprecated, uses dvograbber) + # if not self.getIDsFromDVO(): return False # dec is reserved in MySQL, so STILTS replaces if with dec_, which is not the name of @@ -547,5 +553,5 @@ a.ippObjID = b.ippObjID, \ a.ra = b.ra, \ - a.dec_ = b.dec_, \ + a.dec = b.dec_, \ a.raErr = b.raErr, \ a.decErr = b.decErr, \ @@ -791,6 +797,6 @@ # columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG AP_FLUX KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" - # XXX EAM NOTE : this is fragile : requires PS1_V4 - columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ POSANGLE PLTSCALE" + # XXX EAM NOTE : this is fragile : requires PS1_V5 + columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG POSANGLE PLTSCALE PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_FWHM_MAJ PSF_FWHM_MIN PSF_THETA PSF_CORE PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_FLUX AP_FLUX_SIG AP_NPIX AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2" return super(DetectionBatch, self).importIppTables(columns, regex) Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/dvo.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/dvo.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/dvo.py (revision 37133) @@ -684,7 +684,7 @@ if (useP2 ==1 and useST==0): - #grabd only P2s - cmd += " -photcode-start 10000 -photcode-end 10577" - + #grab only P2s (4000-4500 are test cameras, SIMTEST and SIMMOSAIC) + # XXX EAM 20140729 : this is a terrible hack... + cmd += " -photcode-start 4000 -photcode-end 10577" self.logger.infoPair("Running dvopsps", cmd) Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/gpc1db.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/gpc1db.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/gpc1db.py (revision 37133) @@ -117,4 +117,6 @@ stage = "forcedwarp" sql = "SELECT differentStuff from Stufftable" + + print "SQL: " + sql try: Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/initbatch.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/initbatch.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/initbatch.py (revision 37133) @@ -33,4 +33,5 @@ scratchDb, batchID): + super(InitBatch, self).__init__(logger, config, @@ -42,6 +43,5 @@ batchID, "IN", - None, - 0) + None) self.outputFitsFile = "00000000.FITS"; Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/ipptopsps.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/ipptopsps.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/ipptopsps.py (revision 37133) @@ -155,4 +155,6 @@ while not self.refreshSkychunk() or self.ippToPspsDb.isPaused(self.HOST, self.PID): + print "looping in ipptopsps" + self.ippToPspsDb.updateClient(self.config.programName, self.HOST, self.PID) if self.ippToPspsDb.isKilled(self.HOST, self.PID): @@ -162,4 +164,5 @@ time.sleep(self.PAUSEPERIOD) + print "DONE looping in ipptopsps" ''' Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/ipptopspsdb.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/ipptopspsdb.py (revision 37133) @@ -1136,90 +1136,112 @@ self.skychunk.batchTypes = [] - try: - # print "sql: " + sql - rs = self.executeQuery(sql) - - rs.first() - self.skychunk.datastoreProduct = rs.getString(1) - self.skychunk.datastoreType = rs.getString(2) - if rs.getInt(3) == 1: self.skychunk.datastorePublishing = True - else: self.skychunk.datastorePublishing = False - self.skychunk.dvoLabel = rs.getString(4) - - # if dvoLabel is null is can break queries, so set to something - if not self.skychunk.dvoLabel: self.skychunk.dvoLabel = "none" - self.skychunk.dvoLocation = rs.getString(5) - self.skychunk.minRa = rs.getDouble(6) - self.skychunk.maxRa = rs.getDouble(7) - self.skychunk.minDec = rs.getDouble(8) - self.skychunk.maxDec = rs.getDouble(9) - - self.skychunk.boxSize = rs.getDouble(10) - # self.skychunk.halfBox = self.skychunk.boxSize/2.0 - # self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2) - - self.skychunk.basePath = rs.getString(11) - self.skychunk.dataRelease = rs.getInt(12) - - self.skychunk.deleteLocal = rs.getInt(13) - self.skychunk.deleteDatastore = rs.getInt(14) - self.skychunk.deleteDxLayer = rs.getInt(15) - - self.skychunk.epoch = rs.getString(16) - - self.skychunk.survey = rs.getString(17) - self.skychunk.pspsSurvey = rs.getString(18) - - if rs.getInt(19) == 1: self.skychunk.batchTypes.append("P2") - if rs.getInt(20) == 1: self.skychunk.batchTypes.append("ST") - if rs.getInt(21) == 1: self.skychunk.batchTypes.append("OB") - if rs.getInt(22) == 1: self.skychunk.batchTypes.append("DF") - if rs.getInt(23) == 1: self.skychunk.batchTypes.append("DO") - if rs.getInt(24) == 1: self.skychunk.batchTypes.append("FW") - if rs.getInt(25) == 1: self.skychunk.batchTypes.append("FO") - - self.skychunk.force = True # TODO - self.skychunk.parallel = False # TODO - - if rs.getInt(26) == 1: self.skychunk.parallel = True - self.skychunk.P2_smf_version = rs.getString(27) - # options: "use_new", "not_reproc", "use_original" - - self.skychunk.ST_cmf_version = rs.getString(28) - # options: not used? - - # NOTE : the rs (ResultSet) class is somewhat annoying and finicky esp about timestamp values - # for example, rs.getTimestamp(N) fails silently if the value is not a valid time. this is - # also true to getString() on the same column. Docs for the ResultSet class can be found here: - # http://docs.oracle.com/javase/1.5.0/docs/api/java/sql/ResultSet.html?is-external=true - - # rsmd = rs.getMetaData() - # print "got metadata" - # print "number of columns: " + str(rsmd.getColumnCount()) - # print "label: " + rsmd.getColumnLabel(25) - # print "label: " + rsmd.getColumnLabel(29) - # print "name: " + rsmd.getColumnName(25) - # print "name: " + rsmd.getColumnName(29) - # print "typename: " + rsmd.getColumnTypeName(25) - # print "typename: " + rsmd.getColumnTypeName(29) - - # epochStamp = rs.getTimestamp(16) - # print "epoch: " + epochStamp.toString() - - # print "label: " + rsmd.getColumnLabel(16) - # print "name: " + rsmd.getColumnName(16) - # print "typename: " + rsmd.getColumnTypeName(16) - - self.skychunk.trange_start = rs.getString(29) - # options: "YYYY-MM-DD,HH:MM:SS" or "0000-00-00 00:00:00" - - self.skychunk.trange_end = rs.getString(30) - # options: "YYYY-MM-DD,HH:MM:SS" or "0000-00-00 00:00:00" - - if self.skychunk.parallel: print "USING parallel" - self.skychunk.isLoaded = True - except: + + try: + rs = self.executeQuery("show tables") + except: + self.logger.errorPair("cannot even show tables?", "Boo hoo") + sys.exit(1) + + # while rs.next(): + # print "table row" + # print rs.getString(1) + # + # rs = self.executeQuery("select * from skychunk") + # while rs.next(): + # print "skychunk row" + # print rs.getString(1) + # + # print "SQL (2): " + sql + + try: + rs = self.executeQuery(sql) + + except: + self.logger.errorPair("problem with sql query for skychunk", sql) + sys.exit(1) + + if not rs.next(): self.logger.errorPair("Could not read skychunk with name", self.skychunk.name) self.skychunk.isLoaded = False + return False + + self.skychunk.datastoreProduct = rs.getString(1) + self.skychunk.datastoreType = rs.getString(2) + if rs.getInt(3) == 1: self.skychunk.datastorePublishing = True + else: self.skychunk.datastorePublishing = False + self.skychunk.dvoLabel = rs.getString(4) + + # if dvoLabel is null is can break queries, so set to something + if not self.skychunk.dvoLabel: self.skychunk.dvoLabel = "none" + self.skychunk.dvoLocation = rs.getString(5) + self.skychunk.minRa = rs.getDouble(6) + self.skychunk.maxRa = rs.getDouble(7) + self.skychunk.minDec = rs.getDouble(8) + self.skychunk.maxDec = rs.getDouble(9) + + self.skychunk.boxSize = rs.getDouble(10) + # self.skychunk.halfBox = self.skychunk.boxSize/2.0 + # self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2) + + self.skychunk.basePath = rs.getString(11) + self.skychunk.dataRelease = rs.getInt(12) + + self.skychunk.deleteLocal = rs.getInt(13) + self.skychunk.deleteDatastore = rs.getInt(14) + self.skychunk.deleteDxLayer = rs.getInt(15) + + self.skychunk.epoch = rs.getString(16) + + self.skychunk.survey = rs.getString(17) + self.skychunk.pspsSurvey = rs.getString(18) + + if rs.getInt(19) == 1: self.skychunk.batchTypes.append("P2") + if rs.getInt(20) == 1: self.skychunk.batchTypes.append("ST") + if rs.getInt(21) == 1: self.skychunk.batchTypes.append("OB") + if rs.getInt(22) == 1: self.skychunk.batchTypes.append("DF") + if rs.getInt(23) == 1: self.skychunk.batchTypes.append("DO") + if rs.getInt(24) == 1: self.skychunk.batchTypes.append("FW") + if rs.getInt(25) == 1: self.skychunk.batchTypes.append("FO") + + self.skychunk.force = True # TODO + self.skychunk.parallel = False # TODO + + if rs.getInt(26) == 1: self.skychunk.parallel = True + self.skychunk.P2_smf_version = rs.getString(27) + # options: "use_new", "not_reproc", "use_original" + + self.skychunk.ST_cmf_version = rs.getString(28) + # options: not used? + + # NOTE : the rs (ResultSet) class is somewhat annoying and finicky esp about timestamp values + # for example, rs.getTimestamp(N) fails silently if the value is not a valid time. this is + # also true to getString() on the same column. Docs for the ResultSet class can be found here: + # http://docs.oracle.com/javase/1.5.0/docs/api/java/sql/ResultSet.html?is-external=true + + # rsmd = rs.getMetaData() + # print "got metadata" + # print "number of columns: " + str(rsmd.getColumnCount()) + # print "label: " + rsmd.getColumnLabel(25) + # print "label: " + rsmd.getColumnLabel(29) + # print "name: " + rsmd.getColumnName(25) + # print "name: " + rsmd.getColumnName(29) + # print "typename: " + rsmd.getColumnTypeName(25) + # print "typename: " + rsmd.getColumnTypeName(29) + + # epochStamp = rs.getTimestamp(16) + # print "epoch: " + epochStamp.toString() + + # print "label: " + rsmd.getColumnLabel(16) + # print "name: " + rsmd.getColumnName(16) + # print "typename: " + rsmd.getColumnTypeName(16) + + self.skychunk.trange_start = rs.getString(29) + # options: "YYYY-MM-DD,HH:MM:SS" or "0000-00-00 00:00:00" + + self.skychunk.trange_end = rs.getString(30) + # options: "YYYY-MM-DD,HH:MM:SS" or "0000-00-00 00:00:00" + + if self.skychunk.parallel: print "USING parallel" + self.skychunk.isLoaded = True return self.skychunk.isLoaded @@ -1387,27 +1409,37 @@ # XXX print "sql: ", sql - try: - rs = self.executeQuery(sql) - rs.first() - raCenter = rs.getFloat(1) - rs.close() - except: - self.logger.errorPair("No free stripes", "trying to steal boxes from other client...") - sql = "SELECT ra_center, COUNT(*) AS numPending\ - FROM box \ - JOIN pending ON (id = box_id) \ - WHERE skychunk = '" + self.skychunk.name + "' \ - AND batch_type = '" + batchType + "' \ - GROUP BY ra_center \ - ORDER BY numPending \ - DESC LIMIT 1" - - try: - rs = self.executeQuery(sql) - rs.first() - raCenter = rs.getFloat(1) - rs.close() - except: - self.logger.errorPair("No outstanding boxes to steal", "no stripe obtained") + try: rs = self.executeQuery(sql) + except: + self.logger.errorPair("Problem querying for outstanding boxes", "error in sql?") + print "SQL: " + sql + + print "SQL: " + sql + if not rs.next(): + self.logger.errorPair("No outstanding boxes", "nothing to do here") + rs.close() + self.unlockTables() + return ids + + raCenter = rs.getFloat(1) + rs.close() + + #except: + # self.logger.errorPair("No free stripes", "trying to steal boxes from other client...") + # sql = "SELECT ra_center, COUNT(*) AS numPending\ + # FROM box \ + # JOIN pending ON (id = box_id) \ + # WHERE skychunk = '" + self.skychunk.name + "' \ + # AND batch_type = '" + batchType + "' \ + # GROUP BY ra_center \ + # ORDER BY numPending \ + # DESC LIMIT 1" + # + # try: + # rs = self.executeQuery(sql) + # rs.first() + # raCenter = rs.getFloat(1) + # rs.close() + # except: + # self.logger.errorPair("No outstanding boxes to steal", "no stripe obtained") # tests against ra_center need to use a finite box @@ -1458,26 +1490,36 @@ GROUP BY ra_center LIMIT 1" - try: - rs = self.executeQuery(sql) - rs.first() - raCenter = rs.getFloat(1) - rs.close() - except: - self.logger.errorPair("No free stripes", "trying to steal boxes from other client...") - sql = "SELECT ra_center, COUNT(*) AS numPending\ - FROM box \ - JOIN pending ON (id = box_id) \ - WHERE skychunk = '" + self.skychunk.name + "' \ - GROUP BY ra_center \ - ORDER BY numPending \ - DESC LIMIT 1" - - try: - rs = self.executeQuery(sql) - rs.first() - raCenter = rs.getFloat(1) - rs.close() - except: - self.logger.errorPair("No outstanding boxes to steal", "no stripe obtained") + try: rs = self.executeQuery(sql) + except: + self.logger.errorPair("Problem querying for outstanding boxes", "error in sql?") + print "SQL: " + sql + + print "SQL: " + sql + if not rs.next(): + self.logger.infoPair("No outstanding boxes", "nothing to do here") + rs.close() + self.unlockTables() + return ids + + raCenter = rs.getFloat(1) + rs.close() + + # except: + # self.logger.errorPair("No free stripes", "trying to steal boxes from other client...") + # sql = "SELECT ra_center, COUNT(*) AS numPending\ + # FROM box \ + # JOIN pending ON (id = box_id) \ + # WHERE skychunk = '" + self.skychunk.name + "' \ + # GROUP BY ra_center \ + # ORDER BY numPending \ + # DESC LIMIT 1" + # + # try: + # rs = self.executeQuery(sql) + # rs.first() + # raCenter = rs.getFloat(1) + # rs.close() + # except: + # self.logger.errorPair("No outstanding boxes to steal", "no stripe obtained") # tests against ra_center need to use a finite box Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/loader.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/loader.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/loader.py (revision 37133) @@ -82,4 +82,5 @@ self.skychunk.printAll() + print "done with loader.py init" ''' @@ -174,6 +175,9 @@ self.logger.infoPair("processed","ok") + self.logger.infoPair("done with processing boxes, numAttempts: ", str(numAttempts)) if abort or not self.checkClientStatus(): abort = True elif numAttempts > 1 and not self.waitForPollTime(): break + + print "done with wait?" if self.onePassOnly: self.exitProgram("one pass completed") Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/queue.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/queue.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/queue.py (revision 37133) @@ -63,10 +63,10 @@ ''' def run(self): - print "here" + self.logger.info("------------- starting queue.run -----------------") # this outer while loop simply waits a few minutes then starts queuing again (as more stuff may appear in DVO over time) while True: self.ippToPspsDb.removeAllBoxes() - print "remove boxes" + # queue up batches that are processed but not loaded to datastore for batchType in self.skychunk.batchTypes: @@ -77,10 +77,10 @@ # get totals for whole area to check if there is anything to do processedIDs = self.ippToPspsDb.getProcessedIDs(batchType) - print "got processed ids" consistantlyFailedIDs = self.ippToPspsDb.getConsistentlyFailedIDs(batchType) - print "got here" + # for object batches, get full list of stuff from Dvo # XXX : need to fix 'dvo.setSkyArea' or this may not return the full list for a parallel dvo - #for object-like batches, info comes from dvo + + # for object-like batches, info comes from dvo if (batchType == "OB" or batchType =="DO" or batchType =="FO"): self.dvoObjects.setSkyArea( Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/scratchdb.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/scratchdb.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/scratchdb.py (revision 37133) @@ -229,30 +229,20 @@ def alreadyImportedThisDvoTable(self, path): - print "***** alreadyImportedThisDvoTable *****" - print "path: " + path - fileStat = os.stat(path) - print "path: " + path sql = "SELECT COUNT(*) FROM " + self.dvoDoneTable + " WHERE path = '" + path + "' AND modifiedDate = " + str(fileStat.st_mtime) - print "sql: " + sql - - try: - rs = self.executeQuery(sql) - print "ran sql: " + sql + + try: + rs = self.executeQuery(sql) rs.first() if rs.getInt(1) > 0: - print "***** DID import previously *****" self.logger.debugPair("Already imported up-to-date version of", path) return True else: - print "***** did NOT import previously *****" + self.logger.infoPair("Did NOT already import from", path) return False except: print "***** exception in alreadyImportedThisDvoTable *****" self.logger.exception("Unable to check whether this DVO table has been imported") - - print "***** done with alreadyImportedThisDvoTable *****" - ''' @@ -342,14 +332,24 @@ rs.close() if not haveDone: - print "dvoDone not yet build" + self.logger.info("dvoDone not yet built") return False - except: - print "failed to run query: " + sql + + except Exception, e: + print str(e) + self.logger.errorPair("problem examining table", sql) + sys.exit(1) sql = "SELECT path FROM " + self.dvoDoneTable + " WHERE path LIKE '%SkyTable.fits'" - try: - rs = self.executeQuery(sql) - rs.first() - + try: rs = self.executeQuery(sql) + except Exception, e: + print str(e) + self.logger.errorPair("problem querying for dvo path", path) + sys.exit(1) + + if not rs.next(): + self.logger.info("dvoDone exists, but is empty") + return False + + try: head,tail = os.path.split(rs.getString(1)) @@ -358,11 +358,11 @@ return True else: - self.logger.errorPair("DVO does not match. We have", head) + self.logger.infoPair("DVO does not match. We have", head) return False except Exception, e: print str(e) - self.logger.errorPair("No DVO files ingested for this path", path) - return False + self.logger.errorPair("problem searching for DVO at", path) + sys.exit(1) ''' Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/sqlUtility.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/sqlUtility.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/sqlUtility.py (revision 37133) @@ -16,9 +16,10 @@ def group(self,field,value): + print "group field: " + field + ", value: " + value self.fields.append(field) self.values.append(value) return - def make(self,middleString,endString): + def makeRaw(self,middleString,endString): output = self.startString @@ -40,4 +41,24 @@ return output + def make(self,middleString,endString): + + output = self.startString + + for i in range(len(self.fields)): + output += self.fields[i] + if (i < len(self.fields) - 1): + output += ", " + + output += middleString + + for i in range(len(self.values)): + output += "'" + self.values[i] + "'" + if (i < len(self.values) - 1): + output += ", " + + output += endString + + return output + # when we do a sql insert, we have a command that looks like this: # INSERT INTO Table (field1, field2,...) OPERATION (value1, value2,...) Index: /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py (revision 37132) +++ /branches/eam_branches/ipp-20140717/ippToPsps/jython/stackbatch.py (revision 37133) @@ -1199,7 +1199,7 @@ self.skipBatch = False - # get the IDs from the DVO tables - if not self.getIDsFromDVO(): - return False + # get the IDs from the DVO tables + # XXX deprecated along with dvograbber + # if not self.getIDsFromDVO(): return False # do some basics first: