Index: trunk/ippToPsps/jython/diffbatch.py =================================================================== --- trunk/ippToPsps/jython/diffbatch.py (revision 39048) +++ trunk/ippToPsps/jython/diffbatch.py (revision 39049) @@ -77,5 +77,6 @@ self.expTime = {} self.obsTime = {} - self.fits = {} + self.posFits = {} + self.negFits = {} self.tessID = {} self.tessName = {} @@ -101,39 +102,52 @@ self.logger.infoPair("getting diff" ," metadata") self.logger.infoPair("for diff skyfile id ", str(diffSkyFileID)) - meta = {} + meta = self.gpc1Db.getDiffStageMeta(int(diffSkyFileID)) - # this code is very fragile: it depends on the right order in the output - # of the sql call in another file (see forcedwarpbatch.py for a better method) + # EAM 20151105 : I have changed the function above to return "meta" as a dictionary if not meta: self.logger.errorPair("Could not get", "diff metadata") raise - self.filterName[num] = meta[0]; - - self.difftypeID[num] = meta[1]; - self.ippDiffID[num] = stageid - self.posImageID[num] = meta[2]; - self.negImageID[num] = meta[3]; - self.skycellName[num] = meta[4]; - self.analysisVer[num] = meta[5]; - self.expTime[num] = meta[6]; + + self.ippDiffID[num] = stageid + + self.filterName[num] = meta['filter']; + self.difftypeID[num] = meta['diff_mode']; + self.posImageID[num] = meta['warp1']; + self.negImageID[num] = meta['stack2']; + self.skycellName[num] = meta['skycell_id']; + self.analysisVer[num] = meta['software_ver']; + self.expTime[num] = meta['exp_time']; + self.expID[num] = meta['exp_id']; + self.tessName[num] = meta['tess_id']; + ## MEH -- should be able to use diff header now for mjd (see below) - #self.obsTime[num] = meta[7]; - # tess_id is handled below, that's #8. - self.expID[num] = meta[9]; + # self.obsTime[num] = meta['obsTime']; + + # tessName could also come from the header... + # tessName = None + # tessName = self.getKeyValue(self.header[num], 'TESS_ID') + self.logger.infoPair("loading diff stage cmf", diffSkyFileID) - self.fits[num] = gpc1Db.getDiffStageCmf(skychunk.dvoLabel, int(diffSkyFileID)) + + # look up the path base once + pathBase = gpc1Db.getDiffStagePathBase(skychunk.dvoLabel, int(diffSkyFileID)) + + # select the two (possible) cmf files (.cmf and .inv.cmf) + self.posFits[num] = gpc1Db.getDiffStageCmfForPathBase(pathBase) + self.negFits[num] = gpc1Db.getDiffStageCmfForPathBase(pathBase + ".inv") + self.logger.infoPair("loading diff stage header", diffSkyFileID) - self.header[num] = self.fits[num].getPrimaryHeader() + self.header[num] = self.posFits[num].getPrimaryHeader() + + # obsTime is the center of the exposure. MJD-OBS is the start in days, expTime is the length in seconds self.obsTime[num] = float(self.header[num]['MJD-OBS']) + (float(self.expTime[num]) / 172800.0) - # tessName = None - # tessName = self.getKeyValue(self.header[num], 'TESS_ID') - self.tessName[num]= meta[8]; + tessName = self.tessName[num] + self.logger.infoPair("TessName ", tessName) + self.tessID[num] = self.scratchDb.getTessID(tessName) self.logger.infoPair("TessName ",tessName) - self.tessID[num] = self.scratchDb.getTessID(tessName) - - self.logger.infoPair("TessName ",tessName) - # XXX EAM 20140812 : I am hardwiring the parsing logic for RINGS vs LOCAL + + # XXX EAM 20140812 : I am hardwiring the parsing logic for RINGS vs LOCAL if self.tessName[num] == 'RINGS.V3': # skycell is, eg "skycell.1133.081" @@ -152,7 +166,7 @@ ## MEH md5sum test - self.md5sum[num] = self.fits[num].getMd5sum() - - # drop the existing tables + self.md5sum[num] = self.posFits[num].getMd5sum() + + # drop the existing tables self.dropTableVerbose("DiffMeta_"+str(num)) @@ -162,8 +176,11 @@ if not self.analysisVer[num]: self.analysisVer[num] = -999 + self.logger.infoPair("getPhotoCalID",diffSkyFileID) self.photoCalID[num] = str(self.scratchDb.getPhotoCalID(int(diffSkyFileID))) + self.logger.infoPair("get filterID", diffSkyFileID) self.filterID[num] = self.scratchDb.getFilterID(self.filterName[num][0]) + self.logger.infoPair("insertdiffmeta", diffSkyFileID) self.logger.infoPair("insertdiffmeta", self.filterName[num][0]) @@ -171,33 +188,14 @@ self.ippToPspsDb.insertDiffMeta(self.batchID, self.filterName[num][0], self.difftypeID[num]) - # make some generic tables - #sql = "CREATE TABLE DiffMeta_"+str(num)+" like DiffMeta" - #try: self.scratchDb.execute(sql) - #except: raise - #sql = "CREATE TABLE DiffToImage_"+str(num)+" like DiffToImage" - #try: self.scratchDb.execute(sql) - #except: raise - #sql = "CREATE TABLE DiffDetection_"+str(num)+" like DiffDetection" - #try: self.scratchDb.execute(sql) - #except: raise - - - num = num + 1 # create an output filename, which is {expID}.FITS - - self.outputFitsFile = "%08d.FITS" % self.stageid self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) - - self.processingVersion = skychunk.processingVersion; - # dump stuff to log self.logger.infoPair("stage_id", "%d" % self.stageid) - ''' @@ -257,150 +255,138 @@ Populates the Diff table for this OTA ''' - def populateDiffDetectionTable(self,num): - - pspsTableName = "DiffDetection_"+str(num) - ippTableName = "SkyChip_psf_"+str(num) - diffSkyFileID = self.number[num] - sql = "CREATE TABLE DiffDetection_"+str(num)+" like DiffDetection" - try: self.scratchDb.execute(sql) - except: raise - #results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName) - - # drop then re-create table - #self.scratchDb.dropTable(pspsTableName) - #sql = "CREATE TABLE " + pspsTableName + " LIKE DiffDetection" - #try: self.scratchDb.execute(sql) - #except: pass - + def populateDiffDetectionTable(self, num, isPosFile): + # XXX we no longer delete detections with funny mags (eg PSF_INST_MAG < -17.5) # for examples of code which do this, go to r35097 or earlier - BEFORE = self.scratchDb.getRowCount(ippTableName) - #results['SATDET'] = 0 + if isPosFile: + ippTableName = "SkyChip_psf_"+str(num) + else: + ippTableName = "SkyChip_psf_inv_"+str(num) + + pspsTableName = "DiffDetection_" + str(num) + diffSkyFileID = self.number[num] + + if isPosFile: + sql = "CREATE TABLE DiffDetection_" + str(num) + " like DiffDetection" + self.scratchDb.execute(sql) expTimeString = str(self.expTime[num]) - - # insert all detections into table : do NOT include the IGNORE unless we have to - # sqlLine = sqlUtility("INSERT IGNORE INTO " + pspsTableName + " (") - sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (") - - # XXX WARNING: need to apply platescale to convert sizes to arcsec - # XXX apFluxF or apFluxNpix + apFluxRadius? - - sqlLine.group("ippDetectID", "IPP_IDET") - sqlLine.group("diffImageID", str(diffSkyFileID)) - - sqlLine.group("randomDiffID", "RAND("+str(self.batchID)+")") - sqlLine.group("filterID", str(self.filterID[num])) - sqlLine.group("surveyID", str(self.surveyID)) - sqlLine.group("skycellID", str(self.skycellID[num])) - sqlLine.group("expTime", str(self.expTime[num])) - sqlLine.group("obsTime", str(self.obsTime[num])) - sqlLine.group("xPos", "X_PSF") - sqlLine.group("yPos", "Y_PSF") - sqlLine.group("xPosErr", "X_PSF_SIG") - sqlLine.group("yPosErr", "Y_PSF_SIG") - if (self.id >= 982483): - sqlLine.group("pltScale", "PLTSCALE") - sqlLine.group("posAngle", "POSANGLE") - sqlLine.group("raErr", "X_PSF_SIG * abs(PLTSCALE)") - sqlLine.group("decErr", "Y_PSF_SIG * abs(PLTSCALE)") + if isPosFile: + fluxScaleStr = "1" + detIDoffset = 0 else: - sqlLine.group("pltScale", "0.257") - sqlLine.group("posAngle", "-999") - sqlLine.group("raErr", "X_PSF_SIG * 0.25") - sqlLine.group("decErr", "Y_PSF_SIG * 0.25") - sqlLine.group("DpsfFlux", "PSF_INST_FLUX / " + expTimeString) - sqlLine.group("DpsfFluxErr", "PSF_INST_FLUX_SIG / " + expTimeString) - - if self.havePsfFwhm[num]: - print "I have PSF_FWHM_MAJ, setting a value" - sqlLine.group("DpsfMajorFWHM", "PSF_FWHM_MAJ * abs(PLTSCALE)") - sqlLine.group("DpsfMinorFWHM", "PSF_FWHM_MIN * abs(PLTSCALE)") - sqlLine.group("DpsfCore", "PSF_CORE") - - if self.haveSrcPos[num]: - print "I have SRC_CHIP_X, setting a value" - sqlLine.group("xPosChip", "SRC_CHIP_X") - sqlLine.group("yPosChip", "SRC_CHIP_Y") - sqlLine.group("ccdID" , "SRC_CHIP_NUM") - - sqlLine.group("DpsfTheta", "PSF_THETA") - sqlLine.group("DpsfQf", "PSF_QF") - sqlLine.group("DpsfQfPerfect", "PSF_QF_PERFECT") - sqlLine.group("DpsfChiSq", "PSF_CHISQ") - sqlLine.group("DpsfLikelihood", "psfLikelihood(EXT_NSIGMA)") - sqlLine.group("DmomentXX", "MOMENTS_XX * PLTSCALE * PLTSCALE") - sqlLine.group("DmomentXY", "MOMENTS_XY * PLTSCALE * PLTSCALE") - sqlLine.group("DmomentYY", "MOMENTS_YY * PLTSCALE * PLTSCALE") - sqlLine.group("DmomentR1", "MOMENTS_R1 * abs(PLTSCALE)") - sqlLine.group("DmomentRH", "MOMENTS_RH * sqrt(abs(PLTSCALE))") - if (self.id >= 982483): - sqlLine.group("DapFlux", "AP_FLUX / " + expTimeString) - sqlLine.group("DapFluxErr", "AP_FLUX_SIG / " + expTimeString) - - if self.haveApNpix[num]: - sqlLine.group("DapFillF", "AP_NPIX / (3.14159265359 * AP_MAG_RADIUS * AP_MAG_RADIUS)") - sqlLine.group("DkronFlux", "KRON_FLUX / " + expTimeString) - sqlLine.group("DkronFluxErr", "KRON_FLUX_ERR / " + expTimeString) - sqlLine.group("DkronRad", "MOMENTS_R1 * 2.5 * abs(PLTSCALE)") - sqlLine.group("diffNPos", "DIFF_NPOS") - sqlLine.group("diffFRatio", "DIFF_FRATIO") - sqlLine.group("diffNBad", "DIFF_NRATIO_BAD") - sqlLine.group("diffNMask", "DIFF_NRATIO_MASK") - sqlLine.group("diffNAll", "DIFF_NRATIO_ALL") - sqlLine.group("diffPdist", "DIFF_R_P") - sqlLine.group("diffNdist", "DIFF_R_M") - sqlLine.group("diffPSN", "DIFF_SN_P") - sqlLine.group("diffNSN", "DIFF_SN_M") - sqlLine.group("Dsky", "SKY / PLTSCALE / PLTSCALE / " + expTimeString) - sqlLine.group("DskyErr", "SKY_SIGMA / PLTSCALE / PLTSCALE / " + expTimeString) - sqlLine.group("DinfoFlag", "FLAGS") - sqlLine.group("DinfoFlag2", "FLAGS2") - sqlLine.group("processingVersion", str(self.skychunk.processingVersion)) - - sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) - - try: self.scratchDb.execute(sql) - except: - self.logger.errorPair("failed sql: ", sql) - raise - - # NOTE : Flux limits : in the current PSPS schema, negative fluxes - # cause problems for sql queries which work in mags as SQL cannot do - # something like (f < 0.0) ? -999 : -2.5*log10(f) - # as a result, the negative fluxes here result in floating point errors - # XXX EAM 2014072 : Is this still a problem? - # sql = "UPDATE " + pspsTableName + " SET psfFlux = 1e20 WHERE psfFlux <= 0.0" - # self.scratchDb.execute(sql) - - # sql = "UPDATE " + pspsTableName + " SET apFlux = 1e20 WHERE apFlux <= 0.0" - # self.scratchDb.execute(sql) - - # sql = "UPDATE " + pspsTableName + " SET kronFlux = 1e20 WHERE kronFlux <= 0.0" - # self.scratchDb.execute(sql) - - # we don't delete these anymore - #results['NULLINSTFLUX'] = 0; - - -# insert stuff from dvo + fluxScaleStr = "-1" + detIDoffset = self.posImageDetCount + + if isPosFile or self.haveNegFits[num]: + + # insert all detections into table : do NOT include the IGNORE unless we have to + # sqlLine = sqlUtility("INSERT IGNORE INTO " + pspsTableName + " (") + sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (") + + # XXX WARNING: need to apply platescale to convert sizes to arcsec + # XXX apFluxF or apFluxNpix + apFluxRadius? + + sqlLine.group("ippDetectID", "IPP_IDET + " + str(detIDoffset)) + sqlLine.group("diffImageID", str(diffSkyFileID)) + sqlLine.group("fromPosImage", str(isPosFile)) + + sqlLine.group("randomDiffID", "RAND("+str(self.batchID)+")") + sqlLine.group("filterID", str(self.filterID[num])) + sqlLine.group("surveyID", str(self.surveyID)) + sqlLine.group("skycellID", str(self.skycellID[num])) + sqlLine.group("expTime", str(self.expTime[num])) + sqlLine.group("obsTime", str(self.obsTime[num])) + sqlLine.group("xPos", "X_PSF") + sqlLine.group("yPos", "Y_PSF") + sqlLine.group("xPosErr", "X_PSF_SIG") + sqlLine.group("yPosErr", "Y_PSF_SIG") + if (self.id >= 982483): + sqlLine.group("pltScale", "PLTSCALE") + sqlLine.group("posAngle", "POSANGLE") + sqlLine.group("raErr", "X_PSF_SIG * abs(PLTSCALE)") + sqlLine.group("decErr", "Y_PSF_SIG * abs(PLTSCALE)") + else: + sqlLine.group("pltScale", "0.257") + sqlLine.group("posAngle", "-999") + sqlLine.group("raErr", "X_PSF_SIG * 0.25") + sqlLine.group("decErr", "Y_PSF_SIG * 0.25") + sqlLine.group("DpsfFlux", "PSF_INST_FLUX / " + expTimeString + " * " + fluxScaleStr) + sqlLine.group("DpsfFluxErr", "PSF_INST_FLUX_SIG / " + expTimeString) + + if self.havePsfFwhm[num]: + print "I have PSF_FWHM_MAJ, setting a value" + sqlLine.group("DpsfMajorFWHM", "PSF_FWHM_MAJ * abs(PLTSCALE)") + sqlLine.group("DpsfMinorFWHM", "PSF_FWHM_MIN * abs(PLTSCALE)") + sqlLine.group("DpsfCore", "PSF_CORE") + + if self.haveSrcPos[num]: + print "I have SRC_CHIP_X, setting a value" + sqlLine.group("xPosChip", "SRC_CHIP_X") + sqlLine.group("yPosChip", "SRC_CHIP_Y") + sqlLine.group("ccdID" , "SRC_CHIP_NUM") + + sqlLine.group("DpsfTheta", "PSF_THETA") + sqlLine.group("DpsfQf", "PSF_QF") + sqlLine.group("DpsfQfPerfect", "PSF_QF_PERFECT") + sqlLine.group("DpsfChiSq", "PSF_CHISQ") + sqlLine.group("DpsfLikelihood", "psfLikelihood(EXT_NSIGMA)") + sqlLine.group("DmomentXX", "MOMENTS_XX * PLTSCALE * PLTSCALE") + sqlLine.group("DmomentXY", "MOMENTS_XY * PLTSCALE * PLTSCALE") + sqlLine.group("DmomentYY", "MOMENTS_YY * PLTSCALE * PLTSCALE") + sqlLine.group("DmomentR1", "MOMENTS_R1 * abs(PLTSCALE)") + sqlLine.group("DmomentRH", "MOMENTS_RH * sqrt(abs(PLTSCALE))") + if (self.id >= 982483): + sqlLine.group("DapFlux", "AP_FLUX / " + expTimeString + " * " + fluxScaleStr) + sqlLine.group("DapFluxErr", "AP_FLUX_SIG / " + expTimeString) + + if self.haveApNpix[num]: + sqlLine.group("DapFillF", "AP_NPIX / (3.14159265359 * AP_MAG_RADIUS * AP_MAG_RADIUS)") + + sqlLine.group("DkronFlux", "KRON_FLUX / " + expTimeString + " * " + fluxScaleStr) + sqlLine.group("DkronFluxErr", "KRON_FLUX_ERR / " + expTimeString) + sqlLine.group("DkronRad", "MOMENTS_R1 * 2.5 * abs(PLTSCALE)") + sqlLine.group("diffNPos", "DIFF_NPOS") + sqlLine.group("diffFPosRatio", "DIFF_FRATIO") + sqlLine.group("diffNPosRatio", "DIFF_NRATIO_BAD") + sqlLine.group("diffNPosMask", "DIFF_NRATIO_MASK") + sqlLine.group("diffNPosAll", "DIFF_NRATIO_ALL") + sqlLine.group("diffPosdist", "DIFF_R_P") + sqlLine.group("diffNegdist", "DIFF_R_M") + sqlLine.group("diffPosSN", "DIFF_SN_P") + sqlLine.group("diffNegSN", "DIFF_SN_M") + sqlLine.group("Dsky", "SKY / PLTSCALE / PLTSCALE / " + expTimeString) + sqlLine.group("DskyErr", "SKY_SIGMA / PLTSCALE / PLTSCALE / " + expTimeString) + sqlLine.group("DinfoFlag", "FLAGS") + sqlLine.group("DinfoFlag2", "FLAGS2") + sqlLine.group("processingVersion", str(self.skychunk.processingVersion)) + + sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) + + self.scratchDb.execute(sql) + + # XXX EAM 20151105 : Not sure how this must be modified to handle the inv data: + + ## if we have a negative image (inv.cmf), do the ending chunk after loading the negative image data + if isPosFile and self.haveNegFits[num]: + return + + # insert stuff from dvo self.logger.infoPair("inserting dvo info:find imageID"," from externID") imageID = self.scratchDb.getImageIDFromExternID(diffSkyFileID) self.logger.infoPair("updating","diffdetection_"+str(num)) sqlLine = sqlUtility("UPDATE DiffDetection_"+str(num)+" as a, " + self.scratchDb.dvoDetectionTable + " as b SET ") - sqlLine.group("a.diffObjID", "b.objId") - sqlLine.group("a.diffDetID", "b.detectID") - sqlLine.group("a.ippObjID","b.ippObjID") - sqlLine.group("a.dvoRegionID", "b.catID") - sqlLine.group("a.ra", "b.ra") - sqlLine.group("a.dec", "b.dec_") - sqlLine.group("a.raErr", "b.raErr") - sqlLine.group("a.decErr", "b.decErr") - - sqlLine.group("a.zp","b.zp") + sqlLine.group("a.diffObjID", "b.objId") + sqlLine.group("a.diffDetID", "b.detectID") + sqlLine.group("a.ippObjID", "b.ippObjID") + sqlLine.group("a.dvoRegionID", "b.catID") + sqlLine.group("a.ra", "b.ra") + sqlLine.group("a.dec", "b.dec_") + sqlLine.group("a.raErr", "b.raErr") # NOTE: This is supplied above as well, is this needed?? + sqlLine.group("a.decErr", "b.decErr") # NOTE: This is supplied above as well, is this needed?? + sqlLine.group("a.zp", "b.zp") sqlLine.group("a.telluricExt", "b.telluricExt") - sqlLine.group("a.airmass","b.airmass") - + sqlLine.group("a.airmass", "b.airmass") sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " +str( imageID)) @@ -410,4 +396,5 @@ self.logger.errorPair("failed sql:",sql) raise + self.logger.infoPair("need to cull"," nulls objid") results= self.scratchDb.reportAndDeleteRowsWithNULLS("DiffDetection_"+str(num), "diffObjID") @@ -529,8 +516,4 @@ return True - - - - ''' @@ -640,5 +623,6 @@ self.logger.infoPair("populated","DiffToImage_"+str(num)) self.tablesToExport.append("DiffDetection_"+str(num)) - self.populateDiffDetectionTable(num) + self.populateDiffDetectionTable(num, 1) # populate the positive image data + self.populateDiffDetectionTable(num, 0) # populate the negative image data (if present) self.logger.infoPair("populated","DiffDetection_"+str(num)) self.setMinMaxDiffObjID(["DiffDetection_"+str(num)]) @@ -647,4 +631,5 @@ self.logger.infoPair("populated","DiffDetEffMeta_"+str(num)) # CZW: I appended to tablesToExport inside populateDiffDetEffMeta to be like stack. + #test = self.scratchDb.getRowCount("Object") #return False @@ -682,10 +667,11 @@ self.havePsfFwhm = {} self.haveSrcPos = {} + self.haveNegFits = {} + self.posImageDetCount = 0 for num in self.number: - diffSkyFileID = self.number[num] - fileName = self.fits[num].getPath() + fileNamePos = self.posFits[num].getPath() - self.logger.infoPair("using filename:",fileName) + self.logger.infoPair("using filename:",fileNamePos) ## output table names need to have "_num" appended. @@ -697,11 +683,32 @@ ## *** actually write the FITS data to mysql - self.scratchDb.loadFITStoMYSQL(fileName, fitsOptions) + self.scratchDb.loadFITStoMYSQL(fileNamePos, fitsOptions) + + ## if we have a negative image (inv.cmf), load that as well: + self.haveNegFits[num] = 0 + if not self.negFits[num] == None: + fileNameNeg = self.negFits[num].getPath() + + self.logger.infoPair("using filename:", fileNameNeg) + + ## output table names need to have "_num" appended. + fitsOptions = "-tablesuffix _inv_" + str(num) + + # this option is equivalent to the stilts "addcol table_index $0" call + fitsOptions += " -sequence-column table_index" + fitsOptions += " -sequence-column-start 1" + + ## *** actually write the FITS data to mysql + self.scratchDb.loadFITStoMYSQL(fileNameNeg, fitsOptions) + self.haveNegFits[num] = 1 + + posTableName = "SkyChip_psf_" + str(num) + self.posImageDetCount = self.scratchDb.getRowCount(posTableName) ## use stilts to get a list of the tables from the header try: - tables = stilts.treads(fileName) + tables = stilts.treads(fileNamePos) except: - self.logger.errorPair("STILTS could not import from", fileName) + self.logger.errorPair("STILTS could not import from", fileNamePos) return False @@ -751,5 +758,5 @@ for num in self.number: diffSkyFileID = self.number[num] - fileName = self.fits[num].getPath() + fileName = self.posFits[num].getPath() self.logger.infoPair("using filename:",fileName) Index: trunk/ippToPsps/jython/gpc1db.py =================================================================== --- trunk/ippToPsps/jython/gpc1db.py (revision 39048) +++ trunk/ippToPsps/jython/gpc1db.py (revision 39049) @@ -340,11 +340,4 @@ return addIDs - - - - - - - ''' Gets some stack-stage meta data for this sky_id # TODO this SQL could surely be improved @@ -408,9 +401,8 @@ AND warp_skyfile_id = " + str(warpSkyfileID) - try: - rs = self.executeQuery(sql) - rs.first() - except: + rs = self.executeQuery(sql) + if not rs.first(): self.logger.errorPair("Can't query for", "forced warp meta data") + os._exit(3) try: @@ -435,5 +427,6 @@ self.logger.debug("Querying GPC1 for diff meta data") - meta = [] + meta = {} + sql = "SELECT \ filter,\ @@ -442,7 +435,7 @@ stack2, \ skycell_id, \ - diffRun.software_ver, \ + diffRun.software_ver as software_ver, \ exp_time, \ - (to_days(dateobs)-678941 + time_to_sec(dateobs)/86400.), \ + (to_days(dateobs)-678941 + time_to_sec(dateobs)/86400.) as obsTime, \ diffInputSkyfile.tess_id, \ exp_id \ @@ -454,23 +447,22 @@ JOIN rawExp using (exp_id) \ WHERE diff_skyfile_id = %d" % diffSkyfileID - #this is a crappy way to get the mjd from the diff but we don't save it anywhere - try: - rs = self.executeQuery(sql) - rs.first() - except: + + rs = self.executeQuery(sql) + if not rs.first(): self.logger.infoPair ("this sql is broken:",sql) self.logger.errorPair("Can't query for", "stack meta data") + os._exit(3) try: - meta.append(rs.getString(1)) - meta.append(rs.getString(2)) - meta.append(rs.getString(3)) - meta.append(rs.getString(4)) - meta.append(rs.getString(5)) - meta.append(rs.getString(6)) - meta.append(rs.getString(7)) - meta.append(rs.getString(8)) - meta.append(rs.getString(9)) - meta.append(rs.getString(10)) + meta['filter'] = rs.getString(1) + meta['diff_mode'] = rs.getString(2) + meta['warp1'] = rs.getString(3) + meta['stack2'] = rs.getString(4) + meta['skycell_id'] = rs.getString(5) + meta['software_ver'] = rs.getString(6) + meta['exp_time'] = rs.getString(7) + meta['obsTime'] = rs.getString(8) + meta['tess_id'] = rs.getString(9) + meta['exp_id'] = rs.getString(10) except: @@ -515,4 +507,53 @@ if len(files) < 1: return None + fits = Fits(self.logger, self.config, files[0]) + return fits + + ''' + get diff stage cmf name + ''' + def getDiffStagePathBase(self, dvoDb, diffSkyfileID): + print "got here, diffstagecmf" + sql = "SELECT DISTINCT diffSkyfile.path_base \ + FROM mergedvodbRun join minidvodbRun using (minidvodb_id) \ + JOIN addRun using (minidvodb_name) \ + JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 \ + AND diff_id = stage_id AND stage = 'diff') \ + JOIN diffSkyfile using (diff_id, skycell_id, diff_skyfile_id) \ + WHERE diff_skyfile_id = " + str(diffSkyfileID) + " \ + AND minidvodbRun.state = 'merged' \ + AND mergedvodbRun.state = 'full' \ + AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \ + AND stage = 'diff'" + + rs = self.executeQuery(sql) + + if not rs.first(): + self.logger.errorPair("missing cmf entry diff_skyfile_id:", str(diff_skyfile_id)) + # self.logger.errorPair("no diff cmfs found for diff_skyfile_id = %d" %diff_skyfile_id) + return None + + pathBase = rs.getString(1) + return pathBase + + ''' + get diff stage cmf name + ''' + def getDiffStageCmfForPathBase(self, pathBase): + + files = [] + if pathBase.startswith("neb"): + self.logger.debugPair("searching for nebulous files", pathBase) + f = os.popen("neb-ls -p " + pathBase + ".cmf") + for i in f.readlines(): + self.logger.debugPair("real file", i.rstrip()) + files.append(i.rstrip()) + else: + files = glob.glob(pathBase + ".cmf") + + if len(files) < 1: + self.logger.errorPair ("no cmf file found matching pathBase", pathBase + ".cmf") + return None + fits = Fits(self.logger, self.config, files[0]) return fits