Index: /branches/eam_branches/ipp-20121219/ippToPsps/doc/concerns.txt =================================================================== --- /branches/eam_branches/ipp-20121219/ippToPsps/doc/concerns.txt (revision 35002) +++ /branches/eam_branches/ipp-20121219/ippToPsps/doc/concerns.txt (revision 35003) @@ -51,4 +51,9 @@ (basically, the mods to the sql were wrong) +*** SMF header layout is hard-wired in detectionbatch.py (ie, XYnn) + * need to have a way of defining / loading a camera layout, either + * from the ippconfig info or elsewhere + + ----- Index: /branches/eam_branches/ipp-20121219/ippToPsps/jython/detectionbatch.py =================================================================== --- /branches/eam_branches/ipp-20121219/ippToPsps/jython/detectionbatch.py (revision 35002) +++ /branches/eam_branches/ipp-20121219/ippToPsps/jython/detectionbatch.py (revision 35003) @@ -214,4 +214,7 @@ ," + self.safeDictionaryAccess(self.header, 'PCA2X0Y2') + " \ )" + + print "frame meta sql: ", sql + self.scratchDb.execute(sql) @@ -593,4 +596,31 @@ self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) + ''' + Applies indexes and other constraints to the PSPS tables + ''' + def alterPspsTablesChip(self, chipname, extname, x, y): + + # load corresponding header into memory + header = self.fits.findAndReadHeader(extname) + if not header: + self.logger.errorPair("No header found for chip", chipname) + return False + + # check we have valid sourceID/imageID pair from the header + if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": return False + if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": return False + + # store sourceID/imageID combo in Db so DVO can look up later + if not self.useFullTables: + self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID']) + + # store these for later + self.imageIDs[chipname] = header['IMAGEID'] + + # populate ImageMeta + self.populateImageMetaTable(chipname, header) + self.updateImageID("ImageMeta_" + chipname, x, y) + + return True ''' @@ -623,25 +653,9 @@ ota = "XY%d%d" % (x, y) - # load corresponding header into memory - header = self.fits.findAndReadHeader(ota + ".hdr") - if not header: - self.logger.errorPair("No header found for OTA", ota) - continue - - # check we have valid sourceID/imageID pair from the header - if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": continue - if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": continue - - # store sourceID/imageID combo in Db so DVO can look up later - if not self.useFullTables: - self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID']) - - # store these for later - self.imageIDs[ota] = header['IMAGEID'] - - # populate ImageMeta - self.populateImageMetaTable(ota, header) - self.updateImageID("ImageMeta_" + ota, x, y) - + self.alterPspsTablesChip(ota, ota + ".hdr", x, y) + + # try the test Chip + self.alterPspsTablesChip("Chip", "Chip.hdr", 0, 0) + # now run DVO code to get all IDs if not self.useFullTables: @@ -651,6 +665,4 @@ # the column in PSPS self.scratchDb.execute("ALTER TABLE DetectionCalib CHANGE dec_ `dec` double") - - return True @@ -675,5 +687,9 @@ extension = "XY%d%d_psf" % (x, y) self.scratchDb.createIndex(extension, "IPP_IDET") + # try the test Chip + self.scratchDb.createIndex(Chip_psf, "IPP_IDET") + self.logger.infoPair("created indexes on", "IPP tables") + ''' Updates provided table with DVO IDs from DVO table @@ -693,4 +709,66 @@ self.scratchDb.execute(sql) + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTablesChip(self, chipname, x, y): + #self.logger.infoTitle("Processing " + chipname) + if not self.scratchDb.astrometricSolutionOK(chipname): + self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % chipname) + return False + self.logger.info("populate stuff "); + # populate remainder of tables + self.populateDetectionTable(chipname, results) + self.logger.info("successful populate "); + # now add DVO IDs + self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname]) + self.logger.info("updated dvoids") + results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID") + #self.logger.info("deleted nulls") + self.updateImageID("Detection_" + chipname, x, y) + self.logger.info("updateImageId") + rowCount = self.scratchDb.getRowCount("Detection_" + chipname) + self.logger.info("got row count") + self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", + chipname, + results['ORIGINALTOTAL'], + results['SATDET'], + results['NULLINSTFLUX'], + results['NULLPEAKADU'], + results['NULLOBJID'], + rowCount) + totalOriginal = totalOriginal + results['ORIGINALTOTAL'] + totalSatDet = totalSatDet + results['SATDET'] + totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX'] + totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU'] + totalNullObjID = totalNullObjID + results['NULLOBJID'] + totalDetections = totalDetections + rowCount + #self.logger.info("updated totals") + # check we have something in this Detection table TODO add this to table above + if rowCount < 1: + self.logger.debugPair("Skipping empty table for chipname", chipname) + return False; + + # update ImageMeta with count of detections for this CHIPNAME and photoCodeID + sql = "UPDATE ImageMeta_" + chipname + " \ + SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + chipname), self.scratchDb.getPhotoCalID(self.imageIDs[chipname])) + self.scratchDb.execute(sql) + + #self.logger.info("updated imagedata") + self.populateSkinnyObjectTable(chipname) + #self.logger.info("updated skinnyobject") + #self.populateObjectCalColorTable(chipname) + #self.logger.info("updated objectcalcolor") + self.populateDetectionCalibTable(chipname) + #self.logger.info("updated detectioncalibtable") + + # add these to list of tables to export later + self.tablesToExport.append("ImageMeta_" + chipname) + self.tablesToExport.append("Detection_" + chipname) + self.tablesToExport.append("SkinnyObject_" + chipname) + #self.tablesToExport.append("ObjectCalColor_" + chipname) + self.tablesToExport.append("DetectionCalib_" + chipname) + tables.append("Detection_" + chipname) + return True ''' @@ -721,64 +799,8 @@ if ota not in self.imageIDs: continue - #self.logger.infoTitle("Processing " + ota) - if not self.scratchDb.astrometricSolutionOK(ota): - self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % ota) - continue - # self.logger.info("populate stuff "); - # populate remainder of tables - self.populateDetectionTable(ota, results) - # self.logger.info("successful populate "); - # now add DVO IDs - self.updateDvoIDs("Detection_" + ota, self.imageIDs[ota]) - # self.logger.info("updated dvoids") - results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID") - #self.logger.info("deleted nulls") - self.updateImageID("Detection_" + ota, x, y) - # self.logger.info("updateImageId") - rowCount = self.scratchDb.getRowCount("Detection_" + ota) - # self.logger.info("got row count") - self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", - ota, - results['ORIGINALTOTAL'], - results['SATDET'], - results['NULLINSTFLUX'], - results['NULLPEAKADU'], - results['NULLOBJID'], - rowCount) - totalOriginal = totalOriginal + results['ORIGINALTOTAL'] - totalSatDet = totalSatDet + results['SATDET'] - totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX'] - totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU'] - totalNullObjID = totalNullObjID + results['NULLOBJID'] - totalDetections = totalDetections + rowCount - #self.logger.info("updated totals") - # check we have something in this Detection table TODO add this to table above - if rowCount < 1: - self.logger.debugPair("Skipping empty table for ota", ota) - continue; - - # update ImageMeta with count of detections for this OTA and photoCodeID - sql = "UPDATE ImageMeta_" + ota + " \ - SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.imageIDs[ota])) - self.scratchDb.execute(sql) - - #self.logger.info("updated imagedata") - self.populateSkinnyObjectTable(ota) - #self.logger.info("updated skinnyobject") - #self.populateObjectCalColorTable(ota) - #self.logger.info("updated objectcalcolor") - self.populateDetectionCalibTable(ota) - #self.logger.info("updated detectioncalibtable") - - # add these to list of tables to export later - self.tablesToExport.append("ImageMeta_" + ota) - self.tablesToExport.append("Detection_" + ota) - self.tablesToExport.append("SkinnyObject_" + ota) - #self.tablesToExport.append("ObjectCalColor_" + ota) - self.tablesToExport.append("DetectionCalib_" + ota) - tables.append("Detection_" + ota) - - otaCount = otaCount + 1 - #self.logger.info("end of ota") + if populatePspsTables(ota, x, y): otaCount = otaCount + 1 + + if populatePspsTables("Chip", 0, 0): otaCount = otaCount + 1 + # print totals self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") @@ -833,4 +855,5 @@ else : regex = ".*.psf" + # XXX EAM NOTE : this is fragile : requires PS1_V_ columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" Index: /branches/eam_branches/ipp-20121219/ippToPsps/jython/fits.py =================================================================== --- /branches/eam_branches/ipp-20121219/ippToPsps/jython/fits.py (revision 35002) +++ /branches/eam_branches/ipp-20121219/ippToPsps/jython/fits.py (revision 35003) @@ -137,5 +137,5 @@ # this regex will get param/value pairs for all header cards, ignoring comments and parsing out 'HIERACH' prefixes - match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-_\.:\s@#]+)\'*\\/*', record) + match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-+_\.:\s@#]+)\'*\\/*', record) if match: Index: /branches/eam_branches/ipp-20121219/ippToPsps/jython/ippjython =================================================================== --- /branches/eam_branches/ipp-20121219/ippToPsps/jython/ippjython (revision 35002) +++ /branches/eam_branches/ipp-20121219/ippToPsps/jython/ippjython (revision 35003) @@ -44,3 +44,6 @@ shift +# XXX if you want to run a test on a local file, use this line: +# $JAVA -Xbootclasspath/a:$CLASSPATH -jar $JYTHON $classfile $* + $JAVA -Xbootclasspath/a:$CLASSPATH -jar $JYTHON $JYDIR/$classfile $* Index: /branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopspsdb.py (revision 35002) +++ /branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopspsdb.py (revision 35003) @@ -999,4 +999,5 @@ self.config.epoch = rs.getString(16) + self.config.survey = rs.getString(17) self.config.pspsSurvey = rs.getString(18) @@ -1005,5 +1006,5 @@ if rs.getInt(20) == 1: self.config.batchTypes.append("ST") if rs.getInt(21) == 1: self.config.batchTypes.append("OB") - self.config.force = False # TODO + self.config.force = True # TODO self.config.test = False # TODO self.config.isLoaded = True @@ -1101,4 +1102,6 @@ GROUP BY ra_center LIMIT 1" + # XXX print "sql: ", sql + try: rs = self.executeQuery(sql) Index: /branches/eam_branches/ipp-20121219/ippToPsps/jython/queue.py =================================================================== --- /branches/eam_branches/ipp-20121219/ippToPsps/jython/queue.py (revision 35002) +++ /branches/eam_branches/ipp-20121219/ippToPsps/jython/queue.py (revision 35003) @@ -30,4 +30,5 @@ self.gpc1Db = Gpc1Db(self.logger, self.config) self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + try: self.dvoObjects = DvoObjects(self.logger, self.config) @@ -39,5 +40,4 @@ if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) - ''' Index: /branches/eam_branches/ipp-20121219/ippToPsps/test/mkgpc1data.dvo =================================================================== --- /branches/eam_branches/ipp-20121219/ippToPsps/test/mkgpc1data.dvo (revision 35002) +++ /branches/eam_branches/ipp-20121219/ippToPsps/test/mkgpc1data.dvo (revision 35003) @@ -4,8 +4,4 @@ if (not($?VERBOSE)) set VERBOSE = 0 - -macro test.all - test.relphot PS1_V2 PS1_V2 -end ## this script must be run in this directory, output data goes to 'testdata' @@ -40,6 +36,6 @@ macro mkfull init.db - mkcatdir.cam PS1_V2 PS1_V2 - mkcatdir.stk PS1_V2 PS1_V2 + mkcatdir.cam PS1_V3 PS1_V2 + mkcatdir.stk PS1_V3 PS1_V2 insert.stack.set end @@ -95,14 +91,113 @@ $offset = $word:3 - ctimes -ref 2008/01/01,$i\:00:00 -var mjd + ctimes -abs 2008/01/01,$i\:00:00 -var mjd + + # XXX for a simple test of ippToPsps, i need to generate smf files with some correspondence to gpc1 exposure smfs + # this means: + # 1) a PHU with some basic header data + # 2) add extensions with EXTNAME of XYnn + + # create an empty header and populate with the desired keywords + mcreate dummy 0 0 + keyword dummy MJD-OBS -wf $mjd + keyword dummy FILTERID -w r.00000 + keyword dummy EXPTIME -wf 1.0 + + keyword dummy ZPT_ERR -wf 0.01 + keyword dummy EXPREQ -wf 1.0 + keyword dummy AIRMASS -wf 1.0 + keyword dummy RA -wf $RA + keyword dummy DEC -wf $DEC + keyword dummy CTYPE1 -w "RA---DIS" + keyword dummy CTYPE2 -w "DEC--DIS" + keyword dummy CRVAL1 -wf $RA + keyword dummy CRVAL2 -wf $DEC + keyword dummy CRPIX1 -wf 0.0 + keyword dummy CRPIX2 -wf 0.0 + keyword dummy CDELT1 -wf {1.0/3600} + keyword dummy CDELT2 -wf {1.0/3600} + keyword dummy PC001001 -wf 1.0 + keyword dummy PC001002 -wf 0.0 + keyword dummy PC002001 -wf 0.0 + keyword dummy PC002002 -wf 1.0 + keyword dummy NPLYTERM -wd 0 + keyword dummy PCA1X3Y0 -wf 0.0 + keyword dummy PCA1X2Y1 -wf 0.0 + keyword dummy PCA1X1Y2 -wf 0.0 + keyword dummy PCA1X0Y3 -wf 0.0 + keyword dummy PCA1X2Y0 -wf 0.0 + keyword dummy PCA1X1Y1 -wf 0.0 + keyword dummy PCA1X0Y2 -wf 0.0 + keyword dummy PCA2X3Y0 -wf 0.0 + keyword dummy PCA2X2Y1 -wf 0.0 + keyword dummy PCA2X1Y2 -wf 0.0 + keyword dummy PCA2X0Y3 -wf 0.0 + keyword dummy PCA2X2Y0 -wf 0.0 + keyword dummy PCA2X1Y1 -wf 0.0 + keyword dummy PCA2X0Y2 -wf 0.0 + wd dummy $cmffile + + exec ftable -list $cmffile mkinput $offset $rawfile exec mkcmf $rawfile $cmffile -date 2008/1/1 -time $i\:00:00 -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 - exec rm -f header.dat - output header.dat - echo MJD-OBS = $mjd - echo FILTERID= 'r.00000' - output stdout - exec fits_insert $cmffile header.dat + + exec ftable -list $cmffile + + # keywords which are read from image extension + # MSKY_MN + # MSKY_SIG + # FSATUR + # FLIMIT + # CERROR + # ZPT_OBS + # NASTRO + # NASTRO + # CNAXIS1 + # CNAXIS2 + # FWHM_MAJ + # FWHM_MIN + # ANGLE + # IQ_FW1 + # IQ_FW2 + # IQ_FW1 + # IQ_FW2 + # APMIFIT + # DAPMIFIT + # DETECTOR + # IMAGEID + # DETREND.MASK + # DETREND.DARK + # DETREND.FLAT + # ZPT_OBS + # CTYPE1 + # CTYPE2 + # CRVAL1 + # CRVAL2 + # CRPIX1 + # CRPIX2 + # CDELT1 + # CDELT2 + # PC001001 + # PC001002 + # PC002001 + # PC002002 + # NPLYTERM + # PCA1X3Y0 + # PCA1X2Y1 + # PCA1X1Y2 + # PCA1X0Y3 + # PCA1X2Y0 + # PCA1X1Y1 + # PCA1X0Y2 + # PCA2X3Y0 + # PCA2X2Y1 + # PCA2X1Y2 + # PCA2X0Y3 + # PCA2X2Y0 + # PCA2X1Y1 + # PCA2X0Y2 + + echo addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass exec addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass end