IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 37750


Ignore:
Timestamp:
Dec 16, 2014, 5:30:25 PM (12 years ago)
Author:
heather
Message:

diffbatch is very close - need to add rest of dvo stuffs and then cull the nulls and we are good to go!

Location:
trunk/ippToPsps
Files:
2 added
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/config/tables.DF.vot

    r37748 r37750  
    283283        <DESCRIPTION>flag indicating provenance information</DESCRIPTION>
    284284      </FIELD>
     285      <FIELD name="infoFlag2" arraysize="1" datatype="long" unit="dimensionless" default="-999">
     286        <DESCRIPTION>flag indicating provenance information</DESCRIPTION>
     287      </FIELD>
    285288      <FIELD name="dataRelease" arraysize="1" datatype="unsignedByte" unit="dimensionless" default="NA">
    286289        <DESCRIPTION>Data release when this detection was originally taken. Recalibrations do not affect this value.</DESCRIPTION>

  • trunk/ippToPsps/jython/diffbatch.py

    r37749 r37750  
    66import time
    77import stilts
    8 
     8import re
    99from java.lang import *
    1010from java.sql import *
    … …  
    178178    Populates the Diff table for this OTA
    179179    '''
    180     def populateDiffTable(self, ota, results):
     180    def populateDiffDetectionTable(self):
    181181
    182182        pspsTableName = "DiffDetection"
    183         ippTableName =  "diff_psf"
     183        ippTableName =  "SkyChip_psf"
    184184       
    185         results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName)
     185        #results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName)
    186186
    187187        # drop then re-create table
    188         self.scratchDb.dropTable(pspsTableName)
    189         sql = "CREATE TABLE " + pspsTableName + " LIKE DiffDetection"
    190         try: self.scratchDb.execute(sql)
    191         except: pass
     188        #self.scratchDb.dropTable(pspsTableName)
     189        #sql = "CREATE TABLE " + pspsTableName + " LIKE DiffDetection"
     190        #try: self.scratchDb.execute(sql)
     191        #except: pass
    192192       
    193193        # XXX we no longer delete detections with funny mags (eg PSF_INST_MAG < -17.5)
    … …  
    195195
    196196        BEFORE = self.scratchDb.getRowCount(ippTableName)
    197         results['SATDET'] = 0
     197        #results['SATDET'] = 0
    198198
    199199        extTimeString = str(self.expTime)
    … …  
    206206        # XXX apFluxF or apFluxNpix + apFluxRadius?
    207207
    208         sqlLine.group("ippDetectID",     "IPP_IDET")                                               
     208        sqlLine.group("ippDetectID",     "IPP_IDET")
     209        sqlLine.group("diffDetMetaID",            str(self.diffSkyFileID));                                               
    209210        sqlLine.group("randomDiffID",     "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)")                       
    210211        sqlLine.group("filterID",        str(self.filterID))                                       
    211         sqlLine.group("surveyID",        str(self.surveyID))                                       
    212         ##sqlLine.group("obsTime",         str(self.obsTime))     I have expTime
     212        sqlLine.group("surveyID",        str(self.surveyID))         
     213        sqlLine.group("skycellID",       str(self.skycellID))
     214##        sqlLine.group("obsTime",         str(self.obsTime))     
    213215        sqlLine.group("expTime",         str(self.expTime))                                   
    214216        sqlLine.group("xPos",            "X_PSF")                                                   
    … …  
    228230        sqlLine.group("psfFlux",         "PSF_INST_FLUX / " + extTimeString)
    229231        sqlLine.group("psfFluxErr",      "PSF_INST_FLUX_SIG / " + extTimeString)
    230         if (self.id >= 982483):
    231             sqlLine.group("psfMajorFWHM",    "PSF_FWHM_MAJ")                                               
    232             sqlLine.group("psfMinorFWHM",    "PSF_FWHM_MIN")                                               
    233             sqlLine.group("psfCore",         "PSF_CORE")                                                   
     232#        if (self.id >= 982483):   #we don't have PSF_FWHM_MAJ or MIN or PSF_CORE
     233#            sqlLine.group("psfMajorFWHM",    "PSF_FWHM_MAJ")                                               
     234#            sqlLine.group("psfMinorFWHM",    "PSF_FWHM_MIN")                                               
     235#            sqlLine.group("psfCore",         "PSF_CORE")                                                   
    234236        sqlLine.group("psfTheta",        "PSF_THETA")                                               
    235237        sqlLine.group("psfQf",           "PSF_QF")                                                 
    … …  
    249251            sqlLine.group("apFlux",          "AP_FLUX / " + extTimeString)
    250252            sqlLine.group("apFluxErr",       "AP_FLUX_SIG / " + extTimeString)
    251             sqlLine.group("apFillF",         "AP_NPIX / (3.14159265359 * POW(AP_MAG_RADIUS - 0.5, 2))")
     253 #           sqlLine.group("apFillF",         "AP_NPIX / (3.14159265359 * POW(AP_MAG_RADIUS - 0.5, 2))")
    252254 #       sqlLine.group("apRadius",        "AP_MAG_RADIUS")
    253255        sqlLine.group("kronFlux",        "KRON_FLUX / " + extTimeString)
    … …  
    291293
    292294        # we don't delete these anymore
    293         results['NULLINSTFLUX'] = 0;
     295        #results['NULLINSTFLUX'] = 0;
    294296
    295297
    296298# insert stuff from dvo
     299        self.logger.infoPair("inserting dvo info:find imageID"," from externID")
     300        imageID = self.scratchDb.getImageIDFromExternID(self.diffSkyFileID)
     301        self.logger.infoPair("updating","diffdetection")
     302        sqlLine  = sqlUtility("UPDATE DiffDetection as a, " + self.scratchDb.dvoDetectionTable + " as b SET ")
     303        sqlLine.group("a.diffObjID", "b.objId")
     304       
     305        sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " +str( imageID))   
     306       
     307        try: self.scratchDb.execute(sql)
     308        except:
     309            self.logger.errorPair("failed sql:",sql)
     310            raise
     311        self.logger.infoPair("need to cull"," nulls objid")
     312        self.logger.infoPair("need to add","uniquePSPSDFid")
    297313
    298314
    … …  
    304320        self.logger.infoPair("Procesing table", "DiffToImage")
    305321
    306         imageIDs = self.gpc1Db.getImageIDsForThisDiffID(diffkmetaID)
     322        imageIDs = self.gpc1Db.getImageIDsForThisDiffID(self.diffSkyFileID)
    307323
    308324        for imageID in imageIDs:
    … …  
    310326            sql = "INSERT INTO DiffToImage (diffMetaID, imageID) \
    311327            VALUES (\
    312             " + str(diffmetaID) + ", " + imageID + ")"
     328            " + str(self.diffSkyFileID) + ", " + imageID + ")"
    313329            self.scratchDb.execute(sql)
    314330
    … …  
    335351        self.scratchDb.makeColumnPrimaryKey("DiffDetection", "ippDetectID")
    336352
    337         self.populateDiffMeta()
     353        #self.populateDiffMeta()
    338354     
    339355        # dictionary objects to hold imageIDs for later
    … …  
    355371        self.logger.infoPair("Creating indexes on", "IPP tables")
    356372
    357         self.scratchDb.createIndex(extension, "IPP_IDET")
     373        #self.scratchDb.createIndex(extension, "IPP_IDET")
    358374        # try the test Chip
    359         self.scratchDb.createIndex("Chip_psf", "IPP_IDET")
     375        self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET")
    360376
    361377        self.logger.infoPair("created indexes on", "IPP tables")     
    … …  
    391407
    392408    def updatePspsUniqueIDs(self,table):
    393         sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, objID from \
     409        sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \
    394410        (select @r:=0) r, "+table+" t) as foo using (objID) set \
    395         uniquePspsP2id = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
     411        uniquePspsDFid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
    396412        try: self.scratchDb.execute(sql)
    397413        except:
    … …  
    402418    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
    403419    '''
    404     def populatePspsTablesChip(self, chipname, x, y, results, tables):
    405         # XXX EAM NOTE: drop tables Detection_* here so
    406         # they do not polute the db?
    407         # XXX or put an explicit drop at the end of the loop?
    408 
    409         # self.logger.infoTitle("Processing " + chipname)
    410         # this is a bit crude: if the chip is not present, this test will fail and the chip
    411         # will be (correctly) skipped.  would be better to carry that information explicitly ("chip is missing")
    412 
    413         # XXX keep this in or not?
    414         # if not self.scratchDb.astrometricSolutionOK(chipname):
    415         #     self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
    416         #     return False
    417 
    418         # does this chip exist in the DVO image table (if NOT, then skip it)
    419         if not self.scratchDb.haveThisChip(self.imageIDs[chipname], self.sourceIDs[chipname]):
    420             self.logger.info("| %5s |            ------------- Chip not in DVO : rejecting ------------                    |" % chipname)
    421             return False
    422 
    423         #self.logger.info("populate stuff ");
    424         # populate remainder of tables
    425         self.populateDetectionTable(chipname, results)
    426         #self.logger.info("successful populate ");
    427         # now add DVO IDs
    428         self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname])
    429         #self.logger.info("updated dvoids")
    430         results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID")
    431         #self.logger.info("deleted nulls")
    432         self.logger.info("add psps unique p2 ids")
    433         self.updatePspsUniqueIDs("Detection_" + chipname, self.imageIDs[chipname])
    434         self.updateImageID("Detection_" + chipname, x, y)
    435         #self.logger.info("updateImageId")
    436         rowCount = self.scratchDb.getRowCount("Detection_" + chipname)
    437         #self.logger.info("got row count")
    438         self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d |",
    439                 chipname,
    440                 results['ORIGINALTOTAL'],
    441                 results['SATDET'],
    442                 results['NULLINSTFLUX'],
    443                 results['NULLOBJID'],
    444                 rowCount)
    445         self.totalOriginal     = self.totalOriginal + results['ORIGINALTOTAL']
    446         self.totalSatDet       = self.totalSatDet + results['SATDET']
    447         self.totalNulIInstFlux = self.totalNulIInstFlux + results['NULLINSTFLUX']
    448         self.totalNullObjID    = self.totalNullObjID + results['NULLOBJID']
    449         self.totalDetections   = self.totalDetections + rowCount
    450 
    451         fractionKept = float(rowCount) / float(results['ORIGINALTOTAL'])
    452         # print "kept ", str(fractionKept), " detections from ", chipname
    453 
    454         if fractionKept < 0.8:
    455             # print "Too many rejected detections (", str(fractionKept), ", fail on exposure for", chipname
    456             self.logger.infoPair("Too many rejected detections, fail on exposure for", chipname)
    457             self.skipBatch = True
    458             return False;
    459 
    460         #self.logger.info("updated totals")
    461         # check we have something in this Detection table TODO add this to table above
    462         if rowCount < 1:
    463             self.logger.infoPair("Empty table for chip, fail on exposure", chipname)
    464             self.skipBatch = True
    465             return False;
    466 
    467         # update ImageMeta with count of detections for this CHIPNAME and photoCodeID
    468         sql = "UPDATE ImageMeta_" + chipname + " \
    469                SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + chipname), self.scratchDb.getPhotoCalID(self.imageIDs[chipname]))
    470         self.scratchDb.execute(sql)
    471        
    472         # add these to list of tables to export later
    473         self.tablesToExport.append("ImageMeta_" + chipname)
    474         self.logger.info("export ImageMeta")
    475 
    476         self.tablesToExport.append("Detection_" + chipname)
    477         self.logger.info("export Detection")
    478 
    479         tables.append("Detection_" + chipname)
    480         self.logger.info("updated detection")
    481 
    482         self.validChips.append(chipname)
    483 
    484         return True
    485420
    486421    '''
    … …  
    494429        self.tablesToExport.append("DiffMeta")
    495430        self.populateDiffMeta()
     431        self.logger.infoPair("populated","DiffMeta")
    496432        self.tablesToExport.append("DiffToImage")
    497433        self.populateDiffToImage()
     434        self.logger.infoPair("populated","DiffToImage")
    498435        self.tablesToExport.append("DiffDetection")
    499         self.populateDiffDetection()
    500        
     436        self.populateDiffDetectionTable()
     437        self.logger.infoPair("populated","DiffDetection")
    501438
    502439
    … …  
    575512            else: self.scratchDb.reportNulls(table.name + "_XY33", showPartials)
    576513
    577 
    578     '''
    579     This function reads the cmf/smf file and loads it into the database as its own table
    580     Overriding this method. Filter to only import *.psf extensions
    581     '''
    582     def importIppTables(self, filter=""):
    583        
    584         regex = ".*.psf"
    585         if False and self.config.test and self.config.camera == "gpc1":
    586             regex = "XY01.psf"
     514    '''
     515    Imports IPP tables from FITS file
     516    Accepts a regular expression to match the tables so not all tables need to be imported
     517    (This regex feature is not currently used...)
     518    '''
     519    def importIppTables(self, columns="*", tableRE=""):
     520       
     521        if self.config.retry: return True
     522
     523        self.logger.infoPair("Importing DF tables with table match expression: ", tableRE)
     524
     525        count = 0
     526        fileName = self.fits.getPath()
    587527           
    588         print "my ID: " + str(self.id)
    589 
    590         if (self.id < 982483):
    591             # XXX EAM NOTE : this is fragile : requires PS1_V4
    592             columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG                   PSF_INST_FLUX PSF_INST_FLUX_SIG       PSF_MAJOR PSF_MINOR PSF_THETA          PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S                             AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2"
    593         else:
    594             # XXX EAM NOTE : this is fragile : requires PS1_V5
    595             columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG POSANGLE PLTSCALE PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_FWHM_MAJ PSF_FWHM_MIN PSF_THETA PSF_CORE PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_FLUX AP_FLUX_SIG AP_NPIX AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2"
    596 
    597         return super(DetectionBatch, self).importIppTables(columns, regex)
     528        self.logger.infoPair("using filename:",fileName)
     529       
     530        try:
     531            tables = stilts.treads(fileName)
     532        except:
     533            self.logger.errorPair("STILTS could not import from", fileName)
     534            return False
     535       
     536            # count = 0
     537        for table in tables:
     538             
     539            match = re.match(tableRE, table.name)
     540            if not match: continue
     541
     542            self.logger.infoPair("Reading IPP table", table.name)
     543            table = stilts.tpipe(table, cmd='addcol table_index $0')
     544            table = stilts.tpipe(table, cmd='explodeall')
     545                 
     546            # drop any previous tables before import
     547            self.scratchDb.dropTable(table.name)
     548                 
     549                 
     550            # IPP FITS files are littered with infinities, so remove these
     551            self.logger.info("Removing Infinity values from all columns")
     552            table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
     553            table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
     554            table = stilts.tpipe(table, cmd='replaceval Infinity null *')
     555                 
     556            try:
     557                table.write(self.scratchDb.url + '#' +  table.name)
     558                count = count + 1
     559            except:
     560                self.logger.exception("Problem writing table '" +  table.name + "' to the database")
     561                     
     562        self.logger.infoPair("Done. Imported", "%d tables" % count)
     563        self.indexIppTables()
     564                     
     565        return True
     566
     567
     568
     569
     570
    598571
    599572    '''
    … …  
    601574    '''
    602575    def exportPspsTablesToFits(self, regex="(.*)"):
    603        return super(DetectionBatch, self).exportPspsTablesToFits("([a-zA-Z]+)")
     576       return super(DiffBatch, self).exportPspsTablesToFits("([a-zA-Z]+)")
     577
     578
     579    '''
     580    Drop a table and report hte drop
     581    '''
     582    def dropTableVerbose(self, table):
     583        self.logger.infoPair("dropping table:",table)
     584        self.scratchDb.dropTable(table)
     585

  • trunk/ippToPsps/jython/gpc1db.py

    r37749 r37750  
    354354               FROM warpSkyCellMap \
    355355               JOIN warpRun USING(warp_id) \
    356                JOIN diffInputSkyfile on warp1 = warp_id \
    357                WHERE diffInputSkyfile.diff_skyfile_id = " + str(diffID) + " AS a"
    358 
    359         try:
    360             rs = self.executeQuery(sql)
    361         except:
     356               JOIN diffInputSkyfile on warp1 = warp_id  \
     357               JOIN fakeRun using (fake_id) \
     358               JOIN camRun using (cam_id) \
     359               JOIN chipRun using (chip_id) \
     360               WHERE diffInputSkyfile.diff_skyfile_id = " + str(diffID) + ") AS a"
     361
     362        try:
     363            rs = self.executeQuery(sql)
     364        except:
     365            self.logger.infoPair("failed sql:",sql)
    362366            self.logger.exception("Can't query for imageIDs")
     367           
    363368
    364369        imageIDs = []
Note: See TracChangeset for help on using the changeset viewer.