IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 38890


Ignore:
Timestamp:
Oct 20, 2015, 2:13:13 PM (11 years ago)
Author:
eugene
Message:

clean up some of the indenting; ingest the forced warp tables and index only after all are loaded; add to tablesLoaded

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/jython/forcedwarpbatch.py

    r38881 r38890  
    5757               "FW",
    5858               None)
    59      #          gpc1Db.getForcedWarpStageSmf(forcedWarpID)
     59
     60       # gpc1Db.getForcedWarpStageSmf(forcedWarpID)
    6061       # there is a funny problem with order here: we need to open the file first to read the header to get
    6162       # to skycell ID
    62        #forcedWarpID = self.forcedWarpID   
    63 
    64        #make a set of lists (needs to be outside of the loop
     63       # forcedWarpID = self.forcedWarpID   
     64
     65       # make a set of lists (needs to be outside of the loop
    6566
    6667       stageID = self.id
    … …  
    8990       
    9091       self.forcedWarpIds = gpc1Db.getListOfForcedWarpIdsForForcedWarpBatch(skychunk.dvoLabel,stageID)
     92
    9193       for forcedWarpID in self.forcedWarpIds:
    9294           self.number[num] = forcedWarpID
    … …  
    9698           self.fits[num] = gpc1Db.getForcedWarpStageCmf(skychunk.dvoLabel, int(forcedWarpID))
    9799           
     100           ## print "--- PHU Header ---"
     101           ## myHeader = self.fits[num].getPrimaryHeader()
     102           ## for key in myHeader:
     103           ##     print "--- header: " + key + " = " + myHeader[key]
     104           ## print "--- END Header ---"
     105
    98106           self.header[num] = self.fits[num].getPrimaryHeader()
    99107           self.warpSkyFileID[num] = self.getKeyValue(self.header[num],"IMAGEID")
    … …  
    101109           self.logger.infoPair("found filterName",self.filterName[num])
    102110           self.logger.infoPair("found warp_skyfile_id",self.warpSkyFileID[num])
    103        # get camera meta data
     111
     112           # get camera meta data from gpc1 database
     113           # NOTE: this should all be in the header
    104114           meta ={}
    105115           meta = self.gpc1Db.getForcedWarpStageMeta(forcedWarpID, self.warpSkyFileID[num])
    … …  
    111121           self.expID[num] = meta[0]
    112122           self.expTime[num] = meta[1]
    113 # This isn't populated in the database (it's NULL).  The hack is below.
    114 #          self.analysisVer[num] = meta[2]
     123           # This isn't populated in the database (it's NULL).  The hack is below.
     124           # self.analysisVer[num] = meta[2]
    115125           self.skycellName[num] = meta[3]
    116126           self.tessName[num] = meta[4]
    … …  
    121131           self.logger.infoPair("found tess_id",self.tessName[num])
    122132
    123 
    124        #tessName = self.getKeyValue(self.header, 'TESS_ID')
    125        #self.tessID = self.scratchDb.getTessID(tessName)
     133           # tessName = self.getKeyValue(self.header, 'TESS_ID')
     134           # self.tessID = self.scratchDb.getTessID(tessName)
    126135
    127136           # As the database doesn't have the software_ver entry, pull the photometry version as the analysis version.
    … …  
    131140           if self.analysisVer[num] is None: self.analysisVer[num] = "-999"
    132141           self.logger.infoPair("found analysisVer",self.analysisVer[num])
    133 
     142           
    134143           if self.tessName[num] == 'RINGS.V3':
    135 
    136            # skycell is, eg "skycell.1133.081"
    137            #                 0123456789012345
     144               # skycell is, eg "skycell.1133.081"
     145               #                 0123456789012345
    138146               self.projectionID[num] = self.skycellName[num][8:12]
    139            #self.skycellID = self.skycellName[13:]
    140                self.skycellID[num] = self.skycellName[num][8:]
     147               self.skycellID[num] = self.skycellName[num][13:]
    141148           else:
    142            # skycell is, eg "skycell.081"
    143            #                 01234567890
     149               # skycell is, eg "skycell.081"
     150               #                 01234567890
    144151               self.projectionID[num] = 0
    145                self.skycellID[num] = self.skycellName[num][8:11]
    146                self.skycellID[num]=self.skycellName[num][8:]
    147                if self.skycellID[num] == "":
    148                    self.skycellID[num] = -1
     152               self.skycellID[num] = self.skycellName[num][8:]
     153               
     154           if self.projectionID[num] == "":
     155               self.projectionID[num] = -1
     156           if self.skycellID[num] == "":
     157               self.skycellID[num] = -1
    149158
    150159           self.logger.infoPair("projection cell found ",self.projectionID[num])
    … …  
    164173               # create the tables
    165174
     175           # XXX why are these deleted?
    166176           sql = "CREATE TABLE  ForcedWarpMeasurement_" + str(forcedWarpID) + " like ForcedWarpMeasurement"
    167177           print sql
    … …  
    183193
    184194           self.logger.infoPair("calculating ObsTime","ok")
    185        # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2
    186 
     195           # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2
    187196
    188197           self.obsTime[num] = float(self.header[num]['MJD-OBS']) + (float(self.expTime[num]) / 172800.0)
    … …  
    346355     
    347356        sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (")
    348         sqlLine.group("ippDetectID",     "IPP_IDET")
     357        sqlLine.group("ippDetectID",       "IPP_IDET")
    349358     
    350         sqlLine.group("forcedWarpID",            str(forcedWarpID))
    351         print " got here too, num", num
    352         sqlLine.group("randomWarpID",     "RAND("+str(self.batchID)+")")
    353         print "here" 
    354         sqlLine.group("filterID",        str(self.filterID[num]))
    355         print "here1"
    356         sqlLine.group("surveyID",        str(self.surveyID))
    357         print "here2"
    358         sqlLine.group("skycellID",       str(self.skycellID[num]))
    359         print "here3"
    360         sqlLine.group("expTime",         str(self.expTime[num]))
    361         sqlLine.group("forcedSummaryID", str(self.id))
    362         print "here4"
    363         sqlLine.group("obsTime", str(self.obsTime[num]))
     359        sqlLine.group("forcedWarpID",      str(forcedWarpID))
     360        sqlLine.group("randomWarpID",      "RAND("+str(self.batchID)+")")
     361        sqlLine.group("filterID",          str(self.filterID[num]))
     362        sqlLine.group("surveyID",          str(self.surveyID))
     363        sqlLine.group("skycellID",         str(self.skycellID[num]))
     364        sqlLine.group("expTime",           str(self.expTime[num]))
     365        sqlLine.group("forcedSummaryID",   str(self.id))
     366                                           
     367        sqlLine.group("obsTime",           str(self.obsTime[num]))
    364368        sqlLine.group("FpsfFlux",          "PSF_INST_FLUX / " + expTimeString)
    365369        sqlLine.group("FpsfFluxErr",       "PSF_INST_FLUX_SIG / " + expTimeString)
    … …  
    394398        sqlLine.group("processingVersion", str(self.skychunk.processingVersion))
    395399
    396         print "got here too"
    397 
    398400        sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName)
    399 
    400         try: self.scratchDb.execute(sql)
    401         except:
    402             self.logger.errorPair("failed sql: ", sql)
    403             raise
    404 
     401        self.scratchDb.execute(sql)
    405402
    406403        # insert the per-object data (IDs, random stack ID, processingVersion, etc)
    407         #self.selectDvoObjIDs()
     404        # self.selectDvoObjIDs()
    408405        self.logger.infoPair("inserting dvo info: find imageID","from externID")
    409406        imageID = self.scratchDb.getImageIDFromExternID(self.warpSkyFileID[num])
    … …  
    430427
    431428        sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " +str( imageID))
    432         try: self.scratchDb.execute(sql)
    433         except:
    434             self.logger.errorPair("failed sql:",sql)
    435             raise
     429        self.scratchDb.execute(sql)
     430
    436431        self.logger.infoPair("need to cull"," nulls objid")
    437432        rowCountBefore = self.scratchDb.getRowCount("ForcedWarpMeasurement_"+forcedWarpID)
    … …  
    456451        self.scratchDb.createIndex("ForcedWarpMeasurement_"+forcedWarpID, "objID")
    457452        self.scratchDb.createIndex("ForcedWarpMeasurement_"+forcedWarpID, "ippDetectID")
     453
    458454    '''
    459455    Populates the ForcedWarpExtended table
    … …  
    651647    def importIppTables(self, columns="*", tableRE=""):
    652648
     649        self.tablesLoaded = []
     650
    653651        count = 0
    654652        for num in self.number:
    … …  
    666664                self.logger.errorPair("STILTS could not import from", fileName)
    667665                return False
     666
    668667            for table in tables:
    669668             
    670669                match = re.match(tableRE, table.name)
    671670                if not match: continue
     671
    672672                self.logger.infoPair("Reading IPP table", table.name+"_"+str(forcedWarpID))
    673673                table = stilts.tpipe(table, cmd='addcol table_index $0')
    674674                table = stilts.tpipe(table, cmd='explodeall')
    675675           
    676            # drop any previous tables before import
     676                # drop any previous tables before import
    677677           
    678678                self.scratchDb.dropTable(table.name+"_"+str(forcedWarpID))
    679679                 
    680            # IPP FITS files are littered with infinities, so remove these
     680                # IPP FITS files are littered with infinities, so remove these
    681681                self.logger.info("Removing Infinity values from all columns")
    682682                table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
    … …  
    689689                except:
    690690                    self.logger.exception("Problem writing table '" + table.name+"_"+str(forcedWarpID) + "' to the database")
    691                
    692691
    693692                self.logger.infoPair("Done. Imported", "%d tables" % count)
    694                 self.indexIppTables(forcedWarpID)
     693
     694                self.tablesLoaded.append(table.name)
     695
     696            # after all tables are read for this forcedWarpID, then index the set
     697            self.indexIppTables(forcedWarpID)
     698
    695699        return True
    696700
    … …  
    748752    def indexIppTables(self,forcedWarpID):
    749753
    750         self.logger.infoPair("Creating indexes on"+forcedWarpID, "IPP tables")
    751         self.scratchDb.createIndex("SkyChip_psf_"+forcedWarpID,  "IPP_IDET")
    752         self.scratchDb.createIndex("SkyChip_xfit_"+forcedWarpID, "IPP_IDET")
    753         self.scratchDb.createIndex("SkyChip_xrad_"+forcedWarpID, "IPP_IDET")
    754         self.scratchDb.createIndex("SkyChip_xsrc_"+forcedWarpID, "IPP_IDET")
    755         self.scratchDb.createIndex("SkyChip_xgal_"+forcedWarpID, "IPP_IDET")
     754        self.logger.infoPair("Creating indexes on " + forcedWarpID, "IPP tables")
     755
     756        # it probably should not be an error not to have one of these:
     757        cmfTable = "SkyChip_psf_" + forcedWarpID
     758        if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable,  "IPP_IDET")
     759
     760        cmfTable = "SkyChip_xfit_" + forcedWarpID
     761        if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable,  "IPP_IDET")
     762
     763        cmfTable = "SkyChip_xrad_" + forcedWarpID
     764        if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable,  "IPP_IDET")
     765
     766        cmfTable = "SkyChip_xsrc_" + forcedWarpID
     767        if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable,  "IPP_IDET")
     768
     769        cmfTable = "SkyChip_xgal_" + forcedWarpID
     770        if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable,  "IPP_IDET")
     771
     772        ## self.scratchDb.createIndex("SkyChip_xfit_"+forcedWarpID, "IPP_IDET")
     773        ## self.scratchDb.createIndex("SkyChip_xrad_"+forcedWarpID, "IPP_IDET")
     774        ## self.scratchDb.createIndex("SkyChip_xsrc_"+forcedWarpID, "IPP_IDET")
     775        ## self.scratchDb.createIndex("SkyChip_xgal_"+forcedWarpID, "IPP_IDET")
    756776
    757777        return True
Note: See TracChangeset for help on using the changeset viewer.