IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Oct 10, 2015, 12:40:44 PM (11 years ago)
Author:
eugene
Message:

make code less forgiving: sql execute aborts on failures; deal with various hidden problems: add IF EXISTS to drop table operations to avoid raising unneeded error, make sure tables exist before trying to add indexes, test for existence of the stack _xsrc, _xfit, etc tables (and skip if missing); fix sqlLine.group to cast inputs to string before performing string ops; getKeyFloat now returns a float value which is safe with sqlLine.group, but can be used in a math expression; use python math. operators as needed (sqrt, fabs); fulltest.sh runs for camera, stack, object stages; update dvoSkyTable to use REGION_ID instead of special word INDEX; add existsClient method (avoid using mysql failure as a test for table existence); no need to test for exceptions to sql.execute since we abort on failure; replace esoteric SQL with dropTable calls; trap errors in stilts.treads in setupScratchdb; drop StackObjecThin and StackDetOffMeta along with other stack scratch tables

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/jython/stackbatch.py

    r38826 r38837  
    160160           self.dropTableVerbose(tableName)
    161161
     162       self.dropTableVerbose("StackMeta")
    162163       self.dropTableVerbose("StackObjectThin")
    163164       self.dropTableVerbose("StackObjectAttributes")
    … …  
    176177
    177178       self.dropTableVerbose("StackToImage")
     179       self.dropTableVerbose("StackDetEffMeta")
    178180
    179181       # delete IPP tables
    … …  
    217219
    218220        header = self.headerSet[filter]
    219         tablename = filter + "StackMeta"
     221
    220222        stackID = self.stackIDs[filter]
    221223        filterName = filter + ".00000"
    222224
    223         self.logger.infoPair("Populating table", tablename)
    224 
    225         fwhm_maj    = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')
    226         fwhm_maj_uq = self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ')
    227         psfmodel_name    = self.getKeyValue(header, 'PSFMODEL')
    228         psfmodelID  = self.scratchDb.getFitModelID(psfmodel_name)
    229 
    230         ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX')
    231         ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY')
    232         astroscat = sqrt(ast_cdx^2 + ast_cdy^2)
    233        
    234225        # make a table
    235226        filterID = self.scratchDb.getFilterID(filter)
    … …  
    238229        photoCalID = str(self.scratchDb.getPhotoCalID(stackID))
    239230
     231        tablename = filter + "StackMeta"
     232
     233        sql = "CREATE TABLE " + tablename + " LIKE StackMeta"
     234        self.scratchDb.execute(sql)
     235
     236        self.logger.infoPair("Populating table", tablename)
     237
     238        # fwhm_maj    = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')
     239        # fwhm_maj_uq = self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ')
     240
     241        # XXX hard-wired platescale : 0.25
     242        pltscale = 0.25
     243        pltscale2 = pltscale * pltscale
     244
     245        psfFwhmMajor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')
     246        psfFwhmMinor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MIN')
     247        psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor)
     248
     249        psfmodel_name    = self.getKeyValue(header, 'PSFMODEL')
     250        psfmodelID  = self.scratchDb.getFitModelID(psfmodel_name)
     251
     252        ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX')
     253        ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY')
     254        astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2)
     255       
    240256        # Convert detectionThreshold to appropriate magnitudes
    241257        detectionThreshold = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF')
    242258        expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME')
    243         zp      = self.getKeyFloat(header, "%.8f", 'FPA.ZP')
    244         # CZW check?
     259        zp      = self.getKeyFloat(header, "%.8f", 'ZPT_OBS')
     260
     261        # XXX zp correction should come from DVO
    245262        detectionThreshold = detectionThreshold + zp - 2.5 * math.log10(expTime)
    246263       
    247        
    248         # mysql is sensitive to values which are ambiugously float.  eg
    249         # a warning is raised if we try to insert '25.' into a float field.
    250         # use getKeyFloat(hdr, format, key) to avoid this problem
    251 
    252         sql = "CREATE TABLE " + tablename + " LIKE StackMeta"
    253         try: self.scratchDb.execute(sql)
    254         except: pass
    255 
     264        # insert stack metadata into table
    256265        sqlLine = sqlUtility("INSERT INTO " + tablename + " (")
    257266
    258         sqlLine.group("stackImageID",  str(stackID))       
    259         sqlLine.group("batchID",       str(self.batchID))
    260         sqlLine.group("surveyID",      str(self.surveyID))
    261         sqlLine.group("filterID",      str(filterID))           
    262         sqlLine.group("stackTypeID",   str(self.stackTypeID))
    263         sqlLine.group("tessID",        str(self.tessID))           
    264         sqlLine.group("projectionID",  str(self.projectionID))           
    265         sqlLine.group("skyCellID",     str(self.skycellID))           
    266         sqlLine.group("photoCalID",    photoCalID)
    267         sqlLine.group("analysisVer",   str(self.analysisVer))
    268         sqlLine.group("detectionThreshold",       detectionThreshold)
    269         sqlLine.group("expTime",       self.getKeyFloat(header, "%.5f", 'EXPTIME')) 
    270         sqlLine.group("psfModelID",    psfmodelID)           
    271         sqlLine.group("psfFwhm_mean",  fwhm_maj)     
    272         sqlLine.group("psfFwhm_max",   fwhm_maj_uq)
    273         sqlLine.group("astroScat", astroscat)
    274         sqlLine.group("photoZero",     self.getKeyFloat(header, "%.5f", 'FPA.ZP'))
    275         sqlLine.group("ctype1",        header['CTYPE1']) 
    276         sqlLine.group("ctype2",        header['CTYPE2']) 
    277         sqlLine.group("crval1",        self.getKeyFloat(header, "%.8f", 'CRVAL1'))   
    278         sqlLine.group("crval2",        self.getKeyFloat(header, "%.8f", 'CRVAL2'))   
    279         sqlLine.group("crpix1",        self.getKeyFloat(header, "%.8f", 'CRPIX1'))   
    280         sqlLine.group("crpix2",        self.getKeyFloat(header, "%.8f", 'CRPIX2'))   
    281         sqlLine.group("cdelt1",        self.getKeyFloat(header, "%.8e", 'CDELT1'))   
    282         sqlLine.group("cdelt2",        self.getKeyFloat(header, "%.8e", 'CDELT2'))   
    283         sqlLine.group("pc001001",      self.getKeyFloat(header, "%.8e", 'PC001001'))
    284         sqlLine.group("pc001002",      self.getKeyFloat(header, "%.8e", 'PC001002'))
    285         sqlLine.group("pc002001",      self.getKeyFloat(header, "%.8e", 'PC002001'))
    286         sqlLine.group("pc002002",      self.getKeyFloat(header, "%.8e", 'PC002002'))
    287         sqlLine.group("processingVersion",   str(self.skychunk.processingVersion))
     267        sqlLine.group("stackImageID",       str(stackID))       
     268        sqlLine.group("batchID",            str(self.batchID))
     269        sqlLine.group("surveyID",           str(self.surveyID))
     270        sqlLine.group("filterID",           str(filterID))           
     271        sqlLine.group("stackTypeID",        str(self.stackTypeID))
     272        sqlLine.group("tessID",             str(self.tessID))           
     273        sqlLine.group("projectionID",       str(self.projectionID))           
     274        sqlLine.group("skyCellID",          str(self.skycellID))           
     275        sqlLine.group("photoCalID",         photoCalID)
     276        sqlLine.group("analysisVer",        str(self.analysisVer))
     277        sqlLine.group("detectionThreshold", detectionThreshold)
     278        sqlLine.group("expTime",            self.getKeyFloat(header, "%.5f", 'EXPTIME')) 
     279        sqlLine.group("psfModelID",         psfmodelID)           
     280        sqlLine.group("psfFWHM",            psfFWHM)
     281        sqlLine.group("psfWidMajor",        psfFwhmMajor)
     282        sqlLine.group("psfWidMinor",        psfFwhmMinor)
     283        sqlLine.group("psfTheta",           self.getKeyFloat(header, "%.8f", 'ANGLE'))
     284#       sqlLine.group("psfFwhm_mean",       fwhm_maj)     
     285#       sqlLine.group("psfFwhm_max",        fwhm_maj_uq)
     286        sqlLine.group("astroScat",          astroscat)
     287        sqlLine.group("photoZero",          zp)
     288        sqlLine.group("ctype1",             header['CTYPE1']) 
     289        sqlLine.group("ctype2",             header['CTYPE2']) 
     290        sqlLine.group("crval1",             self.getKeyFloat(header, "%.8f", 'CRVAL1'))   
     291        sqlLine.group("crval2",             self.getKeyFloat(header, "%.8f", 'CRVAL2'))   
     292        sqlLine.group("crpix1",             self.getKeyFloat(header, "%.8f", 'CRPIX1'))   
     293        sqlLine.group("crpix2",             self.getKeyFloat(header, "%.8f", 'CRPIX2'))   
     294        sqlLine.group("cdelt1",             self.getKeyFloat(header, "%.8e", 'CDELT1'))   
     295        sqlLine.group("cdelt2",             self.getKeyFloat(header, "%.8e", 'CDELT2'))   
     296        sqlLine.group("pc001001",           self.getKeyFloat(header, "%.8e", 'PC001001'))
     297        sqlLine.group("pc001002",           self.getKeyFloat(header, "%.8e", 'PC001002'))
     298        sqlLine.group("pc002001",           self.getKeyFloat(header, "%.8e", 'PC002001'))
     299        sqlLine.group("pc002002",           self.getKeyFloat(header, "%.8e", 'PC002002'))
     300        sqlLine.group("processingVersion",  str(self.skychunk.processingVersion))
    288301
    289302        sql = sqlLine.make(") VALUES ( ", ")")
    … …  
    536549                continue
    537550
     551            cmfTable = filter + "SkyChip_xsrc"
     552            if not cmfTable in self.tablesLoaded:
     553                self.logger.infoPair("no xsrc data for filter" , filter)
     554                continue
     555
    538556            # insert all the detections
    539             sqlLine = sqlUtility("UPDATE " + tablename + " AS a , " + filter + "SkyChip_xsrc AS b SET")
     557            sqlLine = sqlUtility("UPDATE " + tablename + " AS a , " + cmfTable + " AS b SET")
    540558
    541559            sqlLine.group("a."+filter+"haveData",       "'1'")
    … …  
    612630            return True
    613631
    614         if self.scratchDb.getRowCount(filter+"SkyChip_xfit") <=0 :
     632        cmfTable = filter + "SkyChip_xfit"
     633        if not cmfTable in self.tablesLoaded:
     634            self.logger.infoPair("no xfit data for filter" , filter)
     635            return True
     636
     637        if self.scratchDb.getRowCount(cmfTable) <=0 :
    615638            self.logger.infoPair("no extended source information, trying to skip",filter)
    616639            return True
    … …  
    621644        magtime = "%.5f" % (2.5*math.log10(float(header['EXPTIME'])))
    622645
    623         sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + filter + "SkyChip_xfit AS b SET")
     646        sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + cmfTable + " AS b SET")
    624647
    625648        sqlLine.group("a."+filter+"haveData",              "'1'")
    … …  
    775798            return True
    776799
    777         if self.scratchDb.getRowCount(filter+"SkyChip_xsrc") <=0 :
    778             self.logger.infoPair("no extended source information, trying to skip",filter)
     800        cmfTable = filter + "SkyChip_xsrc"
     801        if not cmfTable in self.tablesLoaded:
     802            self.logger.infoPair("no xsrc data for filter" , filter)
    779803            return True
    780804
     805        if self.scratchDb.getRowCount(cmfTable) <=0 :
     806            self.logger.infoPair("no extended source information, trying to skip", filter)
     807            return True
    781808
    782809        header  = self.headerSet[filter]
    … …  
    790817
    791818        self.logger.infoPair("Adding", "petrosians for extended sources")
    792         sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + filter + "SkyChip_xsrc AS b SET")
     819        sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + cmfTable + " AS b SET")
    793820
    794821        sqlLine.group("a."+filter+"haveData",         "'1'")
    … …  
    904931
    905932        # properly -999 these to start with.  the default should take
    906         # care of this, but does not
     933        # care of this, but does not?
    907934
    908935        sqlLine = sqlUtility("UPDATE " + tablename + " SET")
    … …  
    925952        # set the flux values from the cmf file
    926953        cmfTable = filter + "SkyChip_xrad"
     954
     955        # skip this table if it was not loaded from the cmf
     956        if not cmfTable in self.tablesLoaded:
     957            self.logger.infoPair("no xrad data for filter" , filter)
     958            return True
    927959
    928960        # we have variable numbers of these columns. find out which are in use
    … …  
    10571089                sql += " AND "
    10581090
    1059         self.logger.infoPair('sql',sql)   
    10601091        self.scratchDb.execute(sql)
    10611092
    … …  
    10841115
    10851116            self.logger.infoPair("Creating indexes on", "IPP tables")
    1086             self.scratchDb.createIndex(filter + "SkyChip_psf",  "IPP_IDET")
    1087             self.scratchDb.createIndex(filter + "SkyChip_xfit", "IPP_IDET")
    1088             self.scratchDb.createIndex(filter + "SkyChip_xrad", "IPP_IDET")
    1089             self.scratchDb.createIndex(filter + "SkyChip_xsrc", "IPP_IDET")
     1117
     1118            cmfTable = filter + "SkyChip_psf"
     1119            if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET")
     1120
     1121            cmfTable = filter + "SkyChip_xfit"
     1122            if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET")
     1123
     1124            cmfTable = filter + "SkyChip_xrad"
     1125            if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET")
     1126
     1127            cmfTable = filter + "SkyChip_xsrc"
     1128            if cmfTable in self.tablesLoaded: self.scratchDb.createIndex(cmfTable, "IPP_IDET")
    10901129
    10911130        return True
    … …  
    13031342        self.logger.infoPair("Importing ST tables with table match expression: ", tableRE)
    13041343
     1344        self.tablesLoaded = []
     1345
    13051346        count = 0
    13061347        for filter in self.filters:
    … …  
    13191360                return False
    13201361           
    1321             # count = 0
    13221362            for table in tables:
    13231363             
    … …  
    13441384                    self.logger.exception("Problem writing table '" + filter + table.name + "' to the database")
    13451385                     
     1386                self.tablesLoaded.append(table.name)
     1387
     1388        # XXX validate required tables (do we need _psf?, _xsrc? etc)
     1389
    13461390        self.logger.infoPair("Done. Imported", "%d tables" % count)
    13471391        self.indexIppTables()
Note: See TracChangeset for help on using the changeset viewer.