IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 39103 for trunk


Ignore:
Timestamp:
Nov 11, 2015, 11:08:06 AM (11 years ago)
Author:
eugene
Message:

use Ohana / fhead to read headers (ex PHU) so we avoid seek problems, etc; merge reading of exptime, zp, zperr, ninjected, magerf into the init methods of diff and stack batches

Location:
trunk/ippToPsps
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/config/tables.IN.vot

    r39088 r39103  
    540540      <DATA>
    541541        <TABLEDATA>
    542           <TR><TD>1</TD><TD>RINGS.V3</TD><TD>Tessellation for the 3pi survey, ex pole regions</TD></TR>
    543           <TR><TD>2</TD><TD>CNP.LAP</TD><TD>Tessellation for the 3pi survey pole regions (LOCAL)</TD></TR>
     542          <TR><TD>1</TD><TD>CNP.LAP</TD><TD>Tessellation for the 3pi survey pole regions (LOCAL)</TD></TR>
     543          <TR><TD>2</TD><TD>RINGS.V3</TD><TD>Tessellation for the 3pi survey, ex pole regions</TD></TR>
    544544          <TR><TD>3</TD><TD>MD01.V3</TD><TD>Tessellation for MD01 field (LOCAL)</TD></TR>
    545545          <TR><TD>4</TD><TD>MD02.V3</TD><TD>Tessellation for MD02 field (LOCAL)</TD></TR>

  • trunk/ippToPsps/jython/diffbatch.py

    r39102 r39103  
    7777       self.md5sum = {}
    7878       self.expTime = {}
     79       self.zpImage = {}
     80       self.magref = {}
     81       self.nInjected = {}
    7982       self.obsTime = {}
    8083       self.posFits = {}
    … …  
    150153           self.obsTime[num] = float(self.header[num]['MJD-OBS']) + (float(self.expTime[num]) / 172800.0)
    151154
     155           # grap zpImage from the PHU header
     156           self.zpImage[num] = self.getKeyFloat(self.header[num],"%.8f","FPA.ZP")
     157
     158           deteffHeader = self.posFits[num].findAndReadHeader("SkyChip.deteff", self.config.test)
     159           if not deteffHeader:
     160               if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter)
     161               return False
     162
     163           # grab the detection threshold reference magnitude
     164           self.magref[num] = self.getKeyFloat(deteffHeader, "%.8f", "DETEFF.MAGREF")
     165
     166           self.nInjected[num] = self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM')
     167
     168           # self.logger.infoPair("Found zeropoint and set magref", str(magref))
     169
    152170           tessName = self.tessName[num]
    153171           self.logger.infoPair("TessName ", tessName)
    … …  
    225243
    226244        # Convert detectionThreshold to appropriate magnitudes
    227         detectionThreshold = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF')
    228         zpImage            = self.getKeyFloat(header, "%.8f", 'FPA.ZP')
    229            
    230         detectionThreshold = detectionThreshold + zpImage + 2.5 * math.log10(float(self.expTime[num]))
     245        # we have read these values from the appropriate locations in init()
     246        zpImage = float(self.zpImage[num])
     247        expTime = float(self.expTime[num])
     248        magref  = float(self.magref[num])
     249
     250        detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime)
    231251
    232252        sql = "CREATE TABLE DiffMeta_"+str(num)+" like DiffMeta"
    … …  
    506526        self.logger.infoPair("Created tmp table:", ippTableName)
    507527
    508         header = self.header[num]
    509         zpImage = self.getKeyFloat(header,"%.8f","FPA.ZP")
    510         expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME')
    511 
    512         if zpImage == 0.0:
    513             self.logger.infoPair("CZW If this value for the zeropoint is zero, the getImageZeroPoint call is likely still broken", str(zpImage))
    514 
    515 # CZW 20151110 This is broken due to findAndReadHeader being broken.
    516 #        deteffHeader = self.posFits[num].findAndReadHeader("SkyChip.deteff", self.config.test)
    517 #        if not deteffHeader:
    518 #            if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter)
    519 #            return False
    520 # This next line is a work around.
    521        
    522         deteffHeader = header
    523         magref = self.getKeyFloat(deteffHeader,"%.8f","DETEFF.MAGREF") + zpImage + 2.5 * math.log10(expTime)
    524         self.logger.infoPair("Found zeropoint and set magref", str(magref))
    525 
    526         nInjected = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM'))
    527        
     528        # we have read these values from the appropriate locations in init()
     529        zpImage = float(self.zpImage[num])
     530        expTime = float(self.expTime[num])
     531        magref  = float(self.magref[num])
     532
     533        detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime)
     534
     535        nInjected = int(self.nInjected[num])
    528536
    529537        sqlLine = sqlUtility("INSERT INTO " + ippTableName + "(")
    530538        sqlLine.group("diffImageID", str(diffSkyFileID))
    531         sqlLine.group("magref",       magref)
     539        sqlLine.group("magref",       detectionThreshold)
    532540        sqlLine.group("nInjected",    nInjected)
    533541        sql = sqlLine.make(") VALUES ( ", ")")
    … …  
    574582        self.scratchDb.makeColumnPrimaryKey("DiffDetection", "ippDetectID")
    575583
    576         #self.populateDiffMeta()
    577      
    578584        # dictionary objects to hold imageIDs for later
    579585        self.imageIDs = {}

  • trunk/ippToPsps/jython/fits.py

    r39085 r39103  
    66import hashlib
    77from subprocess import call, PIPE, Popen
     8
     9import tempfile
    810
    911'''
    … …  
    121123
    122124    '''
     125    Find and read a header extension.
     126    '''
     127    def findAndReadHeader(self, name, VERBOSE):
     128
     129        if False:
     130            status = self.findAndReadHeaderRaw(name, VERBOSE)
     131            return status
     132
     133        status = self.findAndReadHeaderOhana(name, VERBOSE)
     134        return status
     135
     136    '''
    123137    Finds and reads a header extension.
    124138
    … …  
    132146    through STILTS
    133147    '''
    134     def findAndReadHeader(self, name, VERBOSE):
     148    def findAndReadHeaderRaw(self, name, VERBOSE):
    135149
    136150        found = False
    … …  
    167181
    168182    '''
     183    Find and read a header extension.
     184    '''
     185    def findAndReadHeaderOhana(self, name, VERBOSE):
     186
     187        cmd = "fhead -n %s %s" % (name, self.originalPath)
     188        p = Popen(cmd, shell=True, stdout=PIPE)
     189        output = p.communicate()[0]       
     190
     191        header = {}
     192
     193        # split the output header bytes into key/value pairs for each 80-byte line
     194        for i in range(0, len(output), 80):
     195            # print "line %d : %s" % (i, output[i:i+80])
     196            record = output[i:i+80]
     197
     198            match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-+_\.:\s@#\(\)\,]+)\'*\\/*', record)
     199
     200            if match:
     201                param = match.group(2)
     202                value = match.group(3).strip()
     203                if value == "NaN": value = "NULL"
     204                header[param] = value
     205
     206        return header
     207
     208    '''
    169209    Reads FITS header and stores all fields in a dictionary object
    170210    '''

  • trunk/ippToPsps/jython/stackbatch.py

    r39101 r39103  
    6565       self.md5sum = {}
    6666       self.stackEpochs = {}
     67       self.expTime = {}
     68       self.zpImage = {}
     69       self.zpError = {}
     70       self.magref = {}
     71       self.nInjected = {}
    6772
    6873       # we have two sets of imageIDs here:
    … …  
    116121
    117122           self.stackEpochs[filter] = self.getKeyFloat(header, "%.10f", 'MJD-OBS')
     123
     124           self.expTime[filter] = self.getKeyFloat(header, "%.8f", 'EXPTIME')
     125           self.zpImage[filter] = self.getKeyFloat(header, "%.8f", 'ZPT_OBS')
     126           self.zpError[filter] = self.getKeyFloat(header, "%.8f", 'ZPT_ERR')
     127
     128           deteffHeader = self.fits[filter].findAndReadHeader("SkyChip.deteff", self.config.test)
     129           if not deteffHeader:
     130               if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter)
     131               return False
     132
     133           self.magref[filter] = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF')
     134           self.nInjected[filter] = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM'))
    118135
    119136           ## MEH -- md5sum test
    … …  
    268285       
    269286        # Convert detectionThreshold to appropriate magnitudes
    270         # Grab this from the appropriate extension.
    271 
    272 # CZW 20151110 This is broken.
    273 #        deteffHeader = self.fits[filter].findAndReadHeader("SkyChip.deteff", self.config.test)
    274 #        if not deteffHeader:
    275 #            if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter)
    276 #            return False
    277 # CZW This is the workaround
    278 
    279         deteffHeader = header
    280         detectionThreshold = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF')
    281         expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME')
    282         zp      = self.getKeyFloat(header, "%.8f", 'ZPT_OBS')
    283         zpErr   = self.getKeyFloat(header, "%.8f", 'ZPT_ERR')
     287        magref  = self.magref[filter]
     288        expTime = self.expTime[filter]
     289        zp      = self.zpImage[filter]
     290        zpErr   = self.zpError[filter]
    284291
    285292        # zp correction should comes from DVO (unless image is not in DVO)
    … …  
    288295            zpImage = zp
    289296
    290         detectionThreshold = detectionThreshold + zpImage + 2.5 * math.log10(expTime)
    291        
     297        detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime)
     298
    292299        # insert stack metadata into table
    293300        sqlLine = sqlUtility("INSERT INTO " + tablename + " (")
    … …  
    305312        sqlLine.group("md5sum",             str(self.md5sum[filter]))
    306313        sqlLine.group("detectionThreshold", detectionThreshold)
    307         sqlLine.group("expTime",            self.getKeyFloat(header, "%.5f", 'EXPTIME')) 
     314        sqlLine.group("expTime",            expTime)
    308315        sqlLine.group("psfModelID",         psfmodelID)           
    309316        sqlLine.group("psfFWHM",            psfFWHM)
    … …  
    481488
    482489            header = self.headerSet[filter]
    483             exptime = self.getKeyFloat(header, "%.5f", "EXPTIME")
     490            exptime = self.expTime[filter]
    484491            expTimeString = str(exptime)
    485492
    … …  
    850857        ## XXX is this statement true: is FPA.ZP + 2.5log(exptime) applied to PETRO_MAG?
    851858
    852         hdrZP   = "%.5f" % float(header['FPA.ZP'])
    853         magtime = "%.5f" % (2.5*math.log10(float(header['EXPTIME'])))
     859        hdrZP   = str(self.zpImage[filter])
     860        # magtime = "%.5f" % (2.5*math.log10(self.expTime[filter]))
    854861
    855862        # XXX hard-wired platescale : 0.25 (could get this from header, but stacks have constant plate scale
    … …  
    11351142               
    11361143                field = "EXP_%04d" % input
    1137                 expTime = self.getKeyFloat(header, "%6.3f", field)
     1144                expTime = self.getKeyFloat(header, "%6.3f", field) # exptime of INPUT image, not stack
    11381145               
    11391146                field = "AIR_%04d" % input
    … …  
    11831190            except: pass
    11841191
    1185 
    1186             header = self.headerSet[filter]
    1187 
    11881192            # Convert detectionThreshold to appropriate magnitudes
    1189             # Grab this from the appropriate extension.
    1190 # CZW 20151110 This is broken.
    1191 #            deteffHeader = self.fits[filter].findAndReadHeader("SkyChip.deteff", self.config.test)
    1192 #            if not deteffHeader:
    1193 #                if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter)
    1194 #                return False
    1195 # CZW This is the workaround
    1196            
    1197             deteffHeader = header
    1198             detectionThreshold = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF')
    1199             expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME')
    1200             zp      = self.getKeyFloat(header, "%.8f", 'ZPT_OBS')
    1201             zpErr   = self.getKeyFloat(header, "%.8f", 'ZPT_ERR')
    1202 
    1203         # zp correction should comes from DVO (unless image is not in DVO)
    1204             zpImage = self.scratchDb.getImageZeroPoint(stackID)
     1193            magref  = self.magref[filter]
     1194            expTime = self.expTime[filter]
     1195            zp      = self.zpImage[filter]
     1196            zpErr   = self.zpError[filter]
     1197
     1198            # zp correction should comes from DVO (unless image is not in DVO)
     1199            zpImage = self.scratchDb.getImageZeroPoint(stackImageID)
    12051200            if zpImage < 0.0:
    12061201                zpImage = zp
    12071202
    1208             detectionThreshold = detectionThreshold + zpImage + 2.5 * math.log10(expTime)
    1209             nInjected = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM'))
    1210 
    1211             zpImage = self.scratchDb.getImageZeroPoint(stackImageID)
    1212             if zpImage < 0.0:
    1213                 zpImage = 0.0
    1214             magref = self.getKeyFloat(header,"%.8f","DETEFF.MAGREF") + zpImage
     1203            detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime)
     1204            nInjected = self.nInjected[filter]
    12151205
    12161206            sqlLine = sqlUtility("INSERT INTO " + ippTableName + "(")
    12171207            sqlLine.group("stackImageID", str(stackImageID))
    1218             sqlLine.group("magref",       magref)
     1208            sqlLine.group("magref",       detectionThreshold)
    12191209            sqlLine.group("nInjected",    nInjected)
    12201210            sql = sqlLine.make(") VALUES ( ", ")")
    1221            
    12221211
    12231212            try: self.scratchDb.execute(sql)

  • trunk/ippToPsps/jython/testCode.py

    r39068 r39103  
    9696        print "words[1]: " + words[1]
    9797
     98       
     99
    98100        # self.connectMysql(argv)
    99101        # self.importIppTables()
    … …  
    118120    testcode.run(sys.argv)
    119121except: pass
     122
     123        # XXX a test of fits header reading:
     124        ## XX myFits = Fits(self.logger, self.config, sys.argv[2])
     125        ## XX
     126        ## XX for ix in range(7, -1, -1):
     127        ## XX     for iy in range(7, -1, -1):
     128        ## XX         if (ix == 0) and (iy == 0): continue
     129        ## XX         if (ix == 7) and (iy == 0): continue
     130        ## XX         if (ix == 0) and (iy == 7): continue
     131        ## XX         if (ix == 7) and (iy == 7): continue
     132        ## XX         extname = "XY%d%d.hdr" % (ix, iy)
     133        ## XX
     134        ## XX         header = myFits.findAndReadHeader(extname, False)
     135        ## XX         print "read " + extname
     136        ## XX
     137        ## XX os._exit(3)
     138
     139        ## the above works in loader.init()
Note: See TracChangeset for help on using the changeset viewer.