IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 39305 for trunk


Ignore:
Timestamp:
Jan 5, 2016, 3:34:49 PM (11 years ago)
Author:
eugene
Message:

need to handle possibly NULL / NaN values in the headers

Location:
trunk/ippToPsps/jython
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/jython/detectionbatch.py

    r39133 r39305  
    149149        sqlLine.group("analysisVer",           self.analysisVer);
    150150        sqlLine.group("md5sum",                str(self.md5sum))
     151
     152        # these lines should write "NULL" if the header contains NaN -- NULLs are converted below
     153        # to -999 for the sql database (a terrible solution, but mandatory?)
     154
    151155        sqlLine.group("photoScat",             self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR'));
    152156        sqlLine.group("expStart",              self.getKeyFloat(self.header, "%.10f", 'MJD-OBS'));
    … …  
    218222        else: ccdID = 0
    219223
     224        ### any operations below which act on the output of getKeyFloat need to be
     225        ### able to handle NULL results.
     226
    220227        # XXX hard-wired platescale : 0.257
    221228        pltscale = 0.257
    222229        pltscale2 = pltscale * pltscale
    223230
    224         psfFwhmMajor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')
    225         psfFwhmMinor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MIN')
    226         psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor)
     231        # NOTE: getKeyFloat returns the header value if it is a float,
     232        # -999 if the value is missing and "NULL" if the value in the
     233        # header is a NaN.  any math operations below need to handle
     234        # these cases correctly
     235
     236        # NOTE 2: getKeyFloat NOW returns the header value if it is a
     237        # float or "NULL" if the value is missing or if the value in
     238        # the header is a NaN.  any math operations below need to
     239        # handle these cases correctly
     240
     241        psfFwhmMajor = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')
     242        if (psfFwhmMajor != "NULL"): psfFwhmMajor *= pltscale
     243
     244        psfFwhmMinor = self.getKeyFloat(header, "%.8f", 'FWHM_MIN')
     245        if (psfFwhmMinor != "NULL"): psfFwhmMinor *= pltscale
     246
     247        psfFWHM = "NULL"
     248        if (psfFwhmMajor != "NULL") and (psfFwhmMinor != "NULL"):
     249            psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor)
    227250
    228251        psfmodel_name = self.getKeyValue(header, 'PSFMODEL')
    … …  
    231254        ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX')
    232255        ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY')
    233         astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2)
     256
     257        astroscat = "NULL"
     258        if (ast_cdx != "NULL") and (ast_cdy != "NULL"):
     259            astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2)
    234260       
    235261        # Convert detectionThreshold to appropriate magnitudes
    236         detectionThreshold = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF')
     262        magref = self.getKeyFloat(header, "%.8f", 'DETEFF.MAGREF')
    237263        expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME')
    238264        zp      = self.getKeyFloat(header, "%.8f", 'ZPT_OBS')
    239265
     266        # zp correction from DVO
    240267        zpImage = self.scratchDb.getImageZeroPoint(self.imageIDs[ota])
    241268
    242         # XXX zp correction should come from DVO
    243         detectionThreshold = detectionThreshold + zpImage + 2.5 * math.log10(expTime)
    244        
     269        if (zpImage == "NULL"):
     270            zpImage = zp
     271
     272        detectionThreshold = "NULL"
     273        if (magref != "NULL") and (zpImage != "NULL") and (expTime != "NULL"):
     274            detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime)
     275       
     276        # other header fields which depend on pltscale:
     277        momentMajor =        self.getKeyFloat(header, "%.8f", 'IQ_FW1'))
     278        momentMinor =        self.getKeyFloat(header, "%.8f", 'IQ_FW2'))
     279        momentM2C =          self.getKeyFloat(header, "%.8f", 'IQ_M2C'))
     280        momentM2S =          self.getKeyFloat(header, "%.8f", 'IQ_M2S'))
     281        momentM3 =           self.getKeyFloat(header, "%.8f", 'IQ_M3'))
     282        momentM4 =           self.getKeyFloat(header, "%.8f", 'IQ_M4'))
     283
     284        if (momentMajor != "NULL"): momentMajor *= pltscale
     285        if (momentMinor != "NULL"): momentMinor *= pltscale
     286        if (momentM2C   != "NULL"): momentM2C   *= pltscale2
     287        if (momentM2S   != "NULL"): momentM2S   *= pltscale2
     288        if (momentM3    != "NULL"): momentM3    *= pltscale2
     289        if (momentM4    != "NULL"): momentM4    *= pltscale2
     290
    245291        # insert image metadata into table
    246292        sqlLine = sqlUtility("INSERT INTO " + tableName + "(")
    … …  
    265311        sqlLine.group("psfWidMinor",        psfFwhmMinor)
    266312        sqlLine.group("psfTheta",           self.getKeyFloat(header, "%.8f", 'ANGLE'))
    267         sqlLine.group("momentMajor",        pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW1'))
    268         sqlLine.group("momentMinor",        pltscale * self.getKeyFloat(header, "%.8f", 'IQ_FW2'))
    269         sqlLine.group("momentM2C",          pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2C'))
    270         sqlLine.group("momentM2S",          pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M2S'))
    271         sqlLine.group("momentM3",           pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M3'))
    272         sqlLine.group("momentM4",           pltscale2 * self.getKeyFloat(header, "%.8f", 'IQ_M4'))
     313        sqlLine.group("momentMajor",        momentMajor)
     314        sqlLine.group("momentMinor",        momentMinor)
     315        sqlLine.group("momentM2C",          momentM2C)
     316        sqlLine.group("momentM2S",          momentM2S)
     317        sqlLine.group("momentM3",           momentM3)
     318        sqlLine.group("momentM4",           momentM4)
    273319        sqlLine.group("apResid",            self.getKeyFloat(header, "%.8f", 'APMIFIT'))
    274320        sqlLine.group("dapResid",           self.getKeyFloat(header, "%.8f", 'DAPMIFIT'))
    … …  
    374420        # CZW: 20151110 This needs to get the deteff header as well.
    375421        nInjected = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM'))
    376         magref = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF')
    377 
     422        magref  = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF')
    378423        expTime = self.getKeyFloat(header, "%.8f", 'EXPTIME')
    379424        zp      = self.getKeyFloat(header, "%.8f", 'ZPT_OBS')
    … …  
    382427        zpImage = self.scratchDb.getImageZeroPoint(self.imageIDs[ota])
    383428
    384         if zpImage < 0.0:
     429        if (zpImage == "NULL"):
    385430            zpImage = zp
    386431
    387         detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime)
     432        detectionThreshold = "NULL"
     433        if (magref != "NULL") and (zpImage != "NULL") and (expTime != "NULL"):
     434            detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime)
    388435
    389436        sqlLine.group("frameID",          str(self.expID))

  • trunk/ippToPsps/jython/diffbatch.py

    r39249 r39305  
    5656               "DF",
    5757               None)
    58     #           gpc1Db.getDiffStageCmf(skychunk.dvoLabel,diffSkyFileID))
    5958
    6059       # get diff meta data
    … …  
    195194           ## -- XXX hard-wired platescale : 0.25
    196195           pltscale = 0.25
    197            self.psfFwhmMajor[num] = pltscale * self.getKeyFloat(self.header[num], "%.8f", 'FWHM_MAJ')
    198            self.psfFwhmMinor[num] = pltscale * self.getKeyFloat(self.header[num], "%.8f", 'FWHM_MIN')
    199            self.psfFWHM[num] = 0.5*(self.psfFwhmMajor[num] + self.psfFwhmMinor[num])
    200 
    201            self.psfModelName[num]    = self.getKeyValue(self.header[num], 'PSFMODEL')
    202            self.psfModelID[num]  = self.scratchDb.getFitModelID(self.psfModelName[num])
     196           self.psfFwhmMajor[num] = self.getKeyFloat(self.header[num], "%.8f", 'FWHM_MAJ')
     197           if (self.psfFwhmMajor[num] != "NULL"): self.psfFwhmMajor[num] *= pltscale
     198
     199           self.psfFwhmMinor[num] = self.getKeyFloat(self.header[num], "%.8f", 'FWHM_MIN')
     200           if (self.psfFwhmMinor[num] != "NULL"): self.psfFwhmMinor[num] *= pltscale
     201
     202           self.psfFWHM[num] = "NULL"
     203           if (self.psfFwhmMajor[num] != "NULL") and (self.psfFwhmMinor[num] != "NULL"):
     204               self.psfFWHM[num] = 0.5*(self.psfFwhmMajor[num] + self.psfFwhmMinor[num])
     205
     206           self.psfModelName[num] = self.getKeyValue(self.header[num], 'PSFMODEL')
     207           self.psfModelID[num]   = self.scratchDb.getFitModelID(self.psfModelName[num])
    203208
    204209           # drop the existing tables   
    … …  
    244249        # Convert detectionThreshold to appropriate magnitudes
    245250        # we have read these values from the appropriate locations in init()
    246         zpImage = float(self.zpImage[num])
    247         expTime = float(self.expTime[num])
    248         magref  = float(self.magref[num])
    249 
    250         detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime)
     251        zpImage = "NULL" if (self.zpImage[num] == "NULL") else float(self.zpImage[num])
     252        expTime = "NULL" if (self.expTime[num] == "NULL") else float(self.expTime[num])
     253        magref  = "NULL" if (self.magref[num]  == "NULL") else float(self.magref[num])
     254
     255        isNull = (zpImage == "NULL") or (expTime == "NULL") or (magref == "NULL")
     256        detectionThreshold = "NULL" if isNull else magref + zpImage + 2.5 * math.log10(expTime)
    251257
    252258        sql = "CREATE TABLE DiffMeta_"+str(num)+" like DiffMeta"
    … …  
    532538
    533539        # we have read these values from the appropriate locations in init()
    534         zpImage = float(self.zpImage[num])
    535         expTime = float(self.expTime[num])
    536         magref  = float(self.magref[num])
    537 
    538         detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime)
     540        zpImage = "NULL" if (self.zpImage[num] == "NULL") else float(self.zpImage[num])
     541        expTime = "NULL" if (self.expTime[num] == "NULL") else float(self.expTime[num])
     542        magref  = "NULL" if (self.magref[num]  == "NULL") else float(self.magref[num])
     543
     544        isNull = (zpImage == "NULL") or (expTime == "NULL") or (magref == "NULL")
     545        detectionThreshold = "NULL" if isNull else magref + zpImage + 2.5 * math.log10(expTime)
    539546
    540547        nInjected = int(self.nInjected[num])

  • trunk/ippToPsps/jython/forcedwarpbatch.py

    r39123 r39305  
    228228        pltscale2 = pltscale * pltscale
    229229
    230         psfFwhmMajor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')
    231         psfFwhmMinor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MIN')
    232         psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor)
     230        # NOTE: getKeyFloat returns the header value if it is a float,
     231        # -999 if the value is missing and "NULL" if the value in the
     232        # header is a NaN.  any math operations below need to handle
     233        # these cases correctly
     234
     235        # NOTE 2: getKeyFloat NOW returns the header value if it is a
     236        # float or "NULL" if the value is missing or if the value in
     237        # the header is a NaN.  any math operations below need to
     238        # handle these cases correctly
     239
     240        psfFwhmMajor = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')
     241        if (psfFwhmMajor != "NULL"): psfFwhmMajor *= pltscale
     242
     243        psfFwhmMinor = self.getKeyFloat(header, "%.8f", 'FWHM_MIN')
     244        if (psfFwhmMinor != "NULL"): psfFwhmMinor *= pltscale
     245
     246        psfFWHM = "NULL"
     247        if (psfFwhmMajor != "NULL") and (psfFwhmMinor != "NULL"):
     248            psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor)
    233249
    234250        psfmodel_name = self.getKeyValue(header, 'PSFMODEL')
    … …  
    252268        sqlLine.group("ippSkycalID",    self.ippSkycalID[num])
    253269        sqlLine.group("stackMetaID",    self.stackMetaID[num])
    254 # no astrometry calibration for forced warp
    255 #       sqlLine.group("astroScat",      self.getKeyFloat(header, "%.8f", 'CERROR'))
    256 #       sqlLine.group("nAstroRef",      self.getKeyValue(header, 'NASTRO'))
    257 #       sqlLine.group("nPhotoRef",      self.getKeyValue(header, 'NASTRO'))
    258 #       sqlLine.group("photoScat",      self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR'));
    259270        sqlLine.group("psfModelID",     psfmodelID)
    260271        sqlLine.group("psfFWHM",        psfFWHM)
    261272        sqlLine.group("psfWidMajor",    psfFwhmMajor)
    262273        sqlLine.group("psfWidMinor",    psfFwhmMinor)
    263         # sqlLine.group("psfFwhm_mean",   self.getKeyFloat(header, "%.8f", 'FWHM_MAJ'))
    264         # sqlLine.group("psfFwhm_max",    self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ'))
     274
     275        # sqlLine.group("photoZero",      self.getKeyFloat(header, "%.8f", 'FPA.ZP'))
    265276
    266277        sqlLine.group("psfTheta",       self.getKeyFloat(header, "%.8f", 'ANGLE'))
    267 
    268         # sqlLine.group("photoZero",      self.getKeyFloat(header, "%.8f", 'FPA.ZP'))
    269278        sqlLine.group("photoZero",      zpImage)
    270279

  • trunk/ippToPsps/jython/stackbatch.py

    r39250 r39305  
    274274        pltscale2 = pltscale * pltscale
    275275
    276         psfFwhmMajor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')
    277         psfFwhmMinor = pltscale * self.getKeyFloat(header, "%.8f", 'FWHM_MIN')
    278         psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor)
     276        psfFwhmMajor = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')
     277        if (psfFwhmMajor != "NULL"): psfFwhmMajor *= pltscale
     278
     279        psfFwhmMinor = self.getKeyFloat(header, "%.8f", 'FWHM_MIN')
     280        if (psfFwhmMinor != "NULL"): psfFwhmMinor *= pltscale
     281
     282        psfFWHM = "NULL"
     283        if (psfFwhmMajor != "NULL") and (psfFwhmMinor != "NULL"):
     284            psfFWHM = 0.5*(psfFwhmMajor + psfFwhmMinor)
    279285
    280286        psfmodel_name    = self.getKeyValue(header, 'PSFMODEL')
    … …  
    283289        ast_cdx = self.getKeyFloat(header, "%.8f",'AST_CDX')
    284290        ast_cdy = self.getKeyFloat(header, "%.8f",'AST_CDY')
    285         astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2)
     291
     292        astroscat = "NULL"
     293        if (ast_cdx != "NULL") and (ast_cdy != "NULL"):
     294            astroscat = math.sqrt(ast_cdx**2 + ast_cdy**2)
    286295       
    287296        # Convert detectionThreshold to appropriate magnitudes
    288         magref  = self.magref[filter]
    289         expTime = self.expTime[filter]
    290         zp      = self.zpImage[filter]
    291         zpErr   = self.zpError[filter]
     297        magref  = "NULL" if (self.magref[filter]  == "NULL") else float(self.magref[filter])
     298        expTime = "NULL" if (self.expTime[filter] == "NULL") else float(self.expTime[filter])
     299        zp      = "NULL" if (self.zpImage[filter] == "NULL") else float(self.zpImage[filter])
     300        zpErr   = "NULL" if (self.zpError[filter] == "NULL") else float(self.zpError[filter])
    292301
    293302        # zp correction should comes from DVO (unless image is not in DVO)
    294303        zpImage = self.scratchDb.getImageZeroPoint(stackID)
    295         if zpImage < 0.0:
     304
     305        if (zpImage == "NULL"):
    296306            zpImage = zp
    297307
    298         detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime)
     308        detectionThreshold = "NULL"
     309        if (magref != "NULL") and (zpImage != "NULL") and (expTime != "NULL"):
     310            detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime)
    299311
    300312        # insert stack metadata into table
    … …  
    10401052        exptimeString = str(self.getKeyFloat(header, "%.5f", "EXPTIME"))
    10411053
     1054        ## this is a serious problem: not sure we have to handle it.
     1055        if (exptimeString == "NULL"):
     1056            print "**** serious error: EXPTIME is NULL (missing or NAN)"
     1057            os._exit(4)
     1058
    10421059        # generate the sql to do the necessary ops on the columns   
    10431060        sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + cmfTable + " AS b SET ")
Note: See TracChangeset for help on using the changeset viewer.