IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jun 3, 2015, 11:23:52 AM (11 years ago)
Author:
heather
Message:

changes for forced warp / diff

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/jython/forcedwarpbatch.py

    r38083 r38358  
    4343                 ippToPspsDb,
    4444                 scratchDb,
    45                  forcedWarpID,
     45                 id,
    4646                 batchID):
    4747
    … …  
    5353               ippToPspsDb,
    5454               scratchDb,
    55                forcedWarpID,
     55               id,
    5656               batchID,
    5757               "FW",
    … …  
    6060       # there is a funny problem with order here: we need to open the file first to read the header to get
    6161       # to skycell ID
    62        #forcedWarpID = self.forcedWarpID       
    63        self.logger.infoPair("getting the fits file for add_id",forcedWarpID)
    64        self.fits = gpc1Db.getForcedWarpStageCmf(skychunk.dvoLabel, forcedWarpID)
    65        self.forcedWarpID = forcedWarpID
    66        self.header = self.fits.getPrimaryHeader()
    67        self.warpSkyFileID = self.getKeyValue(self.header,"IMAGEID")
    68        self.filterName = self.getKeyValue(self.header,"FPA.FILTERID")
    69        self.logger.infoPair("found filterName",self.filterName)
    70        self.logger.infoPair("found warp_skyfile_id",self.warpSkyFileID)
     62       #forcedWarpID = self.forcedWarpID   
     63
     64       #make a set of lists (needs to be outside of the loop
     65
     66       stageID = self.id
     67       self.fits={}
     68       self.header={}
     69       self.warpSkyFileID={}
     70       self.filterName = {}
     71       self.historyModNum = "0"
     72
     73
     74       self.calibModNum = 0
     75       self.totalNumPhotoRef = 0
     76       self.number={}
     77       self.expID = {}
     78       self.expTime = {}
     79       self.analysisVer = {}
     80       self.tessName = {}
     81       self.obsTime = {}
     82       self.filterID={}           
     83       self.skycellName={}
     84       self.skycellID = {}
     85       self.projectionID = {}
     86
     87       #I give a number to the forcedwarp stuff, I am not sure if this is the best way but it might work
     88
     89       num = 0
     90       
     91       self.forcedWarpIds = gpc1Db.getListOfForcedWarpIdsForForcedWarpBatch(skychunk.dvoLabel,stageID)
     92       for forcedWarpID in self.forcedWarpIds:
     93           self.number[num] = forcedWarpID
     94           self.logger.infoPair("index",num)
     95           self.logger.infoPair("forcedWarpID", forcedWarpID)
     96           self.logger.infoPair("getting the fits file for add_id",forcedWarpID)
     97           self.fits[num] = gpc1Db.getForcedWarpStageCmf(skychunk.dvoLabel, int(forcedWarpID))
     98           
     99           self.header[num] = self.fits[num].getPrimaryHeader()
     100           self.warpSkyFileID[num] = self.getKeyValue(self.header[num],"IMAGEID")
     101           self.filterName[num] = self.getKeyValue(self.header[num],"FPA.FILTERID")
     102           self.logger.infoPair("found filterName",self.filterName[num])
     103           self.logger.infoPair("found warp_skyfile_id",self.warpSkyFileID[num])
    71104       # get camera meta data
    72 
    73        meta = self.gpc1Db.getForcedWarpStageMeta(forcedWarpID, self.warpSkyFileID)
    74 
    75        if not meta:
    76            self.logger.errorPair("Could not get", "forced warp metadata")
    77            raise
    78 
    79        self.expID = meta[0]
    80        self.expTime = meta[1]
    81        self.analysisVer = meta[2]
    82        self.skycellName = meta[3]
    83        self.tessName = meta[4]
     105           meta ={}
     106           meta = self.gpc1Db.getForcedWarpStageMeta(forcedWarpID, self.warpSkyFileID[num])
     107
     108           if not meta:
     109               self.logger.errorPair("Could not get", "forced warp metadata")
     110               raise
     111
     112           self.expID[num] = meta[0]
     113           self.expTime[num] = meta[1]
     114           self.analysisVer[num] = meta[2]
     115           self.skycellName[num] = meta[3]
     116           self.tessName[num] = meta[4]
    84117       
    85        self.logger.infoPair("found exp_id",str(self.expID))
    86        self.logger.infoPair("found exp_time",str(self.expTime))
    87        self.logger.infoPair("found skycell_id",self.skycellName)
    88        self.logger.infoPair("found tess_id",self.tessName)
     118           self.logger.infoPair("found exp_id",str(self.expID[num]))
     119           self.logger.infoPair("found exp_time",str(self.expTime[num]))
     120           self.logger.infoPair("found skycell_id",self.skycellName[num])
     121           self.logger.infoPair("found tess_id",self.tessName[num])
    89122
    90123
    … …  
    92125       #self.tessID = self.scratchDb.getTessID(tessName)
    93126
    94        if not self.analysisVer: self.analysisVer = -999
    95        if self.analysisVer is None: self.analysisVer = "-999"
    96        self.logger.infoPair("found analysisVer",self.analysisVer)
    97 
    98        if self.tessName == 'RINGS.V3':
     127           if not self.analysisVer[num]: self.analysisVer[num] = -999
     128           if self.analysisVer[num] is None: self.analysisVer[num] = "-999"
     129           self.logger.infoPair("found analysisVer",self.analysisVer[num])
     130
     131           if self.tessName[num] == 'RINGS.V3':
    99132
    100133           # skycell is, eg "skycell.1133.081"
    101134           #                 0123456789012345
    102            self.projectionID = self.skycellName[8:12]
     135               self.projectionID[num] = self.skycellName[num][8:12]
    103136           #self.skycellID = self.skycellName[13:]
    104            self.skycellID = self.skycellName[8:]
    105        else:
     137               self.skycellID[num] = self.skycellName[num][8:]
     138           else:
    106139           # skycell is, eg "skycell.081"
    107140           #                 01234567890
    108            self.projectionID = 0
    109            self.skycellID = self.skycellName[8:11]
    110            self.skycellID=self.skycellName[8:]
    111        if self.skycellID == "":
    112            self.skycellID = -1
    113 
    114        self.logger.infoPair("projection cell found ",self.projectionID)
    115        self.logger.infoPair("skycell found", self.skycellID)
    116 
    117 
    118 
     141               self.projectionID[num] = 0
     142               self.skycellID[num] = self.skycellName[num][8:11]
     143               self.skycellID[num]=self.skycellName[num][8:]
     144               if self.skycellID[num] == "":
     145                   self.skycellID[num] = -1
     146
     147           self.logger.infoPair("projection cell found ",self.projectionID[num])
     148           self.logger.infoPair("skycell found", self.skycellID[num])
     149           self.dropTableVerbose("ForcedWarpMeasurement"+"_"+str(forcedWarpID))
     150           self.dropTableVerbose("ForcedWarpExtended"+"_"+str(forcedWarpID))
     151           self.dropTableVerbose("ForcedWarpMeta"+"_"+str(forcedWarpID))
     152           self.dropTableVerbose("ForcedWarpToImage"+"_"+str(forcedWarpID))
     153           self.dropTableVerbose("ForcedWarpLensing"+"_"+str(forcedWarpID))
     154           if not self.config.retry:
     155               self.scratchDb.dropTable("ForcedWarpMeasurement"+"_"+str(forcedWarpID))
     156               self.scratchDb.dropTable("ForcedWarpExtended"+"_"+str(forcedWarpID))
     157               self.scratchDb.dropTable("ForcedWarpMeta"+"_"+str(forcedWarpID))
     158               self.scratchDb.dropTable("ForcedWarpToImage"+"_"+str(forcedWarpID))
     159               self.scratchDb.dropTable("ForcedWarpLensing"+"_"+str(forcedWarpID))
     160       # get a few primary header values. if in test mode, then use defaults
     161               # create the tables
     162
     163           sql = "CREATE TABLE  ForcedWarpMeasurement_" + str(forcedWarpID) + " like ForcedWarpMeasurement"
     164           print sql
     165           #try: self.scratchDb.execute(sql)
     166           #except: raise
     167           sql = "CREATE TABLE  ForcedWarpExtended_" + str(forcedWarpID) + " like ForcedWarpExtended"
     168           #try: self.scratchDb.execute(sql)
     169           #except: raise
     170           sql = "CREATE TABLE  ForcedWarpMeta_" + str(forcedWarpID) + " like ForcedWarpMeta"
     171           #try: self.scratchDb.execute(sql)
     172           #except: raise
     173           sql = "CREATE TABLE  ForcedWarpToImage_" + str(forcedWarpID) + " like ForcedWarpToImage"
     174           #try: self.scratchDb.execute(sql)
     175           #except: raise
     176           sql = "CREATE TABLE  ForcedWarpLensing_" + str(forcedWarpID) + " like ForcedWarpLensing"
     177           #try: self.scratchDb.execute(sql)
     178           #except: raise
     179
     180
     181           self.logger.infoPair("calculating ObsTime","ok")
     182       # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2
     183
     184
     185           self.obsTime[num] = float(self.header[num]['MJD-OBS']) + (float(self.expTime[num]) / 172800.0)
     186
     187           self.logger.infoPair("obstime",self.obsTime[num])
     188           self.filterID[num] = self.scratchDb.getFilterID(self.filterName[num][0])
     189           self.logger.infoPair("filterID is ",self.filterID[num])
     190           self.logger.infoPair("inserting basics into ","ForcedWarpMeta"+"_"+forcedWarpID)
     191           
     192           self.ippToPspsDb.insertForcedWarpMeta( int(self.batchID), int(self.expID[num]), self.filterName[num][0])       
     193
     194           num=num+1
    119195       # create an output filename, which is {expID}.FITS
    120        self.outputFitsFile = "%08d.FITS" % forcedWarpID
     196       self.outputFitsFile = "%08d.FITS" % self.id
    121197       self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
    122        self.dropTableVerbose("ForcedWarpMeasurement")
    123        self.dropTableVerbose("ForcedWarpExtended")
    124        self.dropTableVerbose("ForcedWarpMeta")
    125        self.dropTableVerbose("ForcedWarpToImage")
    126        self.dropTableVerbose("ForcedWarpLensing")
    127        if not self.config.retry:
    128            self.scratchDb.dropTable("ForcedWarpMeasurement")
    129            self.scratchDb.dropTable("ForcedWarpExtended")
    130            self.scratchDb.dropTable("ForcedWarpMeta")
    131            self.scratchDb.dropTable("ForcedWarpToImage")
    132            self.scratchDb.dropTable("ForcedWarpLensing")
    133        self.historyModNum = "0"
    134 
    135        # get a few primary header values. if in test mode, then use defaults
    136        self.logger.infoPair("calculating ObsTime","ok")
    137        # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2
    138        self.obsTime = float(self.header['MJD-OBS']) + (float(self.expTime) / 172800.0)
    139        self.logger.infoPair("obstime",self.obsTime)
    140        # set up some defauts
    141        self.calibModNum = 0
    142        self.totalNumPhotoRef = 0
    143 
    144        self.filterID = self.scratchDb.getFilterID(self.filterName[0])
    145        self.logger.infoPair("filterID is ",self.filterID)
    146 
    147        # insert what we know about this stack batch into the stack table
    148        self.logger.infoPair("inserting basics into ","ForcedWarpMeta")
    149 
    150        self.ippToPspsDb.insertForcedWarpMeta(int(self.batchID), int(self.expID), self.filterName[0])
     198
     199
    151200
    152201       # dump stuff to log
    153202       self.logger.infoPair("add ID", "%d" % self.id)
    154        self.logger.infoPair("Exp ID", self.expID)
     203
     204#       self.logger.infoPair("Exp ID", self.expID)
    155205
    156206
    … …  
    162212    Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header
    163213    '''
    164     def populateForcedWarpMeta(self):
    165         self.logger.infoPair("Processing table", "ForcedWarpMeta")
    166 
    167         header   = self.header
     214    def populateForcedWarpMeta(self,num):
     215        forcedWarpID = self.number[num]
     216        self.logger.infoPair("Processing table", "ForcedWarpMeta_"+forcedWarpID)
     217
     218        sql = "CREATE TABLE ForcedWarpMeta_"+forcedWarpID+" LIKE ForcedWarpMeta"
     219        try: self.scratchDb.execute(sql)
     220        except: pass
     221
     222
     223        header   = self.header[num]
     224        self.logger.infoPair("found header","ok")
    168225#        photoCalID = str(self.scratchDb.getPhotoCalID(forcedWarpID))       
    169226
    170         sqlLine = sqlUtility("INSERT INTO ForcedWarpMeta (")
     227        sqlLine = sqlUtility("INSERT INTO ForcedWarpMeta_"+forcedWarpID+" (")
    171228        #batchID (below)
    172229        #surveyID (below)
    173230        #filterdID (below)
    174         sqlLine.group("forcedWarpID", str(self.forcedWarpID))
    175         sqlLine.group("skyCellID",     str(self.skycellID))
    176 #        sqlLine.group("photoCalID",    photoCalID)
     231
     232#        self.logger.infoPair("building sql ","ok")
     233
     234        sqlLine.group("forcedWarpID", str(forcedWarpID))
     235#        self.logger.infoPair("building sql ","1")
     236        sqlLine.group("skyCellID",     str(self.skycellID[num]))
     237#        sqlLine.group("photoCalID",    photoCalID)
     238#        self.logger.infoPair("building sql ","2")
    177239        sqlLine.group("magSat",           self.getKeyFloat(header, "%.8f", 'FSATUR'))
    178         sqlLine.group("analVer",           str(self.analysisVer))
    179         sqlLine.group("expTime",self.expTime)
     240#        self.logger.infoPair("building sql ","3")
     241        sqlLine.group("analVer",           str(self.analysisVer[num]))
     242 #       self.logger.infoPair("building sql ","3")
     243
     244        sqlLine.group("expTime",self.expTime[num])
     245 #       self.logger.infoPair("building sql ","3")
    180246        sqlLine.group("completMag",       self.getKeyFloat(header, "%.8f", 'FLIMIT'))
    181247#        sqlLine.group("astroScat",        self.getKeyFloat(header, "%.8f", 'CERROR'))
    … …  
    189255#        sqlLine.group("photoZero",        self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS'))
    190256        #photoColor -- how do I set this? it's also not set in stack meta
    191         sqlLine.group("ctype1",                self.getKeyValue(self.header, 'CTYPE1'));
    192         sqlLine.group("ctype2",                self.getKeyValue(self.header, 'CTYPE2'));
    193         sqlLine.group("crval1",                self.getKeyFloat(self.header, "%.8f", 'CRVAL1'));
    194         sqlLine.group("crval2",                self.getKeyFloat(self.header, "%.8f", 'CRVAL2'));
    195         sqlLine.group("crpix1",                self.getKeyFloat(self.header, "%.8f", 'CRPIX1'));
    196         sqlLine.group("crpix2",                self.getKeyFloat(self.header, "%.8f", 'CRPIX2'));
    197         sqlLine.group("cdelt1",                self.getKeyFloat(self.header, "%.8e", 'CDELT1'));
    198         sqlLine.group("cdelt2",                self.getKeyFloat(self.header, "%.8e", 'CDELT2'));
    199         sqlLine.group("pc001001",              self.getKeyFloat(self.header, "%.8e", 'PC001001'));
    200         sqlLine.group("pc001002",              self.getKeyFloat(self.header, "%.8e", 'PC001002'));
    201         sqlLine.group("pc002001",              self.getKeyFloat(self.header, "%.8e", 'PC002001'));
    202         sqlLine.group("pc002002",              self.getKeyFloat(self.header, "%.8e", 'PC002002'));
     257        sqlLine.group("ctype1",                self.getKeyValue(header, 'CTYPE1'));
     258        sqlLine.group("ctype2",                self.getKeyValue(header, 'CTYPE2'));
     259        sqlLine.group("crval1",                self.getKeyFloat(header, "%.8f", 'CRVAL1'));
     260        sqlLine.group("crval2",                self.getKeyFloat(header, "%.8f", 'CRVAL2'));
     261        sqlLine.group("crpix1",                self.getKeyFloat(header, "%.8f", 'CRPIX1'));
     262        sqlLine.group("crpix2",                self.getKeyFloat(header, "%.8f", 'CRPIX2'));
     263        sqlLine.group("cdelt1",                self.getKeyFloat(header, "%.8e", 'CDELT1'));
     264        sqlLine.group("cdelt2",                self.getKeyFloat(header, "%.8e", 'CDELT2'));
     265        sqlLine.group("pc001001",              self.getKeyFloat(header, "%.8e", 'PC001001'));
     266        sqlLine.group("pc001002",              self.getKeyFloat(header, "%.8e", 'PC001002'));
     267        sqlLine.group("pc002001",              self.getKeyFloat(header, "%.8e", 'PC002001'));
     268        sqlLine.group("pc002002",              self.getKeyFloat(header, "%.8e", 'PC002002'));
    203269
    204270
    … …  
    210276            raise
    211277
    212         self.scratchDb.updateAllRows("ForcedWarpMeta", "batchID", str(self.batchID))
    213         self.scratchDb.updateAllRows("ForcedWarpMeta", "surveyID", str(self.surveyID))
     278        self.scratchDb.updateAllRows("ForcedWarpMeta_"+forcedWarpID, "batchID", str(self.batchID))
     279        self.scratchDb.updateAllRows("ForcedWarpMeta_"+forcedWarpID, "surveyID", str(self.surveyID))
    214280#        self.scratchDb.updateFilterID("ForcedWarpMeta", self.filterID)
    215         self.scratchDb.updateFilterID("ForcedWarpMeta",self.filterName[0])
    216         self.scratchDb.updateAllRows("ForcedWarpMeta", "calibModNum", str(self.calibModNum))
    217         self.scratchDb.updateAllRows("ForcedWarpMeta", "dataRelease", str(self.skychunk.dataRelease))
     281        self.scratchDb.updateFilterID("ForcedWarpMeta_"+forcedWarpID,self.filterName[num][0])
     282        self.scratchDb.updateAllRows("ForcedWarpMeta_"+forcedWarpID, "calibModNum", str(self.calibModNum))
     283        self.scratchDb.updateAllRows("ForcedWarpMeta_"+forcedWarpID, "dataRelease", str(self.skychunk.dataRelease))
    218284        #self.tablesToExport.append("ForcedWarpMeta")
    219285       
    … …  
    221287    Populates the ForcedWarpToImage table
    222288    '''
    223     def populateForcedWarpToImage(self):
    224 
     289    def populateForcedWarpToImage(self,num):
     290        forcedwarpID = self.number[num]
    225291        self.logger.infoPair("Procesing table", "ForcedWarpToImage")
    226 
    227         forcedwarpID = self.forcedWarpID
     292        sql = "CREATE TABLE ForcedWarpToImage_"+forcedwarpID+" LIKE ForcedWarpToImage"
     293        try: self.scratchDb.execute(sql)
     294        except: pass
     295
     296        #forcedwarpID = self.forcedWarpID
    228297        if forcedwarpID > 0:
    229298   
    … …  
    232301                for imageID in imageIDs:
    233302
    234                     sql = "INSERT INTO ForcedWarpToImage (forcedwarpID, imageID) \
     303                    sql = "INSERT INTO ForcedWarpToImage_"+str(forcedwarpID)+" (forcedwarpID, imageID) \
    235304                   VALUES (\
    236305                   " + str(forcedwarpID) + ", " + imageID + ")"
    … …  
    250319    Populates the ForcedWarpMeasurement table
    251320    '''
    252     def populateForcedWarpMeasurement(self):
    253 
    254         pspsTableName = "ForcedWarpMeasurement"
    255         ippTableName = "SkyChip_psf"
    256 
    257 
    258         self.logger.infoPair("Procesing table", "ForcedWarpMeasurement")
     321    def populateForcedWarpMeasurement(self,num):
     322        forcedWarpID = self.number[num]
     323        pspsTableName = "ForcedWarpMeasurement_"+forcedWarpID
     324        ippTableName = "SkyChip_psf_"+forcedWarpID
     325
     326        sql = "CREATE TABLE ForcedWarpMeasurement_"+forcedWarpID+" LIKE ForcedWarpMeasurement"
     327        try: self.scratchDb.execute(sql)
     328        except: pass
     329
     330        self.logger.infoPair("Procesing table", "ForcedWarpMeasurement_"+forcedWarpID)
    259331
    260332        BEFORE = self.scratchDb.getRowCount(ippTableName)
    261         extTimeString = str(self.expTime)
     333        extTimeString = str(self.expTime[num])
    262334
    263335#missing are zp, telluricExt, airmass (not easy to get ?)
    264 
     336     
    265337        sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (")
    266338        sqlLine.group("ippDetectID",     "IPP_IDET")
    267         sqlLine.group("forcedWarpID",            str(self.forcedWarpID));
    268         sqlLine.group("randomWarpID",     "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)")   
    269         sqlLine.group("filterID",        str(self.filterID))
     339     
     340        sqlLine.group("forcedWarpID",            str(forcedWarpID))
     341        print " got here too, num", num
     342        sqlLine.group("randomWarpID",     "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)")
     343        print "here" 
     344        sqlLine.group("filterID",        str(self.filterID[num]))
     345        print "here1"
    270346        sqlLine.group("surveyID",        str(self.surveyID))
    271         sqlLine.group("skycellID",       str(self.skycellID))
    272         sqlLine.group("expTime",         str(self.expTime))
    273         sqlLine.group("obsTime", str(self.obsTime))
     347        print "here2"
     348        sqlLine.group("skycellID",       str(self.skycellID[num]))
     349        print "here3"
     350        sqlLine.group("expTime",         str(self.expTime[num]))
     351        sqlLine.group("forcedSummaryID", str(self.id))
     352        print "here4"
     353        sqlLine.group("obsTime", str(self.obsTime[num]))
    274354        sqlLine.group("psfFlux", "PSF_INST_FLUX / "+extTimeString)
    275355        sqlLine.group("psfFluxErr", "PSF_INST_FLUX_SIG / "+extTimeString)
    … …  
    303383        sqlLine.group("dataRelease",str(self.skychunk.dataRelease))
    304384
     385        print "got here too"
     386
    305387        sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName)
    306388
    … …  
    316398        #self.selectDvoObjIDs()
    317399        self.logger.infoPair("inserting dvo info: find imageID","from externID")
    318         imageID = self.scratchDb.getImageIDFromExternID(self.warpSkyFileID)
    319         self.logger.infoPair("updating","forcedWarpMeasurement")
    320         sqlLine = sqlUtility("UPDATE ForcedWarpMeasurement as a, " + self.scratchDb.dvoDetectionTable + " as b SET ")
     400        imageID = self.scratchDb.getImageIDFromExternID(self.warpSkyFileID[num])
     401        self.logger.infoPair("updating","forcedWarpMeasurement_"+forcedWarpID)
     402        sqlLine = sqlUtility("UPDATE ForcedWarpMeasurement_"+forcedWarpID+" as a, " + self.scratchDb.dvoDetectionTable + " as b SET ")
    321403        sqlLine.group("a.objID","b.objID")
    322404        sqlLine.group("a.detectID","b.detectID")
    … …  
    332414            raise
    333415        self.logger.infoPair("need to cull"," nulls objid")
    334         rowCountBefore = self.scratchDb.getRowCount("ForcedWarpMeasurement")
    335         results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpMeasurement", "objID")
    336         rowCountAfter =  self.scratchDb.getRowCount("ForcedWarpMeasurement")
     416        rowCountBefore = self.scratchDb.getRowCount("ForcedWarpMeasurement_"+forcedWarpID)
     417        results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpMeasurement_"+forcedWarpID, "objID")
     418        rowCountAfter =  self.scratchDb.getRowCount("ForcedWarpMeasurement_"+forcedWarpID)
    337419        self.logger.infoPair("row count of ForcedWarpMeasurement",rowCountBefore)
    338         self.logger.infoPair("row count of ForcedWarpMeasurement after cull of null objID", rowCountAfter)
     420        self.logger.infoPair("row count of ForcedWarpMeasurement after cull of null objID ", rowCountAfter)
    339421        if rowCountAfter == 0:
    340422            self.skipBatch = True
    … …  
    343425#        self.generateRandomIDs()
    344426
    345         self.logger.infoPair("Adding 'row' columns to", "ForcedWarpMeasurement")
    346         self.scratchDb.addRowCountColumn("ForcedWarpMeasurement", "row")
    347         self.logger.infoPair("Adding unique ids to", "ForcedWarpMeasurement")
    348         self.generateUniquePspsIDs()
    349         self.logger.infoPair("Dropping row column from", "ForcedWarpMeasurement table")
    350         self.scratchDb.dropColumn("ForcedWarpMeasurement", "row")
     427        self.logger.infoPair("Adding 'row' columns to", "ForcedWarpMeasurement_"+forcedWarpID)
     428        self.scratchDb.addRowCountColumn("ForcedWarpMeasurement_"+forcedWarpID, "row")
     429        self.logger.infoPair("Adding unique ids to", "ForcedWarpMeasurement_"+forcedWarpID)
     430        self.generateUniquePspsIDs(num)
     431        self.logger.infoPair("Dropping row column from", "ForcedWarpMeasurement_"+forcedWarpID+" table")
     432        self.scratchDb.dropColumn("ForcedWarpMeasurement_"+forcedWarpID, "row")
    351433           
    352434        # add indexes ForcedWarpMeasurement
    353         self.scratchDb.createIndex("ForcedWarpMeasurement", "objID")
    354         self.scratchDb.createIndex("ForcedWarpMeasurement", "ippDetectID")
     435        self.scratchDb.createIndex("ForcedWarpMeasurement_"+forcedWarpID, "objID")
     436        self.scratchDb.createIndex("ForcedWarpMeasurement_"+forcedWarpID, "ippDetectID")
    355437    '''
    356438    Populates the ForcedWarpExtended table
    357439    '''
    358     def populateForcedWarpExtended(self):
    359 
    360         pspsTableName = "ForcedWarpExtended"
    361         ippTableName = "SkyChip_xrad"
     440    def populateForcedWarpExtended(self,num):
     441        forcedWarpID = self.number[num]
     442        pspsTableName = "ForcedWarpExtended_"+forcedWarpID
     443        ippTableName = "SkyChip_xrad_"+forcedWarpID
     444        sql = "CREATE TABLE ForcedWarpExtended_"+forcedWarpID+" LIKE ForcedWarpExtended"
     445        try: self.scratchDb.execute(sql)
     446        except: pass
    362447
    363448        #self.tablesToExport.append("ForcedWarpExtended")
    … …  
    366451        sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (")
    367452        sqlLine.group("ippDetectID",     "IPP_IDET")
    368         sqlLine.group("flxR5", "APER_FLUX_5")
    369         sqlLine.group("flxR5Err", "APER_FLUX_ERR_5" )
    370         sqlLine.group("flxR5Std", "APER_FLUX_STDEV_5")
    371         sqlLine.group("flxR5Fill", "APER_FILL_5")
    372         sqlLine.group("flxR6", "APER_FLUX_6")
    373         sqlLine.group("flxR6Err", "APER_FLUX_ERR_6" )
    374         sqlLine.group("flxR6Std", "APER_FLUX_STDEV_6")
    375         sqlLine.group("flxR6Fill", "APER_FILL_6")
     453        sqlLine.group("flxR5", "APER_FLUX_3")
     454        sqlLine.group("flxR5Err", "APER_FLUX_ERR_3" )
     455        sqlLine.group("flxR5Std", "APER_FLUX_STDEV_3")
     456        sqlLine.group("flxR5Fill", "APER_FILL_3")
     457        sqlLine.group("flxR6", "APER_FLUX_4")
     458        sqlLine.group("flxR6Err", "APER_FLUX_ERR_4" )
     459        sqlLine.group("flxR6Std", "APER_FLUX_STDEV_4")
     460        sqlLine.group("flxR6Fill", "APER_FILL_4")
     461        sqlLine.group("flxR7", "APER_FLUX_5")
     462        sqlLine.group("flxR7Err", "APER_FLUX_ERR_5" )
     463        sqlLine.group("flxR7Std", "APER_FLUX_STDEV_5")
     464        sqlLine.group("flxR7Fill", "APER_FILL_5")
    376465
    377466        sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName)
    … …  
    383472
    384473        #insert the things from FOrcedWarpMeasurement
    385         self.logger.infoPair("inserting objID,etc from","ForcedWarpMeasurement")
    386         sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, ForcedWarpMeasurement as b SET ")
     474        self.logger.infoPair("inserting objID,etc from","ForcedWarpMeasurement_"+forcedWarpID)
     475        sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, ForcedWarpMeasurement_"+forcedWarpID+" as b SET ")
    387476        sqlLine.group("a.objID","b.objID")
    388477        sqlLine.group("a.uniquePspsFWid","b.uniquePspsFWid")
    … …  
    403492       
    404493       # add in the psps unique ids from stuff
    405         self.logger.infoPair("adding information from Skychip_xrad into","ForcedWarpExtended")
     494        self.logger.infoPair("adding information from Skychip_xrad_"+forcedWarpID+" into","ForcedWarpExtended_"+forcedWarpID)
    406495       
    407496        # cull bad d
    408497        self.logger.infoPair("need to cull"," nulls objid")
    409         results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpExtended", "objID")
     498        results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpExtended_"+forcedWarpID, "objID")
    410499       
    411         self.scratchDb.createIndex("ForcedWarpExtended", "objID")
     500        self.scratchDb.createIndex("ForcedWarpExtended_"+forcedWarpID, "objID")
    412501               
    413502
    … …  
    416505    Populates the ForcedWarpExtended table                                                                                                                               
    417506    '''
    418     def populateForcedWarpLensing(self):
    419 
    420         pspsTableName = "ForcedWarpLensing"
    421         ippTableName = "SkyChip_psf"
     507    def populateForcedWarpLensing(self, num):
     508        forcedWarpID = self.number[num]
     509        pspsTableName = "ForcedWarpLensing_"+forcedWarpID
     510        ippTableName = "SkyChip_psf_"+forcedWarpID
     511        sql = "CREATE TABLE ForcedWarpLensing_"+forcedWarpID+" LIKE ForcedWarpLensing"
     512        try: self.scratchDb.execute(sql)
     513        except: pass
    422514
    423515        #self.tablesToExport.append("ForcedWarpExtended")                                                                                                               
    … …  
    446538        sqlLine.group("lensPSFShearE1", "LENS_E1_SH_PSF")
    447539        sqlLine.group("lensPSFShearE2", "LENS_E2_SH_PSF")
     540        sqlLine.group("psfE1", "LENS_E1_PSF")
     541        sqlLine.group("psfE2", "LENS_E2_PSF")
    448542        sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName)
    449543
    … …  
    454548
    455549        #insert the things from FOrcedWarpMeasurement                                                                                                                   
    456         self.logger.infoPair("inserting objID,etc from","ForcedWarpMeasurement")
    457         sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, ForcedWarpMeasurement as b SET ")
     550        self.logger.infoPair("inserting objID,etc from","ForcedWarpMeasurement_"+forcedWarpID)
     551        sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, ForcedWarpMeasurement_"+forcedWarpID+" as b SET ")
    458552        sqlLine.group("a.objID","b.objID")
    459553        sqlLine.group("a.uniquePspsFWid","b.uniquePspsFWid")
    … …  
    474568
    475569       # add in the psps unique ids from stuff                                                                                                                           
    476         self.logger.infoPair("adding information from Skychip_psf into","ForcedWarpLensing")
     570        self.logger.infoPair("adding information from Skychip_psf_"+forcedWarpID+" into","ForcedWarpLensing_"+forcedWarpID)
    477571
    478572        # cull bad d                                                                                                                                                     
    479573        self.logger.infoPair("need to cull"," nulls objid")
    480         results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpLensing", "objID")
    481 
    482         self.scratchDb.createIndex("ForcedWarpLensing", "objID")
     574        results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpLensing_"+forcedWarpID, "objID")
     575
     576        self.scratchDb.createIndex("ForcedWarpLensing_"+forcedWarpID, "objID")
    483577
    484578
    … …  
    494588   
    495589        #this should be done
    496         self.logger.infoPair("Populating","ForcedWarpMeta")
    497         self.populateForcedWarpMeta()         
    498         self.tablesToExport.append("ForcedWarpMeta")
    499 
    500         self.logger.infoPair("Populating","ForcedWarpMeasurement")
    501         self.populateForcedWarpMeasurement()         
    502         self.tablesToExport.append("ForcedWarpMeasurement")
    503 
    504         self.logger.infoPair("Populating","ForcedWarpExtended")
    505         self.populateForcedWarpExtended()
    506         self.tablesToExport.append("ForcedWarpExtended")
    507 
    508         self.logger.infoPair("Populating","ForcedWarpLensing")
    509         self.populateForcedWarpLensing()
    510         self.tablesToExport.append("ForcedWarpLensing")
    511 
    512 
    513         self.logger.infoPair("Populating","ForcedWarpToImage")
    514         self.populateForcedWarpToImage()
    515         self.tablesToExport.append("ForcedWarpToImage")             
    516 
    517 
    518         self.logger.infoPair("setting min/max objid from","ForcedWarpMeasurement")
    519         self.setMinMaxObjID(["ForcedWarpMeasurement"])
    520 
    521         self.logger.infoPair("finishing","populatePspsTables");
     590        #number  = 0
     591        #for forcedWarpID in self.forcedWarpIds:
     592        for num in self.number:   
     593            print "hawaht?"
     594            forcedWarpID = self.number[num]
     595            self.logger.infoPair("Populating","ForcedWarpMeta_"+forcedWarpID)
     596            self.populateForcedWarpMeta(num)         
     597            self.tablesToExport.append("ForcedWarpMeta_"+forcedWarpID)
     598           
     599            self.logger.infoPair("Populating","ForcedWarpMeasurement_"+forcedWarpID)
     600            self.populateForcedWarpMeasurement(num)         
     601            self.tablesToExport.append("ForcedWarpMeasurement_"+forcedWarpID)
     602           
     603            self.logger.infoPair("Populating","ForcedWarpExtended_"+forcedWarpID)
     604            self.populateForcedWarpExtended(num)
     605            self.tablesToExport.append("ForcedWarpExtended_"+forcedWarpID)
     606
     607            self.logger.infoPair("Populating","ForcedWarpLensing_"+forcedWarpID)
     608            self.populateForcedWarpLensing(num)
     609            self.tablesToExport.append("ForcedWarpLensing_"+forcedWarpID)
     610
     611
     612            self.logger.infoPair("Populating","ForcedWarpToImage_"+forcedWarpID)
     613            self.populateForcedWarpToImage(num)
     614            self.tablesToExport.append("ForcedWarpToImage_"+forcedWarpID)             
     615
     616
     617            self.logger.infoPair("setting min/max objid from","ForcedWarpMeasurement_"+forcedWarpID)
     618            self.setMinMaxObjID(["ForcedWarpMeasurement_"+forcedWarpID])
     619           
     620            self.logger.infoPair("finishing","populatePspsTables")
     621
     622
    522623        return True
    523624         
    … …  
    528629    '''
    529630    def importIppTables(self, columns="*", tableRE=""):
     631
    530632        count = 0
    531         if self.config.retry: return True
    532         self.logger.infoPair("Importing FW tables with table match expression: ", tableRE)
    533         fileName = self.fits.getPath()
    534 
    535         self.logger.infoPair("using filename:",fileName)
     633        for num in self.number:
     634            forcedWarpID = self.number[num]
     635            if self.config.retry: return True
     636            self.logger.infoPair("loading for add id", forcedWarpID)
     637            self.logger.infoPair("Importing FW tables with table match expression: ", tableRE)
     638            fileName = self.fits[num].getPath()
    536639           
    537         try:
    538            tables = stilts.treads(fileName)
    539         except:
    540            self.logger.errorPair("STILTS could not import from", fileName)
    541            return False
    542         for table in tables:
     640            self.logger.infoPair("using filename:",fileName)
     641           
     642            try:
     643                tables = stilts.treads(fileName)
     644            except:
     645                self.logger.errorPair("STILTS could not import from", fileName)
     646                return False
     647            for table in tables:
    543648             
    544            match = re.match(tableRE, table.name)
    545            if not match: continue
    546            self.logger.infoPair("Reading IPP table", table.name)
    547            table = stilts.tpipe(table, cmd='addcol table_index $0')
    548            table = stilts.tpipe(table, cmd='explodeall')
     649                match = re.match(tableRE, table.name)
     650                if not match: continue
     651                self.logger.infoPair("Reading IPP table", table.name+"_"+str(forcedWarpID))
     652                table = stilts.tpipe(table, cmd='addcol table_index $0')
     653                table = stilts.tpipe(table, cmd='explodeall')
    549654           
    550655           # drop any previous tables before import
    551656           
    552            self.scratchDb.dropTable(table.name)
     657                self.scratchDb.dropTable(table.name+"_"+str(forcedWarpID))
    553658                 
    554659           # IPP FITS files are littered with infinities, so remove these
    555            self.logger.info("Removing Infinity values from all columns")
    556            table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
    557            table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
    558            table = stilts.tpipe(table, cmd='replaceval Infinity null *')
     660                self.logger.info("Removing Infinity values from all columns")
     661                table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
     662                table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
     663                table = stilts.tpipe(table, cmd='replaceval Infinity null *')
    559664                 
    560            try:
    561                table.write(self.scratchDb.url + '#' + table.name)
    562                count = count + 1
    563            except:
    564                self.logger.exception("Problem writing table '" + table.name + "' to the database")
    565                      
    566         self.logger.infoPair("Done. Imported", "%d tables" % count)
    567         self.indexIppTables()
     665                try:
     666                    table.write(self.scratchDb.url + '#' + table.name+"_"+str(forcedWarpID))
     667                    count = count + 1
     668                except:
     669                    self.logger.exception("Problem writing table '" + table.name+"_"+str(forcedWarpID) + "' to the database")
     670               
     671
     672                self.logger.infoPair("Done. Imported", "%d tables" % count)
     673                self.indexIppTables(forcedWarpID)
    568674        return True
    569675
    570676
    571     def generateRandomIDs(self):
    572         sql = "UPDATE ForcedWarpMeasurement set randomWarpID = FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)"
     677    def generateRandomIDs(self,forcedWarpID):
     678        sql = "UPDATE ForcedWarpMeasurement_"+forcedWarpID+" set randomWarpID = FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)"
    573679        try:
    574680            self.scratchDb.execute(sql)
    … …  
    582688    '''
    583689
    584     def generateUniquePspsIDs(self):
    585         sql = "UPDATE ForcedWarpMeasurement set uniquePspsFWid = (("+str(self.batchID)+"*1000000000 ) + row)"
     690    def generateUniquePspsIDs(self,num):
     691       
     692        sql = "UPDATE ForcedWarpMeasurement_"+str(self.number[num])+" set uniquePspsFWid = (("+str(self.batchID)+"*1000000000 ) + ("+str(num)+"*1000000 ) + row)"
    586693        try: self.scratchDb.execute(sql)
    587694        except:
    … …  
    594701
    595702    def exportPspsTablesToFits(self, regex="(.*)"):
    596        return super(ForcedWarpBatch, self).exportPspsTablesToFits("(ForcedWarp.*)")
     703#       return super(ForcedWarpBatch, self).exportPspsTablesToFits("(ForcedWarp.*)")
     704       return super(ForcedWarpBatch, self).exportPspsTablesToFits("([a-zA-Z]+)")
    597705
    598706
    … …  
    617725    Applies indexes to the IPP tables
    618726    '''
    619     def indexIppTables(self):
    620 
    621         self.logger.infoPair("Creating indexes on", "IPP tables")
    622         self.scratchDb.createIndex("SkyChip_psf",  "IPP_IDET")
    623         self.scratchDb.createIndex("SkyChip_xfit", "IPP_IDET")
    624         self.scratchDb.createIndex("SkyChip_xrad", "IPP_IDET")
    625         self.scratchDb.createIndex("SkyChip_xsrc", "IPP_IDET")
    626         self.scratchDb.createIndex("SkyChip_xgal", "IPP_IDET")
     727    def indexIppTables(self,forcedWarpID):
     728
     729        self.logger.infoPair("Creating indexes on"+forcedWarpID, "IPP tables")
     730        self.scratchDb.createIndex("SkyChip_psf_"+forcedWarpID,  "IPP_IDET")
     731        self.scratchDb.createIndex("SkyChip_xfit_"+forcedWarpID, "IPP_IDET")
     732        self.scratchDb.createIndex("SkyChip_xrad_"+forcedWarpID, "IPP_IDET")
     733        self.scratchDb.createIndex("SkyChip_xsrc_"+forcedWarpID, "IPP_IDET")
     734        self.scratchDb.createIndex("SkyChip_xgal_"+forcedWarpID, "IPP_IDET")
    627735
    628736        return True
    … …  
    633741
    634742    def updatePspsUniqueIDs(self,table):
    635         sql = "UPDATE ForcedWarpMeasurement set uniquePspsFWid = (("+str(self.batchID)+"*1000000000 ) + row)"
    636         try: self.scratchDb.execute(sql)
    637         except:
    638             self.logger.errorPair('failed sql',sql)
    639             raise
     743        sql = "UPDATE ForcedWarpMeasurement_"+forcedWarpID+" set uniquePspsFWid = (("+str(self.batchID)+"*1000000000 ) + (str(number)*1000000 ) + row)"
     744        try: self.scratchDb.execute(sql)
     745        except:
     746           self.logger.errorPair('failed sql',sql)
     747           raise
Note: See TracChangeset for help on using the changeset viewer.