IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Mar 28, 2015, 8:28:32 PM (11 years ago)
Author:
eugene
Message:

merge changes from trunk

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20150112/ippToPsps/jython/forcedobjectbatch.py

    r37551 r38052  
    1414from ipptopspsdb import IppToPspsDb
    1515from scratchdb import ScratchDb
    16 from dvoforcedobjects import DvoForcedObjects # need one specific for cpy ?
     16from dvoforcedobjects import DvoForcedObjects # need one specific for cpy.
    1717from sqlUtility import sqlUtility
    1818
    … …  
    2525
    2626'''
    27 class ForcedMeanObjectBatch(Batch):
     27class ForcedObjectBatch(Batch):
    2828
    2929    '''
    … …  
    4040                 batchID):
    4141
    42        super(ForcedMeanObjectBatch, self).__init__(
     42       super(ForcedObjectBatch, self).__init__(
    4343               logger,
    4444               config,
    … …  
    7171
    7272        self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id)
    73         self.ippToPspsDb.insertForcedMeanObjectMeta(self.batchID, self.region)
     73        self.ippToPspsDb.insertForcedObjectMeta(self.batchID, self.region)
    7474        self.dvoForcedObjects.nativeIngestRegion(self.region)
    7575
    … …  
    8686            return False
    8787
     88        self.logger.infoPair("checking if we have anything in"," cpy/cpt")
     89        rowcountcpt = self.scratchDb.getRowCount(cptTableName);
     90        rowcountcpy = self.scratchDb.getRowCount(cpyTableName);
     91        if (rowcountcpt == 0):
     92            self.logger.infoPair("0 rows for ","cpt table")
     93            return False
     94        if (rowcountcpy == 0):
     95            self.logger.infoPair("o rows for ","cpy table")
     96            return False
    8897        return True
    8998
    … …  
    104113
    105114        return True
     115
     116
     117    '''
     118    Updates table and generates pspsuniqueids
     119    '''
     120   
     121    def updatePspsUniqueIDs(self):
     122        sql = "UPDATE ForcedMeanObject set uniquePspsFOid = (("+str(self.batchID)+"*1000000000 ) + row)"
     123        try: self.scratchDb.execute(sql)
     124        except:
     125            self.logger.errorPair('failed sql',sql)
     126            raise
     127
     128
     129
    106130
    107131    '''
    … …  
    146170                   SET \
    147171                    ForcedMeanObject." + filter[1] + "nTotal         = NWARP \
    148                    ,ForcedMeanObject." + filter[1] + "nIncPSFMag     = NUSED_WRP \
    149                    ,ForcedMeanObject." + filter[1] + "nIncKronMag    = NUSED_KRON_WRP \
    150                    ,ForcedMeanObject." + filter[1] + "nIncApMag      = NUSED_AP_WRP \
    151                    ,ForcedMeanObject." + filter[1] + "PSFMag         = MAG_PSF_WRP \
    152                    ,ForcedMeanObject." + filter[1] + "PSFMagErr      = MAG_PSF_WRP_ERR \
    153                    ,ForcedMeanObject." + filter[1] + "PSFMagStd      = MAG_PSF_WRP_STD \
    154                    ,ForcedMeanObject." + filter[1] + "KronMag        = MAG_KRON_WRP \
    155                    ,ForcedMeanObject." + filter[1] + "KronMagErr     = MAG_KRON_WRP_ERR \
    156                    ,ForcedMeanObject." + filter[1] + "KronMagStd     = MAG_KRON_WRP_STD \
    157                    ,ForcedMeanObject." + filter[1] + "ApMag          = MAG_AP_WRP \
    158                    ,ForcedMeanObject." + filter[1] + "ApMagErr       = MAG_AP_WRP_ERR \
    159                    ,ForcedMeanObject." + filter[1] + "ApMagStd       = MAG_AP_WRP_STD \
     172                   ,ForcedMeanObject." + filter[1] + "nIncPSFFlux     = NUSED_WRP \
     173                   ,ForcedMeanObject." + filter[1] + "nIncKronFlux    = NUSED_KRON_WRP \
     174                   ,ForcedMeanObject." + filter[1] + "nIncApFlux      = NUSED_AP_WRP \
     175                   ,ForcedMeanObject." + filter[1] + "PSFFlux         = FLUX_PSF_WRP \
     176                   ,ForcedMeanObject." + filter[1] + "PSFFluxErr      = FLUX_PSF_WRP_ERR \
     177                   ,ForcedMeanObject." + filter[1] + "PSFFluxStd      = FLUX_PSF_WRP_STDEV \
     178                   ,ForcedMeanObject." + filter[1] + "KronFlux        = FLUX_KRON_WRP \
     179                   ,ForcedMeanObject." + filter[1] + "KronFluxErr     = FLUX_KRON_WRP_ERR \
     180                   ,ForcedMeanObject." + filter[1] + "KronFluxStd     = FLUX_KRON_WRP_STDEV \
     181                   ,ForcedMeanObject." + filter[1] + "ApFlux          = FLUX_AP_WRP \
     182                   ,ForcedMeanObject." + filter[1] + "ApFluxErr       = FLUX_AP_WRP_ERR \
     183                   ,ForcedMeanObject." + filter[1] + "ApFluxStd       = FLUX_AP_WRP_STDEV \
    160184                   ,ForcedMeanObject." + filter[1] + "Flags          = FLAGS "
    161185
    … …  
    163187            try: self.scratchDb.execute(sql)
    164188            except:
    165                 self.logger.errorPair("failed update MeanObject", sql)
     189                self.logger.errorPair("failed update ForcedMeanObject", sql)
    166190                raise
    167191
    … …  
    175199                   SET " + filter[1] + "PSFMagErr = null \
    176200                   WHERE " + filter[1] + "PSFMagErr > " + str(cut)
    177             self.scratchDb.execute(sql)
    178 
     201# taking out the cut since this is for mag, not flux
     202
     203    #       try: self.scratchDb.execute(sql)
     204     #       except:
     205     #           self.logger.errorPair("failed setting null meanmagerr","ForcedMeanObject")
     206     #           raise
    179207        self.logger.infoPair("Calculating nDetections from", "n[filters]")
    180208        for filter in filters:
    181209            # now do a sum of n[filters], but do not include the ones with -999
    182             sql  = "UPDATE ObjectThin "
    183             sql += "SET nDetections = nDetections + n" + filter[1]
    184             sql += " WHERE n" + filter[1] + " != -999"
    185             self.scratchDb.execute(sql)
     210            sql  = "UPDATE ForcedMeanObject "
     211            sql += "SET nDetections = nDetections + " + filter[1] +"nTotal "
     212            sql += " WHERE " + filter[1] + "nTotal  != -999"
     213            #self.scratchDb.execute(sql)
     214            try: self.scratchDb.execute(sql)
     215            except:
     216                self.logger.errorPair("failed update MeanObject nDetections", sql)
     217                raise
    186218
    187219    '''
    188220    Inserts stuff for all mags from the cpy files
    189221    '''
    190     def updateForcedMeanObjectFromCpy(self, cpyTable):
     222    def updateForcedMeanTablesFromCpy(self, cpyTable):
    191223
    192224        # list of all filters PSPS is interested in
    … …  
    225257                   " + cpyTable + " AS cpy ON (cpy.row = (ForcedMeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
    226258                   SET \
    227                     ForcedMeanObject." + filter[1] + "nIncR5          =  NMEAS \
     259                   ForcedMeanObject." + filter[1] + "nIncR5          =  NMEAS \
    228260                   ,ForcedMeanObject." + filter[1] + "nIncR6          =  NMEAS \
    229261                   ,ForcedMeanObject." + filter[1] + "FmeanflxR5      =  FLUX_AP_R5 \
    … …  
    235267                   ,ForcedMeanObject." + filter[1] + "FmeanflxR6Std   =  FLUX_STD_AP_R6 \
    236268                   ,ForcedMeanObject." + filter[1] + "FmeanflxR6Fill  =  FLUX_FIL_AP_R6 \
    237                    ,ForcedMeanObject." + filter[1] + "LensObjSmearX11 =  X11_SM_OBJ \
    238                    ,ForcedMeanObject." + filter[1] + "LensObjSmearX12 =  X12_SM_OBJ \
    239                    ,ForcedMeanObject." + filter[1] + "LensObjSmearX22 =  X22_SM_OBJ \
    240                    ,ForcedMeanObject." + filter[1] + "LensObjSmearE1  =  E1_SM_OBJ \
    241                    ,ForcedMeanObject." + filter[1] + "LensObjSmearE2  =  E2_SM_OBJ \
    242                    ,ForcedMeanObject." + filter[1] + "LensObjShearX11 =  X11_SH_OBJ \
    243                    ,ForcedMeanObject." + filter[1] + "LensObjShearX12 =  X12_SH_OBJ \
    244                    ,ForcedMeanObject." + filter[1] + "LensObjShearX22 =  X22_SH_OBJ \
    245                    ,ForcedMeanObject." + filter[1] + "LensObjShearE1  =  E1_SH_OBJ \
    246                    ,ForcedMeanObject." + filter[1] + "LensObjShearE2  =  E2_SH_OBJ \
    247                    ,ForcedMeanObject." + filter[1] + "LensPSFSmearX11 =  X11_SM_PSF \
    248                    ,ForcedMeanObject." + filter[1] + "LensPSFSmearX12 =  X12_SM_PSF \
    249                    ,ForcedMeanObject." + filter[1] + "LensPSFSmearX22 =  X22_SM_PSF \
    250                    ,ForcedMeanObject." + filter[1] + "LensPSFSmearE1  =  E1_SM_PSF \
    251                    ,ForcedMeanObject." + filter[1] + "LensPSFSmearE2  =  E2_SM_PSF \
    252                    ,ForcedMeanObject." + filter[1] + "LensPSFShearX11 =  X11_SH_PSF \
    253                    ,ForcedMeanObject." + filter[1] + "LensPSFShearX12 =  X12_SH_PSF \
    254                    ,ForcedMeanObject." + filter[1] + "LensPSFShearX22 =  X22_SH_PSF \
    255                    ,ForcedMeanObject." + filter[1] + "LensPSFShearE1  =  E1_SH_PSF \
    256                    ,ForcedMeanObject." + filter[1] + "LensPSFShearE2  =  E2_SH_PSF \
    257269                   ,ForcedMeanObject." + filter[1] + "Gamma           =  GAMMA \
    258270                   ,ForcedMeanObject." + filter[1] + "E1              =  E1 \
    259271                   ,ForcedMeanObject." + filter[1] + "E2              =  E2 "
    260272
    261 
    262273            try: self.scratchDb.execute(sql)
    263274            except:
    264                 self.logger.errorPair("failed update on ForcedMeanObject using cpy table", sql)
     275                self.logger.errorPair("failed update on ForcedMeanObject using cpy table",sql)
     276                raise
     277
     278            sql = "UPDATE ForcedMeanLensing JOIN \
     279                   " + cpyTable + " AS cpy ON (cpy.row = (ForcedMeanLensing.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
     280                   SET \
     281                   ForcedMeanLensing." + filter[1] + "LensObjSmearX11 =  X11_SM_OBJ \
     282                   ,ForcedMeanLensing." + filter[1] + "LensObjSmearX12 =  X12_SM_OBJ \
     283                   ,ForcedMeanLensing." + filter[1] + "LensObjSmearX22 =  X22_SM_OBJ \
     284                   ,ForcedMeanLensing." + filter[1] + "LensObjSmearE1  =  E1_SM_OBJ \
     285                   ,ForcedMeanLensing." + filter[1] + "LensObjSmearE2  =  E2_SM_OBJ \
     286                   ,ForcedMeanLensing." + filter[1] + "LensObjShearX11 =  X11_SH_OBJ \
     287                   ,ForcedMeanLensing." + filter[1] + "LensObjShearX12 =  X12_SH_OBJ \
     288                   ,ForcedMeanLensing." + filter[1] + "LensObjShearX22 =  X22_SH_OBJ \
     289                   ,ForcedMeanLensing." + filter[1] + "LensObjShearE1  =  E1_SH_OBJ \
     290                   ,ForcedMeanLensing." + filter[1] + "LensObjShearE2  =  E2_SH_OBJ \
     291                   ,ForcedMeanLensing." + filter[1] + "LensPSFSmearX11 =  X11_SM_PSF \
     292                   ,ForcedMeanLensing." + filter[1] + "LensPSFSmearX12 =  X12_SM_PSF \
     293                   ,ForcedMeanLensing." + filter[1] + "LensPSFSmearX22 =  X22_SM_PSF \
     294                   ,ForcedMeanLensing." + filter[1] + "LensPSFSmearE1  =  E1_SM_PSF \
     295                   ,ForcedMeanLensing." + filter[1] + "LensPSFSmearE2  =  E2_SM_PSF \
     296                   ,ForcedMeanLensing." + filter[1] + "LensPSFShearX11 =  X11_SH_PSF \
     297                   ,ForcedMeanLensing." + filter[1] + "LensPSFShearX12 =  X12_SH_PSF \
     298                   ,ForcedMeanLensing." + filter[1] + "LensPSFShearX22 =  X22_SH_PSF \
     299                   ,ForcedMeanLensing." + filter[1] + "LensPSFShearE1  =  E1_SH_PSF \
     300                   ,ForcedMeanLensing." + filter[1] + "LensPSFShearE2  =  E2_SH_PSF "
     301
     302            try: self.scratchDb.execute(sql)
     303            except:
     304                self.logger.errorPair("failed update on ForcedMeanLensing using cpy table", sql)
    265305                raise
    266306
    … …  
    295335        except:
    296336            self.logger.errorPair("Couldn't populate Object table from cpt", sql)
    297             return False
     337            raise
    298338 
    299339        # add row count columns so we can perform joins to get colors
    300         self.logger.infoPair("Adding 'row' columns to", "Object and cps tables")
     340        self.logger.infoPair("Adding 'row' columns to", "Object ,cps, cpy tables")
    301341        self.scratchDb.addRowCountColumn("ForcedMeanObject", "row")
    302342        self.scratchDb.addRowCountColumn(cpsTableName, "row")
     343        self.scratchDb.addRowCountColumn(cpyTableName, "row")
     344
     345        self.logger.infoPair("generating unique ids for ","ForcedMeanObject")
     346        self.updatePspsUniqueIDs()
    303347
    304348
    305349        self.logger.infoPair("update ForcedMeanObject from ","cps table")
    306 
    307350        self.updateForcedMeanObjectFromCps(cpsTableName)
    308351
    309         self.logger.infoPair("update ForcedMeanObject from ","cpy table")
    310 
    311         self.updateForcedMeanObjectFromCpy(cpyTableName)
     352        sqlLine = sqlUtility("INSERT INTO ForcedMeanLensing (")
     353
     354        sqlLine.group("objID",           "objID")
     355        sqlLine.group("uniquePspsFOid","uniquePspsFOid")
     356        sqlLine.group("ippObjID",        "ippObjID")
     357        sqlLine.group("batchID",         "batchID")
     358        sqlLine.group("nDetections",      "nDetections")
     359        sqlLine.group("gStackID",      "gStackID")
     360        sqlLine.group("rStackID",      "rStackID")
     361        sqlLine.group("iStackID",      "iStackID")
     362        sqlLine.group("zStackID",      "zStackID")
     363        sqlLine.group("yStackID",      "yStackID")
     364
     365        sql = sqlLine.makeRaw(") SELECT ", " FROM ForcedMeanObject")
     366
     367        try:
     368            self.scratchDb.execute(sql)
     369        except:
     370            self.logger.errorPair("Couldn't populate ForcedMeanLensing table from ForcedMeanObject", sql)
     371            raise
     372
     373        self.logger.infoPair("adding row to ","ForcedMeanLensing")
     374        self.scratchDb.addRowCountColumn("ForcedMeanLensing", "row")
     375
     376        self.logger.infoPair("updating ForcedMeanObject and ForcedMeanLensing from ","cpy tables")
     377        self.updateForcedMeanTablesFromCpy(cpyTableName)
    312378
    313379
    314380        self.logger.infoPair("Dropping row column from", "ForcedMeanObject table")
    315381        self.scratchDb.dropColumn("ForcedMeanObject", "row")
    316 
     382        self.logger.infoPair("Dropping row column from", "ForcedMeanLensing table")
     383        self.scratchDb.dropColumn("ForcedMeanLensing", "row")
    317384        ##self.logger.infoPair("Purging from scratch Db", self.region + " region")
    318385
    319386        ##Don't do this till after MeanObject
    320387        ##self.dvoForcedObjects.purgeRegion(self.region)
    321 
     388        self.logger.infoPair("setting minmaxobjid for ", "ForcedMeanObject table")
    322389        self.setMinMaxObjID(["ForcedMeanObject"])
    323 
    324         return True
     390        self.logger.infoPair("checking if we have detctions", "FOrcedMeanObject table")
     391        rowcount = self.scratchDb.getRowCount("ForcedMeanObject")
     392        if rowcount == 0:
     393            self.logger.infoPair("skipping this one","ForcedMeanObject ")
     394            self.skipBatch  = False
     395            return True
     396        else:
     397            return True
    325398
    326399 
Note: See TracChangeset for help on using the changeset viewer.