IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 33155


Ignore:
Timestamp:
Jan 25, 2012, 5:10:25 PM (14 years ago)
Author:
rhenders
Message:

now storing size and modified date of each file ingested from DVO

Location:
trunk/ippToPsps/jython
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/jython/dvo.py

    r33154 r33155  
    4747    '''
    4848    def __del__(self):
    49 
    5049        self.logger.debug("Dvo destructor")
    51 
    5250
    5351    '''
    … …  
    6260    def loadImages(self):
    6361
    64         # import Images.dat table
     62        # check if we have up-to-date version
     63        path = self.dvoLocation + "/Images.dat"
     64        if self.scratchDb.alreadyImportedThisDvoTable(path): return
     65
     66        # first, delete old dvoMeta table
    6567        self.logger.infoPair("Deleting from table", self.scratchDb.dvoMetaTable)
    6668        sql = "DELETE FROM " + self.scratchDb.dvoMetaTable
    6769        self.scratchDb.execute(sql)
    6870
    69         self.imagesTableName = self.importFits(
    70                 "Images.dat",
    71                 "SOURCE_ID IMAGE_ID CCDNUM EXTERN_ID FLAGS PHOTCODE NSTAR")
    72         self.scratchDb.createIndex(self.imagesTableName, "EXTERN_ID")
     71        self.importFits(
     72                path,
     73                "SOURCE_ID IMAGE_ID CCDNUM EXTERN_ID FLAGS PHOTCODE NSTAR",
     74                self.scratchDb.dvoImageTable)
     75        self.scratchDb.createIndex(self.scratchDb.dvoImageTable, "EXTERN_ID")
    7376
    7477        # insert into dvoMetaFull
    … …  
    8891               FLAGS, \
    8992               PHOTCODE \
    90                FROM " + self.imagesTableName
    91         self.scratchDb.execute(sql)
    92 
     93               FROM " + self.scratchDb.dvoImageTable
     94
     95        try:
     96            self.scratchDb.execute(sql)
     97        except:
     98            self.logger.errorPair("Could not insert into", self.scratchDb.dvoMetaTable)
     99            return
     100
     101        self.scratchDb.setImportedThisDvoTable(path)
    93102
    94103    '''
    … …  
    96105    '''
    97106    def loadSkyTable(self):
     107       
     108        path =  self.dvoLocation + "/SkyTable.fits"
     109        if self.scratchDb.alreadyImportedThisDvoTable(path): return
     110        self.importFits(
     111                path,
     112                "R_MIN R_MAX D_MIN D_MAX INDEX NAME",
     113                self.scratchDb.dvoSkyTable)
     114
     115        self.logger.infoPair("Adding index to", self.scratchDb.dvoSkyTable)
     116
     117        self.scratchDb.createIndex(self.scratchDb.dvoSkyTable, "INDEX")
    98118       
    99         self.skyTableName = self.importFits(
    100                 "SkyTable.fits",
    101                 "R_MIN R_MAX D_MIN D_MAX INDEX NAME")
    102 
    103         self.logger.infoPair("Adding index to", self.skyTableName)
    104 
    105         self.scratchDb.createIndex(self.skyTableName, "INDEX")
    106        
     119        self.scratchDb.setImportedThisDvoTable(path)
    107120        self.logger.infoPair("Finished importing SkyTable at", self.dvoLocation)
    108121
    109122
    110123    '''
    111     Populates dvoDetections with FITS tables from DVO for the whole sky
    112     '''
    113     def loadRegion(self, minRa, maxRa, minDec, maxDec):
    114         self.loadRegion(-1, 361, -91, 91)
    115 
    116     '''
    117124    Populates dvoDetections and dvoMeta with FITS tables from DVO for the defined region
    118     '''
    119     def loadRegion(self, minRa, maxRa, minDec, maxDec):
    120 
     125    Defaults to whole sky if limits are missing
     126    '''
     127    def loadRegion(self, minRa=-1., maxRa=361., minDec=-91., maxDec=91):
     128
     129        if (maxRa - minRa <= 0) or (maxDec - minDec) <= 0:
     130            self.logger.errorPair("Zero range in either RA or Dec",
     131                    "RA: %.2f -> %.2f, Dec: %.2f -> %.2f" % (minRa, maxRa, minDec, maxDec))
     132            return False       
     133
     134        self.logger.infoPair("Finding DVO files for range",
     135                "RA: %.2f -> %.2f, Dec: %.2f -> %.2f" % (minRa, maxRa, minDec, maxDec))
    121136        files = self.scratchDb.getDvoFilesCoveringThisRegion(minRa, maxRa, minDec, maxDec)
    122137
    123138        count = 0
     139
     140        self.logger.infoSeparator()
    124141        for file in files:
    125            path = self.dvoLocation + "/" + file + ".cpm"
    126            if not os.path.isfile(path): continue
     142
     143           cpmPath = self.dvoLocation + "/" + file + ".cpm"
     144           cptPath = self.dvoLocation + "/" + file + ".cpt"
     145
     146           if not os.path.isfile(cpmPath): continue
    127147           count = count + 1
    128            print path
    129 
    130            if self.scratchDb.alreadyImportedThisDvoTable(file): continue
     148
     149           # only if we have already imported up-to-date versions of both the cpm and ctp
     150           # will we skip this one
     151           if self.scratchDb.alreadyImportedThisDvoTable(cpmPath) and self.scratchDb.alreadyImportedThisDvoTable(cptPath): continue
    131152
    132153           # import cpm table and index
    133154           cpmTableName = self.importFits(
    134                    file + ".cpm",
     155                   cpmPath,
    135156                   "IMAGE_ID DET_ID OBJ_ID CAT_ID EXT_ID DB_FLAGS")
    136157           self.scratchDb.createIndex(cpmTableName, "IMAGE_ID")
    … …  
    140161           # import cpt table and index
    141162           cptTableName = self.importFits(
    142                    file + ".cpt",
     163                   cptPath,
    143164                   "OBJ_ID CAT_ID EXT_ID")
    144165           self.scratchDb.createIndex(cptTableName, "IMAGE_ID")
    … …  
    150171           sql = "ALTER TABLE " + cpmTableName + " ADD COLUMN (SOURCE_ID SMALLINT)"
    151172           self.scratchDb.execute(sql)
    152            sql = "UPDATE " + cpmTableName + " AS a, " + self.imagesTableName + " AS b \
     173           sql = "UPDATE " + cpmTableName + " AS a, " + self.scratchDb.dvoImageTable + " AS b \
    153174                  SET a.SOURCE_ID = b.SOURCE_ID \
    154175                  WHERE a.IMAGE_ID = b.IMAGE_ID"
    … …  
    196217           # now drop what we don't need
    197218           self.logger.infoPair("Dropping table", cpmTableName)
    198            #self.scratchDb.dropTable(cpmTableName)
     219           self.scratchDb.dropTable(cpmTableName)
    199220           self.logger.infoPair("Dropping table", cptTableName)
    200            #self.scratchDb.dropTable(cptTableName)
    201 
    202            self.scratchDb.setImportedThisDvoTable(file)
     221           self.scratchDb.dropTable(cptTableName)
     222
     223           self.scratchDb.setImportedThisDvoTable(cpmPath)
     224           self.scratchDb.setImportedThisDvoTable(cptPath)
    203225
    204226        self.logger.infoPair("Found", "%d region files" % count)
    205227
     228        return True
     229
    206230
    207231    '''
    208232    Imports a FITS file. A regex filter lets you choose which tables to pull from the file
    209     '''
    210     def importFits(self, file, columns):
    211 
    212       fullPath = self.dvoLocation + "/" + file
    213       self.logger.infoPair("Importing file", fullPath)
    214 
    215       tableName = file
    216       tableName = tableName.replace('.', '_')
    217       tableName = tableName.replace('/', '_')
     233    An optional final argument lets you name the resulatant database table
     234    '''
     235    def importFits(self, path, columns, tableName=None):
     236
     237      self.logger.infoPair("Importing file", path)
     238
     239      if not tableName:
     240          head,tail = os.path.split(path)
     241          tableName = tail
     242          tableName = tableName.replace('.', '_')
     243          tableName = tableName.replace('/', '_')
    218244
    219245      self.logger.infoPair("Writing to database table", tableName)
    220246
    221       tables = stilts.treads(fullPath)
     247      tables = stilts.treads(path)
    222248
    223249      count = 0
    … …  
    263289dvo = Dvo(logger, configDoc)
    264290#dvo.resetAllTables()
    265 #dvo.loadImages()
    266 #dvo.loadSkyTable()
    267 dvo.loadRegion(330, 330.1, 0.3, 0.4)
     291dvo.loadImages()
     292dvo.loadSkyTable()
     293dvo.loadRegion(330, 330.1, 0.2, 0.7)
    268294
    269295logger.infoPair("Program...", "complete")

  • trunk/ippToPsps/jython/scratchdb.py

    r33147 r33155  
    2626            self.dvoDetectionTable = "dvoDetectionFull"
    2727            self.dvoDoneTable = "dvoDone"
    28             self.dvoSkyTable = "SkyTable_fits"
     28            self.dvoSkyTable = "dvoSkyTable"
     29            self.dvoImageTable = "dvoImages"
    2930        else:
    3031            self.dvoMetaTable = "dvoMeta"
    … …  
    183184
    184185    '''
     186    Have we already imported this DVO table?
     187    '''
     188    def alreadyImportedThisDvoTable(self, path):
     189
     190        fileStat = os.stat(path)
     191
     192        sql = "SELECT COUNT(*) FROM " + self.dvoDoneTable + " WHERE path = '" + path + "' AND modifiedDate = " + str(fileStat.st_mtime)
     193
     194        try:
     195            rs = self.executeQuery(sql)
     196            rs.first()
     197            if rs.getInt(1) > 0:
     198                self.logger.errorPair("Already imported up-to-date version of",  path)
     199                return True
     200            else:
     201                return False
     202        except:
     203            self.logger.exception("Unable to check whether this DVO table has been imported")
     204
     205
     206    '''
    185207    Updates dvoDone table with this DVO table
    186208    '''
    187     def setImportedThisDvoTable(self, name):
    188 
    189         sql = "INSERT INTO dvoDone (name) VALUES ('" + name + "')"
     209    def setImportedThisDvoTable(self, path):
     210
     211        fileStat = os.stat(path)
     212
     213        # first delete any old version we have ingested
     214        sql = "DELETE FROM " + self.dvoDoneTable + " WHERE path = '" + path + "'"
     215        self.execute(sql)
     216
     217        # now insert new version with up-to-date size and date
     218        sql = "INSERT INTO " + self.dvoDoneTable + " (path, modifiedDate, size) VALUES ('" + path + "', " + str(str(fileStat.st_mtime)) + ", " + str(str(fileStat.st_size)) + ")"
    190219        self.execute(sql)
    191220       
    … …  
    210239
    211240        return True
    212 
    213     '''
    214     Have we already imported this DVO table?
    215     '''
    216     def alreadyImportedThisDvoTable(self, name):
    217 
    218         sql = "SELECT COUNT(*) FROM dvoDone WHERE name = '" + name + "'"
    219 
    220         try:
    221             rs = self.executeQuery(sql)
    222             rs.first()
    223             if rs.getInt(1) > 0:
    224                 self.logger.errorPair("Already imported DVO tables for",  name)
    225                 return True
    226             else:
    227                 return False
    228         except:
    229             self.logger.exception("Unable to check whether this DVO table has been imported")
    230241
    231242
    … …  
    298309
    299310       self.logger.infoPair("Creating table", self.dvoDoneTable )
    300        sql = "CREATE TABLE " + self.dvoDoneTable + " (name VARCHAR(100))"
     311       sql = "CREATE TABLE " + self.dvoDoneTable + " (path VARCHAR(1000), modifiedDate BIGINT, size BIGINT)"
    301312       try: self.execute(sql)
    302313       except: self.logger.errorPair("Unable to create table", self.dvoDoneTable)
Note: See TracChangeset for help on using the changeset viewer.