Changeset 33676 for trunk

Timestamp:

Apr 3, 2012, 4:31:34 PM (14 years ago)

Author:

rhenders

Message:

added methos to ingest a region based on DVO index; purgeRegions now takes list of regions as arg; added methid to ingest a single region; methos to ingest Photcode table

File:

• trunk/ippToPsps/jython/dvo.py (modified) (15 diffs)

trunk/ippToPsps/jython/dvo.py

@@ -19 +19 @@
 
 '''
-A class that encapsulates a DVO database
-
-Pulls relavant bits and pieces into the scratch MySQL database and then provides access queries
+Abstract base-class for DVO classes
+
+Provides generic functionality to pull relevant stuff from a DVO database into a scratch MySQL database while also providing access queries
 '''
 class Dvo(object):
@@ -34 +34 @@
 
     '''
-    def __init__(self, logger, config, scratchDbName):
+    def __init__(self, logger, config, scratchDbName="ipptopsps_scratch"):
 
         # set up logging
@@ -92 +92 @@
 
     '''
+    Loads the DVO Photcodes.dat FITS file
+    '''
+    def loadPhotcodes(self):
+
+        # go no further if we've already partly ingested a different DVO
+        if not self.correctDvo: return
+
+        # check if we have up-to-date version
+        path = self.config.dvoLocation + "/Photcodes.dat"
+        if self.scratchDb.alreadyImportedThisDvoTable(path): 
+            self.logger.debugPair("DVO Photcodes.dat file", "up-to-date")
+            return
+        
+        self.logger.infoSeparator()
+        self.logger.infoPair("DVO Photcodes.dat file", "NOT up-to-date")
+
+        self.importFits(
+                path,
+                "CODE NAME",
+                self.scratchDb.dvoPhotcodesTable)
+        self.logger.infoPair("Adding primary key to", self.scratchDb.dvoPhotcodesTable)
+        self.scratchDb.makeColumnPrimaryKey(self.scratchDb.dvoPhotcodesTable, "CODE")
+
+        self.scratchDb.setImportedThisDvoTable(path)
+
+    '''
     Loads the DVO Images.dat FITS file
     '''
@@ -189 +215 @@
         self.regionsToPurge = []
         self.allPopulatedRegions = []
+        self.allPopulatedRegionInfo = []
         self.regionsAlreadyIngested = []
         self.regionsIngestedButOutOfDate = []
@@ -194 +221 @@
         # loop through all regions in DVO inside this bounding-box
         for region in allRegions:
+
+           regionId = region[0]
+           regionRa = region[1]
+           regionDec = region[2]
+           regionPath = region[3]
 
            # get paths for this region for the file types we are interested in
            paths = []
            for fileType in self.ingestFileTypes:
-               paths.append(self.config.dvoLocation + "/" + region + "." + fileType)
+               paths.append(self.config.dvoLocation + "/" + regionPath + "." + fileType)
 
            # check for the existence of all interested file types
@@ -210 +242 @@
 
            # ok, have real, populated files so add this region to the list
-           self.allPopulatedRegions.append(region)
+           self.allPopulatedRegions.append(regionPath)
+           self.allPopulatedRegionInfo.append([regionId, regionRa, regionDec])
 
            # if we have already imported up-to-date versions of all interested files, then we skip this region
@@ -220 +253 @@
     
            if alreadyIngested:
-               self.regionsAlreadyIngested.append(region)
+               self.regionsAlreadyIngested.append(regionPath)
                continue
 
@@ -230 +263 @@
                    break
 
-           if outOfDate: self.regionsIngestedButOutOfDate.append(region)
-
-           self.regionsToIngest.append(region)
+           if outOfDate: self.regionsIngestedButOutOfDate.append(regionPath)
+
+           self.regionsToIngest.append(regionPath)
 
         # add out-of-range regions for purge list
@@ -241 +274 @@
         for item in self.regionsIngestedButOutOfDate: self.regionsToPurge.append(item)
 
-        self.sizeOfAllPopulatedRegions = self.getDiskSizeOfDvoRegions(self.allPopulatedRegions)
-        self.sizeOfRegionsAlreadyIngested = self.getDiskSizeOfDvoRegions(self.regionsAlreadyIngested)
-        self.sizeOfRegionsIngestedButOutOfDate = self.getDiskSizeOfDvoRegions(self.regionsIngestedButOutOfDate)
-        self.sizeOfRegionsToPurge = self.getDiskSizeOfDvoRegions(self.regionsToPurge)
-        self.sizeOfRegionsToIngest = self.getDiskSizeOfDvoRegions(self.regionsToIngest)
+        self.sizeOfAllPopulatedRegions = self.getDiskSizeOfRegions(self.allPopulatedRegions)
+        self.sizeOfRegionsAlreadyIngested = self.getDiskSizeOfRegions(self.regionsAlreadyIngested)
+        self.sizeOfRegionsIngestedButOutOfDate = self.getDiskSizeOfRegions(self.regionsIngestedButOutOfDate)
+        self.sizeOfRegionsToPurge = self.getDiskSizeOfRegions(self.regionsToPurge)
+        self.sizeOfRegionsToIngest = self.getDiskSizeOfRegions(self.regionsToIngest)
 
     '''
@@ -288 +321 @@
 
     '''
+    Purges a single region from the database
+    '''
+    def purgeRegion(self, region):
+
+        regions = []
+        regions.append(region)
+        self.purgeRegions(regions)
+
+    '''
     Purges the provided regions from the database
     '''
-    def purgeRegions(self):
+    def purgeRegions(self, regions):
 
         # go no further if we've already partly ingested a different DVO
         if not self.correctDvo: return False
 
-        if len(self.regionsToPurge) < 1: return False
-
-        self.logger.infoPair("Purging", "%d regions" % len(self.regionsToPurge)) 
-
-        detectionDeleteCount = self.scratchDb.purgeTheseDvoRegions(self.regionsToPurge, self.ingestFileTypes)
+        if len(regions) < 1: return False
+
+        self.logger.infoPair("Purging", "%d regions" % len(regions)) 
+
+        detectionDeleteCount = self.scratchDb.purgeTheseDvoRegions(regions, self.ingestFileTypes)
         if detectionDeleteCount > 0: self.logger.infoPair("Deleted", "%d detections" % detectionDeleteCount)
 
@@ -327 +369 @@
     Returns the disk size of these DVO regions, i.e. the size on disk of all relevant DVO FITS files
     '''
-    def getDiskSizeOfDvoRegions(self, regions):
+    def getDiskSizeOfRegions(self, regions):
 
         size = 0.0
@@ -338 +380 @@
         return size
 
+
+    '''
+    Ingests object files for this region, given an INDEX from the SkyTable
+    '''
+    def ingestRegionUsingIndex(self, index):
+        return self.ingestRegion(self.scratchDb.getRegionNameFromThisDvoIndex(index))
+
     '''
     Abstract method for ingesting a particular reegion. Must be implemented in all subclasses
     '''
     def ingestRegion(self, region):
-        self.logger.errorPair("ingestRegion()", "not implemented for this class") 
-
+        return self.logger.errorPair("ingestRegion()", "not implemented for this class") 
 
     '''
@@ -364 +412 @@
 
         # purge stuff
-        self.purgeRegions()
+        self.purgeRegions(self.regionsToPurge)
 
         totalRegions = len(self.regionsToIngest)
@@ -389 +437 @@
     '''
     Imports a FITS file. A regex filter lets you choose which tables to pull from the file
-    An optional final argument lets you name the resulatant database table
+    An optional final argument lets you name the resultant database table
     '''
     def importFits(self, path, columns, tableName):
@@ -404 +452 @@
           table = stilts.tpipe(table, cmd='explodeall')
      
-          # IPP FITS files are littered with infinities, so remove these
+          # IPP FITS files are littered with infinity values. Remove them
           self.logger.debugPair("Removing", "infinity values")
           table = stilts.tpipe(table, cmd='replaceval -Infinity null *')