Changeset 33170 for trunk

Timestamp:

Jan 27, 2012, 12:31:28 PM (14 years ago)

Author:

rhenders

Message:

using more meaningful name of 'region' as opposed to 'file' to refer to DVO regions

File:

• trunk/ippToPsps/jython/dvo.py (modified) (11 diffs)

trunk/ippToPsps/jython/dvo.py

@@ -144 +144 @@
     Gets a bunch of region files from DVO for this RA/Dec box
     '''
-    def getRegionFiles(self, minRa=-1., maxRa=361., minDec=-91., maxDec=91):
+    def getRegions(self, regionsToIngest, minRa=-1., maxRa=361., minDec=-91., maxDec=91):
     
         if (maxRa - minRa <= 0) or (maxDec - minDec) <= 0:
@@ -154 +154 @@
                 "RA: %.2f -> %.2f, Dec: %.2f -> %.2f" % (minRa, maxRa, minDec, maxDec))
 
-        allFiles = self.scratchDb.getDvoFilesCoveringThisRegion(minRa, maxRa, minDec, maxDec)
-
-        files = []
-
-        total = 0
+        allRegions = self.scratchDb.getDvoRegionsForThisBox(minRa, maxRa, minDec, maxDec)
+
+        allPopulatedRegions = []
         alreadyIngested = 0
-        toIngest = 0
-        for file in allFiles:
-
-           cpmPath = self.dvoLocation + "/" + file + ".cpm"
-           cptPath = self.dvoLocation + "/" + file + ".cpt"
+        for region in allRegions:
+
+           cpmPath = self.dvoLocation + "/" + region + ".cpm"
+           cptPath = self.dvoLocation + "/" + region + ".cpt"
 
            # check for existence of cpm and cpt files
@@ -170 +167 @@
            if not os.path.isfile(cptPath): continue
 
-           total = total + 1
+           allPopulatedRegions.append(region)
 
            # if we have already imported up-to-date versions of both the cpm and cpt then we skip this region
@@ -177 +174 @@
                continue
 
-           files.append(file)
-
-           toIngest = toIngest + 1
-
-        self.logger.infoPair("Total DVO regions to ingest", "%d" % total) 
-        self.logger.infoPair("DVO regions already ingested", "%d" % alreadyIngested) 
-        self.logger.infoPair("DVO regions to ingest", "%d" % toIngest) 
-
-        return files
+           regionsToIngest.append(region)
+
+
+
+
+        #regionsToDelete = list(set(allIDs) - set(processedIDs))
+
+
+        self.logger.infoPair("Total DVO regions", "%d" % len(allPopulatedRegions)) 
+        self.logger.infoPair("Total DVO regions already ingested", "%d" % alreadyIngested) 
+        self.logger.infoPair("Total DVO regions to ingest", "%d" % len(regionsToIngest)) 
+
 
     '''
@@ -194 +194 @@
 
         # format a string for the user -smaller than one MB
-        if byteSize < 1048576: return "%.1f Kb" % (byteSize/1024.0)
+        if byteSize < 1048576: return "%.1f K" % (byteSize/1024.0)
         # smaller than one GB
-        elif byteSize < 1073741824: return "%.1f Mb" % (byteSize/1048576.0)
+        elif byteSize < 1073741824: return "%.1fM" % (byteSize/1048576.0)
         # more than a GB
-        else: return "%.1f Gb" % (byteSize/1073741824.0)
-
-    '''
-    Returns the disk size of this region of data in DVO
-    '''
-    def getSizeOfDvoRegionFromFiles(self, files):
+        else: return "%.1fG" % (byteSize/1073741824.0)
+
+    '''
+    Returns the disk size of these DVO regions, i.e. the size on disk of all cpm and cpt files
+    '''
+    def getDiskSizeOfDvoRegions(self, regions):
 
         size = 0.0
-        for file in files:
+        for region in regions:
 
             # get combined size of cpm and cpt files
-            size = size + os.stat(self.dvoLocation + "/" + file + ".cpm").st_size
-            size = size + os.stat(self.dvoLocation + "/" + file + ".cpt").st_size
+            size = size + os.stat(self.dvoLocation + "/" + region + ".cpm").st_size
+            size = size + os.stat(self.dvoLocation + "/" + region + ".cpt").st_size
 
         return size
@@ -232 +232 @@
         if not self.correctDvo: return
 
-        files = self.getRegionFiles(minRa, maxRa, minDec, maxDec)
-        totalSize = self.getSizeOfDvoRegionFromFiles(files)
+        regionsToIngest = []
+        self.getRegions(regionsToIngest, minRa, maxRa, minDec, maxDec)
+        totalSize = self.getDiskSizeOfDvoRegions(regionsToIngest)
         self.logger.infoPair("Total size of cpm/cpt files to ingest", self.getFileSizeStr(totalSize))
         count = 0
@@ -239 +240 @@
         self.logger.infoSeparator()
 
-        # now loop through all files
-        for file in files:
-
-           cpmPath = self.dvoLocation + "/" + file + ".cpm"
-           cptPath = self.dvoLocation + "/" + file + ".cpt"
-
-           cpmTableName = self.getDatabaseFriendlyTableName(file + ".cpm")
-           cptTableName = self.getDatabaseFriendlyTableName(file + ".cpt")
+        # now loop through all regions and ingest to MySQL
+        for region in regionsToIngest:
+
+           cpmPath = self.dvoLocation + "/" + region + ".cpm"
+           cptPath = self.dvoLocation + "/" + region + ".cpt"
+
+           cpmTableName = self.getDatabaseFriendlyTableName(region + ".cpm")
+           cptTableName = self.getDatabaseFriendlyTableName(region + ".cpt")
 
            # if we have not already ingested the cpm table, then do it now
@@ -271 +272 @@
 
                # we can report that we have imported this table
-               self.scratchDb.setImportedThisDvoTable(cpmPath)
+               self.scratchDb.setImportedThisDvoTable(cpmPath, region)
+
+           # get the id for this cpm file so we can insert it in the dvoDetection table
+           fileId = self.scratchDb.getDvoFileId(cpmPath)
 
            # import cpt table and index
@@ -307 +311 @@
                   ,objID \
                   ,flags \
+                  ,fileID \
                   ) SELECT \
                   SOURCE_ID \
@@ -315 +320 @@
                   ,PSPS_OBJ_ID \
                   ,DB_FLAGS \
+                  , " + str(fileId) + " \
                   FROM " + cpmTableName
            try:
@@ -328 +334 @@
            self.scratchDb.dropTable(cptTableName)
 
-           self.scratchDb.setImportedThisDvoTable(cptPath)
+           self.scratchDb.setImportedThisDvoTable(cptPath, region)
            count = count + 1
 
         self.logger.infoSeparator()
 
-        self.logger.infoPair("Ingested", "%d DVO region files" % count)
+        self.logger.infoPair("Ingested", "%d DVO regions" % count)
 
         return True