Changeset 33176

Timestamp:

Jan 27, 2012, 3:48:57 PM (14 years ago)

Author:

rhenders

Message:

now figuring out which currently ingested regions can be deleted; if writing to database files, keep re-trying

File:

• trunk/ippToPsps/jython/dvo.py (modified) (6 diffs)

trunk/ippToPsps/jython/dvo.py

@@ -13 +13 @@
 from scratchdb import ScratchDb
 
+import time
 from java.lang import *
 from java.sql import *
@@ -144 +145 @@
     Gets a bunch of region files from DVO for this RA/Dec box
     '''
-    def getRegions(self, regionsToIngest, minRa=-1., maxRa=361., minDec=-91., maxDec=91):
+    def getRegions(self, regionsToIngest, regionsToPurge, minRa=-1., maxRa=361., minDec=-91., maxDec=91):
     
         if (maxRa - minRa <= 0) or (maxDec - minDec) <= 0:
@@ -154 +155 @@
                 "RA: %.2f -> %.2f, Dec: %.2f -> %.2f" % (minRa, maxRa, minDec, maxDec))
 
+
         allRegions = self.scratchDb.getDvoRegionsForThisBox(minRa, maxRa, minDec, maxDec)
+        allIngestedRegions = self.scratchDb.getIngestedDvoRegions()
 
         allPopulatedRegions = []
         alreadyIngested = 0
+
         for region in allRegions:
 
@@ -176 +180 @@
            regionsToIngest.append(region)
 
-
-
-
-        #regionsToDelete = list(set(allIDs) - set(processedIDs))
-
-
-        self.logger.infoPair("Total DVO regions", "%d" % len(allPopulatedRegions)) 
+        # get list of ingested regions to remove
+        items = list(set(allIngestedRegions) - set(allPopulatedRegions))
+        for item in items: regionsToPurge.append(item)
+
+        self.logger.infoPair("Total populated DVO regions", "%d" % len(allPopulatedRegions)) 
         self.logger.infoPair("Total DVO regions already ingested", "%d" % alreadyIngested) 
+        self.logger.infoPair("Total DVO regions to purge", "%d" % len(regionsToPurge)) 
         self.logger.infoPair("Total DVO regions to ingest", "%d" % len(regionsToIngest)) 
 
+
+    '''
+    Purges the provided regions from the database
+    '''
+    def purgeRegions(self, regions):
+
+        self.logger.infoPair("Purging", "%d regions" % len(regions)) 
+        deleteCount = self.scratchDb.purgeTheseDvoRegions(regions)
+        self.logger.infoPair("Deleted", "%d detections" % deleteCount) 
 
     '''
@@ -233 +245 @@
 
         regionsToIngest = []
-        self.getRegions(regionsToIngest, minRa, maxRa, minDec, maxDec)
+        regionsToPurge = []
+        self.getRegions(regionsToIngest, regionsToPurge, minRa, maxRa, minDec, maxDec)
+
+
         totalSize = self.getDiskSizeOfDvoRegions(regionsToIngest)
-        self.logger.infoPair("Total size of cpm/cpt files to ingest", self.getFileSizeStr(totalSize))
+        self.logger.infoPair("Total size of cpm/cpt files to ingest", self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsToIngest)))
+        self.logger.infoPair("Total size of cpm/cpt files to purge", self.getFileSizeStr(self.getDiskSizeOfDvoRegions(regionsToPurge)))
         count = 0
+
+        self.purgeRegions(regionsToPurge)
 
         self.logger.infoSeparator()
@@ -377 +395 @@
           table = stilts.tpipe(table, cmd='replaceval Infinity null *')
 
-          #try:
-          self.logger.infoPair("Writing " + table.name + " table to", "database")
-          table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName)
-          #except:
-          #    self.logger.exception("   Problem writing table '" + table.name + "' to the database")
+          attempts = 0
+          self.logger.infoPair("Writing table to database",  table.name)
+          while True:
+    
+              try:
+                  self.logger.infoPair("          attempt", "%d" % attempts)
+                  table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName)
+                  break
+              except:
+                  time.sleep(2)
+                  attempts = attempts + 1
+
           count = count + 1