Changeset 33487 for trunk

Timestamp:

Mar 13, 2012, 9:48:05 AM (14 years ago)

Author:

rhenders

Message:

now deletes an item from the pending table once started working on it; now pauses if no stripes are available after 2 attemps; Dvo object no created in constructor and scratch Db only changed if necessary; changes to way IN batches are handled; using new argument in base-class printUsage() method

File:

• trunk/ippToPsps/jython/loader.py (modified) (10 diffs)

trunk/ippToPsps/jython/loader.py

@@ -49 +49 @@
         if not self.scratchDb.connected: 
             self.logger.errorPair("Cannot connect to a", "scratch database")
-            raise Exception("No scratch Db")
+            exitProgram("no available scratch database")
         
         # now pass scratch Db to dvo object
+        self.dvo = Dvo(self.logger, self.config)
         self.dvo.setScratchDb(self.scratchDb.dbName)
 
         # if an IN batch is requested, create and quit
         if len(sys.argv) > 2 and sys.argv[2] == "init":
-           batchID = self.ippToPspsDb.createNewBatch("IN", 0, self.config)
+           batchID = self.ippToPspsDb.createNewBatch("IN", 0)
            if batchID > 0:
                batch = InitBatch(self.logger, 
@@ -65 +66 @@
                        batchID)
                batch.run()
-    
+           else:
+               self.logger.errorPair("Could not create batch ID", "%d" % batchID)
+               self.exitProgram("failed to create init batch")
+               
            self.exitProgram("init batch created")
     
@@ -80 +84 @@
 
         ret = super(Loader, self).refreshConfig()
-        self.dvo = Dvo(self.logger, self.config)
+        
         try: self.dvo.setScratchDb(self.scratchDb.dbName)
         except: pass
@@ -92 +96 @@
     def run(self):
 
+        numAttempts = 0
         while True:
 
-            boxIds = self.ippToPspsDb.getBoxIds(self.config.name)
-   
-            self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds))
-
-            for boxId in boxIds:
-      
-                self.checkClientStatus()
-
-                # get box info. if boxes have changed, break and start again
-                try:
-                    boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
-                except:
-                   self.logger.infoPair("Can't get details for this box", "%d" % boxId)
-                   break
-                 
-                for batchType in self.config.batchTypes:
-                    
-                    ids = self.ippToPspsDb.getPendingIds(boxId, batchType)
-            
+            for batchType in self.config.batchTypes:
+
+                # get a stripe's worth of box IDs
+                boxIds = self.ippToPspsDb.getStripeBoxIds(self.HOST, self.PID, batchType)
+
+                numAttempts += 1
+
+                self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds))
+    
+                if len(boxIds) > 0: numAttempts = 0
+
+                for boxId in boxIds:
+          
+                    self.checkClientStatus()
+    
+                    # get box info. if boxes have changed, break and start again
+                    try:
+                        boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
+                    except:
+                       self.logger.infoPair("Can't get details for this box", "%d" % boxId)
+                       break
+     
+                    ids = self.ippToPspsDb.getPendingIdsForThisBox(boxId, batchType)
+                
                     if len(ids) < 1: 
                         self.logger.debugPair("No " + batchType + " items found in this box", "skipping")
                         continue
-            
+                
                     self.logger.infoSeparator()
+                    self.logger.infoTitle("New box")
                     self.logger.infoPair("Box dimensions", "%.1f / %.1f / %.1f" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE']))
                     self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids))
                     boxSizeWithBorder = boxDim['SIDE'] + (self.config.BORDER * 2)
-   
+       
                     # look in DVO for this box (with extra border)
                     self.dvo.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
@@ -138 +149 @@
                         useFullTables = 1
                     else: useFullTables = 0
-            
+                
                     self.logger.infoBool("Using pre-ingested DVO data?", useFullTables)
                     self.processTheseItems(batchType, ids, useFullTables)
 
             self.checkClientStatus()
-            if not self.waitForPollTime(): break
+            if numAttempts > 1 and not self.waitForPollTime(): break
 
         
@@ -158 +169 @@
         for id in ids:
     
-            batchID = self.ippToPspsDb.createNewBatch(batchType, id, self.config)
+            batchID = self.ippToPspsDb.createNewBatch(batchType, id)
             
             if batchID < 0: 
@@ -165 +176 @@
     
             self.ippToPspsDb.unlockTables()
-    
+   
+            self.ippToPspsDb.deletePendingItem(batchType, id)
+
             if batchType == "P2":
                 batch = DetectionBatch(self.logger,
@@ -194 +207 @@
             self.logger.infoSeparator()
     
-            # if in test mode, then quit after one batch
+            # if in test mode, then quit after one batch # TODO remove
             if self.config.test: break
     
@@ -204 +217 @@
     '''
     def printUsage(self):
-        super(Cleanup, self).printUsage()
-        print " [init]"
+        super(Cleanup, self).printUsage("<[init]>")
 
     
@@ -217 +229 @@
 except Exception, e:
     print str(e)
-    traceback.print_exc() 
+    traceback.print_exc()