Changeset 33677 for trunk/ippToPsps/jython/loader.py

Timestamp:

Apr 3, 2012, 4:33:49 PM (14 years ago)

Author:

rhenders

Message:

added new OB batch type

File:

• trunk/ippToPsps/jython/loader.py (modified) (8 diffs)

trunk/ippToPsps/jython/loader.py

@@ -16 +16 @@
 from datastore import Datastore
 from batch import Batch
-from dvo import Dvo
+from dvodetections import DvoDetections
 
 from initbatch import InitBatch
 from stackbatch import StackBatch
 from detectionbatch import DetectionBatch
+from objectbatch import ObjectBatch
 
 '''
@@ -50 +51 @@
             self.logger.errorPair("Cannot connect to a", "scratch database")
        
-        # create Dvo and Datastore objects
-        self.dvo = Dvo(self.logger, self.config, self.scratchDb.dbName)
+        # create Datastore objects
         self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb)
 
@@ -72 +72 @@
     
 
-        # set a poll time of about 1 minutes
+        # set a poll time of about 1 minute
         self.parsePollTimeArg("0.0166")
 
@@ -83 +83 @@
 
         ret = super(Loader, self).refreshConfig()
-        try: self.dvo.checkDvoLocation()
-        except: 
-            self.logger.errorPair("can't check", "dvo.checkDvoLocation()")
-            return ret
+        try: self.scratchDb
+        except: return ret
+           
+        self.dvoDetections = DvoDetections(self.logger, self.config, self.scratchDb.dbName)
 
         return ret
@@ -134 +134 @@
                         boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
                     except:
-                       self.logger.infoPair("Can't get details for this box", "%d" % boxId)
-                       break
+                        self.logger.infoPair("Can't get details for this box", "%d" % boxId)
+                        break
      
                     ids = self.ippToPspsDb.getPendingIdsForThisBox(boxId, batchType)
@@ -148 +148 @@
                     self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids))
                     boxSizeWithBorder = boxDim['SIDE'] + (self.config.BORDER * 2)
-       
-                    # look in DVO for this box (with extra border)
-                    #self.dvo.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
-                    #sizeToBeIngested = self.dvo.getDiskSizeOfRegionsToBeIngested()
-                    #if sizeToBeIngested == 0.0: smfsPerGB = 999999999
-                    #else: smfsPerGB = len(ids)/sizeToBeIngested
-                    #self.logger.infoPair("DVO to be ingested", "%.1f GB" % sizeToBeIngested)
-                    #self.logger.infoPair("smfs-per-GB", "%.1f" % smfsPerGB)
-                    smfsPerGB = 0
-                    # should do we pre-ingest stuff from DVO?
-                    if smfsPerGB > 40:
-                        if not self.dvo.sync():
-                            self.logger.errorPair("Could not sync DVO with MySQL", "skipping")
-                            continue
      
-                        useFullTables = 1
-                    else: useFullTables = 0
+                    useFullTables = 0
+                    if batchType != "OB":
+                        # look in DVO for this box (with extra border)
+                        #self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
+                        self.dvoDetections.setSkyArea()
+                        sizeToBeIngested = self.dvoDetections.getDiskSizeOfRegionsToBeIngested()
+                        if sizeToBeIngested == 0.0: smfsPerGB = 999999999
+                        else: smfsPerGB = len(ids)/sizeToBeIngested
+                        self.logger.infoPair("DVO to be ingested", "%.1f GB" % sizeToBeIngested)
+                        self.logger.infoPair("smfs-per-GB", "%.1f" % smfsPerGB)
+                        # should do we pre-ingest stuff from DVO?
+                        if smfsPerGB > 30:
+                            if not self.dvoDetections.sync():
+                                self.logger.errorPair("Could not sync DVO with MySQL", "skipping")
+                                continue
+     
+                            useFullTables = 1
                 
                     self.logger.infoBool("Using pre-ingested DVO data?", useFullTables)
@@ -216 +217 @@
                         useFullTables)
     
+            elif batchType == "OB":
+                batch = ObjectBatch(self.logger,
+                        self.config,
+                        self.gpc1Db,
+                        self.ippToPspsDb,
+                        self.scratchDb,
+                        id,
+                        batchID,
+                        useFullTables)
+    
             batch.run()
     
@@ -223 +234 @@
     
             self.logger.infoSeparator()
-    
-            # if in test mode, then quit after one batch # TODO remove
-            if self.config.test: break
     
         self.ippToPspsDb.unlockTables()