Changeset 35235 for branches/eam_branches/ipp-20130207/ippToPsps/jython/loader.py

Timestamp:

Mar 5, 2013, 2:23:09 PM (13 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20130207/ippToPsps

Files:

• . (modified) (1 prop)
• jython/loader.py (modified) (8 diffs)

branches/eam_branches/ipp-20130207/ippToPsps/jython/loader.py

@@ -8 +8 @@
 import sys
 import os
+import math
 import traceback
 import logging.config
@@ -43 +44 @@
 
         # connect to scratch database
-        scratchDbs = ['1', '2', '3']
-        for dbVersion in scratchDbs:
-            self.scratchDb = ScratchDb(self.logger, self.config, dbVersion)
-            if not self.config.test and self.scratchDb.anyOtherConnections():
-                self.logger.errorPair("This scratch Db is already in use", self.scratchDb.dbName)
-                self.scratchDb.disconnect()
-                continue
-            break
-
-        if not self.scratchDb.connected:
+        self.scratchDb = ScratchDb(self.logger, self.config)
+        if not self.config.test and self.scratchDb.anyOtherConnections():
+            self.logger.errorPair("This scratch Db is already in use", self.scratchDb.dbName)
+            self.scratchDb.disconnect()
             self.exitProgram("Cannot connect to a scratch database")
             raise
@@ -97 +92 @@
         except: return ret
            
-        self.dvoDetections = DvoDetections(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb.dbName)
+        self.dvoDetections = DvoDetections(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
 
         return ret
@@ -123 +118 @@
 
         numAttempts = 0
+        abort = False
         while True:
 
-            abort = False
-            for batchType in self.skychunk.batchTypes:
-
-                # get a stripe's worth of box IDs
-                boxIds = self.ippToPspsDb.getStripeBoxIds(self.HOST, self.PID, batchType)
-
-                numAttempts += 1
-
-                self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds))
-                self.logger.infoPair("numAttempts", str(numAttempts))
-                if len(boxIds) > 0: numAttempts = 0
-
-                for boxId in boxIds:
+            # get a stripe's worth of box IDs
+            boxIds = self.ippToPspsDb.getStripeBoxIdsAnyBatch(self.HOST, self.PID)
+            self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds))
+            self.logger.infoPair("numAttempts", str(numAttempts))
+
+            if len(boxIds) > 0: numAttempts = 0
+            numAttempts += 1
+
+            for boxId in boxIds:
+
+                # get box info. if boxes have changed, break and start again
+                try:
+                    boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
+                except:
+                    self.logger.infoPair("Can't get details for this box", "%d" % boxId)
+                    break
+                self.logger.infoPair("client status", "ok")
+
+                for batchType in self.skychunk.batchTypes:
+
                     self.logger.infoPair("Check","client status")
                     if not self.checkClientStatus():
@@ -143 +146 @@
                         break
                     self.logger.infoPair("client status", "ok")
-                    # get box info. if boxes have changed, break and start again
-                    try:
-                        boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
-                    except:
-                        self.logger.infoPair("Can't get details for this box", "%d" % boxId)
-                        break
-                    self.logger.infoPair("client status", "ok")
+
                     ids = self.ippToPspsDb.getPendingIdsForThisBox(boxId, batchType)
-                
+               
                     # ids are the stage_ids for items to be processed (eg, stack_id for a stack CMF, cam_id for P2 smf)
 
@@ -160 +157 @@
                     self.logger.infoSeparator()
                     self.logger.infoTitle("New box")
-                    self.logger.infoPair("Box dimensions", "%.1f / %.1f / %.1f" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE']))
+                    self.logger.infoPair("Box dimensions: ", "ra / dec / size : %.1f / %.1f / %.1f   --   sky bounds : (%.2f - %2.f, %2.f - %2.f)" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE'], boxDim['minRA'], boxDim['maxRA'], boxDim['minDEC'], boxDim['maxDEC']))
                     self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids))
-                    boxSizeWithBorder = boxDim['SIDE'] + (self.skychunk.BORDER * 2)
-                    boxSizeSansBorder = boxDim['SIDE']
-                    self.logger.infoPair("got here", "ok")
+
                     useFullTables = 0
                     if batchType != "OB":
@@ -171 +166 @@
                             # look in DVO for this box (with extra border)
                             self.logger.infoPair("Querying DVO for this sky area", "")
-                            self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
+
+                            # XXXX EAM : This code should not be used anymore
+                            boxSize = 0.0
+                            self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSize)
                             #self.dvoDetections.setSkyArea()
                             sizeToBeIngested = self.dvoDetections.getDiskSizeOfRegionsToBeIngested()
@@ -191 +189 @@
                                 useFullTables = 1
                         else:
-                            # XXX EAM : this is not currently an
-                            # option. either remove the non-native
-                            # ingest code or make it an option
-                            # if we are using the native loader, always use it
-                            # need to work out a good boundary / region strategy in coordination with 
-                            # impact of mysql insertion
-                            if not self.ippToPspsDb.isBoxIngested(boxId):
-                                self.dvoDetections.nativeIngestDetections(boxId, boxDim['RA'], boxDim['DEC'], boxSizeSansBorder)
+                            # XXX EAM : this is not currently optional
+                            # (it is the only path). either remove the
+                            # non-native ingest code or make it an
+                            # option if we are using the native
+                            # loader, always use it need to work out a
+                            # good boundary / region strategy in
+                            # coordination with impact of mysql
+                            # insertion
+                            if not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost):
+                                self.dvoDetections.nativeIngestDetections(boxId, boxDim)
 
                             useFullTables = 1