Changeset 35235 for branches/eam_branches/ipp-20130207/ippToPsps/jython/dvo.py

Timestamp:

Mar 5, 2013, 2:23:09 PM (13 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20130207/ippToPsps

Files:

• . (modified) (1 prop)
• jython/dvo.py (modified) (16 diffs)

branches/eam_branches/ipp-20130207/ippToPsps/jython/dvo.py

@@ -6 +6 @@
 import sys
 import os
+import math
 import logging
 import glob
@@ -33 +34 @@
 
     '''
-    def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName=None):
+    def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb):
 
         # set up logging
@@ -40 +41 @@
         self.skychunk = skychunk
         self.ippToPspsDb = ippToPspsDb
+        self.scratchDb = scratchDb
 
         # set up empty lists
@@ -48 +50 @@
         self.regionsIngestedButOutOfDate = []
        
-        # connect to the specified scratch database, if a name is given, otherwise use the first version
-        try:
-            self.scratchDb = ScratchDb(self.logger, self.config, '1', scratchDbName)
-        except: raise
-
         # we have (still) 3 modes of ingest from dvo cpt/cps/cpm files 
         # 1) dvograbber (useFullTables == FALSE), 
@@ -211 +208 @@
     def setSkyAreaAsStripe(self, ra=0., width=4.):
 
+        print "this function should not reached now, error"
+        raise
+
         self.setSkyArea(ra, ra+width, -91., 91.)
 
@@ -217 +217 @@
     '''
     def setSkyAreaAsBox(self, ra=180., dec=0., side=181.0):
+
+        print "this function should not reached now, error"
+        raise
 
         halfSide = side/2.0
@@ -223 +226 @@
     '''
     Determines what DVO files need to be ingested to the defines region
+    XXX EAM : this function has been substantially reduced / stripped down to work with the parallel dvo
+    XXX EAM : some additional review of the goals would be useful
     '''
     def setSkyArea(self, minRa=-1., maxRa=361., minDec=-91., maxDec=91):
@@ -263 +268 @@
 
            # get paths for this region for the file types we are interested in
-           paths = []
-           for fileType in self.ingestFileTypes:
-               paths.append(self.skychunk.dvoLocation + "/" + regionPath + "." + fileType)
+           # XXX ipptopsps cannot check files -- it does not know where they are
+           ## paths = []
+           ## for fileType in self.ingestFileTypes:
+           ##     paths.append(self.skychunk.dvoLocation + "/" + regionPath + "." + fileType)
 
            # check for the existence of all interested file types
-           skipThisRegion = False
-           for path in paths:
-               if not os.path.isfile(path):
-                   skipThisRegion = True
-                   break
-
-           if skipThisRegion: continue
+           # XXX ipptopsps cannot check files -- it does not know where they are
+           ## skipThisRegion = False
+           ## for path in paths:
+           ##     if not os.path.isfile(path):
+           ##         skipThisRegion = True
+           ##         break
+
+           ## if skipThisRegion: continue
 
            # ok, have real, populated files so add this region to the list
@@ -281 +288 @@
 
            # if we have already imported up-to-date versions of all interested files, then we skip this region
-           alreadyIngested = True
-           for path in paths:
-               if not self.scratchDb.alreadyImportedThisDvoTable(path): 
-                   alreadyIngested = False
-                   break
-    
+           ## alreadyIngested = True
+           ## for path in paths:
+           ##     if not self.scratchDb.alreadyImportedThisDvoTable(path): 
+           ##         alreadyIngested = False
+           ##         break
+
+           alreadyIngested = False
            if alreadyIngested:
                self.regionsAlreadyIngested.append(regionPath)
@@ -299 +307 @@
 
            # check if we have out-of-date versions of any interested files, if so, add region to the purge list
-           outOfDate = False
-           for path in paths:
-               if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(path):
-                   outOfDate = True
-                   break
-
-           if outOfDate: self.regionsIngestedButOutOfDate.append(regionPath)
+           ## outOfDate = False
+           ## for path in paths:
+           ##     if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(path):
+           ##         outOfDate = True
+           ##         break
+           ## 
+           ## if outOfDate: self.regionsIngestedButOutOfDate.append(regionPath)
+
+           print "using region: ", regionPath
 
            self.regionsToIngest.append(regionPath)
@@ -389 +399 @@
         self.logger.infoPair("Purging", "%d regions" % len(regions)) 
 
-        detectionDeleteCount = self.scratchDb.purgeTheseDvoRegions(regions, self.ingestFileTypes)
-        if detectionDeleteCount > 0: self.logger.infoPair("Deleted", "%d detections" % detectionDeleteCount)
+        self.scratchDb.purgeTheseDvoRegions(regions, self.ingestFileTypes)
+
+        ## detectionDeleteCount = self.scratchDb.purgeTheseDvoRegions(regions, self.ingestFileTypes)
+        ## if detectionDeleteCount > 0: self.logger.infoPair("Deleted", "%d detections" % detectionDeleteCount)
 
         return True
@@ -420 +432 @@
 
         size = 0.0
+        return size
+
         for region in regions:
              self.logger.infoPair("sizes for region", region)
              # get combined size of all interested files
              for fileType in self.ingestFileTypes:
+                 print "trying ", fileType
                  size = size + self.getDiskSize(self.skychunk.dvoLocation + "/" + region + "." + fileType)
                  # EAM TEST I/O
@@ -574 +589 @@
     includes purging detections outside sky area
     '''
-    def nativeIngestDetections(self, boxId, raCenter, decCenter, boxSize):
+    def nativeIngestDetections(self, boxId, boxDim):
 
         # XXX put the chunk below in a separate method 
         # blow away existing dvoDetection table
 
-        # clear the 'ingested' field for all boxes
-        self.ippToPspsDb.clearIngestedBoxes()
+        # clear the 'ingested' field for all boxes owned by this host
+        self.ippToPspsDb.clearIngestedBoxes(self.scratchDb.dbHost)
         
+        ## print "*******************************************************************************"
+        ## self.logger.warning("about to delete detection table and re-ingest; is this correct?")
+        ## response = raw_input("(y/n) ")
+        ## if response != "y":
+        ##     raise
+        ## self.logger.warning("Are you ABSOLUTELY sure you want to do this?")
+        ## response = raw_input("(y/n) ")
+        ## if response != "y":
+        ##     raise
+
         # drop detections table
         self.logger.infoPair("Dropping table", self.scratchDb.dvoDetectionTable)
@@ -608 +633 @@
         self.loadImages()
 
-        # dvopsps -D catdir CATDIR -region .... -dbhost xx -dbname xx -dbuser xx -dbpass xx
-        halfSize = boxSize / 2.0
-        # -region raCenter-halfSize raCenter+halfSize decCenter-halfSize decCenter+halfSize
+        # the box dimensions are the area used to select the items of
+        # interest from the gpc1 database (camera smfs / skycal cmfs).
+        # for dvopsps, we need to include a border region large enough
+        # to grab all detections / objects which are contributed by an
+        # item with the center in the given box
+        dR_border = self.skychunk.BORDER / math.cos(math.radians(boxDim['DEC']))
+        dD_border = self.skychunk.BORDER
+
+        minRA  = boxDim['minRA']  - dR_border
+        maxRA  = boxDim['maxRA']  + dR_border
+        minDEC = boxDim['minDEC'] - dD_border
+        maxDEC = boxDim['maxDEC'] + dD_border
 
         # TODO path to DVO prog hardcoded temporarily
@@ -620 +654 @@
         cmd += " -D CATDIR " + self.skychunk.dvoLocation
         cmd += " -region "
-        cmd += " " + str(raCenter-halfSize)
-        cmd += " " + str(raCenter+halfSize)
-        cmd += " " + str(decCenter-halfSize)
-        cmd += " " + str(decCenter+halfSize)
+        cmd += " " + str(minRA)
+        cmd += " " + str(maxRA)
+        cmd += " " + str(minDEC)
+        cmd += " " + str(maxDEC)
 
         if self.skychunk.parallel:
@@ -632 +666 @@
         p.wait()
 
-        self.ippToPspsDb.setIngestedBox(boxId)
+        ## dvopsps can take a very long time. try 2 or 3 times to re-establish communication with the db before proceeding?
+
+        self.ippToPspsDb.setIngestedBox(boxId, self.scratchDb.dbHost)
 
         # add fileID column