Changeset 35174

Timestamp:

Feb 16, 2013, 3:03:50 PM (13 years ago)

Author:

eugene

Message:

do not test file sizes & dates for validity (ipptopsps does not have access to this info)

Location:

trunk/ippToPsps/jython

Files:

• batch.py (modified) (1 diff)
• dvo.py (modified) (4 diffs)
• loader.py (modified) (1 diff)
• objectbatch.py (modified) (1 diff)

trunk/ippToPsps/jython/batch.py

@@ -544 +544 @@
         # for OB, this imports object data from DVO
         if not self.importIppTables():
+            if self.batchType == "OB":
+                self.logger.errorPair("skipping this batch", "no dvo tables for this region")
+                self.ippToPspsDb.updateProcessed(self.batchID, 1)
+                return True
+
             self.logger.errorPair("Aborting this batch", "could not import IPP tables")
             self.ippToPspsDb.updateProcessed(self.batchID, -1)

trunk/ippToPsps/jython/dvo.py

@@ -263 +263 @@
 
            # get paths for this region for the file types we are interested in
-           paths = []
-           for fileType in self.ingestFileTypes:
-               paths.append(self.skychunk.dvoLocation + "/" + regionPath + "." + fileType)
+           # XXX ipptopsps cannot check files -- it does not know where they are
+           ## paths = []
+           ## for fileType in self.ingestFileTypes:
+           ##     paths.append(self.skychunk.dvoLocation + "/" + regionPath + "." + fileType)
 
            # check for the existence of all interested file types
-           skipThisRegion = False
-           for path in paths:
-               if not os.path.isfile(path):
-                   skipThisRegion = True
-                   break
-
-           if skipThisRegion: continue
+           # XXX ipptopsps cannot check files -- it does not know where they are
+           ## skipThisRegion = False
+           ## for path in paths:
+           ##     if not os.path.isfile(path):
+           ##         skipThisRegion = True
+           ##         break
+
+           ## if skipThisRegion: continue
 
            # ok, have real, populated files so add this region to the list
@@ -281 +283 @@
 
            # if we have already imported up-to-date versions of all interested files, then we skip this region
-           alreadyIngested = True
-           for path in paths:
-               if not self.scratchDb.alreadyImportedThisDvoTable(path): 
-                   alreadyIngested = False
-                   break
-    
+           ## alreadyIngested = True
+           ## for path in paths:
+           ##     if not self.scratchDb.alreadyImportedThisDvoTable(path): 
+           ##         alreadyIngested = False
+           ##         break
+
+           alreadyIngested = False
            if alreadyIngested:
                self.regionsAlreadyIngested.append(regionPath)
@@ -299 +302 @@
 
            # check if we have out-of-date versions of any interested files, if so, add region to the purge list
-           outOfDate = False
-           for path in paths:
-               if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(path):
-                   outOfDate = True
-                   break
-
-           if outOfDate: self.regionsIngestedButOutOfDate.append(regionPath)
+           ## outOfDate = False
+           ## for path in paths:
+           ##     if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(path):
+           ##         outOfDate = True
+           ##         break
+           ## 
+           ## if outOfDate: self.regionsIngestedButOutOfDate.append(regionPath)
+
+           print "using region: ", regionPath
 
            self.regionsToIngest.append(regionPath)
@@ -420 +425 @@
 
         size = 0.0
+        return size
+
         for region in regions:
              self.logger.infoPair("sizes for region", region)
              # get combined size of all interested files
              for fileType in self.ingestFileTypes:
+                 print "trying ", fileType
                  size = size + self.getDiskSize(self.skychunk.dvoLocation + "/" + region + "." + fileType)
                  # EAM TEST I/O

trunk/ippToPsps/jython/loader.py

@@ -198 +198 @@
                             # impact of mysql insertion
                             if not self.ippToPspsDb.isBoxIngested(boxId):
-                                self.dvoDetections.nativeIngestDetections(boxId, boxDim['RA'], boxDim['DEC'], boxSizeSansBorder)
+                                self.dvoDetections.nativeIngestDetections(boxId, boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
 
                             useFullTables = 1

trunk/ippToPsps/jython/objectbatch.py

@@ -77 +77 @@
         else:
             self.dvoObjects.ingestRegion(self.region)
+
+        cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
+        if not self.scratchDb.tableExists(cptTableName):
+            return False
 
         return True