Changeset 39148 for trunk

Timestamp:

Nov 16, 2015, 3:00:27 PM (11 years ago)

Author:

eugene

Message:

recreate the dvo table templates whenever we reset the dvo tables; load photcodes and images if we are loading a new dvo

Location:

trunk/ippToPsps/jython

Files:

• batch.py (modified) (1 diff)
• dvo.py (modified) (7 diffs)
• dvoobjects.py (modified) (1 diff)
• scratchdb.py (modified) (1 diff)

trunk/ippToPsps/jython/batch.py

@@ -104 +104 @@
 
         # create DVO tables (scratch database)
-        self.scratchDb.createDvoTables()
+        ### XXX self.scratchDb.createDvoTables() <-- remove this
 
         # dump stuff to the log

trunk/ippToPsps/jython/dvo.py

@@ -72 +72 @@
             #     response = raw_input("(y/n) ")
             #     if response == "y":
-            self.resetAllTables()
+            self.resetAllTables() ## called if we need to change DVOs
 
     '''
@@ -95 +95 @@
         self.correctDvo = True
         if not self.loadSkyTable():
+            self.correctDvo = False
+            return
+        if not self.loadPhotcodes():
+            self.correctDvo = False
+            return
+        if not self.loadImages():
             self.correctDvo = False
             return
@@ -298 +304 @@
             return False        
 
-        # update the two meta tables, Images and SkyTable
-        self.loadSkyTable()
+        # update the SkyTable
+        self.loadSkyTable() ## in setSkyArea
 
         # set bounding-box coords
@@ -508 +514 @@
     def sync(self):
 
-        # go no further if we've already partly ingested a different DVO
-        if not self.correctDvo: return False
-
-        # check we have something to do
-        if len(self.regionsToPurge) < 1 and len(self.regionsToIngest) < 1:
-            self.logger.infoPair("Nothing to ingest or purge", "finished ingest")
-            return True
-
-        # make sure we have an up-to-date Images table
-        self.loadImages()
-
-        # purge stuff
-        self.purgeRegions(self.regionsToPurge)
-
-        totalRegions = len(self.regionsToIngest)
-        if totalRegions > 0: self.logger.infoSeparator()
-
-        # now loop through all regions and ingest to MySQL
-        ingestedRegionsCount = 0
-        regionCount = 1
-        for region in self.regionsToIngest:
-
-           self.logger.infoTitle("Region %d of %d: %s" % (regionCount, totalRegions, region))
-           if self.ingestRegion(region): ingestedRegionsCount += 1
-           regionCount += 1
-
-        self.logger.infoSeparator()
-        self.logger.infoPair("Ingested", "%d DVO regions (out of %d)" % (ingestedRegionsCount, len(self.regionsToIngest)))
-
-        # update lists after attempted sync
-        self.setSkyArea(self.minRa, self.maxRa, self.minDec, self.maxDec)
-        self.printSummary()
+        print "---- sync is disabled ----"
+        os._exit(3)
+
+        ### XXX # go no further if we've already partly ingested a different DVO
+        ### XXX if not self.correctDvo: return False
+        ### XXX 
+        ### XXX # check we have something to do
+        ### XXX if len(self.regionsToPurge) < 1 and len(self.regionsToIngest) < 1:
+        ### XXX     self.logger.infoPair("Nothing to ingest or purge", "finished ingest")
+        ### XXX     return True
+        ### XXX 
+        ### XXX # make sure we have an up-to-date Images table
+        ### XXX self.loadImages()
+        ### XXX 
+        ### XXX # purge stuff
+        ### XXX self.purgeRegions(self.regionsToPurge)
+        ### XXX 
+        ### XXX totalRegions = len(self.regionsToIngest)
+        ### XXX if totalRegions > 0: self.logger.infoSeparator()
+        ### XXX 
+        ### XXX # now loop through all regions and ingest to MySQL
+        ### XXX ingestedRegionsCount = 0
+        ### XXX regionCount = 1
+        ### XXX for region in self.regionsToIngest:
+        ### XXX 
+        ### XXX    self.logger.infoTitle("Region %d of %d: %s" % (regionCount, totalRegions, region))
+        ### XXX    if self.ingestRegion(region): ingestedRegionsCount += 1
+        ### XXX    regionCount += 1
+        ### XXX 
+        ### XXX self.logger.infoSeparator()
+        ### XXX self.logger.infoPair("Ingested", "%d DVO regions (out of %d)" % (ingestedRegionsCount, len(self.regionsToIngest)))
+        ### XXX 
+        ### XXX # update lists after attempted sync
+        ### XXX self.setSkyArea(self.minRa, self.maxRa, self.minDec, self.maxDec)
+        ### XXX self.printSummary()
 
         return True
@@ -688 +697 @@
 
         # make sure we have an up-to-date Images table
-        self.loadImages()
+        self.loadImages() # this is in nativeIngestDetections (remove?)
 
         # the box dimensions are the area used to select the items of
@@ -698 +707 @@
         dD_border = self.skychunk.BORDER
         
-        #This is a terrible hack but I'm tired of it taking a decade to ingest the FW skycells.
+        # This is a terrible hack but I'm tired of it taking a decade to ingest the FW skycells.
 
         #if self.batchType == "ST":
@@ -706 +715 @@
         #    dR_border = .4 / math.cos(math.radians(boxDim['DEC']))
         #    dD_border = .4
-
-
 
         minRA  = boxDim['minRA']  - dR_border

trunk/ippToPsps/jython/dvoobjects.py

@@ -28 +28 @@
         # declare DVO file types of interest
         self.ingestFileTypes = ['cpt', 'cps']
-        self.loadPhotcodes()
+        self.loadPhotcodes() # need to load photcodes when generating OB and other object-like batches
 
     '''

trunk/ippToPsps/jython/scratchdb.py

@@ -655 +655 @@
     def resetAllDvoTables(self):
 
-       # drop all dvo tables - NB detections have to be deleted prior to dvoDone due to foreign key constraint
-       self.logger.infoPair("Dropping table", self.dvoImagesTable)
-       self.dropTable(self.dvoImagesTable) 
-       self.logger.infoPair("Dropping table", self.dvoSkyTable)
-       self.dropTable(self.dvoSkyTable) 
-       self.logger.infoPair("Dropping table", self.dvoDoneTable)
-       self.dropTable(self.dvoDoneTable) 
-
-       # blow away existing dvoDetection table & re-crate
-       if not self.resetDvoDetectionTable():
-           self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
-           return False
-
-       # create dvoDone table
-       self.logger.infoPair("Creating table", self.dvoDoneTable)
-       sql = "CREATE TABLE " + self.dvoDoneTable + " (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, region VARCHAR(100), path VARCHAR(1000), modifiedDate BIGINT, size BIGINT)"
-       try: self.execute(sql)
-       except: 
-           self.logger.errorPair("Unable to create table", self.dvoDoneTable)
-           return False
-
-       self.changeEngineToInnoDB(self.dvoDoneTable)
-
-       return True
+        # re-create the definition tables
+        self.createDvoTables()
+
+        # drop all dvo tables - NB detections have to be deleted prior to dvoDone due to foreign key constraint
+        self.logger.infoPair("Dropping table", self.dvoImagesTable)
+        self.dropTable(self.dvoImagesTable) 
+        self.logger.infoPair("Dropping table", self.dvoSkyTable)
+        self.dropTable(self.dvoSkyTable) 
+        self.logger.infoPair("Dropping table", self.dvoDoneTable)
+        self.dropTable(self.dvoDoneTable) 
+
+        # blow away existing dvoDetection table & re-crate
+        if not self.resetDvoDetectionTable():
+            self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
+            return False
+
+        # create dvoDone table
+        self.logger.infoPair("Creating table", self.dvoDoneTable)
+        sql = "CREATE TABLE " + self.dvoDoneTable + " (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, region VARCHAR(100), path VARCHAR(1000), modifiedDate BIGINT, size BIGINT)"
+        try: 
+            self.execute(sql)
+        except: 
+            self.logger.errorPair("Unable to create table", self.dvoDoneTable)
+            return False
+
+        self.changeEngineToInnoDB(self.dvoDoneTable)
+            
+        return True
 
     '''