Changeset 35077

Timestamp:

Jan 31, 2013, 5:47:48 PM (13 years ago)

Author:

eugene

Message:

track the boxes for which dvo detections have been ingested

Location:

branches/eam_branches/ipp-20121219/ippToPsps

Files:

• config/ippToPspsDbSchema.sql (modified) (1 diff)
• doc/upgrade.txt (modified) (2 diffs)
• jython/dvo.py (modified) (7 diffs)
• jython/dvodetections.py (modified) (1 diff)
• jython/dvoobjects.py (modified) (1 diff)
• jython/ipptopspsdb.py (modified) (2 diffs)
• jython/loader.py (modified) (3 diffs)
• jython/objectbatch.py (modified) (1 diff)
• jython/queue.py (modified) (2 diffs)
• test/fulltest.sh (modified) (1 diff)

branches/eam_branches/ipp-20121219/ippToPsps/config/ippToPspsDbSchema.sql

@@ -62 +62 @@
   `dec_center` float default NULL,
   `box_side` float default NULL,
+  `ingested` tinyint(1) default '0',
   PRIMARY KEY  (`id`),
   KEY `fk_skychunk` (`skychunk`),

branches/eam_branches/ipp-20121219/ippToPsps/doc/upgrade.txt

@@ -1 +1 @@
 
-2012.01.29
+2013.0
+
+2013.01.29
 
 I'm trying to re-work the operational model.  First, make sure we have correct names for concepts:
@@ -30 +32 @@
 
 
-2012.01.27
+2013.01.27
 
 remaining issues:

branches/eam_branches/ipp-20121219/ippToPsps/jython/dvo.py

@@ -33 +33 @@
 
     '''
-    def __init__(self, logger, config, skychunk, scratchDbName=None):
+    def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName=None):
 
         # set up logging
@@ -39 +39 @@
         self.config = config
         self.skychunk = skychunk
+        self.ippToPspsDb = ippToPspsDb
 
         # set up empty lists
@@ -47 +48 @@
         self.regionsIngestedButOutOfDate = []
        
-        # XXX use Stilts for ingest in certain cases or not?
-        self.useStilts = 0
-
         # connect to the specified scratch database, if a name is given, otherwise use the first version
         try:
@@ -55 +53 @@
         except: raise
 
+        # we have (still) 3 modes of ingest from dvo cpt/cps/cpm files 
+        # 1) dvograbber (useFullTables == FALSE), 
+        # 2) jython/stilts (useStilts == TRUE, useFullTables == TRUE)
+        # 3) dvopsps (useStilts == FALSE, useFullTables == TRUE)
+        # my goal (EAM, 2013/01/31) is to migrate to only dvopsps
+
+        self.useStilts = 0
         self.scratchDb.setUseFullTables(True)
 
@@ -569 +574 @@
     includes purging detections outside sky area
     '''
-    def nativeIngestDetections(self, raCenter, decCenter, boxSize):
-
-        # XXX put the skychunk below in a separate method 
+    def nativeIngestDetections(self, boxId, raCenter, decCenter, boxSize):
+
+        # XXX put the chunk below in a separate method 
+        # blow away existing dvoDetection table
+
+        # clear the 'ingested' field for all boxes
+        self.ippToPspsDb.clearIngestedBoxes()
         
         # drop detections table
@@ -599 +608 @@
         self.loadImages()
 
-        # XXX blow away existing dvoDetection table?
-
         # dvopsps -D catdir CATDIR -region .... -dbhost xx -dbname xx -dbuser xx -dbpass xx
         halfSize = boxSize / 2.0
@@ -625 +632 @@
         p.wait()
 
+        self.ippToPspsDb.setIngestedBox(boxId)
+
         # update lists after attempted sync
         # self.printSummary()

branches/eam_branches/ipp-20121219/ippToPsps/jython/dvodetections.py

@@ -20 +20 @@
     Constructor
     '''
-    def __init__(self, logger, config, skychunk, scratchDbName):
+    def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName):
 
-        super(DvoDetections, self).__init__(logger, config, skychunk, scratchDbName)
+        super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDbName)
 
         # declare DVO file types of interest

branches/eam_branches/ipp-20121219/ippToPsps/jython/dvoobjects.py

@@ -20 +20 @@
     Constructor
     '''
-    def __init__(self, logger, config, skychunk, scratchDbName=None):
+    def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName=None):
 
-        super(DvoObjects, self).__init__(logger, config, skychunk, scratchDbName)
+        super(DvoObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDbName)
 
         # declare DVO file types of interest

branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopspsdb.py

@@ -894 +894 @@
            if field == 'timestamp': continue
            if field == 'name': continue
+           if field == 'ingested_det': continue
+           if field == 'ingested_obj': continue
 
            # get current value for this field, if any
@@ -1075 +1077 @@
 
     '''
+    Inserts new box into box table. If it already exisits, it returns existing box id 
+    '''
+    def isBoxIngested(self, id):
+
+        # set the field 'ingested' to 1 for this box
+        sql = "select ingested from box where id = " + str(id)
+        try:
+            rs = self.executeQuery(sql)
+            if not rs.next():
+                print "no matching boxes for ", str(id)
+                raise
+            status = rs.getInt(1)
+        except:
+            print "failed to set box ", str(id), "as ingested"
+            raise
+
+        return status
+    '''
+    Inserts new box into box table. If it already exisits, it returns existing box id 
+    '''
+    def setIngestedBox(self, id):
+
+        # set the field 'ingested' to 1 for this box
+        sql = "update box set ingested = 1 where id = " + str(id)
+        try:
+            self.execute(sql)
+        except:
+            print "failed to set box ", str(id), "as ingested"
+            raise
+
+    '''
+    Inserts new box into box table. If it already exisits, it returns existing box id 
+    '''
+    def clearIngestedBoxes(self):
+
+        # set the field 'ingested' to 1 for this box
+        sql = "update box set ingested = 0"
+        try:
+            self.execute(sql)
+        except:
+            print "failed to clear boxes as ingested"
+            raise
+
+    '''
     Inserts new box ids into the pending table 
     '''

branches/eam_branches/ipp-20121219/ippToPsps/jython/loader.py

@@ -97 +97 @@
         except: return ret
            
-        self.dvoDetections = DvoDetections(self.logger, self.config, self.skychunk, self.scratchDb.dbName)
+        self.dvoDetections = DvoDetections(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb.dbName)
 
         return ret
@@ -197 +197 @@
                             # need to work out a good boundary / region strategy in coordination with 
                             # impact of mysql insertion
-                            self.dvoDetections.nativeIngestDetections(boxDim['RA'], boxDim['DEC'], boxSizeSansBorder)
+                            if not self.ippToPspsDb.isBoxIngested(boxId):
+                                self.dvoDetections.nativeIngestDetections(boxId, boxDim['RA'], boxDim['DEC'], boxSizeSansBorder)
+
                             useFullTables = 1
                 
@@ -215 +217 @@
                         break
                     self.logger.infoPair("processed","ok")
+
             if abort or not self.checkClientStatus(): abort = True
             elif numAttempts > 1 and not self.waitForPollTime():  break

branches/eam_branches/ipp-20121219/ippToPsps/jython/objectbatch.py

@@ -54 +54 @@
 
        try:
-           self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.scratchDb.dbName)
+           self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb.dbName)
        except:
            self.logger.errorPair("Unable to create instance of", "DvoObjects")

branches/eam_branches/ipp-20121219/ippToPsps/jython/queue.py

@@ -32 +32 @@
 
         try:
-            self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk)
+            self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb)
         except:
             self.exitProgram("Unable to create instance of DvoObject")
@@ -64 +64 @@
 
                 # for object batches, get full list of stuff from Dvo
+                # XXX : need to fix 'dvo.setSkyArea' or this may not return the full list for a parallel dvo
                 if batchType == "OB":
                     self.dvoObjects.setSkyArea(

branches/eam_branches/ipp-20121219/ippToPsps/test/fulltest.sh

@@ -29 +29 @@
 set mkdummy      = 0
 set initscratch  = 0
-set camqueue     = 0
+set camqueue     = 1
 set initbatch    = 0
-set cambatch     = 0
-set stackqueue   = 1
-set stackbatch   = 1
-set objectqueue  = 1
-set objectbatch  = 1
+set cambatch     = 1
+set stackqueue   = 0
+set stackbatch   = 0
+set objectqueue  = 0
+set objectbatch  = 0
 
 set OUTDIR = "testdata"