Changeset 35056 for branches/eam_branches/ipp-20121219

Timestamp:

Jan 25, 2013, 4:48:08 PM (14 years ago)

Author:

eugene

Message:

add native dvo ingest option for objects

Location:

branches/eam_branches/ipp-20121219/ippToPsps

Files:

• config/settings.xml (modified) (1 diff)
• jython/batch.py (modified) (1 diff)
• jython/dvoobjects.py (modified) (2 diffs)
• jython/loader.py (modified) (2 diffs)
• jython/objectbatch.py (modified) (1 diff)
• jython/stackbatch.py (modified) (1 diff)

branches/eam_branches/ipp-20121219/ippToPsps/config/settings.xml

@@ -18 +18 @@
   <!-- GPC1 Db section -->
   <gpc1database>
-    <!-- TEST VERSION
+    <!-- TEST VERSION -->
     <name>gpc1test</name>
     <host>localhost</host>
     <user>dvo</user>
-    <password>dvo</password> -->
-    <!-- REAL VERSION -->
+    <password>dvo</password>
+    <!-- REAL VERSION
     <name>gpc1</name>
     <host>ippdb01</host>
     <user>ipp</user>
-    <password>ipp</password>
+    <password>ipp</password> -->
   </gpc1database>
 

branches/eam_branches/ipp-20121219/ippToPsps/jython/batch.py

@@ -543 +543 @@
             raise
 
-        # this reads the detection tables from the CMF/SMF file(s)
+        # for P2/ST, this reads the detection tables from the CMF/SMF file(s)
+        # for OB, this imports object data from DVO
         if not self.importIppTables():
             self.logger.errorPair("Aborting this batch", "could not import IPP tables")

branches/eam_branches/ipp-20121219/ippToPsps/jython/dvoobjects.py

@@ -8 +8 @@
 import logging
 import glob
+from subprocess import call, PIPE, Popen
 
 from dvo import Dvo
@@ -47 +48 @@
 
 
+    '''
+    ingest object data into MySQL database using the native DVO program dvopsps
+    Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this
+    includes purging detections outside sky area
+    '''
+    def nativeIngestRegion(self, region):
+
+        # drop cpt/cps/object table?
+        # create detections table?
+
+        # TODO path to DVO prog hardcoded temporarily
+        cmd = "dvopsps objects"
+        cmd += " -dbhost " + self.scratchDb.dbHost
+        cmd += " -dbname " + self.scratchDb.dbName
+        cmd += " -dbuser " + self.scratchDb.dbUser
+        cmd += " -dbpass " + self.scratchDb.dbPass
+        cmd += " -D CATDIR " + self.config.dvoLocation
+        cmd += " -cpt " + region
+
+        self.logger.infoPair("Running dvopsps", cmd)
+        p = Popen(cmd, shell=True, stdout=PIPE)
+        p.wait()
+
+        # update lists after attempted sync
+        # self.printSummary()
+
+        return True

branches/eam_branches/ipp-20121219/ippToPsps/jython/loader.py

@@ -190 +190 @@
                                 useFullTables = 1
                         else:
+                            # XXX EAM : this is not currently an
+                            # option. either remove the non-native
+                            # ingest code or make it an option
                             # if we are using the native loader, always use it
                             # need to work out a good boundary / region strategy in coordination with 
@@ -196 +199 @@
                             useFullTables = 1
                 
+                    ## else:
+                    ##     # if we are using the native loader, always use it
+                    ##     # need to work out a good boundary / region strategy in coordination with 
+                    ##     # impact of mysql insertion
+                    ##     self.dvoObjects.nativeIngestObjects(boxDim['RA'], boxDim['DEC'], boxSizeSansBorder)
+                    ##     useFullTables = 1
+
                     '''
                     NOTE EAM : "TheseItems" refers to each of the stack CMFs, camera SMFs, or object batches

branches/eam_branches/ipp-20121219/ippToPsps/jython/objectbatch.py

@@ -71 +71 @@
         self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id)
         self.ippToPspsDb.insertObjectMeta(self.batchID, self.region)
-        self.dvoObjects.ingestRegion(self.region)
+        if True:
+            self.dvoObjects.nativeIngestRegion(self.region)
+        else:
+            self.dvoObjects.ingestRegion(self.region)
 
         return True

branches/eam_branches/ipp-20121219/ippToPsps/jython/stackbatch.py

@@ -714 +714 @@
     '''    
     Populates the StackDetectionCalib table
+    XXX this can probably get a big speed increase by using 'SELECT () INTO OUTFILE '/tmp/name' and then
+    calling the load data infile '/tmp/name' into table;
     '''
     def populateStackDetectionCalib(self):