Changeset 35097 for trunk/ippToPsps/jython/stackbatch.py

Timestamp:

Feb 6, 2013, 3:16:35 PM (13 years ago)

Author:

eugene

Message:

upgrade to ippToPsps (see doc/upgrade.txt): adds native dvo to mysql ingest operations, adds autogen configuration & installation, splits out global config information from new "skychunk" information (current region on the sky being processed), adds test suites

Location:

trunk/ippToPsps

Files:

• . (modified) (1 prop)
• jython (modified) (1 prop)
• jython/stackbatch.py (modified) (24 diffs)

trunk/ippToPsps/jython

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in

trunk/ippToPsps/jython/stackbatch.py

@@ -2 +2 @@
 
 import os.path
+import glob
 import sys
 
@@ -32 +33 @@
                  logger, 
                  config,
+                 skychunk,
                  gpc1Db,
                  ippToPspsDb,
@@ -42 +44 @@
                logger,
                config,
+               skychunk,
                gpc1Db,
                ippToPspsDb,
@@ -48 +51 @@
                batchID,
                "ST", 
-               gpc1Db.getStackStageCmf(config.dvoLabel, stackID),
+               gpc1Db.getStackStageCmf(skychunk.dvoLabel, stackID),
                useFullTables)
+
+       # self.printline = 0
+       # # self.testprint()
 
        self.stackType = "DEEP_STACK" # TODO
@@ -63 +69 @@
        self.filterID = self.scratchDb.getFilterID(self.filter)
        self.skycell = meta[1];
+
+       # self.testprint()
 
        # skycell is, eg "skycell.1133.081"
@@ -76 +84 @@
        self.skycell = self.skycell[8:12]
        self.projectioncell = self.skycell
+
+       # self.testprint()
 
        # proposed new values. Need to coordinate with the SkyCell table
@@ -96 +106 @@
        self.scratchDb.dropTable("StackDetectionCalib")
 
+       # self.testprint()
+
        # delete IPP tables
        self.scratchDb.dropTable("SkyChip_psf")
@@ -115 +127 @@
        if not self.useFullTables:
            self.scratchDb.insertNewDvoExternID(self.header['SOURCEID'], self.header['IMAGEID'])
+
+       # self.testprint()
 
        # dump stuff to log
@@ -139 +153 @@
         self.scratchDb.execute(sql)
 
+
+    def testprint(self):
+      print "here ", self.printline
+      self.printline += 1
 
     '''
@@ -348 +366 @@
         self.logger.infoPair("Procesing table", "StackMeta")
 
+        self.fwhm_maj    = self.safeDictionaryAccess(self.header, 'FWHM_MAJ')
+        self.fwhm_maj_uq = self.safeDictionaryAccess(self.header, 'FW_MJ_UQ')
+        self.psfmodel    = self.safeDictionaryAccess(self.header, 'PSFMODEL')
+        if (self.fwhm_maj    == "NULL"): self.fwhm_maj    = -999
+        if (self.fwhm_maj_uq == "NULL"): self.fwhm_maj_uq = -999
+
+        # print "fwhm_maj    = ", self.fwhm_maj
+        # print "fwhm_maj_uq = ", self.fwhm_maj_uq
+        # print "psfmodel    = ", self.psfmodel
+
         sql = "INSERT INTO StackMeta (\
         stackMetaID \
@@ -375 +403 @@
         ," + self.header['FPA.ZP'] + " \
         ," + self.expTime + " \
-        ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \
-        ,'" + self.safeDictionaryAccess(self.header, 'FWHM_MAJ') + "' \
-        ,'" + self.safeDictionaryAccess(self.header, 'FW_MJ_UQ') + "' \
+        ,'" + self.psfmodel   + "' \
+        ," + str(self.fwhm_maj)    + " \
+        ," + str(self.fwhm_maj_uq) + " \
         ,'" + self.header['CTYPE1'] + "' \
         ,'" + self.header['CTYPE2'] + "' \
@@ -395 +423 @@
         self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID))
         self.scratchDb.updateFilterID("StackMeta", self.filter)
-        self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.config.dataRelease))
+        self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.skychunk.dataRelease))
         self.updateStackTypeID("StackMeta")
 
@@ -486 +514 @@
                ," + self.historyModNum + " \
                FROM SkyChip_psf"
-        self.scratchDb.execute(sql)
-        
+
+        # print "sql: ", sql
+        # response = raw_input("ready to insert stack det ")
+
+        self.scratchDb.execute(sql)
+
         #it is possible to drop some detections from dvo (that are present in the cmf). when that happens we get a 0 for objid
         #we drop those...
@@ -498 +530 @@
         self.scratchDb.updateFilterID("StackDetection", self.filter)
         
-        self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.config.dataRelease))
+        self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.skychunk.dataRelease))
         
         self.scratchDb.updateAllRows("StackDetection", "primaryF", "0")
@@ -509 +541 @@
         
         self.updateDvoIDsAndFlags("StackDetection")
+        # response = raw_input("updated dvo ")
         
         sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)"
         
         self.scratchDb.execute(sql)
+        # response = raw_input("add primary key? ")
   
         if self.stackType == "DEEP_STACK": 
@@ -524 +558 @@
             
             self.scratchDb.execute(sql)
+            # response = raw_input("add psf flux ")
             
         #leave null instflux in
@@ -529 +564 @@
         
         self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID")
+        # response = raw_input("delete nulls ")
         
         sql="DELETE FROM StackDetection where objID = 0"
         
         self.scratchDb.execute(sql)
-        self.logger.infoPair("Delleting", "entries with StackDetection.objID = 0")
+        self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0")
+        # response = raw_input("deleted objID is 0 ")
         
 
@@ -577 +614 @@
         self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID))
         self.scratchDb.updateFilterID("StackApFlx", self.filter)
-        self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.config.dataRelease))
+        self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.skychunk.dataRelease))
         self.scratchDb.updateAllRows("StackApFlx", "primaryF", "0")
         self.scratchDb.updateAllRows("StackApFlx", "activeFlag", "0")
@@ -611 +648 @@
         self.scratchDb.updateAllRows("StackModelFit", "surveyID", str(self.surveyID))
         self.scratchDb.updateFilterID("StackModelFit", self.filter)
-        self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.config.dataRelease))
+        self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.skychunk.dataRelease))
         self.scratchDb.updateAllRows("StackModelFit", "primaryF", "0")
         self.scratchDb.updateAllRows("StackModelFit", "activeFlag", "0")
@@ -676 +713 @@
 
         self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID))
-        self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.config.dataRelease))
+        self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.skychunk.dataRelease))
 
     '''    
     Populates the StackDetectionCalib table
-    '''
-    def populateStackDetectionCalib(self):
+    XXX this can probably get a big speed increase by using 'SELECT () INTO OUTFILE '/tmp/name' and then
+    calling the load data infile '/tmp/name' into table;
+    '''
+    def populateStackDetectionCalibInsertUpdate(self):
         self.logger.infoPair("Processing table", "StackDetectionCalib")
         tableName = "StackDetectionCalib"
@@ -702 +741 @@
                FROM StackDetection"
         self.scratchDb.execute(sql)
+
+        imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID'])
+        self.logger.infoPair("obtained","imageID")
+
         # insert calibration information from dvoDetections into the Table
         sql = "UPDATE " + tableName + " AS a, "  + self.scratchDb.dvoDetectionTable + " AS b \
@@ -712 +755 @@
             a.expTime = b.expTime, \
             a.airMass = b.airMass   \
-            WHERE a.stackDetectID = b.detectID"
-        self.scratchDb.execute(sql)
-        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
-        
-    
-
+            WHERE a.stackDetectID = b.detectID \
+            AND b.imageID = " + str(imageID)
+        self.scratchDb.execute(sql)
+        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease))
+        
+    def populateStackDetectionCalib(self):
+
+        tableName = "StackDetectionCalib"
+        self.logger.infoPair("Processing table", tableName)
+
+        imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID'])
+        self.logger.infoPair("obtained","imageID")
+
+        # check for & create output directory first
+        datadumpDir = "/tmp/datadump"
+        try:
+            statinfo = os.stat(datadumpDir)
+            # check on the stat results?
+        except:
+            print "making the data dump directory ", datadumpDir
+            os.mkdir(datadumpDir)
+            os.chmod(datadumpDir, 0777)
+            statinfo = os.stat(datadumpDir)
+
+        dumpFile = datadumpDir + "/genetest.xx.dat"
+        files = glob.glob(dumpFile)
+        if len(files) > 0:
+            os.unlink(dumpFile)
+
+        # insert all detections into table
+        sql = "SELECT \
+          a.objID,    \
+          a.stackDetectID, \
+          a.ippObjID,      \
+          a.ippDetectID,   \
+          a.filterID,      \
+          a.surveyID,      \
+          b.ra,            \
+          b.dec_,          \
+          b.raErr,         \
+          b.decErr,        \
+          b.zp,            \
+          b.zpErr,         \
+          b.expTime,       \
+          b.airMass,       \
+          " + str(self.skychunk.dataRelease) + " \
+         FROM              \
+           StackDetection as a \
+         JOIN " + self.scratchDb.dvoDetectionTable + " as b \
+         ON (a.stackDetectID = b.detectID) where b.imageID = " + str(imageID) + \
+         " INTO OUTFILE '" + dumpFile + "'"
+        print "sql: ", sql
+        self.scratchDb.execute(sql)
+
+        sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName
+        print "sql: ", sql
+        self.scratchDb.execute(sql)
+
+        ## XXX write this with the select/insert
+        # self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease))
 
     '''
@@ -737 +834 @@
 
         self.scratchDb.updateFilterID("ObjectCalColor", self.filter)
-        self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.config.dataRelease))
+        self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.skychunk.dataRelease))
 
 
@@ -802 +899 @@
                WHERE a.ippDetectID = b.ippDetectID \
                AND b.imageID = " + str(imageID)
+
+        # print "update dvo sql: ", sql
+        # response = raw_input("update dvo...")
         self.scratchDb.execute(sql)
     

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in