Changeset 36723 for trunk

Timestamp:

May 1, 2014, 9:46:08 PM (12 years ago)

Author:

heather

Message:

first pass of stackbatch stackObject - has objID + basic filter dvo data. woot.

Location:

trunk/ippToPsps/jython

Files:

• batch.py (modified) (5 diffs)
• stackbatch.py (modified) (4 diffs)

trunk/ippToPsps/jython/batch.py

@@ -454 +454 @@
 
       filters = ["g","r","i","z","y"]
-
+      count = 0
       for f in filters:
+          skip = 0
           if (f == "g"):
-              fileName = self.gfits.getPath()
+              if self.gstackID > 0:  
+                  fileName = self.gfits.getPath()
+              else:
+                  skip = 1
           if (f == "r"):
-              fileName = self.rfits.getPath()
+              if self.rstackID > 0:
+                  fileName = self.rfits.getPath()
+              else:
+                  skip = 1
           if (f == "i"):
-              fileName = self.ifits.getPath()
+              if self.istackID > 0:
+                  fileName = self.ifits.getPath()
+              else:
+                  skip = 1
           if (f == "z"):
-              fileName = self.zfits.getPath()
+              if self.zstackID > 0:
+                  fileName = self.zfits.getPath()
+              else:
+                  skip = 1
           if (f == "y"):
-              fileName = self.yfits.getPath()
-
-
-          try:
+              if self.ystackID > 0:
+                  fileName = self.yfits.getPath()
+              else:
+                  skip = 1
+
+          if skip == 1:
+              self.logger.infoPair("no files for filter" , f) 
+          else:
+             self.logger.infoPair("using filename:",fileName)
+          
+             try:
               tables = stilts.treads(fileName)
-          except:
+             except:
               self.logger.errorPair("STILTS could not import from", fileName)
               return False
               
-          count = 0
-          for table in tables:
+             #count = 0
+             for table in tables:
               
               # print "import smf table ", table
@@ -481 +501 @@
               
               if not match: continue
-              self.logger.debugPair("Reading IPP table", table.name)
+              self.logger.infoPair("Reading IPP table", f + table.name)
               table = stilts.tpipe(table, cmd='addcol table_index $0')
                   
@@ -506 +526 @@
                   self.logger.exception("Problem writing table '" + f + table.name + "' to the database")
                       
-              self.logger.infoPair("Done. Imported", "%d tables" % count)
-              self.indexIppTables()
+      self.logger.infoPair("Done. Imported", "%d tables" % count)
+      self.indexIppTables()
                       
-              return True
+      return True
 
     '''
@@ -622 +642 @@
         # for OB, this imports object data from DVO
         if self.batchType == "ST":
+            self.logger.infoPair("did we get here","?")  
             if not self.importIppSTTables():
                 self.logger.errorPair("skipping this batch", "could not import ST IPP tables")
                 self.ippToPspsDb.updateProcessed(self.batchID, -1)
                 raise
+            self.logger.infoPair("we did the ST","yes")
         else:    
             # if not ST we do this 
@@ -636 +658 @@
                 self.ippToPspsDb.updateProcessed(self.batchID, -1)
                 raise
-
+        self.logger.infoPair("populatePspsTables","ok")
         if not self.populatePspsTables():
             if self.skipBatch:

trunk/ippToPsps/jython/stackbatch.py

@@ -1015 +1015 @@
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease))
         
-    def populateStackObjectCalib(self):
+    def populateStackObjectFromDvo(self):
 
         tableName = "StackObject"
         self.logger.infoPair("Processing table", tableName)
 
-        imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID'])
-        self.logger.infoPair("obtained","imageID")
-        sql = "UPDATE " + tableName + " AS a, "  + self.scratchDb.dvoDetectionTable + " AS b \
-            SET a.ra = b.ra, \
-            a.dec = b.dec_, \
-            a.raErr = b.raErr, \
-            a.decErr = b.decErr, \
-            a.zp = b.zp, \
-            a.expTime = b.expTime, \
-            a.airMass = b.airMass   \
-            WHERE a.stackDetectID = b.detectID \
-            AND b.imageID = " + str(imageID)
-        self.scratchDb.execute(sql)
-        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease))
+        filters = ['g','r','i','z','y']
+        
+        for f in filters:
+            skip = 0
+            #first: do we skip this filter
+            if f == 'g':
+                self.header = self.gheader
+                if self.gstackID < 0:
+                    skip = 1
+            elif f == 'r':
+                self.header = self.rheader
+                if self.rstackID < 0:
+                    skip = 1
+            elif f == 'i':
+                self.header = self.iheader
+                if self.istackID < 0:
+                    skip = 1
+            elif f == 'z':
+                self.header = self.zheader
+                if self.zstackID < 0:
+                    skip = 1
+            elif f == 'y':
+                self.header = self.yheader
+                if self.ystackID < 0:
+                    skip = 1
+            if skip == 0:        
+                imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID'])
+                self.logger.infoPair("obtained","imageID")
+                sql = "UPDATE " + tableName + " AS a, "  + self.scratchDb.dvoDetectionTable + " AS b \
+                      SET a." + f + "ra = b.ra, \
+                      a." + f + "dec = b.dec_, \
+                      a." + f + "raErr = b.raErr, \
+                      a." + f + "decErr = b.decErr, \
+                      a." + f + "zp = b.zp, \
+                      a." + f + "expTime = b.expTime, \
+                      a." + f + "stackDetectID = b.detectID   \
+                      WHERE a.objID = b.objID \
+                      AND b.imageID = " + str(imageID)
+            self.logger.infoPair('sql',sql)    
+            self.scratchDb.execute(sql)
+            self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease))
 
         ## XXX write this with the select/insert
@@ -1091 +1118 @@
         sql = "INSERT IGNORE INTO " + table + " ( \
                objID ) \
-               SELECT (objID) FROM " + self.scratchDb.dvoDetectionTable + " ( \
-               WHERE b.imageID = " + str(imageID)
+               SELECT (objID) FROM " + self.scratchDb.dvoDetectionTable + "  \
+               WHERE " + self.scratchDb.dvoDetectionTable + ".imageID = " + str(imageID)
+        self.logger.infoPair("sql",sql)
         self.scratchDb.execute(sql)
     
@@ -1139 +1167 @@
     '''
     def populatePspsTables(self):
-        self.infoPair("starting","populatePspsTables");
+        self.logger.infoPair("starting","populatePspsTables");
         self.skipBatch = False
      
@@ -1182 +1210 @@
             self.insertDvoIDs("StackApFlx","y.00000")
 
-        self.logger.infoPair("populating","StackObject") 
+        self.logger.infoPair("populating","StackObject")
+        # need to index on objid or it is stupidly slow
+        self.scratchDb.createIndex("StackObject", "objID")
+        self.scratchDb.createIndex("StackModelFit", "objID")
+        self.scratchDb.createIndex("StackApFlx", "objID")
+
+
+
+        #ok, now we have the shell stackObjects - these need to be filled.
+        self.populateStackObjectFromDvo()
         self.populateStackObject()