Changeset 37066 for branches/eam_branches/ipp-ops-20130712/ippToPsps/jython/gpc1db.py

Timestamp:

Jul 17, 2014, 12:30:45 PM (12 years ago)

Author:

eugene

Message:

merge changes (from past YEAR) into trunk

Location:

branches/eam_branches/ipp-ops-20130712/ippToPsps

Files:

• . (modified) (1 prop)
• jython/gpc1db.py (modified) (8 diffs)

branches/eam_branches/ipp-ops-20130712/ippToPsps/jython/gpc1db.py

@@ -49 +49 @@
     NB this uses Bill's potentially temporary 'skycells' database when finding RA/Dec for stacks.
     '''
-    def getItemsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999):
+    def getItemsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999, tstart="0000-00-00 00:00:00", tend = "0000-00-00 00:00:00"):
 
         self.logger.debugPair("GPC1 RA range", "%.2f -> %.2f" % (minRA, maxRA))
         self.logger.debugPair("GPC1 Dec range", "%.2f -> %.2f" % (minDec, maxDec))
-
+        
         rows = []
 
@@ -80 +80 @@
                    AND addRun.state = 'full' \
                    AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \
-                   AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ")"
+                   AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ") \
+                   AND dateobs >= '" + tstart + "'\
+                   AND dateobs <= '" + tend  + "'"
 
         elif batchType == "ST":       
 
             stage = "skycal"
-            sql = "SELECT DISTINCT stack_id, radeg, decdeg FROM skycalRun \
+            sql = "SELECT DISTINCT sky_id, radeg, decdeg FROM skycalRun \
                    JOIN addRun ON(skycalRun.skycal_id = addRun.stage_id) \
                    JOIN minidvodbRun USING(minidvodb_name) \
@@ -91 +93 @@
                    JOIN mergedvodbRun USING(minidvodb_id) \
                    JOIN mergedvodbProcessed USING (merge_id) \
-                   JOIN  stackRun USING(stack_id) JOIN skycell USING(skycell_id) \
+                   JOIN stackRun USING(stack_id) \
+                   JOIN stackSumSkyfile using (stack_id) \
+                   JOIN skycell USING(skycell_id) \
                    WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \
                    AND minidvodbRun.state = 'merged' \
@@ -100 +104 @@
                    AND addRun.state = 'full' \
                    AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
-                   AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) 
+                   AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+                   AND mjd_obs >= (to_days('" + tstart + "')-678941) \
+                   AND mjd_obs <= (to_days('" + tend + "') - 678941) "
+           # self.logger.infoPair("sql",sql)      
+        elif batchType == "DF":
+
+            stage = "diff"
+            sql = "SELECT stuff from stufftable"
+
+        elif batchType == "FW":
             
+            stage = "forcedwarp"
+            sql = "SELECT differentStuff from Stufftable"
+
         try:
             # XXX EAM : test output
@@ -212 +228 @@
     Gets a camera-stage smf for this cam_id
     '''
-    def getCameraStageSmf(self, camID):
-
+    def getCameraStageSmf(self, camID, smfversion):
+        self.logger.infoPair("using", smfversion)
         self.logger.debug("Querying GPC1 for camera smf files with cam_id = " + str(camID))
 
@@ -237 +253 @@
         # list all smf files if a neb path
         files = []
-        if path.startswith("neb"):
-
-            # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right?
-            f=os.popen("neb-ls -p "+path+"%smf")
-            for i in f.readlines():
-                files.append(i.rstrip())
-
-        # or not a neb path
-        else:
-            files = glob.glob(path + ".smf")
+
+
+        #there are a couple of states for smfversion
+
+        # "use_new" => we have reprocessed the data, use the .smf file
+
+        # "not_reproc" => this has not been reprocessed, use the .smf file
+
+        # "use_original" => we *may* have reprocessed the data, 
+        #    use (file).smf.original if camRun.state = 'full' and if there is a .smf and .orig
+        #    use (file).smf if camRun.state = 'full' and if there is only a .smf
+        #    fault out if camRun.state != 'full' -- that is a race condition we don't want.
+       
+        
+        if (smfversion == "not_reproc") or (smfversion == "use_new"):
+            if path.startswith("neb"):
+
+                # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right?
+                f=os.popen("neb-ls -p "+path+"%smf")
+                for i in f.readlines():
+                    files.append(i.rstrip())
+
+                # or not a neb path
+            else:
+                files = glob.glob(path + ".smf")
+
+        if (smfversion == "use_original"):
+            # get state of camRun
+            sql = "SELECT state \
+               FROM camProcessedExp \
+               JOIN camRun USING(cam_id) \
+               WHERE camRun.cam_id = %d" % camID
+            try:
+                rs = self.executeQuery(sql)
+                rs.first()
+            except:
+                self.logger.errorPair("Can't query for camRun.state for cam_id",  str(camID))
+                return None
+
+            try:
+                # get state of camRun.cam_id
+                camrunstate = rs.getString(1)
+            except:
+                self.logger.errorPair("No state found for cam_id", str(camID) )
+                return None
+            
+            if (camrunstate == "full"):
+              self.logger.infoPair("camRun.state is ",camrunstate)   
+                 # if full find smf files
+              if path.startswith("neb"):
+
+                 # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right?
+                 fsmf=os.popen("neb-ls -p "+path+"%smf")
+                 fsmfs = fsmf.readlines()
+                 fsmforig=os.popen("neb-ls -p "+path+"%smf.original")
+                 fsmforigs = fsmforig.readlines()
+                 smfCount = len(fsmfs)
+                 smfOrigCount = len (fsmforigs)
+
+                 if (smfCount > 0  and smfOrigCount > 0):
+                     #smf and original  so use original
+                     self.logger.infoPair("using:", path+"%smf.original")
+                     for i in fsmforigs:
+                         files.append(i.rstrip())
+
+                 if (smfCount > 0 and smfOrigCount == 0):
+                     self.logger.infoPair("using:", path+"%smf")
+                     #smf file and no smf.original use .smf
+                     for i in fsmfs:
+                         files.append(i.rstrip())
+
+                 #if not smf dont' find files ??)        
+                 # or not a neb path
+
+              else:
+                # this is broken for non neb - on the other hand we probably won't do .original if so
+                files = glob.glob(path + ".smf")
+
+
+
+            # if .smf and .original use .original
+            else:
+                self.logger.infoPair("camRun.state not full, can't ingest", str(camID) )
+                return None
+
 
         # XXX EAM : test output
@@ -307 +398 @@
 
         # print "staring stack stage cmf"
-
+        #self.logger.infoPair("in stackstagecmf - sql is: ", sql) 
         # get single path base for the directory containing all cmf files, one of which is hopefully for our stack_id
         try:
@@ -408 +499 @@
 
 
+    '''
+    grabs the stackID for a particular skyId, filter, and dvodb
+    the mapping of stack_id from sky_id is not unique unless
+    joined to the dvodb 
+    '''
+    def getStackIDFromSkyIDAndFilter(self,dvoDb,skyID,filter):
+
+        self.logger.infoPair("finding stack_ids from gpc1 for sky_id ", skyID)
+
+        sql = "SELECT DISTINCT stack_id from skycalRun \
+               JOIN addRun ON(skycalRun.skycal_id = addRun.stage_id) \
+               JOIN minidvodbRun USING(minidvodb_name) \
+               JOIN minidvodbProcessed USING(minidvodb_id) \
+               JOIN mergedvodbRun USING(minidvodb_id) \
+               JOIN mergedvodbProcessed USING(merge_id) \
+               JOIN stackRun USING (stack_id) \
+               WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \
+               AND minidvodbRun.state = 'merged' \
+               AND minidvodbProcessed.fault = 0 \
+               AND mergedvodbRun.state = 'full' \
+               AND mergedvodbProcessed.fault = 0  \
+               AND addRun.stage = 'skycal' \
+               AND addRun.state = 'full' \
+               AND filter = '" + filter + "' \
+               AND sky_id = %d" % skyID 
+
+#        self.logger.infoPair("sql",sql)
+
+        try:
+            rs = self.executeQuery(sql)
+                         
+        except:    
+            self.logger.exception("can't query for stackid in DVO")
+            self.logger.infoPair("failed sql:", sql)
+            return 0
+
+        rs.first()
+        try:
+            stackid = rs.getInt(1)
+            self.logger.infoPair("found stack_ID", stackid)
+        except:
+            stackid = -999
+            self.logger.infoPair("using stack_ID",stackid)
+            
+        return stackid