Changeset 33415 for branches/meh_branches/ppstack_test/ippToPsps/jython/gpc1db.py

Timestamp:

Mar 5, 2012, 5:19:48 PM (14 years ago)

Author:

mhuber

Message:

merging latest r33407 trunk changes to meh_branches/ppstack_test

Location:

branches/meh_branches/ppstack_test

Files:

• . (modified) (1 prop)
• ippToPsps/jython/gpc1db.py (modified) (11 diffs)

branches/meh_branches/ppstack_test/ippToPsps/jython/gpc1db.py

@@ -20 +20 @@
     Constructor
     '''
-    def __init__(self, logger, doc):
-        super(Gpc1Db, self).__init__(logger, doc, "gpc1database")
+    def __init__(self, logger, config):
+        super(Gpc1Db, self).__init__(logger, config, "gpc1database")
 
     '''
@@ -32 +32 @@
 
     '''
-    TODO
-    '''
-    def getIDsInThisDVODbForThisStageFudge(self):
-
-        sql = "SELECT staticskyRun.sky_id \
-               FROM staticskyInput, staticskyRun, stackRun, staticskyResult \
-               WHERE staticskyRun.sky_id = staticskyInput.sky_id \
-               AND staticskyInput.stack_id = stackRun.stack_id \
-               AND staticskyInput.sky_id = staticskyResult.sky_id \
-               AND staticskyRun.label like 'MD04.staticsky' \
-               AND stackRun.filter like 'i%'"
-
-        try:
-            rs = self.executeQuery(sql)
-        except:
-            self.logger.exception("Can't query for ids in DVO")
+    Gets a list of ids in this DVO database for this stage, could be cam or staticsky, for a box with defined center and side length
+    '''
+    def getIDsInThisDVODbForThisStageInThisBox(self, dvoDb, batchType, ra=180., dec=0., side=181.):
+
+        halfSide = side/2.0
+        return self.getIDsInThisDVODbForThisStage(dvoDb, batchType, ra-halfSide, ra+halfSide, dec-halfSide, dec+halfSide)
+
+
+    '''
+    Gets a list of ids in this DVO database for this stage, could be cam or staticsky, and a box can be defined with equatorial coordinates
+    '''
+    def getIDsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999):
+
+        self.logger.debugPair("GPC1 RA range", "%.2f -> %.2f" % (minRA, maxRA))
+        self.logger.debugPair("GPC1 Dec range", "%.2f -> %.2f" % (minDec, maxDec))
 
         ids = []
-        while (rs.next()):
-            ids.append(rs.getInt(1))
-
-        rs.close()
-
-        self.logger.debug("Found %d items in DVO database '" % (len(ids)))
-
-        return ids
-
-    '''
-    Gets a list of ids in this DVO database for this stage, could be cam or staticsky (so far)
-    '''
-    def getIDsInThisDVODbForThisStage(self, dvoDb, batchType):
 
         if batchType == "P2": # TODO define these someplace 
@@ -71 +57 @@
                    JOIN minidvodbRun USING(minidvodb_name) \
                    JOIN minidvodbProcessed USING(minidvodb_id) \
-                   WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \
+                   JOIN camRun ON (stage_id = cam_id) \
+                   JOIN chipRun USING (chip_id) \
+                   JOIN rawExp USING (exp_id) \
+                   JOIN mergedvodbRun using (minidvodb_id) \
+                   WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \
                    AND minidvodbRun.state = 'merged' \
+                   AND mergedvodbRun.state = 'full' \
                    AND minidvodbProcessed.fault = 0 \
                    AND addRun.stage = '" + stage + "' \
-                   AND addRun.state = 'full'"
+                   AND addRun.state = 'full' \
+                   AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \
+                   AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ")"
 
         elif batchType == "ST":       
 
-            stage = "staticsky"
-            sql = "SELECT DISTINCT stack_id \
-                   FROM staticskyInput \
+            stage = "staticsky_multi"
+            sql = "SELECT DISTINCT stack_id FROM staticskyInput \
                    JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \
                    JOIN minidvodbRun USING(minidvodb_name) \
@@ -90 +82 @@
                    AND addRun.stage = '" + stage + "' \
                    AND addRun.state = 'full'"
-            
-        try:
-            rs = self.executeQuery(sql)
+
+        try:
+            rs = self.executeQuery(sql)
+            while (rs.next()): ids.append(rs.getInt(1))
+            rs.close()
         except:
             self.logger.exception("Can't query for ids in DVO")
-
-        ids = []
-        while (rs.next()):
-            ids.append(rs.getInt(1))
-
-        rs.close()
+            return ids
+
 
         self.logger.debug("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage))
@@ -154 +144 @@
             rs = self.executeQuery(sql)
             rs.first()
+        except:
+            self.logger.errorPair("Can't query for", "stack meta data")
+           
+        try:
             meta.append(rs.getString(1))
             meta.append(rs.getString(2))
             meta.append(rs.getString(3))
         except:
-            self.logger.exception("Can't query for stack meta")
-
+            self.logger.errorPair("getStackStageMeta()", "empty meta data")
+     
         return meta
 
@@ -178 +172 @@
             rs = self.executeQuery(sql)
             rs.first()
+        except:
+            self.logger.errorPair("Can't query for", "camera meta data")
+
+        try:
             meta.append(rs.getInt(1))
             meta.append(rs.getString(2))
@@ -185 +183 @@
             meta.append(rs.getString(6))
         except:
-            self.logger.exception("Can't query for camera meta")
+            self.logger.errorPair("getCameraStageMeta()", "empty meta data")
 
         return meta
@@ -205 +203 @@
             rs.first()
         except:
-            self.logger.exception("Can't query for camera smfs")
-
-        # get path to base dir of cmf files
-        path = rs.getString(1)
-        path = path[0:path.rfind("/")]
+            self.logger.errorPair("Can't query for smf file for cam_id",  str(camID))
+            return None
+
+        try:
+            # get path to base dir of cmf files
+            path = rs.getString(1)
+        except:
+            self.logger.errorPair("No smf files found for cam_id", str(camID) )
+            return None
        
         # list all smf files if a neb path
@@ -215 +217 @@
         if path.startswith("neb"):
 
-            f=os.popen("neb-ls -p "+path+"/%smf")
+            f=os.popen("neb-ls -p "+path+"%smf")
             for i in f.readlines():
                 files.append(i.rstrip())
@@ -221 +223 @@
         # or not a neb path
         else:
-            files = glob.glob(path + "/*.smf")
+            files = glob.glob(path + ".smf")
 
         if len(files) < 1: return None
 
-        return Fits(self.logger, self.doc, files[0]) # TODO just returning first file - check
+        return Fits(self.logger, self.config, files[0]) # TODO just returning first file - check
 
 
     '''
     Gets the path to the cmf file for this stack_id
-    '''
-    def getStackStageCmf(self, stackID):
-      
-        sql = "SELECT staticskyResult.path_base \
-               FROM staticskyRun \
-               JOIN staticskyInput USING(sky_id) \
+    
+    The logic here is fairly ridiculous due to issues with how GPC1 stores info regarding multi-filter stacks. In partcular, it
+    is impossible to find a full path for a cmf file for a given stack_id
+
+    Instead, we get a path_base to a directory with a collection of cmfs. neb-ls will list 5 files, but a number < 5 may actually exist
+    meaning that neb-ls -p will fail.
+
+    our logic is as follows:
+    
+    - use stack_id and the DVO label to find a path base for all cmfs
+    - loop through all the potential cmfs, run neb-ls -p on each
+    - if neb-ls -p fails, then the file does not exists and we ignore it
+    - if neb-ls -p succeeds, we open that cmf file and check the STK_ID value in its header to see if it is the one we want
+    - if it is, great, exit
+    - if not, try the next one in the list
+
+    '''
+    def getStackStageCmf(self, dvoDb, stackID):
+     
+        '''
+        sql = "SELECT DISTINCT staticskyResult.path_base \
+               FROM staticskyRun  \
+               JOIN staticskyInput USING(sky_id)  \
                JOIN staticskyResult USING(sky_id) \
-               JOIN stackRun USING(stack_id) \
+               JOIN stackRun USING(stack_id)  \
                JOIN stackSumSkyfile USING(stack_id) \
-               WHERE stack_id = %s" % stackID
-
+               JOIN addRun ON sky_id = stage_id \
+               JOIN minidvodbRun USING (minidvodb_name) \
+               WHERE stack_id = " + str(stackID) + " \
+               AND minidvodbRun.minidvodb_group = '" + dvoDb + "' \
+               AND minidvodbRun.state = 'merged' \
+               AND stage = 'staticsky_multi'"
+        '''
+
+        sql = "SELECT DISTINCT staticskyResult.path_base \
+               FROM staticskyRun  \
+               JOIN staticskyInput USING(sky_id)  \
+               JOIN staticskyResult USING(sky_id) \
+               JOIN stackRun USING(stack_id)  \
+               JOIN stackSumSkyfile USING(stack_id) \
+               JOIN addRun ON sky_id = stage_id \
+               JOIN minidvodbRun USING (minidvodb_name) \
+               JOIN mergedvodbRun using (minidvodb_id) \
+               WHERE stack_id = " + str(stackID) + " \
+               AND minidvodbRun.state = 'merged' \
+               AND mergedvodbRun.state = 'full' \
+               AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \
+               AND stage = 'staticsky_multi'"
+
+
+        # get single path base for the directory containing all cmf files, one of which is hopefully for our stack_id
         try:
             rs = self.executeQuery(sql)
@@ -246 +288 @@
             pathBase = rs.getString(1)
 
-            # now find ALL cmf files at this path_base as there can be more than one
-            f=os.popen("neb-ls -p " + pathBase + "%cmf")
-            for i in f.readlines():
-
-                path = i.rstrip()
-                fits = Fits(self.logger, self.doc, path)
-
-                # we need to check if this is the correct cmf file, and if so then break and return
-                if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
-                    return fits
-
-        except:
-            self.logger.exception("Can't query for stack cmfs")
-
-        self.logger.error("Could not find stack cmf")
+            # now find and loop through all cmf files at this path_base
+            files=os.popen("neb-ls " + pathBase + "%cmf")
+            for i in files.readlines():
+
+                nebPath = i.rstrip()
+
+                # now attempt to run 'neb-ls -p' to actually read this cmf file
+                try:
+                    self.logger.debug("Trying to neb-ls -p on '" + nebPath + "'")
+                    paths=os.popen("neb-ls -p " + nebPath)
+                    path = paths.readline().rstrip()
+                    if len(path) < 1:
+                       self.logger.debug("zero length path - skipping")
+                       raise
+
+                    # if we get here, then the cmf is readable, now check the stack_id
+                    fits = Fits(self.logger, self.config, path)
+                    if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
+                        # we have the right file!
+                        self.logger.debug("SUCCESS!: %s == %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
+                        return fits
+                    else:
+                        # this is not the correct file
+                        self.logger.debug("STACK ID: %s != %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
+
+                # an exception here means that nebulous could not read this file, so we just skip it
+                except:
+                    self.logger.debug("NEB FAILED SKIPPING")
+                    pass
+
+        except:
+            self.logger.errorPair("Can't query for stack cmfs", "stack_id = %d" % stackID)
+
+        self.logger.errorPair("Couldn't find stack cmf for", "stack_id = %d" % stackID)
         return None