Changeset 33144

Timestamp:

Jan 25, 2012, 1:30:33 PM (14 years ago)

Author:

rhenders

Message:

Big changes to SQL for getting both camera and stack stage files

File:

• trunk/ippToPsps/jython/gpc1db.py (modified) (8 diffs)

trunk/ippToPsps/jython/gpc1db.py

@@ -46 +46 @@
                    JOIN chipRun USING (chip_id) \
                    JOIN rawExp USING (exp_id) \
-                   WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \
+                   JOIN mergedvodbRun using (minidvodb_id) \
+                   WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \
                    AND minidvodbRun.state = 'merged' \
+                   AND mergedvodbRun.state = 'full' \
                    AND minidvodbProcessed.fault = 0 \
                    AND addRun.stage = '" + stage + "' \
@@ -56 +58 @@
         elif batchType == "ST":       
 
-            stage = "staticsky"
+            stage = "staticsky_multi"
             sql = "SELECT DISTINCT stack_id FROM staticskyInput \
                    JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \
@@ -67 +69 @@
                    AND addRun.state = 'full'"
 
+        print sql
         try:
             rs = self.executeQuery(sql)
@@ -193 +196 @@
 
         try:
-        # get path to base dir of cmf files
+            # get path to base dir of cmf files
             path = rs.getString(1)
-            path = path[0:path.rfind("/")]
         except:
             self.logger.errorPair("No smf files found for cam_id", str(camID) )
@@ -204 +206 @@
         if path.startswith("neb"):
 
-            f=os.popen("neb-ls -p "+path+"/%smf")
+            f=os.popen("neb-ls -p "+path+"%smf")
             for i in f.readlines():
                 files.append(i.rstrip())
@@ -210 +212 @@
         # or not a neb path
         else:
-            files = glob.glob(path + "/*.smf")
+            files = glob.glob(path + ".smf")
 
         if len(files) < 1: return None
@@ -219 +221 @@
     '''
     Gets the path to the cmf file for this stack_id
-    '''
-    def getStackStageCmf(self, stackID):
-      
-        sql = "SELECT staticskyResult.path_base \
-               FROM staticskyRun \
-               JOIN staticskyInput USING(sky_id) \
+    
+    The logic here is fairly ridiculous due to issues with how GPC1 stores info regarding multi-filter stacks. In partcular, it
+    is impossible to find a full path for a cmf file for a given stack_id
+
+    Instead, we get a path_base to a directory with a collection of cmfs. neb-ls will list 5 files, but a number < 5 may actually exist
+    meaning that neb-ls -p will fail.
+
+    our logic is as follows:
+    
+    - use stack_id and the DVO label to find a path base for all cmfs
+    - loop through all the potential cmfs, run neb-ls -p on each
+    - if neb-ls -p fails, then the file does not exists and we ignore it
+    - if neb-ls -p succeeds, we open that cmf file and check the STK_ID value in its header to see if it is the one we want
+    - if it is, great, exit
+    - if not, try the next one in the list
+
+    '''
+    def getStackStageCmf(self, dvoDb, stackID):
+     
+        '''
+        sql = "SELECT DISTINCT staticskyResult.path_base \
+               FROM staticskyRun  \
+               JOIN staticskyInput USING(sky_id)  \
                JOIN staticskyResult USING(sky_id) \
-               JOIN stackRun USING(stack_id) \
+               JOIN stackRun USING(stack_id)  \
                JOIN stackSumSkyfile USING(stack_id) \
-               WHERE stack_id = %s" % stackID
-
+               JOIN addRun ON sky_id = stage_id \
+               JOIN minidvodbRun USING (minidvodb_name) \
+               WHERE stack_id = " + str(stackID) + " \
+               AND minidvodbRun.minidvodb_group = '" + dvoDb + "' \
+               AND minidvodbRun.state = 'merged' \
+               AND stage = 'staticsky_multi'"
+        '''
+
+        sql = "SELECT DISTINCT staticskyResult.path_base \
+               FROM staticskyRun  \
+               JOIN staticskyInput USING(sky_id)  \
+               JOIN staticskyResult USING(sky_id) \
+               JOIN stackRun USING(stack_id)  \
+               JOIN stackSumSkyfile USING(stack_id) \
+               JOIN addRun ON sky_id = stage_id \
+               JOIN minidvodbRun USING (minidvodb_name) \
+               JOIN mergedvodbRun using (minidvodb_id) \
+               WHERE stack_id = " + str(stackID) + " \
+               AND minidvodbRun.state = 'merged' \
+               AND mergedvodbRun.state = 'full' \
+               AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \
+               AND stage = 'staticsky_multi'"
+
+
+        # get single path base for the directory containing all cmf files, one of which is hopefully for our stack_id
         try:
             rs = self.executeQuery(sql)
@@ -235 +277 @@
             pathBase = rs.getString(1)
 
-            # now find ALL cmf files at this path_base as there can be more than one
-            f=os.popen("neb-ls -p " + pathBase + "%cmf")
-            for i in f.readlines():
-
-                path = i.rstrip()
-                fits = Fits(self.logger, self.doc, path)
-
-                # we need to check if this is the correct cmf file, and if so then break and return
-                if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
-                    return fits
-
-        except:
-            self.logger.exception("Can't query for stack cmfs")
-
-        self.logger.error("Could not find stack cmf")
+            # now find and loop through all cmf files at this path_base
+            files=os.popen("neb-ls " + pathBase + "%cmf")
+            for i in files.readlines():
+
+                nebPath = i.rstrip()
+
+                # now attempt to run 'neb-ls -p' to actually read this cmf file
+                try:
+                    self.logger.debug("Trying to neb-ls -p on '" + nebPath + "'")
+                    paths=os.popen("neb-ls -p " + nebPath)
+                    path = paths.readline().rstrip()
+                    if len(path) < 1:
+                       self.logger.debug("zero length path - skipping")
+                       raise
+
+                    # if we get here, then the cmf is readable, now check the stack_id
+                    fits = Fits(self.logger, self.doc, path)
+                    if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
+                        # we have the right file!
+                        self.logger.debug("SUCCESS!: %s == %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
+                        return fits
+                    else:
+                        # this is not the correct file
+                        self.logger.debug("STACK ID: %s != %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
+
+                # an exception here means that nebulous could not read this file, so we just skip it
+                except:
+                    self.logger.debug("NEB FAILED SKIPPING")
+                    pass
+
+        except:
+            self.logger.errorPair("Can't query for stack cmfs", "stack_id = %d" % stackID)
+
+        self.logger.errorPair("Couldn't find stack cmf for", "stack_id = %d" % stackID)
         return None