Changeset 33638 for branches/eam_branches/ipp-20111122/ippToPsps/jython/gpc1db.py

Timestamp:

Mar 30, 2012, 2:49:37 PM (14 years ago)

Author:

eugene

Message:

merge changes from trunk

File:

• branches/eam_branches/ipp-20111122/ippToPsps/jython/gpc1db.py (modified) (8 diffs)

branches/eam_branches/ipp-20111122/ippToPsps/jython/gpc1db.py

@@ -20 +20 @@
     Constructor
     '''
-    def __init__(self, logger, doc):
-        super(Gpc1Db, self).__init__(logger, doc, "gpc1database")
+    def __init__(self, logger, config):
+        super(Gpc1Db, self).__init__(logger, config, "gpc1database")
 
     '''
@@ -32 +32 @@
 
     '''
-    Gets a list of ids in this DVO database for this stage, could be cam or staticsky, and a box can be defined with equatorial coordinates
-    '''
-    def getIDsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999):
-
-        if batchType == "P2": # TODO define these someplace 
+    Gets a list of ids, ra and dec in this DVO database for this stage, could be cam or staticsky, for a box with defined center and side length
+    '''
+    def getItemsInThisDVODbForThisStageInThisBox(self, dvoDb, batchType, ra=180., dec=0., side=181.):
+
+        halfSide = side/2.0
+        return self.getItemsInThisDVODbForThisStage(dvoDb, batchType, ra-halfSide, ra+halfSide, dec-halfSide, dec+halfSide)
+
+    '''
+    Gets a list of ids, ra and dec in this DVO database for this stage, could be cam or staticsky, and a box can be defined with equatorial coordinates
+    NB this uses Bill's potentially temporary 'skycells' database when finding RA/Dec for stacks.
+    '''
+    def getItemsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999):
+
+        self.logger.debugPair("GPC1 RA range", "%.2f -> %.2f" % (minRA, maxRA))
+        self.logger.debugPair("GPC1 Dec range", "%.2f -> %.2f" % (minDec, maxDec))
+
+        rows = []
+
+        if batchType == "P2": # TODO define these some place 
 
             stage = "cam"
-            sql = "SELECT DISTINCT stage_id \
+            sql = "SELECT DISTINCT stage_id, DEGREES(ra), DEGREES(decl) \
                    FROM addRun \
                    JOIN minidvodbRun USING(minidvodb_name) \
@@ -46 +60 @@
                    JOIN chipRun USING (chip_id) \
                    JOIN rawExp USING (exp_id) \
+                   JOIN mergedvodbRun using (minidvodb_id) \
+                   WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \
+                   AND minidvodbRun.state = 'merged' \
+                   AND mergedvodbRun.state = 'full' \
+                   AND minidvodbProcessed.fault = 0 \
+                   AND addRun.stage = '" + stage + "' \
+                   AND addRun.state = 'full' \
+                   AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \
+                   AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ")"
+
+        elif batchType == "ST":       
+
+            stage = "staticsky_multi"
+            sql = "SELECT DISTINCT stack_id, ra, decl FROM staticskyInput \
+                   JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \
+                   JOIN minidvodbRun USING(minidvodb_name) \
+                   JOIN minidvodbProcessed USING(minidvodb_id) \
+                   JOIN  stackRun USING(stack_id) JOIN skycells.skycell USING(skycell_id) \
                    WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \
                    AND minidvodbRun.state = 'merged' \
@@ -51 +83 @@
                    AND addRun.stage = '" + stage + "' \
                    AND addRun.state = 'full' \
-                   AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \
-                   AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ")"
-
-        elif batchType == "ST":       
-
-            stage = "staticsky"
-            sql = "SELECT DISTINCT stack_id FROM staticskyInput \
-                   JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \
-                   JOIN minidvodbRun USING(minidvodb_name) \
-                   JOIN minidvodbProcessed USING(minidvodb_id) \
-                   WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \
-                   AND minidvodbRun.state = 'merged' \
-                   AND minidvodbProcessed.fault = 0 \
-                   AND addRun.stage = '" + stage + "' \
-                   AND addRun.state = 'full'"
-
-        try:
-            rs = self.executeQuery(sql)
+                   AND decl BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
+                   AND ra BETWEEN " + str(minRA) + " AND " + str(maxRA) 
+
+        try:
+            rs = self.executeQuery(sql)
+            while (rs.next()): 
+               rows.append([rs.getInt(1), rs.getFloat(2), rs.getFloat(3)])
+            rs.close()
         except:
             self.logger.exception("Can't query for ids in DVO")
-
-        ids = []
-        while (rs.next()):
-            ids.append(rs.getInt(1))
-
-        rs.close()
-
-        self.logger.debug("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage))
-
-        return ids
+            return rows
+
+        self.logger.debug("Found %d items in DVO database '%s' for stage='%s'" % (len(rows), dvoDb, stage))
+
+        return rows
 
     '''
@@ -193 +210 @@
 
         try:
-        # get path to base dir of cmf files
+            # get path to base dir of cmf files
             path = rs.getString(1)
-            path = path[0:path.rfind("/")]
         except:
             self.logger.errorPair("No smf files found for cam_id", str(camID) )
@@ -204 +220 @@
         if path.startswith("neb"):
 
-            f=os.popen("neb-ls -p "+path+"/%smf")
+            f=os.popen("neb-ls -p "+path+"%smf")
             for i in f.readlines():
                 files.append(i.rstrip())
@@ -210 +226 @@
         # or not a neb path
         else:
-            files = glob.glob(path + "/*.smf")
+            files = glob.glob(path + ".smf")
 
         if len(files) < 1: return None
 
-        return Fits(self.logger, self.doc, files[0]) # TODO just returning first file - check
+        return Fits(self.logger, self.config, files[0]) # TODO just returning first file - check
 
 
     '''
     Gets the path to the cmf file for this stack_id
-    '''
-    def getStackStageCmf(self, stackID):
-      
-        sql = "SELECT staticskyResult.path_base \
-               FROM staticskyRun \
-               JOIN staticskyInput USING(sky_id) \
+    
+    The logic here is fairly ridiculous due to issues with how GPC1 stores info regarding multi-filter stacks. In partcular, it
+    is impossible to find a full path for a cmf file for a given stack_id
+
+    Instead, we get a path_base to a directory with a collection of cmfs. neb-ls will list 5 files, but a number < 5 may actually exist
+    meaning that neb-ls -p will fail.
+
+    our logic is as follows:
+    
+    - use stack_id and the DVO label to find a path base for all cmfs
+    - loop through all the potential cmfs, run neb-ls -p on each
+    - if neb-ls -p fails, then the file does not exists and we ignore it
+    - if neb-ls -p succeeds, we open that cmf file and check the STK_ID value in its header to see if it is the one we want
+    - if it is, great, exit
+    - if not, try the next one in the list
+
+    '''
+    def getStackStageCmf(self, dvoDb, stackID):
+     
+        '''
+        sql = "SELECT DISTINCT staticskyResult.path_base \
+               FROM staticskyRun  \
+               JOIN staticskyInput USING(sky_id)  \
                JOIN staticskyResult USING(sky_id) \
-               JOIN stackRun USING(stack_id) \
+               JOIN stackRun USING(stack_id)  \
                JOIN stackSumSkyfile USING(stack_id) \
-               WHERE stack_id = %s" % stackID
-
+               JOIN addRun ON sky_id = stage_id \
+               JOIN minidvodbRun USING (minidvodb_name) \
+               WHERE stack_id = " + str(stackID) + " \
+               AND minidvodbRun.minidvodb_group = '" + dvoDb + "' \
+               AND minidvodbRun.state = 'merged' \
+               AND stage = 'staticsky_multi'"
+        '''
+
+        sql = "SELECT DISTINCT staticskyResult.path_base \
+               FROM staticskyRun  \
+               JOIN staticskyInput USING(sky_id)  \
+               JOIN staticskyResult USING(sky_id) \
+               JOIN stackRun USING(stack_id)  \
+               JOIN stackSumSkyfile USING(stack_id) \
+               JOIN addRun ON sky_id = stage_id \
+               JOIN minidvodbRun USING (minidvodb_name) \
+               JOIN mergedvodbRun using (minidvodb_id) \
+               WHERE stack_id = " + str(stackID) + " \
+               AND minidvodbRun.state = 'merged' \
+               AND mergedvodbRun.state = 'full' \
+               AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \
+               AND stage = 'staticsky_multi'"
+
+
+        # get single path base for the directory containing all cmf files, one of which is hopefully for our stack_id
         try:
             rs = self.executeQuery(sql)
@@ -235 +291 @@
             pathBase = rs.getString(1)
 
-            # now find ALL cmf files at this path_base as there can be more than one
-            f=os.popen("neb-ls -p " + pathBase + "%cmf")
-            for i in f.readlines():
-
-                path = i.rstrip()
-                fits = Fits(self.logger, self.doc, path)
-
-                # we need to check if this is the correct cmf file, and if so then break and return
-                if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
-                    return fits
-
-        except:
-            self.logger.exception("Can't query for stack cmfs")
-
-        self.logger.error("Could not find stack cmf")
+            # now find and loop through all cmf files at this path_base
+            files=os.popen("neb-ls " + pathBase + "%cmf")
+            for i in files.readlines():
+
+                nebPath = i.rstrip()
+
+                # now attempt to run 'neb-ls -p' to actually read this cmf file
+                try:
+                    self.logger.debug("Trying to neb-ls -p on '" + nebPath + "'")
+                    paths=os.popen("neb-ls -p " + nebPath)
+                    path = paths.readline().rstrip()
+                    if len(path) < 1:
+                       self.logger.debug("zero length path - skipping")
+                       raise
+
+                    # if we get here, then the cmf is readable, now check the stack_id
+                    fits = Fits(self.logger, self.config, path)
+                    if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
+                        # we have the right file!
+                        self.logger.debug("SUCCESS!: %s == %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
+                        return fits
+                    else:
+                        # this is not the correct file
+                        self.logger.debug("STACK ID: %s != %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID)))
+
+                # an exception here means that nebulous could not read this file, so we just skip it
+                except:
+                    self.logger.debug("NEB FAILED SKIPPING")
+                    pass
+
+        except:
+            self.logger.errorPair("Can't query for stack cmfs", "stack_id = %d" % stackID)
+
+        self.logger.errorPair("Couldn't find stack cmf for", "stack_id = %d" % stackID)
         return None