Index: trunk/ippToPsps/jython/gpc1db.py =================================================================== --- trunk/ippToPsps/jython/gpc1db.py (revision 32934) +++ trunk/ippToPsps/jython/gpc1db.py (revision 33144) @@ -46,6 +46,8 @@ JOIN chipRun USING (chip_id) \ JOIN rawExp USING (exp_id) \ - WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \ + JOIN mergedvodbRun using (minidvodb_id) \ + WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \ AND minidvodbRun.state = 'merged' \ + AND mergedvodbRun.state = 'full' \ AND minidvodbProcessed.fault = 0 \ AND addRun.stage = '" + stage + "' \ @@ -56,5 +58,5 @@ elif batchType == "ST": - stage = "staticsky" + stage = "staticsky_multi" sql = "SELECT DISTINCT stack_id FROM staticskyInput \ JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \ @@ -67,4 +69,5 @@ AND addRun.state = 'full'" + print sql try: rs = self.executeQuery(sql) @@ -193,7 +196,6 @@ try: - # get path to base dir of cmf files + # get path to base dir of cmf files path = rs.getString(1) - path = path[0:path.rfind("/")] except: self.logger.errorPair("No smf files found for cam_id", str(camID) ) @@ -204,5 +206,5 @@ if path.startswith("neb"): - f=os.popen("neb-ls -p "+path+"/%smf") + f=os.popen("neb-ls -p "+path+"%smf") for i in f.readlines(): files.append(i.rstrip()) @@ -210,5 +212,5 @@ # or not a neb path else: - files = glob.glob(path + "/*.smf") + files = glob.glob(path + ".smf") if len(files) < 1: return None @@ -219,15 +221,55 @@ ''' Gets the path to the cmf file for this stack_id - ''' - def getStackStageCmf(self, stackID): - - sql = "SELECT staticskyResult.path_base \ - FROM staticskyRun \ - JOIN staticskyInput USING(sky_id) \ + + The logic here is fairly ridiculous due to issues with how GPC1 stores info regarding multi-filter stacks. In partcular, it + is impossible to find a full path for a cmf file for a given stack_id + + Instead, we get a path_base to a directory with a collection of cmfs. neb-ls will list 5 files, but a number < 5 may actually exist + meaning that neb-ls -p will fail. + + our logic is as follows: + + - use stack_id and the DVO label to find a path base for all cmfs + - loop through all the potential cmfs, run neb-ls -p on each + - if neb-ls -p fails, then the file does not exists and we ignore it + - if neb-ls -p succeeds, we open that cmf file and check the STK_ID value in its header to see if it is the one we want + - if it is, great, exit + - if not, try the next one in the list + + ''' + def getStackStageCmf(self, dvoDb, stackID): + + ''' + sql = "SELECT DISTINCT staticskyResult.path_base \ + FROM staticskyRun \ + JOIN staticskyInput USING(sky_id) \ JOIN staticskyResult USING(sky_id) \ - JOIN stackRun USING(stack_id) \ + JOIN stackRun USING(stack_id) \ JOIN stackSumSkyfile USING(stack_id) \ - WHERE stack_id = %s" % stackID - + JOIN addRun ON sky_id = stage_id \ + JOIN minidvodbRun USING (minidvodb_name) \ + WHERE stack_id = " + str(stackID) + " \ + AND minidvodbRun.minidvodb_group = '" + dvoDb + "' \ + AND minidvodbRun.state = 'merged' \ + AND stage = 'staticsky_multi'" + ''' + + sql = "SELECT DISTINCT staticskyResult.path_base \ + FROM staticskyRun \ + JOIN staticskyInput USING(sky_id) \ + JOIN staticskyResult USING(sky_id) \ + JOIN stackRun USING(stack_id) \ + JOIN stackSumSkyfile USING(stack_id) \ + JOIN addRun ON sky_id = stage_id \ + JOIN minidvodbRun USING (minidvodb_name) \ + JOIN mergedvodbRun using (minidvodb_id) \ + WHERE stack_id = " + str(stackID) + " \ + AND minidvodbRun.state = 'merged' \ + AND mergedvodbRun.state = 'full' \ + AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \ + AND stage = 'staticsky_multi'" + + + # get single path base for the directory containing all cmf files, one of which is hopefully for our stack_id try: rs = self.executeQuery(sql) @@ -235,19 +277,38 @@ pathBase = rs.getString(1) - # now find ALL cmf files at this path_base as there can be more than one - f=os.popen("neb-ls -p " + pathBase + "%cmf") - for i in f.readlines(): - - path = i.rstrip() - fits = Fits(self.logger, self.doc, path) - - # we need to check if this is the correct cmf file, and if so then break and return - if fits.getPrimaryHeaderValue("STK_ID") == str(stackID): - return fits - - except: - self.logger.exception("Can't query for stack cmfs") - - self.logger.error("Could not find stack cmf") + # now find and loop through all cmf files at this path_base + files=os.popen("neb-ls " + pathBase + "%cmf") + for i in files.readlines(): + + nebPath = i.rstrip() + + # now attempt to run 'neb-ls -p' to actually read this cmf file + try: + self.logger.debug("Trying to neb-ls -p on '" + nebPath + "'") + paths=os.popen("neb-ls -p " + nebPath) + path = paths.readline().rstrip() + if len(path) < 1: + self.logger.debug("zero length path - skipping") + raise + + # if we get here, then the cmf is readable, now check the stack_id + fits = Fits(self.logger, self.doc, path) + if fits.getPrimaryHeaderValue("STK_ID") == str(stackID): + # we have the right file! + self.logger.debug("SUCCESS!: %s == %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID))) + return fits + else: + # this is not the correct file + self.logger.debug("STACK ID: %s != %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID))) + + # an exception here means that nebulous could not read this file, so we just skip it + except: + self.logger.debug("NEB FAILED SKIPPING") + pass + + except: + self.logger.errorPair("Can't query for stack cmfs", "stack_id = %d" % stackID) + + self.logger.errorPair("Couldn't find stack cmf for", "stack_id = %d" % stackID) return None