Index: trunk/ippToPsps/config/changes.txt =================================================================== --- trunk/ippToPsps/config/changes.txt (revision 35994) +++ trunk/ippToPsps/config/changes.txt (revision 35994) @@ -0,0 +1,7 @@ +--these are because we do the update process + +alter table skychunk add column P2_smf_version varchar(16) after parallel; +alter table skychunk add column ST_cmf_version varchar(16) after P2_smf_version; + + +-- not yet alter table batch add column smf_version varchar(16) after comment; Index: trunk/ippToPsps/config/ippToPspsDbSchema.sql =================================================================== --- trunk/ippToPsps/config/ippToPspsDbSchema.sql (revision 35780) +++ trunk/ippToPsps/config/ippToPspsDbSchema.sql (revision 35994) @@ -127,4 +127,6 @@ `active` tinyint(1) default '1', `parallel` tinyint(1) default '0', + `P2_smf_version` varchar(16) default NULL, + `ST_cmf_version` varchar(16) default NULL, UNIQUE KEY `name` (`name`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; Index: trunk/ippToPsps/jython/Makefile.am =================================================================== --- trunk/ippToPsps/jython/Makefile.am (revision 35780) +++ trunk/ippToPsps/jython/Makefile.am (revision 35994) @@ -30,4 +30,5 @@ pslogger.py \ queue.py \ + removepending.py \ scratchdb.py \ setupScratchDb.py \ Index: trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- trunk/ippToPsps/jython/detectionbatch.py (revision 35780) +++ trunk/ippToPsps/jython/detectionbatch.py (revision 35994) @@ -54,5 +54,5 @@ batchID, "P2", - gpc1Db.getCameraStageSmf(camID), + gpc1Db.getCameraStageSmf(camID,skychunk.P2_smf_version), useFullTables) Index: trunk/ippToPsps/jython/gpc1db.py =================================================================== --- trunk/ippToPsps/jython/gpc1db.py (revision 35780) +++ trunk/ippToPsps/jython/gpc1db.py (revision 35994) @@ -212,6 +212,6 @@ Gets a camera-stage smf for this cam_id ''' - def getCameraStageSmf(self, camID): - + def getCameraStageSmf(self, camID, smfversion): + self.logger.infoPair("using", smfversion) self.logger.debug("Querying GPC1 for camera smf files with cam_id = " + str(camID)) @@ -237,14 +237,89 @@ # list all smf files if a neb path files = [] - if path.startswith("neb"): - - # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right? - f=os.popen("neb-ls -p "+path+"%smf") - for i in f.readlines(): - files.append(i.rstrip()) - - # or not a neb path - else: - files = glob.glob(path + ".smf") + + + #there are a couple of states for smfversion + + # "use_new" => we have reprocessed the data, use the .smf file + + # "not_reproc" => this has not been reprocessed, use the .smf file + + # "use_original" => we *may* have reprocessed the data, + # use (file).smf.original if camRun.state = 'full' and if there is a .smf and .orig + # use (file).smf if camRun.state = 'full' and if there is only a .smf + # fault out if camRun.state != 'full' -- that is a race condition we don't want. + + + if (smfversion == "not_reproc") or (smfversion == "use_new"): + if path.startswith("neb"): + + # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right? + f=os.popen("neb-ls -p "+path+"%smf") + for i in f.readlines(): + files.append(i.rstrip()) + + # or not a neb path + else: + files = glob.glob(path + ".smf") + + if (smfversion == "use_original"): + # get state of camRun + sql = "SELECT state \ + FROM camProcessedExp \ + JOIN camRun USING(cam_id) \ + WHERE camRun.cam_id = %d" % camID + try: + rs = self.executeQuery(sql) + rs.first() + except: + self.logger.errorPair("Can't query for camRun.state for cam_id", str(camID)) + return None + + try: + # get state of camRun.cam_id + camrunstate = rs.getString(1) + except: + self.logger.errorPair("No state found for cam_id", str(camID) ) + return None + + if (camrunstate == "full"): + self.logger.infoPair("camRun.state is ",camrunstate) + # if full find smf files + if path.startswith("neb"): + + # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right? + fsmf=os.popen("neb-ls -p "+path+"%smf") + fsmfs = fsmf.readlines() + fsmforig=os.popen("neb-ls -p "+path+"%smf.original") + fsmforigs = fsmforig.readlines() + smfCount = len(fsmfs) + smfOrigCount = len (fsmforigs) + + if (smfCount > 0 and smfOrigCount > 0): + #smf and original so use original + self.logger.infoPair("using:", path+"%smf.original") + for i in fsmforigs: + files.append(i.rstrip()) + + if (smfCount > 0 and smfOrigCount == 0): + self.logger.infoPair("using:", path+"%smf") + #smf file and no smf.original use .smf + for i in fsmfs: + files.append(i.rstrip()) + + #if not smf dont' find files ??) + # or not a neb path + + else: + # this is broken for non neb - on the other hand we probably won't do .original if so + files = glob.glob(path + ".smf") + + + + # if .smf and .original use .original + else: + self.logger.infoPair("camRun.state not full, can't ingest", str(camID) ) + return None + # XXX EAM : test output Index: trunk/ippToPsps/jython/ipptopspsdb.py =================================================================== --- trunk/ippToPsps/jython/ipptopspsdb.py (revision 35780) +++ trunk/ippToPsps/jython/ipptopspsdb.py (revision 35994) @@ -1109,4 +1109,6 @@ ,queue_OB \ ,parallel \ + ,P2_smf_version \ + ,ST_cmf_version \ FROM skychunk \ WHERE name = '" + self.skychunk.name + "'" @@ -1155,4 +1157,6 @@ if rs.getInt(22) == 1: self.skychunk.parallel = True + self.skychunk.P2_smf_version = rs.getString(23) + self.skychunk.ST_cmf_version = rs.getString(24) if self.skychunk.parallel: print "USING parallel" Index: trunk/ippToPsps/jython/queue.py =================================================================== --- trunk/ippToPsps/jython/queue.py (revision 35780) +++ trunk/ippToPsps/jython/queue.py (revision 35994) @@ -71,6 +71,6 @@ for batchType in self.skychunk.batchTypes: self.logger.infoPair("BatchType",batchType) - # self.logger.infoTitle("Previous failed datastore loads") - # self.publishAnyUnpublishedBatches(batchType) + #self.logger.infoTitle("Previous failed datastore loads") + #self.publishAnyUnpublishedBatches(batchType) # get totals for whole area to check if there is anything to do Index: trunk/ippToPsps/jython/removepending.py =================================================================== --- trunk/ippToPsps/jython/removepending.py (revision 35994) +++ trunk/ippToPsps/jython/removepending.py (revision 35994) @@ -0,0 +1,86 @@ +#!/usr/bin/env jython + +# +# This is a way to easily delete pending skychunks +# + +import time +import sys +import os +import math +import logging.config + +from ipptopsps import IppToPsps +from gpc1db import Gpc1Db +from datastore import Datastore +from dvoobjects import DvoObjects +from batch import Batch +from scratchdb import ScratchDb +# basically, "ippjython removepending.py skychunk_name " will remove pending batches + + +''' +Queue class +''' +class Queue(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(Queue, self).__init__(argv) + + # print "inited ipptopsps" + + # create various objects + self.gpc1Db = Gpc1Db(self.logger, self.config) + # print "connect to gpc1" + + self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb) + + print "made datastore" + + # connect to scratch database + self.scratchDb = ScratchDb(self.logger, self.config) + + print "connect to scratchdb" + + try: + self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb) + except: + self.exitProgram("Unable to create instance of DvoObject") + raise + + print "got objects" + + self.skychunk.printAll() + + if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) + + ''' + Main processing loop. + Tiled area is defined in 'skychunk' and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing + by one or more loading clients that may be running on any host + ''' + def run(self): + + self.ippToPspsDb.removeAllBoxes() + print "remove boxes" + + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Queue, self).printUsage("[]") + + +''' +Start of program. +''' +try: + queue = Queue(sys.argv) + queue.run() + queue.exitProgram("completed") +except: pass +