Index: trunk/ippToPsps/jython/load.py =================================================================== --- trunk/ippToPsps/jython/load.py (revision 33237) +++ trunk/ippToPsps/jython/load.py (revision 33259) @@ -10,6 +10,6 @@ import socket import logging.config -from xml.etree.ElementTree import ElementTree, Element, tostring - + +from config import Config from pslogger import PSLogger from gpc1db import Gpc1Db @@ -28,5 +28,5 @@ def publishAnyUnpublishedBatches(batchType): - batchIDs = ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(EPOCH, DVOGPC1LABEL, batchType) + batchIDs = ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(config.epoch, config.dvoLabel, batchType) logger.infoPair("%s batches" % batchType, "%d" % len(batchIDs)) @@ -34,5 +34,5 @@ batchName = Batch.getNameFromID(batchID) - subDir = Batch.getSubDir(BASEPATH, batchType, DVOGPC1LABEL) + subDir = Batch.getSubDir(config.basePath, batchType, config.dvoLabel) tarballFile = Batch.getTarballFile(batchID) logger.infoPair("Batch name", batchName) @@ -48,12 +48,5 @@ logger.infoPair("Box center and size", "%f/%f/%f" %(ra, dec, boxsize)) - allIDs = [] - - # see if there are any IDs listed in the config - try: - for element in configDoc.findall('options/ids/id'): - allIDs.append(int(element.text)) - except: - logger.errorPair("Problem loading IDs from config under", "options/ids/id") + allIDs = config.ids if len(allIDs) > 0: @@ -63,5 +56,5 @@ allIDs = gpc1Db.getIDsInThisDVODbForThisStageInThisBox( - DVOGPC1LABEL, + config.dvoLabel, batchType, ra, @@ -71,11 +64,11 @@ logger.infoPair("All %s items in DVO" % batchType, "%d" % len(allIDs)) - # if in FORCE mode, then queue full list - if FORCE: ids = allIDs - - # if not in TEST mode, then only queue un-processed items + # if in force mode, then queue full list + if config.force: ids = allIDs + + # if not in test mode, then only queue un-processed items else: - processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOGPC1LABEL) - consistaentlyFailedIDs = ippToPspsDb.getConsistentlyFailedIDs(batchType, EPOCH, DVOGPC1LABEL) + processedIDs = ippToPspsDb.getProcessedIDs(batchType, config.epoch, config.dvoLabel) + consistaentlyFailedIDs = ippToPspsDb.getConsistentlyFailedIDs(batchType, config.epoch, config.dvoLabel) ids = list(set(allIDs) - set(processedIDs) - set(consistaentlyFailedIDs)) logger.infoPair("Processed items", "%d" % len(processedIDs)) @@ -97,12 +90,5 @@ for id in ids: - batchID = ippToPspsDb.createNewBatch( - batchType, - id, - SURVEY, - EPOCH, - DVOGPC1LABEL, - datastore.product, - FORCE) + batchID = ippToPspsDb.createNewBatch(batchType, id, config) if batchID < 0: continue; @@ -112,6 +98,5 @@ if batchType == "P2": batch = DetectionBatch(logger, - CONFIG, - configDoc, + config, gpc1Db, ippToPspsDb, @@ -121,6 +106,5 @@ elif batchType == "ST": batch = StackBatch(logger, - CONFIG, - configDoc, + config, gpc1Db, ippToPspsDb, @@ -135,6 +119,6 @@ logger.infoSeparator() - # if in TEST mode, then quit after one batch - if TEST: break + # if in test mode, then quit after one batch + if config.test: break ippToPspsDb.unlockTables() @@ -143,5 +127,5 @@ Start of program. ''' -if len(sys.argv) > 1: CONFIG = sys.argv[1] +if len(sys.argv) > 1: CONFIGPATH = sys.argv[1] else: print "** Usage: " + sys.argv[0] + " [init]" @@ -154,24 +138,13 @@ # open config file -configDoc = ElementTree(file=CONFIG) -TEST = int(configDoc.find("options/testMode").text) - -# set up logging -logging.setLoggerClass(PSLogger) -logger = logging.getLogger("ippToPspsLog") - -# get hostnamee and PID for unique log naming -HOST = socket.gethostname() -PID = os.getpid() - -# if in test mode, print log to screen, otherwise, only to file -#logger.setup(configDoc, "ippToPsps", TEST, not TEST, HOST, PID) TODO put back -logger.setup(configDoc, "ippToPsps") +config = Config(CONFIGPATH) +#logger = config.getLogger("load", 0, 1) +logger = config.getLogger("load") # create various objects -dvo = Dvo(logger, configDoc) -gpc1Db = Gpc1Db(logger, configDoc) -ippToPspsDb = IppToPspsDb(logger, configDoc) -datastore = Datastore(logger, configDoc, ippToPspsDb) +dvo = Dvo(logger, config) +gpc1Db = Gpc1Db(logger, config) +ippToPspsDb = IppToPspsDb(logger, config) +datastore = Datastore(logger, config, ippToPspsDb) # check we connected ok @@ -181,35 +154,12 @@ # get values from the configutaion file POLLPERIOD = 600 -DVOGPC1LABEL = configDoc.find("dvo/gpc1Label").text -FORCE = int(configDoc.find("options/force").text) -EPOCH = configDoc.find("options/epoch").text -PUBLISH = int(configDoc.find("options/publishToDatastore").text) -BASEPATH = configDoc.find("localOutPath").text -SURVEY = configDoc.find("options/survey").text - -# get batch types from config -batchTypes = [] -if int(configDoc.find("options/queueP2").text) == 1: batchTypes.append("P2") -if int(configDoc.find("options/queueST").text) == 1: batchTypes.append("ST") - -# get equatorial coord limits, if any -try: MINRA = float(configDoc.find("dvo/minRA").text) -except: MINRA = 0.0 -try: MAXRA = float(configDoc.find("dvo/maxRA").text) -except: MAXRA = 360.0 -try: MINDEC = float(configDoc.find("dvo/minDec").text) -except: MINDEC = -30.0 -try: MAXDEC = float(configDoc.find("dvo/maxDec").text) -except: MAXDEC = 90.0 -try: BOXSIZE = float(configDoc.find("dvo/boxSize").text) -except: BOXSIZE = 4.0 - -# prompt user: FORCE and PUBLISH is a dangerous combination -if FORCE and PUBLISH and not QUEUE_IN: + +# prompt user: force and publishing is a dangerous combination +if config.force and config.datastorePublishing and not QUEUE_IN: response = raw_input("\n*** Are you sure you want to publish data with the 'force' option enabled (y/n)? ") if response != "y": sys.exit(1) -# prompt user: TEST and PUBLISH is a dangerous combination -if TEST and PUBLISH and not QUEUE_IN: +# prompt user: test and publishing is a dangerous combination +if config.test and config.datastorePublishing and not QUEUE_IN: response = raw_input("\n*** Are you sure you want to publish data with the 'testMode' option enabled (y/n)? ") if response != "y": sys.exit(1) @@ -218,24 +168,10 @@ logger.infoSeparator() logger.infoTitle("ippToPsps loading started") -logger.infoPair("Configuration file", CONFIG) -logger.infoPair("Loading epoch", EPOCH) -logger.infoPair("DVO gpc1 label", DVOGPC1LABEL) -for batchType in batchTypes: logger.infoPair("Queuing batch type", batchType) -logger.infoBool("Forcing?", FORCE) -logger.infoBool("Test mode?", TEST) -logger.infoBool("Publishing?", PUBLISH) # if an IN batch is requested, create and quit if QUEUE_IN: - batchID = ippToPspsDb.createNewBatch( - "IN", - 0, - SURVEY, - EPOCH, - DVOGPC1LABEL, - datastore.product, - 1) + batchID = ippToPspsDb.createNewBatch("IN", 0, config) if batchID > 0: - batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb, batchID) + batch = InitBatch(logger, config, gpc1Db, ippToPspsDb, batchID) batch.run() @@ -251,23 +187,23 @@ ''' -BORDER = 1.60 -HALFBOX = BOXSIZE/2.0 -BOXSIZEWITHBORDER = BOXSIZE + (BORDER * 2) # this outer while loop simply waits a few minutes then starts queuing againas more stuff may appear in DVO over time while True: + config.refresh() + config.printAll() + # starting positions - ra = MINRA + HALFBOX - dec = MINDEC + HALFBOX + ra = config.minRa + config.halfBox + dec = config.minDec + config.halfBox # queue up batches that are processed but not loaded to datastore logger.infoTitle("Previous failed datastore loads") - for batchType in batchTypes: publishAnyUnpublishedBatches(batchType) - - # loop through full range of RA/Dec queueing stuff in boxes of size BOXSIZE - while ra <= MAXRA: - - while dec <= MAXDEC: + for batchType in config.batchTypes: publishAnyUnpublishedBatches(batchType) + + # loop through full range of RA/Dec queueing stuff in boxes of size config.boxSize + while ra <= config.maxRa: + + while dec <= config.maxDec: # for each batch type @@ -275,6 +211,7 @@ # - check if we should pre-load DVO region # - process the items - for batchType in batchTypes: - ids = queueItemsInBox(batchType, ra, dec, BOXSIZE) + for batchType in config.batchTypes: + + ids = queueItemsInBox(batchType, ra, dec, config.boxSize) if len(ids) < 1: @@ -282,5 +219,5 @@ continue - dvo.setSkyAreaAsBox(ra, dec, BOXSIZEWITHBORDER) + dvo.setSkyAreaAsBox(ra, dec, config.boxSizeWithBorder) dvo.printSummary() @@ -291,5 +228,5 @@ # do we pre-ingest stuff from DVO? - if smfsPerGB > 40: + if smfsPerGB > 0: if not dvo.sync(): logger.errorPair("Could not sync DVO with MySQL", "skipping") @@ -301,14 +238,15 @@ logger.infoBool("Using pre-ingested DVO data?", useFullTables) processTheseItems(batchType, ids, useFullTables) - - # in the TEST mode, we quit after submitting one batch - if TEST: break - - dec = dec + BOXSIZE - if TEST: break - - dec = MINDEC + HALFBOX - ra = ra + BOXSIZE - if TEST: break + sys.exit(1) + + # in the test mode, we quit after submitting one batch + if config.test: break + + dec = dec + config.boxSize + if config.test: break + + dec = config.minDec + config.halfBox + ra = ra + config.boxSize + if config.test: break # wait for the POLLPERIOD before checking for new ids