Changeset 33259 for trunk/ippToPsps/jython/load.py

Timestamp:

Feb 14, 2012, 12:02:48 PM (14 years ago)

Author:

rhenders

Message:

Big changes to support a new Config class that encapuslates the xml config file

File:

• trunk/ippToPsps/jython/load.py (modified) (19 diffs)

trunk/ippToPsps/jython/load.py

@@ -10 +10 @@
 import socket
 import logging.config
-from xml.etree.ElementTree import ElementTree, Element, tostring
-
+
+from config import Config
 from pslogger import PSLogger
 from gpc1db import Gpc1Db
@@ -28 +28 @@
 def publishAnyUnpublishedBatches(batchType):
 
-    batchIDs = ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(EPOCH, DVOGPC1LABEL, batchType)
+    batchIDs = ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(config.epoch, config.dvoLabel, batchType)
     logger.infoPair("%s batches" % batchType, "%d" % len(batchIDs))
 
@@ -34 +34 @@
 
         batchName = Batch.getNameFromID(batchID)
-        subDir = Batch.getSubDir(BASEPATH, batchType, DVOGPC1LABEL)
+        subDir = Batch.getSubDir(config.basePath, batchType, config.dvoLabel)
         tarballFile = Batch.getTarballFile(batchID)
         logger.infoPair("Batch name", batchName)
@@ -48 +48 @@
     logger.infoPair("Box center and size", "%f/%f/%f" %(ra, dec, boxsize))
 
-    allIDs = []
-
-    # see if there are any IDs listed in the config
-    try:
-        for element in configDoc.findall('options/ids/id'):
-            allIDs.append(int(element.text))
-    except:
-        logger.errorPair("Problem loading IDs from config under", "options/ids/id") 
+    allIDs = config.ids
 
     if len(allIDs) > 0:
@@ -63 +56 @@
 
         allIDs = gpc1Db.getIDsInThisDVODbForThisStageInThisBox(
-                DVOGPC1LABEL, 
+                config.dvoLabel, 
                 batchType, 
                 ra, 
@@ -71 +64 @@
         logger.infoPair("All %s items in DVO" % batchType, "%d" % len(allIDs))
 
-    # if in FORCE mode, then queue full list
-    if FORCE: ids = allIDs
-
-    # if not in TEST mode, then only queue un-processed items
+    # if in force mode, then queue full list
+    if config.force: ids = allIDs
+
+    # if not in test mode, then only queue un-processed items
     else:
-        processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOGPC1LABEL)
-        consistaentlyFailedIDs = ippToPspsDb.getConsistentlyFailedIDs(batchType, EPOCH, DVOGPC1LABEL)
+        processedIDs = ippToPspsDb.getProcessedIDs(batchType, config.epoch, config.dvoLabel)
+        consistaentlyFailedIDs = ippToPspsDb.getConsistentlyFailedIDs(batchType, config.epoch, config.dvoLabel)
         ids = list(set(allIDs) - set(processedIDs) - set(consistaentlyFailedIDs))
         logger.infoPair("Processed items", "%d" % len(processedIDs))
@@ -97 +90 @@
     for id in ids:
 
-        batchID = ippToPspsDb.createNewBatch(
-                batchType,
-                id,
-                SURVEY,
-                EPOCH,
-                DVOGPC1LABEL,
-                datastore.product,
-                FORCE)
+        batchID = ippToPspsDb.createNewBatch(batchType, id, config)
         
         if batchID < 0: continue;
@@ -112 +98 @@
         if batchType == "P2":
             batch = DetectionBatch(logger,
-                    CONFIG,
-                    configDoc,
+                    config,
                     gpc1Db,
                     ippToPspsDb,
@@ -121 +106 @@
         elif batchType == "ST":
             batch = StackBatch(logger,
-                    CONFIG,
-                    configDoc,
+                    config,
                     gpc1Db,
                     ippToPspsDb,
@@ -135 +119 @@
         logger.infoSeparator()
 
-        # if in TEST mode, then quit after one batch
-        if TEST: break
+        # if in test mode, then quit after one batch
+        if config.test: break
 
     ippToPspsDb.unlockTables()
@@ -143 +127 @@
 Start of program.
 '''
-if len(sys.argv) > 1: CONFIG = sys.argv[1]
+if len(sys.argv) > 1: CONFIGPATH = sys.argv[1]
 else:
     print "** Usage: " + sys.argv[0] + " <configPath> [init]"
@@ -154 +138 @@
 
 # open config file
-configDoc = ElementTree(file=CONFIG)
-TEST = int(configDoc.find("options/testMode").text)
-
-# set up logging
-logging.setLoggerClass(PSLogger)
-logger = logging.getLogger("ippToPspsLog")
-
-# get hostnamee and PID for unique log naming
-HOST = socket.gethostname()
-PID = os.getpid()
-
-# if in test mode, print log to screen, otherwise, only to file
-#logger.setup(configDoc, "ippToPsps", TEST, not TEST, HOST, PID) TODO put back
-logger.setup(configDoc, "ippToPsps")
+config = Config(CONFIGPATH)
+#logger = config.getLogger("load", 0, 1)
+logger = config.getLogger("load")
 
 # create various objects
-dvo = Dvo(logger, configDoc)
-gpc1Db = Gpc1Db(logger, configDoc)
-ippToPspsDb = IppToPspsDb(logger, configDoc)
-datastore = Datastore(logger, configDoc, ippToPspsDb)
+dvo = Dvo(logger, config)
+gpc1Db = Gpc1Db(logger, config)
+ippToPspsDb = IppToPspsDb(logger, config)
+datastore = Datastore(logger, config, ippToPspsDb)
 
 # check we connected ok
@@ -181 +154 @@
 # get values from the configutaion file
 POLLPERIOD = 600
-DVOGPC1LABEL = configDoc.find("dvo/gpc1Label").text
-FORCE = int(configDoc.find("options/force").text)
-EPOCH = configDoc.find("options/epoch").text
-PUBLISH = int(configDoc.find("options/publishToDatastore").text)
-BASEPATH = configDoc.find("localOutPath").text
-SURVEY = configDoc.find("options/survey").text
-
-# get batch types from config
-batchTypes = []
-if int(configDoc.find("options/queueP2").text) == 1: batchTypes.append("P2")
-if int(configDoc.find("options/queueST").text) == 1: batchTypes.append("ST")
-
-# get equatorial coord limits, if any
-try: MINRA = float(configDoc.find("dvo/minRA").text)
-except: MINRA = 0.0
-try: MAXRA = float(configDoc.find("dvo/maxRA").text)
-except: MAXRA = 360.0
-try: MINDEC = float(configDoc.find("dvo/minDec").text)
-except: MINDEC = -30.0 
-try: MAXDEC = float(configDoc.find("dvo/maxDec").text)
-except: MAXDEC = 90.0
-try: BOXSIZE = float(configDoc.find("dvo/boxSize").text)
-except: BOXSIZE = 4.0
-
-# prompt user: FORCE and PUBLISH is a dangerous combination
-if FORCE and PUBLISH and not QUEUE_IN:
+
+# prompt user: force and publishing is a dangerous combination
+if config.force and config.datastorePublishing and not QUEUE_IN:
    response = raw_input("\n*** Are you sure you want to publish data with the 'force' option enabled (y/n)? ")
    if response != "y": sys.exit(1)
 
-# prompt user: TEST and PUBLISH is a dangerous combination
-if TEST and PUBLISH and not QUEUE_IN:
+# prompt user: test and publishing is a dangerous combination
+if config.test and config.datastorePublishing and not QUEUE_IN:
    response = raw_input("\n*** Are you sure you want to publish data with the 'testMode' option enabled (y/n)? ")
    if response != "y": sys.exit(1)
@@ -218 +168 @@
 logger.infoSeparator()
 logger.infoTitle("ippToPsps loading started")
-logger.infoPair("Configuration file", CONFIG)
-logger.infoPair("Loading epoch", EPOCH)
-logger.infoPair("DVO gpc1 label", DVOGPC1LABEL)
-for batchType in batchTypes: logger.infoPair("Queuing batch type", batchType)
-logger.infoBool("Forcing?", FORCE)
-logger.infoBool("Test mode?", TEST)
-logger.infoBool("Publishing?", PUBLISH)
 
 # if an IN batch is requested, create and quit
 if QUEUE_IN:
-   batchID = ippToPspsDb.createNewBatch(
-             "IN",
-             0,
-             SURVEY,
-             EPOCH,
-             DVOGPC1LABEL,
-             datastore.product,
-             1)
+   batchID = ippToPspsDb.createNewBatch("IN", 0, config)
    if batchID > 0:
-       batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb, batchID)
+       batch = InitBatch(logger, config, gpc1Db, ippToPspsDb, batchID)
        batch.run()
 
@@ -251 +187 @@
 
 '''
-BORDER = 1.60
-HALFBOX = BOXSIZE/2.0
-BOXSIZEWITHBORDER = BOXSIZE + (BORDER * 2)
 
 # this outer while loop simply waits a few minutes then starts queuing againas more stuff may appear in DVO over time
 while True:
 
+    config.refresh()
+    config.printAll()
+
     # starting positions
-    ra = MINRA + HALFBOX
-    dec = MINDEC + HALFBOX
+    ra = config.minRa + config.halfBox
+    dec = config.minDec + config.halfBox
 
     # queue up batches that are processed but not loaded to datastore
     logger.infoTitle("Previous failed datastore loads")
-    for batchType in batchTypes: publishAnyUnpublishedBatches(batchType)
-
-    # loop through full range of RA/Dec queueing stuff in boxes of size BOXSIZE
-    while ra <= MAXRA:
-
-       while dec <= MAXDEC:
+    for batchType in config.batchTypes: publishAnyUnpublishedBatches(batchType)
+
+    # loop through full range of RA/Dec queueing stuff in boxes of size config.boxSize
+    while ra <= config.maxRa:
+
+       while dec <= config.maxDec:
 
            # for each batch type
@@ -275 +211 @@
            # - check if we should pre-load DVO region
            # - process the items
-           for batchType in batchTypes: 
-               ids = queueItemsInBox(batchType, ra, dec, BOXSIZE)
+           for batchType in config.batchTypes: 
+
+               ids = queueItemsInBox(batchType, ra, dec, config.boxSize)
 
                if len(ids) < 1: 
@@ -282 +219 @@
                    continue
 
-               dvo.setSkyAreaAsBox(ra, dec, BOXSIZEWITHBORDER)
+               dvo.setSkyAreaAsBox(ra, dec, config.boxSizeWithBorder)
                dvo.printSummary()
 
@@ -291 +228 @@
 
                # do we pre-ingest stuff from DVO?
-               if smfsPerGB > 40:
+               if smfsPerGB > 0:
                    if not dvo.sync():
                        logger.errorPair("Could not sync DVO with MySQL", "skipping")
@@ -301 +238 @@
                logger.infoBool("Using pre-ingested DVO data?", useFullTables)
                processTheseItems(batchType, ids, useFullTables)
-
-               # in the TEST mode, we quit after submitting one batch
-               if TEST: break
-
-           dec = dec + BOXSIZE
-           if TEST: break
-
-       dec = MINDEC + HALFBOX
-       ra = ra + BOXSIZE
-       if TEST: break
+               sys.exit(1)
+               
+               # in the test mode, we quit after submitting one batch
+               if config.test: break
+
+           dec = dec + config.boxSize
+           if config.test: break
+
+       dec = config.minDec + config.halfBox
+       ra = ra + config.boxSize
+       if config.test: break
 
     # wait for the POLLPERIOD before checking for new ids