Index: unk/ippToPsps/jython/ipptopsps.py
===================================================================
--- /trunk/ippToPsps/jython/ipptopsps.py	(revision 32222)
+++ 	(revision )
@@ -1,198 +1,0 @@
-#!/usr/bin/env jython
-
-# The starting point for ippTopsps. See usage below.
-
-import time
-import sys
-import logging.config
-from xml.etree.ElementTree import ElementTree, Element, tostring
-
-from pslogger import PSLogger
-from gpc1db import Gpc1Db
-from ipptopspsdb import IppToPspsDb
-from datastore import Datastore
-from batch import Batch
-
-from initbatch import InitBatch
-from stackbatch import StackBatch
-from detectionbatch import DetectionBatch
-
-'''
-Finds batches that processed ok, but did not get loaded to the datastore, then tries to load them
-'''
-def publishAnyUnpublishedBatches(batchType):
-
-    batchIDs = ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(EPOCH, DVOGPC1LABEL, batchType)
-    logger.infoPair("Failed loads to datastore", "%d %s batches" % (len(batchIDs), batchType))
-
-    for batchID in batchIDs:
-
-        batchName = Batch.getNameFromID(batchID)
-        subDir = Batch.getSubDir(BASEPATH, batchType, DVOGPC1LABEL)
-        tarballFile = Batch.getTarballFile(batchID)
-        logger.infoPair("Batch name", batchName)
-
-        Batch.publishToDatastore(datastore, ippToPspsDb, batchID, batchName, subDir, tarballFile)
-
-
-'''
-Queues up all available stuff for this batch type
-'''
-def queueItems(batchType):
-
-    logger.infoTitle("Queuing " + batchType + " items")
-
-    allIDs = []
-
-    # see if there are any IDs listed in the config
-    try:
-        for element in configDoc.findall('options/ids/id'):
-            allIDs.append(int(element.text))
-    except:
-        logger.errorPair("Problem loading IDs from config under", "options/ids/id") 
-
-    # no config dictated IDs, so get from DVO instead
-    if len(allIDs) > 0:
-        logger.infoPair("Using config IDs (not DVO)", "%d" % len(allIDs))
-    else:
-        allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType)
-        logger.infoPair("All %s items in DVO" % batchType, "%d" % len(allIDs))
-
-    # if in FORCE mode, then queue full list
-    if FORCE: ids = allIDs
-
-    # if not in TEST mode, then only queue un-processed items
-    else:
-        processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOGPC1LABEL)
-        logger.infoPair("Processed %s items" % batchType, "%d" % len(processedIDs))
-        ids = list(set(allIDs) - set(processedIDs))
-        logger.infoPair("Unprocessed %s items" % batchType, "%d" % len(ids))
-
-    logger.infoPair("%s items queued" % batchType, "%d" % len(ids))
-
-    # loop round IDs of all items to be processed
-    for id in ids:
-
-        if not FORCE and ippToPspsDb.alreadyProcessed(batchType, id, EPOCH, DVOGPC1LABEL): continue
-        if not FORCE and ippToPspsDb.consistentlyFailed(batchType, id, EPOCH, DVOGPC1LABEL, 3): continue
-
-        if batchType == "P2":
-            batch = DetectionBatch(logger,
-                    CONFIG,
-                    configDoc,
-                    gpc1Db,
-                    ippToPspsDb,
-                    id)
-        elif batchType == "ST":
-            batch = StackBatch(logger,
-                    CONFIG,
-                    configDoc,
-                    gpc1Db,
-                    ippToPspsDb,
-                    id)
-
-        batch.run()
-
-        # if in TEST mode, then quit after one batch
-        if TEST: break
-
-
-'''
-Start of program.
-'''
-
-if len(sys.argv) > 1: CONFIG = sys.argv[1]
-else:
-    print "** Usage: " + sys.argv[0] + " <configPath> [init]"
-    sys.exit(1)
-
-if len(sys.argv) > 2 and sys.argv[2] == "init":
-    QUEUE_IN = 1
-else:
-    QUEUE_IN = 0
-
-# open config file
-configDoc = ElementTree(file=CONFIG)
-TEST = int(configDoc.find("options/testMode").text)
-
-# set up logging
-logging.setLoggerClass(PSLogger)
-logger = logging.getLogger("ippToPspsLog")
-
-# if in test mode, print log to screen, otherwise, only to file
-logger.setup(configDoc, "ippToPsps", TEST, not TEST)
-
-# create database objects
-gpc1Db = Gpc1Db(logger, configDoc)
-ippToPspsDb = IppToPspsDb(logger, configDoc)
-datastore = Datastore(logger, configDoc)
-
-# check we connected ok
-if not gpc1Db.everythingOK: sys.exit(1)
-if not ippToPspsDb.everythingOK: sys.exit(1)
-
-# get values from the configutaion file
-POLLPERIOD = 600
-DVOGPC1LABEL = configDoc.find("dvo/gpc1Label").text
-FORCE = int(configDoc.find("options/force").text)
-EPOCH = configDoc.find("options/epoch").text
-PUBLISH = int(configDoc.find("options/publishToDatastore").text)
-BASEPATH = configDoc.find("localOutPath").text
-QUEUE_P2 = int(configDoc.find("options/queueP2").text)
-QUEUE_ST = int(configDoc.find("options/queueST").text)
-
-# prompt user: FORCE and PUBLISH is a dangerous combination
-if FORCE and PUBLISH and not QUEUE_IN:
-   response = raw_input("\n*** Are you sure you want to publish data with the 'force' option enabled (y/n)? ")
-   if response != "y": sys.exit(1)
-
-# prompt user: TEST and PUBLISH is a dangerous combination
-if TEST and PUBLISH and not QUEUE_IN:
-   response = raw_input("\n*** Are you sure you want to publish data with the 'testMode' option enabled (y/n)? ")
-   if response != "y": sys.exit(1)
-
-# print info to log
-logger.infoSeparator()
-logger.infoTitle("ippToPsps started")
-logger.infoPair("Configuration file", CONFIG)
-logger.infoPair("Loading epoch", EPOCH)
-logger.infoPair("DVO gpc1 label", DVOGPC1LABEL)
-logger.infoBool("Queuing P2 batches?", QUEUE_P2)
-logger.infoBool("Queuing ST batches?", QUEUE_ST)
-logger.infoBool("Forcing?", FORCE)
-logger.infoBool("Test mode?", TEST)
-logger.infoBool("Publishing?", PUBLISH)
-
-'''
-Main processing loop:
-
- 1) creates an IN batch if requested
- 2) queues available P2 batches (if requested)
- 3) queues available ST batches (if requested)
- 4) if in test mode, quits
- 5) otherwise, waits then repeats from 2
-
-'''
-while True:
-
-    # queue all items for all batch types, sequentially
-    if QUEUE_IN:
-       batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb)
-       batch.run()
-       break;
-
-    # queue up batches that are processed but not loaded to datastore
-    if QUEUE_P2: publishAnyUnpublishedBatches("P2")
-    if QUEUE_ST: publishAnyUnpublishedBatches("ST")
-
-    if QUEUE_P2: queueItems("P2")
-    if QUEUE_ST: queueItems("ST")
-
-    # in the TEST mode, we quit after submitting one batch
-    if TEST: break
-
-    # wait for the POLLPERIOD before checking for new ids
-    logger.infoPair("Finished. Waiting for", "%.1f minutes before checking DVO for new items" % (POLLPERIOD/60.0))
-    time.sleep(POLLPERIOD)
-
-
Index: /trunk/ippToPsps/jython/load.py
===================================================================
--- /trunk/ippToPsps/jython/load.py	(revision 32223)
+++ /trunk/ippToPsps/jython/load.py	(revision 32223)
@@ -0,0 +1,198 @@
+#!/usr/bin/env jython
+
+# The starting point for ippTopsps. See usage below.
+
+import time
+import sys
+import logging.config
+from xml.etree.ElementTree import ElementTree, Element, tostring
+
+from pslogger import PSLogger
+from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
+from datastore import Datastore
+from batch import Batch
+
+from initbatch import InitBatch
+from stackbatch import StackBatch
+from detectionbatch import DetectionBatch
+
+'''
+Finds batches that processed ok, but did not get loaded to the datastore, then tries to load them
+'''
+def publishAnyUnpublishedBatches(batchType):
+
+    batchIDs = ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(EPOCH, DVOGPC1LABEL, batchType)
+    logger.infoPair("Failed loads to datastore", "%d %s batches" % (len(batchIDs), batchType))
+
+    for batchID in batchIDs:
+
+        batchName = Batch.getNameFromID(batchID)
+        subDir = Batch.getSubDir(BASEPATH, batchType, DVOGPC1LABEL)
+        tarballFile = Batch.getTarballFile(batchID)
+        logger.infoPair("Batch name", batchName)
+
+        Batch.publishToDatastore(datastore, ippToPspsDb, batchID, batchName, subDir, tarballFile)
+
+
+'''
+Queues up all available stuff for this batch type
+'''
+def queueItems(batchType):
+
+    logger.infoTitle("Queuing " + batchType + " items")
+
+    allIDs = []
+
+    # see if there are any IDs listed in the config
+    try:
+        for element in configDoc.findall('options/ids/id'):
+            allIDs.append(int(element.text))
+    except:
+        logger.errorPair("Problem loading IDs from config under", "options/ids/id") 
+
+    # no config dictated IDs, so get from DVO instead
+    if len(allIDs) > 0:
+        logger.infoPair("Using config IDs (not DVO)", "%d" % len(allIDs))
+    else:
+        allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType)
+        logger.infoPair("All %s items in DVO" % batchType, "%d" % len(allIDs))
+
+    # if in FORCE mode, then queue full list
+    if FORCE: ids = allIDs
+
+    # if not in TEST mode, then only queue un-processed items
+    else:
+        processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOGPC1LABEL)
+        logger.infoPair("Processed %s items" % batchType, "%d" % len(processedIDs))
+        ids = list(set(allIDs) - set(processedIDs))
+        logger.infoPair("Unprocessed %s items" % batchType, "%d" % len(ids))
+
+    logger.infoPair("%s items queued" % batchType, "%d" % len(ids))
+
+    # loop round IDs of all items to be processed
+    for id in ids:
+
+        if not FORCE and ippToPspsDb.alreadyProcessed(batchType, id, EPOCH, DVOGPC1LABEL): continue
+        if not FORCE and ippToPspsDb.consistentlyFailed(batchType, id, EPOCH, DVOGPC1LABEL, 3): continue
+
+        if batchType == "P2":
+            batch = DetectionBatch(logger,
+                    CONFIG,
+                    configDoc,
+                    gpc1Db,
+                    ippToPspsDb,
+                    id)
+        elif batchType == "ST":
+            batch = StackBatch(logger,
+                    CONFIG,
+                    configDoc,
+                    gpc1Db,
+                    ippToPspsDb,
+                    id)
+
+        batch.run()
+
+        # if in TEST mode, then quit after one batch
+        if TEST: break
+
+
+'''
+Start of program.
+'''
+
+if len(sys.argv) > 1: CONFIG = sys.argv[1]
+else:
+    print "** Usage: " + sys.argv[0] + " <configPath> [init]"
+    sys.exit(1)
+
+if len(sys.argv) > 2 and sys.argv[2] == "init":
+    QUEUE_IN = 1
+else:
+    QUEUE_IN = 0
+
+# open config file
+configDoc = ElementTree(file=CONFIG)
+TEST = int(configDoc.find("options/testMode").text)
+
+# set up logging
+logging.setLoggerClass(PSLogger)
+logger = logging.getLogger("ippToPspsLog")
+
+# if in test mode, print log to screen, otherwise, only to file
+logger.setup(configDoc, "ippToPsps", TEST, not TEST)
+
+# create database objects
+gpc1Db = Gpc1Db(logger, configDoc)
+ippToPspsDb = IppToPspsDb(logger, configDoc)
+datastore = Datastore(logger, configDoc)
+
+# check we connected ok
+if not gpc1Db.everythingOK: sys.exit(1)
+if not ippToPspsDb.everythingOK: sys.exit(1)
+
+# get values from the configutaion file
+POLLPERIOD = 600
+DVOGPC1LABEL = configDoc.find("dvo/gpc1Label").text
+FORCE = int(configDoc.find("options/force").text)
+EPOCH = configDoc.find("options/epoch").text
+PUBLISH = int(configDoc.find("options/publishToDatastore").text)
+BASEPATH = configDoc.find("localOutPath").text
+QUEUE_P2 = int(configDoc.find("options/queueP2").text)
+QUEUE_ST = int(configDoc.find("options/queueST").text)
+
+# prompt user: FORCE and PUBLISH is a dangerous combination
+if FORCE and PUBLISH and not QUEUE_IN:
+   response = raw_input("\n*** Are you sure you want to publish data with the 'force' option enabled (y/n)? ")
+   if response != "y": sys.exit(1)
+
+# prompt user: TEST and PUBLISH is a dangerous combination
+if TEST and PUBLISH and not QUEUE_IN:
+   response = raw_input("\n*** Are you sure you want to publish data with the 'testMode' option enabled (y/n)? ")
+   if response != "y": sys.exit(1)
+
+# print info to log
+logger.infoSeparator()
+logger.infoTitle("ippToPsps started")
+logger.infoPair("Configuration file", CONFIG)
+logger.infoPair("Loading epoch", EPOCH)
+logger.infoPair("DVO gpc1 label", DVOGPC1LABEL)
+logger.infoBool("Queuing P2 batches?", QUEUE_P2)
+logger.infoBool("Queuing ST batches?", QUEUE_ST)
+logger.infoBool("Forcing?", FORCE)
+logger.infoBool("Test mode?", TEST)
+logger.infoBool("Publishing?", PUBLISH)
+
+'''
+Main processing loop:
+
+ 1) creates an IN batch if requested
+ 2) queues available P2 batches (if requested)
+ 3) queues available ST batches (if requested)
+ 4) if in test mode, quits
+ 5) otherwise, waits then repeats from 2
+
+'''
+while True:
+
+    # queue all items for all batch types, sequentially
+    if QUEUE_IN:
+       batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb)
+       batch.run()
+       break;
+
+    # queue up batches that are processed but not loaded to datastore
+    if QUEUE_P2: publishAnyUnpublishedBatches("P2")
+    if QUEUE_ST: publishAnyUnpublishedBatches("ST")
+
+    if QUEUE_P2: queueItems("P2")
+    if QUEUE_ST: queueItems("ST")
+
+    # in the TEST mode, we quit after submitting one batch
+    if TEST: break
+
+    # wait for the POLLPERIOD before checking for new ids
+    logger.infoPair("Finished. Waiting for", "%.1f minutes before checking DVO for new items" % (POLLPERIOD/60.0))
+    time.sleep(POLLPERIOD)
+
+