Changeset 32206 for trunk/ippToPsps/jython/ipptopsps.py

Timestamp:

Aug 29, 2011, 9:26:53 AM (15 years ago)

Author:

rhenders

Message:

new method to publish batches that previously failed to load to the datastore; added some log messages when a batch fails;

File:

• trunk/ippToPsps/jython/ipptopsps.py (modified) (4 diffs)

trunk/ippToPsps/jython/ipptopsps.py

@@ -11 +11 @@
 from gpc1db import Gpc1Db
 from ipptopspsdb import IppToPspsDb
+from datastore import Datastore
+from batch import Batch
 
 from initbatch import InitBatch
 from stackbatch import StackBatch
 from detectionbatch import DetectionBatch
+
+'''
+Finds batches that processed ok, but did not get loaded to the datastore, then tries to load them
+'''
+def publishAnyUnpublishedBatches(batchType):
+
+    batchIDs = ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(EPOCH, DVOGPC1LABEL, batchType)
+    logger.infoPair("Failed load to datastore", "%d %s batches" % (len(batchIDs), batchType))
+
+    for batchID in batchIDs:
+
+        batchName = Batch.getNameFromID(batchID)
+        subDir = Batch.getSubDir(BASEPATH, batchType, DVOGPC1LABEL)
+        tarballFile = Batch.getTarballFile(batchID)
+        logger.infoPair("Batch name", batchName)
+
+        Batch.publishToDatastore(datastore, ippToPspsDb, batchID, batchName, subDir, tarballFile)
+
 
 '''
@@ -105 +125 @@
 gpc1Db = Gpc1Db(logger, configDoc)
 ippToPspsDb = IppToPspsDb(logger, configDoc)
+datastore = Datastore(logger, configDoc)
 
 # check we connected ok
@@ -116 +137 @@
 EPOCH = configDoc.find("options/epoch").text
 PUBLISH = int(configDoc.find("options/publishToDatastore").text)
+BASEPATH = configDoc.find("localOutPath").text
 QUEUE_P2 = int(configDoc.find("options/queueP2").text)
 QUEUE_ST = int(configDoc.find("options/queueST").text)
@@ -159 +181 @@
        break;
 
+    # queue up batches that are processed but not loaded to datastore
+    if QUEUE_P2: publishAnyUnpublishedBatches("P2")
+    if QUEUE_ST: publishAnyUnpublishedBatches("ST")
+
     if QUEUE_P2: queueItems("P2")
     if QUEUE_ST: queueItems("ST")