Index: trunk/ippToPsps/jython/queue.py
===================================================================
--- trunk/ippToPsps/jython/queue.py	(revision 33347)
+++ trunk/ippToPsps/jython/queue.py	(revision 33347)
@@ -0,0 +1,137 @@
+#!/usr/bin/env jython
+
+#
+# The main loading program for ippToPsps. See usage below.
+#
+
+import time
+import sys
+import os
+import logging.config
+
+from ipptopsps import IppToPsps
+from gpc1db import Gpc1Db
+from datastore import Datastore
+from batch import Batch
+
+'''
+Queue class
+'''
+class Queue(IppToPsps):
+
+    '''
+    Constructor
+    '''
+    def __init__(self, argv):
+        super(Queue, self).__init__(argv)
+
+        # create various objects
+        self.gpc1Db = Gpc1Db(self.logger, self.config)
+
+        # set a poll time in hours
+        self.parsePollTimeArg("12")
+    
+
+    '''
+    Main processing loop.
+    Tiles area defined in config and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing
+    by one or more loading clients that may be running on any host
+    '''
+    def run(self):
+
+        # this outer while loop simply waits a few minutes then starts queuing again as more stuff may appear in DVO over time
+        while True:
+        
+            # queue up batches that are processed but not loaded to datastore
+            self.logger.infoTitle("Previous failed datastore loads")
+            for batchType in self.config.batchTypes: self.publishAnyUnpublishedBatches(batchType)
+
+            # get totals for whole area to check if there is anything to do
+            processedIDs = self.ippToPspsDb.getProcessedIDs(batchType, self.config.epoch, self.config.dvoLabel)
+            consistantlyFailedIDs = self.ippToPspsDb.getConsistentlyFailedIDs(batchType, self.config.epoch, self.config.dvoLabel)
+            allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType, self.config.minRa, self.config.maxRa, self.config.minDec, self.config.maxDec)
+            #allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType)
+            ids = list(set(allIDs) - set(processedIDs) - set(consistantlyFailedIDs))
+
+            # loop through full range of RA/Dec queueing stuff in boxes of size self.config.boxSize
+            self.logger.info("+---------------------------------------------------------------------+")
+            self.logger.info("|                All unprocessed items in area: %12d          |" % len(ids))
+
+            if len(ids) > 0:
+
+                self.logger.info("+-------------+-------------+-------------+-------------+-------------+")
+                self.logger.info("|     RA      |     Dec     |   box size  |    in DVO   | unprocessed |")
+                self.logger.info("+-------------+-------------+-------------+-------------+-------------+")
+    
+                # starting positions
+                ra = self.config.minRa + self.config.halfBox
+                dec = self.config.minDec + self.config.halfBox
+    
+                while ra <= self.config.maxRa:
+                   while dec <= self.config.maxDec:
+            
+                       self.checkClientStatus()
+                       box_id = self.ippToPspsDb.insertBox(self.config.name, ra, dec, self.config.boxSize)
+            
+                       # for each batch type look for items to queue in this box
+                       for batchType in self.config.batchTypes: 
+            
+                           allIDs = self.gpc1Db.getIDsInThisDVODbForThisStageInThisBox(
+                                   self.config.dvoLabel, 
+                                   batchType, 
+                                   ra, 
+                                   dec, 
+                                   self.config.boxSize)
+    
+                           if len(allIDs) < 1:
+                               self.logger.info("|    %5.1f    |    %5.1f    |  %9d  |  %9d  |  %9d  |" % (
+                                           ra, 
+                                           dec, 
+                                           self.config.boxSize, 
+                                           len(allIDs), 
+                                           0))
+                               continue    
+    
+                           ids = list(set(allIDs) - set(processedIDs) - set(consistantlyFailedIDs))
+                           self.logger.info("|    %5.1f    |    %5.1f    |  %9d  |  %9d  |  %9d  |" % (
+                                       ra, 
+                                       dec, 
+                                       self.config.boxSize, 
+                                       len(allIDs), 
+                                       len(ids)))
+                           if len(ids) < 1: continue
+            
+                           #self.logger.infoPair("write IDs to database", "NOW")
+                           self.ippToPspsDb.insertPending(box_id, batchType, ids)
+
+                       dec = dec + self.config.boxSize
+                   dec = self.config.minDec + self.config.halfBox
+                   ra = ra + self.config.boxSize
+            
+            self.logger.info("+-------------+-------------+-------------+-------------+-------------+")
+
+            if not self.waitForPollTime(): break
+
+    '''
+    Finds batches that processed ok, but did not get loaded to the datastore, then tries to load them
+    '''
+    def publishAnyUnpublishedBatches(self, batchType):
+    
+        batchIDs = self.ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(self.config.epoch, self.config.dvoLabel, batchType)
+        self.logger.infoPair("%s batches" % batchType, "%d" % len(batchIDs))
+    
+        for batchID in batchIDs:
+    
+            batchName = Batch.getNameFromID(batchID)
+            subDir = Batch.getSubDir(self.config.basePath, batchType, self.config.dvoLabel)
+            tarballFile = Batch.getTarballFile(batchID)
+            self.logger.infoPair("Batch name", batchName)
+    
+            Batch.publishToDatastore(self.datastore, batchID, batchName, subDir, tarballFile)
+    
+'''
+Start of program.
+'''
+queue = Queue(sys.argv)
+queue.run()
+queue.exitProgram("finished")