Index: trunk/ippToPsps/jython/queue.py
===================================================================
--- trunk/ippToPsps/jython/queue.py	(revision 33488)
+++ trunk/ippToPsps/jython/queue.py	(revision 33489)
@@ -28,4 +28,7 @@
         # create various objects
         self.gpc1Db = Gpc1Db(self.logger, self.config)
+        self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb)
+
+        self.config.printAll()
 
         # set a poll time in hours
@@ -35,79 +38,83 @@
     '''
     Main processing loop.
-    Tiles area defined in config and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing
+    Tiled area is defined in config and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing
     by one or more loading clients that may be running on any host
     '''
     def run(self):
 
-        # this outer while loop simply waits a few minutes then starts queuing again as more stuff may appear in DVO over time
+        # this outer while loop simply waits a few minutes then starts queuing again (as more stuff may appear in DVO over time)
         while True:
+
+            self.ippToPspsDb.removeAllBoxes()
         
             # queue up batches that are processed but not loaded to datastore
-            self.logger.infoTitle("Previous failed datastore loads")
-            for batchType in self.config.batchTypes: self.publishAnyUnpublishedBatches(batchType)
+            for batchType in self.config.batchTypes: 
+                self.logger.infoTitle("Previous failed datastore loads")
+                self.publishAnyUnpublishedBatches(batchType)
 
-            # get totals for whole area to check if there is anything to do
-            processedIDs = self.ippToPspsDb.getProcessedIDs(batchType, self.config.epoch, self.config.dvoLabel)
-            consistantlyFailedIDs = self.ippToPspsDb.getConsistentlyFailedIDs(batchType, self.config.epoch, self.config.dvoLabel)
-            allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType, self.config.minRa, self.config.maxRa, self.config.minDec, self.config.maxDec)
-            #allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType)
-            ids = list(set(allIDs) - set(processedIDs) - set(consistantlyFailedIDs))
+                # get totals for whole area to check if there is anything to do
+                processedIDs = self.ippToPspsDb.getProcessedIDs(batchType, self.config.epoch, self.config.dvoLabel)
+                consistantlyFailedIDs = self.ippToPspsDb.getConsistentlyFailedIDs(batchType, self.config.epoch, self.config.dvoLabel)
+                rows = self.gpc1Db.getItemsInThisDVODbForThisStage(
+                        self.config.dvoLabel, 
+                        batchType, 
+                        self.config.minRa, 
+                        self.config.maxRa, 
+                        self.config.minDec, 
+                        self.config.maxDec)
 
-            # loop through full range of RA/Dec queueing stuff in boxes of size self.config.boxSize
-            self.logger.info("+---------------------------------------------------------------------+")
-            self.logger.info("|                All unprocessed items in area: %12d          |" % len(ids))
+                # first report total stuff
+                allIDs = []
+                for row in rows:
+                    try: allIDs.append(row[0])
+                    except: pass
 
-            if len(ids) > 0:
+                ids = list(set(allIDs) - set(processedIDs) - set(consistantlyFailedIDs))
+                self.logger.infoPair("All unprocessed items", "%d" % len(ids))
 
-                self.logger.info("+-------------+-------------+-------------+-------------+-------------+")
-                self.logger.info("|     RA      |     Dec     |   box size  |    in DVO   | unprocessed |")
-                self.logger.info("+-------------+-------------+-------------+-------------+-------------+")
+                # now trim full list down to only pending stuff
+                pending = [[], [], []]
+                for row in rows:
+                    try:
+                        findId = row[0]
+                    except: continue
+                    if findId in ids: pending.append([row[0], row[1], row[2]])
+
+                # store all items in temp table
+                self.logger.infoPair("Inserting all pending items into", "ippToPsps database")
+                self.ippToPspsDb.storeAllItems(pending)
+
+                # loop through full range of RA/Dec queueing stuff in boxes of size self.config.boxSize
+                if len(ids) > 0:
+
+                    self.logger.info("+-------------+-------------+-------------+-------------+")
+                    self.logger.info("|     RA      |     Dec     |   box size  | unprocessed |")
+                    self.logger.info("+-------------+-------------+-------------+-------------+")
     
-                # starting positions
-                ra = self.config.minRa + self.config.halfBox
-                dec = self.config.minDec + self.config.halfBox
+                    # starting positions
+                    ra = self.config.minRa + self.config.halfBox
+                    dec = self.config.minDec + self.config.halfBox
     
-                while ra <= self.config.maxRa:
-                   while dec <= self.config.maxDec:
+                    while ra <= self.config.maxRa:
+                       while dec <= self.config.maxDec:
             
-                       self.checkClientStatus()
-                       box_id = self.ippToPspsDb.insertBox(self.config.name, ra, dec, self.config.boxSize)
+                           self.checkClientStatus()
+                           box_id = self.ippToPspsDb.insertBox(ra, dec)
             
-                       # for each batch type look for items to queue in this box
-                       for batchType in self.config.batchTypes: 
-            
-                           allIDs = self.gpc1Db.getIDsInThisDVODbForThisStageInThisBox(
-                                   self.config.dvoLabel, 
-                                   batchType, 
-                                   ra, 
-                                   dec, 
-                                   self.config.boxSize)
+                           ids = self.ippToPspsDb.getItemsInThisThisBox(ra, dec)
     
-                           if len(allIDs) < 1:
-                               self.logger.info("|    %5.1f    |    %5.1f    |  %9d  |  %9d  |  %9d  |" % (
-                                           ra, 
-                                           dec, 
-                                           self.config.boxSize, 
-                                           len(allIDs), 
-                                           0))
-                               continue    
-    
-                           ids = list(set(allIDs) - set(processedIDs) - set(consistantlyFailedIDs))
-                           self.logger.info("|    %5.1f    |    %5.1f    |  %9d  |  %9d  |  %9d  |" % (
+                           self.logger.info("|    %5.1f    |    %5.1f    |  %9d  |  %9d  |" % (
                                        ra, 
                                        dec, 
                                        self.config.boxSize, 
-                                       len(allIDs), 
                                        len(ids)))
-                           if len(ids) < 1: continue
+                
+                           if len(ids) > 0: self.ippToPspsDb.insertPending(box_id, batchType, ids)
+
+                           dec = dec + self.config.boxSize
+                       dec = self.config.minDec + self.config.halfBox
+                       ra = ra + self.config.boxSize
             
-                           #self.logger.infoPair("write IDs to database", "NOW")
-                           self.ippToPspsDb.insertPending(box_id, batchType, ids)
-
-                       dec = dec + self.config.boxSize
-                   dec = self.config.minDec + self.config.halfBox
-                   ra = ra + self.config.boxSize
-            
-            self.logger.info("+-------------+-------------+-------------+-------------+-------------+")
+                self.logger.info("+-------------+-------------+-------------+-------------+")
 
             if not self.waitForPollTime(): break