Changeset 32250 for trunk/ippToPsps/jython/ipptopspsdb.py

Timestamp:

Aug 31, 2011, 1:20:14 PM (15 years ago)

Author:

rhenders

Message:

various changes to stage ID getters; new method to get batch IDs; now setting processed to -1 when a batch fails to process; some method name changes

File:

• trunk/ippToPsps/jython/ipptopspsdb.py (modified) (11 diffs)

trunk/ippToPsps/jython/ipptopspsdb.py

@@ -59 +59 @@
     Returns a list of processed batch IDs that are merged but not yet deleted
     '''
-    def getMergedButNotDeletedBatchIDs(self, batchType, epoch, dvoGpc1Label, column):
+    def getLoadedToODMButNotDeletedBatchIDs(self, batchType, epoch, dvoGpc1Label, column):
 
         sql = "SELECT DISTINCT batch_id \
@@ -66 +66 @@
                AND batch_type = '" + batchType + "' \
                AND dvo_db = '" + dvoGpc1Label + "' \
-               AND merged = 1\
+               AND loaded_to_ODM != 0 \
                AND " + column + " = 0"
 
@@ -86 +86 @@
     Returns a list of processed batch IDs that are merged but not deleted from local disk
     '''
-    def getMergedButNotDeletedFromLocalDisk(self, batchType, epoch, dvoGpc1Label):
-        return self.getMergedButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_local")
+    def getLoadedToODMButNotDeletedFromLocalDisk(self, batchType, epoch, dvoGpc1Label):
+        return self.getLoadedToODMButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_local")
 
     '''
     Returns a list of processed batch IDs that are merged but not deleted from datastore
     '''
-    def getMergedButNotDeletedFromDatastore(self, batchType, epoch, dvoGpc1Label):
-        return self.getMergedButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_datastore")
+    def getLoadedToODMButNotDeletedFromDatastore(self, batchType, epoch, dvoGpc1Label):
+        return self.getLoadedToODMButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_datastore")
 
     '''
     Returns a list of processed batch IDs that are merged but not deleted from DXLayer
     '''
-    def getMergedButNotDeletedFromDXLayer(self, batchType, epoch, dvoGpc1Label):
-        return self.getMergedButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_dxlayer")
+    def getLoadedToODMButNotDeletedFromDXLayer(self, batchType, epoch, dvoGpc1Label):
+        return self.getLoadedToODMButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_dxlayer")
 
 
@@ -130 +130 @@
 
     '''
-    Returns a list of processed batch IDs that are not yet merged for this epoch and dvo label
-    '''
-    def getProcessedButUnmergedBatchIDs(self, epoch, dvoGpc1Label):
+    Returns a list of processed batch IDs that have not failed to load and are not yet merged, for this epoch and dvo label
+    '''
+    def getUnfinishedBatchIDs(self, epoch, dvoGpc1Label):
 
         sql = "SELECT DISTINCT batch_id \
@@ -139 +139 @@
                AND dvo_db = '" + dvoGpc1Label + "' \
                AND loaded_to_datastore = 1 \
-               AND merged = 0 OR merged = -1"
+               AND merged != 1 \
+               AND loaded_to_ODM != -1"
+
 
         ids = []
@@ -147 +149 @@
                 ids.append(rs.getInt(1))
         except:
-            self.logger.exception("Can't query for processed batch ids in ipptopsps Db")
+            self.logger.exception("Can't query for unfinished batch ids in ipptopsps Db")
 
         rs.close()
 
-        self.logger.debug("Found %d processed and un-merged items" % len(ids))
+        self.logger.debug("Found %d unfinished items" % len(ids))
 
         return ids
 
     '''
+    Returns a list of batch IDs for this epoch and dvo label for which the specified column is set to value
+    '''
+    def getBatchIDs(self, epoch, dvoGpc1Label, column, value):
+
+        sql = "SELECT DISTINCT batch_id \
+               FROM batch \
+               WHERE timestamp > '" + epoch + "' \
+               AND dvo_db = '" + dvoGpc1Label + "' \
+               AND " + column + " = " + str(value)
+
+        ids = []
+        try:
+            rs = self.executeQuery(sql)
+            while (rs.next()):
+                ids.append(rs.getInt(1))
+        except:
+            self.logger.exception("Can't query for batch ids with " + column + " = " + str(value) + " in ipptopsps Db")
+
+        rs.close()
+
+        self.logger.debug("Found %d batches with %s = %d" % (len(ids), column, value))
+
+        return ids
+
+    '''
     Returns a list of IDs for this batch type, epoch and dvo label (i.e. either 
-            cam_ids or stack_ids) for which the spefified column is true (=1)
-    '''
-    def getStageIDs(self, batchType, epoch, dvoGpc1Label, column):
+            cam_ids or stack_ids) for which the specified column is set to value
+    '''
+    def getStageIDs(self, batchType, epoch, dvoGpc1Label, column, value):
 
         sql = "SELECT DISTINCT stage_id \
@@ -166 +193 @@
                AND timestamp > '" + epoch + "' \
                AND dvo_db = '" + dvoGpc1Label + "' \
-               AND " + column + " = 1"
+               AND " + column + " = " + str(value)
 
         ids = []
@@ -174 +201 @@
                 ids.append(rs.getInt(1))
         except:
-            self.logger.exception("Can't query for ids with " + column + " = 1 in ipptopsps Db")
+            self.logger.exception("Can't query for ids with " + column + " = " + str(value) + " in ipptopsps Db")
 
         rs.close()
@@ -183 +210 @@
 
     '''
+    Series of getter methods that utilise getBatchIDs() method above
+    '''
+    def getloadedToDatastoreBatchIDs(self, epoch, dvoGpc1Label):
+        return self.getBatchIDs(epoch, dvoGpc1Label, "loaded_to_datastore", 1)
+    def getFailedLoadedToODMBatchIDs(self, epoch, dvoGpc1Label):
+        return self.getBatchIDs(epoch, dvoGpc1Label, "loaded_to_ODM", -1)
+
+    '''
     Series of getter methods that utilise getStageIDs() method above
     '''
     def getProcessedIDs(self, batchType, epoch, dvoGpc1Label):
-        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "processed")
+        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "processed", 1)
     def getloadedToDatastoreIDs(self, batchType, epoch, dvoGpc1Label):
-        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_datastore")
+        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_datastore", 1)
     def getLoadedToODMIDs(self, batchType, epoch, dvoGpc1Label):
-        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_ODM")
+        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_ODM", 1)
     def getMergeWothyIDs(self, batchType, epoch, dvoGpc1Label):
-        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merge_worthy")
+        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merge_worthy", 1)
     def getMergedIDs(self, batchType, epoch, dvoGpc1Label):
-        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merged")
+        return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merged", 1)
 
     '''
@@ -340 +375 @@
     def updateOdmStatus(self, batchID, odmStatus):
 
+        if odmStatus['LOADFAILED'] == 1: loadedToODM = -1
+        else: loadedToODM = odmStatus['LOADEDTOODM']
         sql = "UPDATE batch \
-               SET loaded_to_ODM = " + str(odmStatus['LOADEDTOODM']) + ", \
+               SET loaded_to_ODM = " + str(loadedToODM) + ", \
                merge_worthy = " + str(odmStatus['MERGEWORTHY']) + ", \
                merged = " + str(odmStatus['MERGED']) + " \
@@ -384 +421 @@
                AND timestamp > '" + epoch + "' \
                AND dvo_db = '" + dvoGpc1Label + "' \
-               AND processed != 1"
+               AND processed == -1"
 
         try: