Changeset 32496

Timestamp:

Oct 5, 2011, 5:04:07 PM (15 years ago)

Author:

rhenders

Message:

new method that returns the batch type for the provided batch ID; using getLong(), rather than getInt(), when finding total detections; changes to methods that return unmerged or not yet merge_worthy batches

File:

• trunk/ippToPsps/jython/ipptopspsdb.py (modified) (9 diffs)

trunk/ippToPsps/jython/ipptopspsdb.py

@@ -23 +23 @@
 
     '''
-    Returns a list of processed batch IDs that are merged but not yet deleted
+    Returns a list of merged batch IDs that are merged but not yet deleted
+    '''
+    def getMergedButNotDeletedBatchIDs(self, batchType, epoch, dvoGpc1Label, column):
+
+        sql = "SELECT DISTINCT batch_id \
+               FROM batch \
+               WHERE timestamp > '" + epoch + "' \
+               AND batch_type = '" + batchType + "' \
+               AND dvo_db = '" + dvoGpc1Label + "' \
+               AND merged = 1 \
+               AND " + column + " = 0"
+
+        ids = []
+        try:
+            rs = self.executeQuery(sql)
+            while (rs.next()):
+                ids.append(rs.getInt(1))
+        except:
+            self.logger.exception("Can't query for merged batch ids in ipptopsps Db")
+
+        rs.close()
+
+        self.logger.debug("Found %d merged but un-deleted items" % len(ids))
+
+        return ids
+
+    '''
+    Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted
     '''
     def getLoadedToODMButNotDeletedBatchIDs(self, batchType, epoch, dvoGpc1Label, column):
@@ -50 +77 @@
 
     '''
-    Returns a list of processed batch IDs that are merged but not deleted from local disk
+    Returns a list of merged batch IDs that are not deleted from local disk
+    '''
+    def getMergedButNotDeletedFromLocalDisk(self, batchType, epoch, dvoGpc1Label):
+        return self.getMergedButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_local")
+
+    '''
+    Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from local disk
     '''
     def getLoadedToODMButNotDeletedFromLocalDisk(self, batchType, epoch, dvoGpc1Label):
@@ -56 +89 @@
 
     '''
-    Returns a list of processed batch IDs that are merged but not deleted from datastore
+    Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from datastore
     '''
     def getLoadedToODMButNotDeletedFromDatastore(self, batchType, epoch, dvoGpc1Label):
@@ -62 +95 @@
 
     '''
-    Returns a list of processed batch IDs that are merged but not deleted from DXLayer
+    Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from DXLayer
     '''
     def getLoadedToODMButNotDeletedFromDXLayer(self, batchType, epoch, dvoGpc1Label):
@@ -97 +130 @@
     '''
     Returns a list of processed batch IDs that have not yet been loaded to the ODM
+    NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to 
+    loading to ODM if there is a known problem with the batch
     '''
     def getUnloadedBatchIDs(self, epoch, dvoGpc1Label, batchType):
@@ -106 +141 @@
                AND loaded_to_datastore = 1 \
                AND batch_type = '" + batchType + "' \
-               AND loaded_to_ODM = 0"
-
+               AND loaded_to_ODM = 0 \
+               AND deleted_datastore = 0"
 
         ids = []
@@ -192 +227 @@
             rs = self.executeQuery(sql)
             rs.first()
-            total =  rs.getInt(1)
+            total =  rs.getLong(1)
         except:
             self.logger.exception("Can't query for total detections published")
@@ -266 +301 @@
     '''
     def getStages(self):
-        return ['processed', 'loaded_to_datastore', 'loaded_to_ODM', 'merge_worthy', 'deleted_datastore', 'merged', 'deleted_dxlayer', 'deleted_local']
+        return ['processed', 'loaded_to_datastore', 'loaded_to_ODM', 'merge_worthy', 'deleted_datastore', 'merged', 'deleted_local', 'deleted_dxlayer']
 
     '''
@@ -414 +449 @@
         except:
             self.logger.exception("Unable to check whether this batch is being processed")
+
+    '''
+    Gets the batch type of this batch
+    '''
+    def getBatchType(self, batch_id):
+
+        sql = "SELECT batch_type \
+               FROM batch \
+               WHERE batch_id = " + str(batch_id)
+
+        batch_type = "NA"
+        try:
+            rs = self.executeQuery(sql)
+            rs.first()
+            batch_type = rs.getString(1) 
+        except:
+            self.logger.exception("Unable to check whether this batch has already been processed")
+ 
+        return batch_type
 
     '''