Changeset 32541

Timestamp:

Oct 13, 2011, 9:50:28 AM (15 years ago)

Author:

rhenders

Message:

no need for 'DISTINCT batch_id as that column is an index, so they are all distinct anyway; new methid to get batches that are not merge-worthy; new method to get all 'unfinished' batches, i.e. everything that is not merged and not failed;

File:

• trunk/ippToPsps/jython/ipptopspsdb.py (modified) (3 diffs)

trunk/ippToPsps/jython/ipptopspsdb.py

@@ -59 +59 @@
                AND batch_type = '" + batchType + "' \
                AND dvo_db = '" + dvoGpc1Label + "' \
-               AND loaded_to_ODM != 0 \
+               AND (loaded_to_ODM = -1 OR merge_worthy = 1) \
                AND " + column + " = 0"
 
@@ -129 +129 @@
 
     '''
-    Returns a list of processed batch IDs that have not yet been loaded to the ODM
-    NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to 
-    loading to ODM if there is a known problem with the batch
+    Returns a list of processed batch IDs that either not yet loaded to the ODM or not merge_worthy or not merged
+    NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to loading to ODM if there is a known problem with the batch
+    '''
+    def getAllUnfinishedBatchIDs(self, epoch, dvoGpc1Label, batchType):
+
+        sql = "SELECT batch_id \
+               FROM batch \
+               WHERE timestamp > '" + epoch + "' \
+               AND dvo_db = '" + dvoGpc1Label + "' \
+               AND loaded_to_datastore = 1 \
+               AND batch_type = '" + batchType + "' \
+               AND deleted_datastore = 0 \
+               AND loaded_to_ODM != -1 \
+               AND merge_worthy != -1 \
+               AND merged = 0"
+
+        ids = []
+        try:
+            rs = self.executeQuery(sql)
+            while (rs.next()):
+                ids.append(rs.getInt(1))
+        except:
+            self.logger.exception("Can't query for unloaded batch ids in ipptopsps Db")
+
+        rs.close()
+
+        self.logger.debug("Found %d unloaded items" % len(ids))
+
+        return ids
+
+    '''
+    Returns a list of processed batch IDs that are not yet loaded to the ODM
+    NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to loading to ODM if there is a known problem with the batch
     '''
     def getUnloadedBatchIDs(self, epoch, dvoGpc1Label, batchType):
 
-        sql = "SELECT DISTINCT batch_id \
+        sql = "SELECT batch_id \
                FROM batch \
                WHERE timestamp > '" + epoch + "' \
@@ -159 +189 @@
 
     '''
+    Returns a list of processed batch IDs that are not yet merge_worthy and that have not failed to load
+    NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to loading to ODM if there is a known problem with the batch
+    '''
+    def getUnmergeWorthyBatchIDs(self, epoch, dvoGpc1Label, batchType):
+
+        sql = "SELECT batch_id \
+               FROM batch \
+               WHERE timestamp > '" + epoch + "' \
+               AND dvo_db = '" + dvoGpc1Label + "' \
+               AND loaded_to_datastore = 1 \
+               AND batch_type = '" + batchType + "' \
+               AND merge_worthy = 0 \
+               AND loaded_to_ODM != -1 \
+               AND deleted_datastore = 0"
+
+        ids = []
+        try:
+            rs = self.executeQuery(sql)
+            while (rs.next()):
+                ids.append(rs.getInt(1))
+        except:
+            self.logger.exception("Can't query for un-mergeworthy batch ids in ipptopsps Db")
+
+        rs.close()
+
+        self.logger.debug("Found %d unloaded items" % len(ids))
+
+        return ids
+
+    '''
     Returns a list of processed batch IDs that are merge_worthy, but not yet merged, for this epoch and dvo label
     '''
     def getUnmergedBatchIDs(self, epoch, dvoGpc1Label, batchType):
 
-        sql = "SELECT DISTINCT batch_id \
-               FROM batch \
-               WHERE timestamp > '" + epoch + "' \
-               AND dvo_db = '" + dvoGpc1Label + "' \
-               AND loaded_to_ODM = 1 \
+        sql = "SELECT batch_id \
+               FROM batch \
+               WHERE timestamp > '" + epoch + "' \
+               AND dvo_db = '" + dvoGpc1Label + "' \
+               AND merge_worthy = 1 \
                AND batch_type = '" + batchType + "' \
                AND merged != 1"