Changeset 32694 for branches/eam_branches/ipp-20111110/ippToPsps

Timestamp:

Nov 17, 2011, 2:02:08 PM (15 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20111110

Files:

• . (modified) (1 prop)
• ippToPsps/jython/ipptopspsdb.py (modified) (1 diff)
• ippToPsps/jython/pollOdm.py (modified) (4 diffs)

branches/eam_branches/ipp-20111110/ippToPsps/jython/ipptopspsdb.py

@@ -125 +125 @@
 
         self.logger.debug("Found %d processed but-not-on-datastore items" % len(ids))
-
-        return ids
-
-    '''
-    Returns a list of processed batch IDs that either not yet loaded to the ODM or not merge_worthy or not merged
-    NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to loading to ODM if there is a known problem with the batch
-    '''
-    def getAllUnfinishedBatchIDs(self, epoch, dvoGpc1Label, batchType):
-
-        sql = "SELECT batch_id \
-               FROM batch \
-               WHERE timestamp > '" + epoch + "' \
-               AND dvo_db = '" + dvoGpc1Label + "' \
-               AND loaded_to_datastore = 1 \
-               AND batch_type = '" + batchType + "' \
-               AND deleted_datastore = 0 \
-               AND loaded_to_ODM != -1 \
-               AND merge_worthy != -1 \
-               AND merged = 0"
-
-        ids = []
-        try:
-            rs = self.executeQuery(sql)
-            while (rs.next()):
-                ids.append(rs.getInt(1))
-        except:
-            self.logger.exception("Can't query for unloaded batch ids in ipptopsps Db")
-
-        rs.close()
-
-        self.logger.debug("Found %d unloaded items" % len(ids))
 
         return ids

branches/eam_branches/ipp-20111110/ippToPsps/jython/pollOdm.py

@@ -17 +17 @@
 Method to actually poll ODM
 '''
-def poll(batchType):
+def poll(batchType, stage):
   
+    if stage == "unloaded":
+        ids = ippToPspsDb.getUnloadedBatchIDs(EPOCH, DVOLABEL, batchType)
+    elif stage == "unmergeworthy":
+        ids = ippToPspsDb.getUnmergeWorthyBatchIDs(EPOCH, DVOLABEL, batchType)
+    elif stage == "unmerged":
+        ids = ippToPspsDb.getUnmergedBatchIDs(EPOCH, DVOLABEL, batchType)
 
-    lookingForUnloaded = 0
-    lookingForUnmergeworthy = 0
-    lookingForUnmerged = 0
-    lookingForAll = 0
-
-    if TYPE == "unloaded":
-        lookingForUnloaded = 1
-        ids = ippToPspsDb.getUnloadedBatchIDs(EPOCH, DVOLABEL, batchType)
-    elif TYPE == "unmergeworthy":
-        lookingForUnmergeworthy = 1
-        ids = ippToPspsDb.getUnmergeWorthyBatchIDs(EPOCH, DVOLABEL, batchType)
-    elif TYPE == "unmerged":
-        lookingForUnmerged = 1
-        ids = ippToPspsDb.getUnmergedBatchIDs(EPOCH, DVOLABEL, batchType)
-    elif TYPE == "unfinished":
-        lookingForAll = 1
-        ids = ippToPspsDb.getAllUnfinishedBatchIDs(EPOCH, DVOLABEL, batchType)
-
-    logger.infoTitle("Checking %d %s %s batches" % (len(ids), TYPE, batchType))
+    logger.infoTitle("Checking %d %s %s batches" % (len(ids), stage, batchType))
 
     results = {}
@@ -53 +41 @@
            ippToPspsDb.updateOdmStatus(id, results)
 
-           if lookingForUnloaded:
+           if stage == "unloaded":
                if results['LOADEDTOODM'] == 1: newLoaded = newLoaded + 1
                if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1
                if results['MERGED'] == 1: newMerged = newMerged + 1
 
-           elif lookingForUnmergeworthy:
+           elif stage == "unmergeworthy":
                if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1
                if results['MERGED'] == 1: newMerged = newMerged + 1
 
-           elif lookingForUnmerged:
+           elif stage == "unmerged":
                if results['MERGED'] == 1: newMerged = newMerged + 1
 
@@ -88 +76 @@
     CONFIG  = sys.argv[1]
     BATCHTYPE = sys.argv[2]
-    TYPE = sys.argv[3]
+    STAGE = sys.argv[3]
 else:
-    print "\n** Usage: " + sys.argv[0] + " <configPath> <P2|ST|all> <unloaded|unmergeworthy|unmerged|unfinished> [hours]\n"
+    print "\n** Usage: " + sys.argv[0] + " <configPath> <P2|ST|all> <unloaded|unmergeworthy|unmerged|all> [hours]\n"
     sys.exit(1)
 
@@ -113 +101 @@
 EPOCH = configDoc.find("options/epoch").text
 
+if STAGE == "all": stages = ['unloaded', 'unmergeworthy', 'unmerged']
+else: stages = [STAGE]
+
+if BATCHTYPE == "all": batchTypes = ['P2', 'ST']
+else: batchTypes = [BATCHTYPE]
+
+for stage in stages: logger.infoPair("Checking stage", stage)
+for batchType in batchTypes: logger.infoPair("Checking batch type", batchType)
+
 while True:
 
-    if BATCHTYPE == "all":
-        poll("P2")
-        poll("ST")
-    else:    
-        poll(BATCHTYPE)
+    for stage in stages: 
+        for batchType in batchTypes:
+            poll(batchType, stage)
 
     if SECONDS: