Changeset 32354 for trunk/ippToPsps

Timestamp:

Sep 6, 2011, 1:37:06 PM (15 years ago)

Author:

rhenders

Message:

now reporting batches-per-hour and last P2 and St batch published; much cleaner code to determine figures and report them in a more readable way (including colour)

File:

• trunk/ippToPsps/jython/metrics.py (modified) (4 diffs)

trunk/ippToPsps/jython/metrics.py

@@ -4 +4 @@
 import sys
 import os
+import datetime
 
 from xml.etree.ElementTree import ElementTree, Element, tostring
@@ -39 +40 @@
 
 '''
-Prepares a string of the format:
-
-n-processed(n-pending)
-
+Formats an int to a string
 '''
-def prepareStr(masterList, prevList, thisList, FILE, name):
-
-    unsuccessful = list(set(prevList) - set(thisList))
-
-    countTotal = len(masterList)
-    countSuccessful = len(thisList)
-    countUnsuccessful = len(unsuccessful)
-
-    str = "%d" % countSuccessful
-    if countUnsuccessful: str = str + "(%d)" % countUnsuccessful
-
-    print >> FILE, name, countSuccessful, countUnsuccessful, (countTotal - countSuccessful - countUnsuccessful)
-    #;sys.stdout.softspace=0; 
-    return str
+def getIntAsString(value):
+    if value == 0: return ""
+    return str(value)
 
 '''
 Prints (to the log) one row of data (one row being one batch-type)
 '''
-def printTableRow(batchType):
+def printTableRow(stages, batchType):
 
+    # get a master list of IDs in DVO for this batch type
     allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType)
-    processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOLABEL)
-    datastoreIDs = ippToPspsDb.getloadedToDatastoreIDs(batchType, EPOCH, DVOLABEL)
-    loadedToODMIDs = ippToPspsDb.getLoadedToODMIDs(batchType, EPOCH, DVOLABEL)
-    mergeWorthyIDs = ippToPspsDb.getMergeWorthyIDs(batchType, EPOCH, DVOLABEL)
-    mergedIDs = ippToPspsDb.getMergedIDs(batchType, EPOCH, DVOLABEL)
+    prevList = allIDs
+    numEverything = len(allIDs)
+
+    sys.stdout.write("| %2s |%6d " % (batchType, numEverything))
 
     tempFilename = batchType + '.dat'
     DATFILE = open(tempFilename,'w')
-    prepareStr(allIDs, allIDs, allIDs, DATFILE, "inDVO"),
+    print >> DATFILE, "DVO", numEverything, 0, 0
+    for stage in stages:
 
-    logger.info("|  %2s  |  %-12d  |  %-12s  |  %-12s  |  %-12s  |  %-12s  |  %-12s  |  %-12s  |  %-19s |" % (
-                batchType, 
-                len(allIDs), 
-                prepareStr(allIDs, allIDs, processedIDs, DATFILE, "processed"),
-                prepareStr(allIDs, processedIDs, datastoreIDs, DATFILE, "onDatastore"),
-                prepareStr(allIDs, datastoreIDs, loadedToODMIDs, DATFILE, "loadedToODM"),
-                prepareStr(allIDs, loadedToODMIDs, mergeWorthyIDs, DATFILE, "MergeWorthy"),
-                prepareStr(allIDs, mergeWorthyIDs, mergedIDs, DATFILE, "Merged"),
-                len(list(set(allIDs) - set(mergedIDs))),
-                ippToPspsDb.getTimeOfLastBatchPublished(batchType, EPOCH, DVOLABEL)))
+       # get lists. Use unions with prev list to make sure the right stuff is included
+       success = list(set(ippToPspsDb.getStageIDs(batchType, EPOCH, DVOLABEL, stage, 1)) & set(prevList)) 
+       fail = list(set(ippToPspsDb.getStageIDs(batchType, EPOCH, DVOLABEL, stage, -1)) & set(prevList) - set(success))
+       pending = list(set(prevList) - set(success) - set(fail))
 
+       # count stuff
+       numSuccess = len(success)
+       numFail = len(fail)
+       numPending = len(pending)
+
+       str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m  " % (getIntAsString(numPending), getIntAsString(numSuccess), getIntAsString(numFail))
+       sys.stdout.write("|%-19s" % str)
+
+       prevList = success
+       print >> DATFILE, stage, numSuccess, numFail, numPending
+
+    sys.stdout.write("|\n")
     plotMe(batchType, tempFilename)
     DATFILE.close()
-    #os.remove(tempFilename)
 
-    unprocessedIDs = list(set(allIDs) - set(processedIDs))
-    if len(unprocessedIDs) > 0:
-        UNPROCESSEDFILE = open("unprocessed" + DVOLABEL + "_" + batchType,'w')
-        for id in unprocessedIDs: print >> UNPROCESSEDFILE, id
-        UNPROCESSEDFILE.close()
+'''
+Writes table separator
+'''
+def writetableSeparator(colCount):
+
+    sys.stdout.write("+----+-------")
+    for i in range(0, colCount): sys.stdout.write("+-------------------")
+    sys.stdout.write("+\n")
 
 '''
@@ -103 +99 @@
     print "** Usage: " + sys.argv[0] + " <configPath>"
     sys.exit(1)
+
+now = datetime.datetime.now()
 
 # open config file
@@ -118 +116 @@
 
 logger.infoTitle("ippToPsps loading summary")
+logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
 logger.infoPair("Loading epoch", EPOCH)
 logger.infoPair("DVO label", DVOLABEL)
+logger.infoPair("P2 batches per hour", "%.1f" % ippToPspsDb.getBatchesPerHour("P2", EPOCH, DVOLABEL, 2))
+logger.infoPair("ST batches per hour", "%.1f" % ippToPspsDb.getBatchesPerHour("ST", EPOCH, DVOLABEL, 2))
+logger.infoPair("Last P2 batch published", ippToPspsDb.getTimeOfLastBatchPublished("P2", EPOCH, DVOLABEL))
+logger.infoPair("Last ST batch published", ippToPspsDb.getTimeOfLastBatchPublished("ST", EPOCH, DVOLABEL))
 logger.info("")
-logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+")
-logger.info("| Type |  Total in DVO  |   Processed    |  On datastore  | Loaded to ODM  |  Merge-worthy  |     Merged     |  Total missing |    Last published    |")
-logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+")
-printTableRow("P2")
-printTableRow("ST")
-logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+")
 
+stages = ippToPspsDb.getStages()
+colCount = len(stages)
+writetableSeparator(colCount)
+sys.stdout.write("|Type|  DVO  ")
+for stage in stages: sys.stdout.write("|%19s" % stage)
+sys.stdout.write("|\n")
+sys.stdout.write("|    |       ")
+for stage in stages: sys.stdout.write("| Pend  Succ  Fail  ")
+sys.stdout.write("|\n")
+writetableSeparator(colCount)
+printTableRow(stages, "P2")
+printTableRow(stages, "ST")
+writetableSeparator(colCount)
 
-