Changeset 32092

Timestamp:

Aug 12, 2011, 4:07:06 PM (15 years ago)

Author:

rhenders

Message:

now reporting much more information, specifically, the ODM status (merge-worthy, merged etc); using new logging method from pslogger; printing processed and pending in table

File:

• trunk/ippToPsps/jython/metrics.py (modified) (3 diffs)

trunk/ippToPsps/jython/metrics.py

@@ -12 +12 @@
 from gpc1db import Gpc1Db
 
+'''
+Prepares a string of the format:
 
+n-processed(n-pending)
 
+'''
+def prepareStr(prevList, thisList):
+
+     unsuccessful = list(set(prevList) - set(thisList))
+
+     str = "%d" % len(thisList)
+     if len(unsuccessful) > 0: str = str + "(%d)" % len(unsuccessful)
+
+     return str
+
+'''
+Prints (to the log) one row of data (one row being one batch-type)
+'''
 def printTableRow(batchType):
 
-    allIDs = gpc1Db.getIDsInThisDVODbForThisStage(dvoGpc1Label, batchType)
-    processedIDs = ippToPspsDb.getProcessedIDsForThisStage(batchType, epoch, dvoGpc1Label)
-    failedIDs = ippToPspsDb.getFailedBatches(batchType, epoch, dvoGpc1Label)
-    unprocessedIDs = list(set(allIDs) - set(processedIDs))
+    allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType)
+    processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOLABEL)
+    datastoreIDs = ippToPspsDb.getloadedToDatastoreIDs(batchType, EPOCH, DVOLABEL)
+    loadedToODMIDs = ippToPspsDb.getLoadedToODMIDs(batchType, EPOCH, DVOLABEL)
+    mergeWorthyIDs = ippToPspsDb.getMergeWothyIDs(batchType, EPOCH, DVOLABEL)
+    mergedIDs = ippToPspsDb.getMergedIDs(batchType, EPOCH, DVOLABEL)
 
-    logger.info("|  %2s  |  %7d     |  %7d        |   %7d    | %7d    | %6.1f hours ago |" % (
+    logger.info("|  %2s  |  %-12d  |  %-12s  |  %-12s  |  %-12s  |  %-12s  |  %-12s  |  %-12s  |  %-19s |" % (
                 batchType, 
                 len(allIDs), 
-                len(processedIDs), 
-                len(list(set(failedIDs) - set(processedIDs))), 
-                len(unprocessedIDs), 
-                ippToPspsDb.getLastBatchPublished(batchType)))
+                prepareStr(allIDs, processedIDs),
+                prepareStr(processedIDs, datastoreIDs),
+                prepareStr(datastoreIDs, loadedToODMIDs),
+                prepareStr(loadedToODMIDs, mergeWorthyIDs),
+                prepareStr(mergeWorthyIDs, mergedIDs),
+                len(list(set(allIDs) - set(mergedIDs))),
+                ippToPspsDb.getTimeOfLastBatchPublished(batchType, EPOCH, DVOLABEL)))
 
 
@@ -33 +54 @@
 Start of program.
 '''
-
 if len(sys.argv) > 1: CONFIG  = sys.argv[1]
 else:
@@ -42 +62 @@
 configDoc = ElementTree(file=CONFIG)
 
-
-logging.config.fileConfig("logging.conf")
 logging.setLoggerClass(PSLogger)
-logger = logging.getLogger("ippToPspsLog")
-logger.setLevel(logging.INFO)
+logger = logging.getLogger("metrics")
+logger.setup(configDoc, "metrics")
 
 ippToPspsDb = IppToPspsDb(logger, configDoc)
 gpc1Db = Gpc1Db(logger, configDoc)
 
-epoch = configDoc.find("options/epoch").text
-dvoGpc1Label = configDoc.find("dvo/gpc1Label").text
+DVOLABEL = configDoc.find("dvo/gpc1Label").text
+EPOCH = configDoc.find("options/epoch").text
 
-logger.info("Loading epoch  " + epoch)
-logger.info("+------+--------------+-----------------+--------------+------------+------------------+")
-logger.info("| Type | Total in DVO | Total published | Total failed | Total left |  Last published  |")
-logger.info("+------+--------------+-----------------+--------------+------------+------------------+")
+logger.infoTitle("ippToPsps loading summary")
+logger.infoPair("Loading epoch", EPOCH)
+logger.infoPair("DVO label", DVOLABEL)
+logger.info("")
+logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+")
+logger.info("| Type |  Total in DVO  |   Processed    |  On datastore  | Loaded to ODM  |  Merge-worthy  |     Merged     |  Total missing |    Last published    |")
+logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+")
 printTableRow("P2")
 printTableRow("ST")
-logger.info("+------+--------------+-----------------+--------------+------------+------------------+")
+logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+")
 
 #logger.info("Total batches published      %d" % totalP2Published)
 #logger.info("Total batches left           %d" % (totalP2 - totalP2Published))
-#logger.info("Total failed batches         %d" % ippToPspsDb.getTotalFailedBatches("P2", epoch))
-#logger.info("Average time per batch       %.2f minutes" % ippToPspsDb.getAverageTimePerBatch("P2", epoch))
+#logger.info("Total failed batches         %d" % ippToPspsDb.getTotalFailedBatches("P2", EPOCH))
+#logger.info("Average time per batch       %.2f minutes" % ippToPspsDb.getAverageTimePerBatch("P2", EPOCH))
 #logger.info("Last published               %.2f hours ago" % ippToPspsDb.getLastBatchPublished("P2"))
-#logger.info("Average batches per day      %.2f" % ippToPspsDb.getBatchesPerDay("P2", epoch))
+#logger.info("Average batches per day      %.2f" % ippToPspsDb.getBatchesPerDay("P2", EPOCH))
 #logger.info("-----------------------------------------------------")