Changeset 33415 for branches/meh_branches/ppstack_test/ippToPsps/jython/metrics.py

Timestamp:

Mar 5, 2012, 5:19:48 PM (14 years ago)

Author:

mhuber

Message:

merging latest r33407 trunk changes to meh_branches/ppstack_test

Location:

branches/meh_branches/ppstack_test

Files:

• . (modified) (1 prop)
• ippToPsps/jython/metrics.py (modified) (1 diff)

branches/meh_branches/ppstack_test/ippToPsps/jython/metrics.py

@@ -1 +1 @@
 import math
+import time
 import logging
 import smtplib
 import sys
 import os
+import datetime
 
-from java.lang import Math
-from org.apache.commons.math.special import Erf
-from xml.etree.ElementTree import ElementTree, Element, tostring
-
-from pslogger import PSLogger
-from ipptopspsdb import IppToPspsDb
+from ipptopsps import IppToPsps
+from czardb import CzarDb
 from gpc1db import Gpc1Db
 
 '''
-Makes a gnuplot of the data
+Metrics class
 '''
-def plotMe(batchType, file):
+class Metrics(IppToPsps):
 
-    OUTPUTFILE = "plots/" + DVOLABEL + "_" + batchType + ".png"
-    f=os.popen('gnuplot', 'w')
+    '''
+    Constructor
+    '''
+    def __init__(self, argv):
+        super(Metrics, self).__init__(argv)
 
-    if 0:
-        TERM = "X11"
-    else:
-        TERM = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8"
+        if len(sys.argv) > 2: self.parsePollTimeArg(sys.argv[2])
+    
+        # create database objects
+        self.czarDb = CzarDb(self.logger, self.config)
+        self.gpc1Db = Gpc1Db(self.logger, self.config)
+   
+    '''
+    Run...
+    '''
+    def run(self):
+
+        while True:
+    
+            self.logger.infoTitle("ippToPsps loading summary")
+            now = datetime.datetime.now()
+            self.logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
+            self.logger.infoPair("Loading epoch", self.config.epoch)
+            self.logger.infoPair("DVO label", self.config.dvoLabel)
+            self.config.printBoxCoords()
         
-    print >> f, "set term " + TERM + "; \
-              set output \"" + OUTPUTFILE + "\"; \
-              set title \"ippToPsps : " + batchType + " loading for " + DVOLABEL + "\"; \
-              set boxwidth; \
-              set xtic rotate by -90 scale 0; \
-              set style data histogram; \
-              set style histogram rowstacked; \
-              set style fill solid border -1; \
-              set ylabel \"Items processed per loading stage\"; \
-              set boxwidth 0.75; \
-              plot '" + file + "' using 2:xtic(1) notitle lt 2, '' using 3 notitle lt 1, '' using 4 notitle lt rgb \"#808080\""
+            self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
+            self.logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |")
+            self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
+            rateP2 = self.printStats("P2")
+            rateST = self.printStats("ST")
+            self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
+        
+            self.logger.info("")
+        
+            stages = self.ippToPspsDb.getStages()
+            colCount = len(stages)
+            self.writeTableSeparator(colCount)
+            sys.stdout.write("|Type|  DVO  ")
+        
+            for stage in stages: sys.stdout.write("|%19s" % stage)
+            sys.stdout.write("|\n")
+            sys.stdout.write("|    |       ")
+            for stage in stages: sys.stdout.write("| Pend  Succ  Fail  ")
+            sys.stdout.write("|\n")
+            self.writeTableSeparator(colCount)
+            pendP2Processed = self.printTableRow(stages, "P2")
+            pendSTProcessed = self.printTableRow(stages, "ST")
+            self.writeTableSeparator(colCount)
 
-    f.flush()
+            self.logger.info("")
+            try: self.logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2)) 
+            except: pass
+            try: self.logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST)) 
+            except: pass
+       
+            self.checkClientStatus()
+            if not self.waitForPollTime(): break
 
-'''
-Prepares a string of the format:
 
-n-processed(n-pending)
+    '''
+    Formats an int to a string
+    '''
+    def getIntAsString(self, value):
+        if value == 0: return ""
+        return str(value)
+    
+    '''
+    Prints (to the log) one row of data (one row being one batch-type)
+    '''
+    def printTableRow(self, stages, batchType):
+    
+        # get a master list of IDs in DVO for this batch type
+        #allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType, self.config.minRa, self.config.maxRa, self.config.minDec, self.config.maxDec)
+        allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType)
+        prevList = allIDs
+        numEverything = len(allIDs)
+    
+        sys.stdout.write("| %2s |%6d " % (batchType, numEverything))
+    
+        tempFilename = batchType + '.dat'
+        DATFILE = open(tempFilename,'w')
+        print >> DATFILE, "DVO", numEverything, 0, 0
+        for stage in stages:
+    
+           # get lists. Use unions with prev list to make sure the right stuff is included
+           success = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, 1)) & set(prevList)) 
+           fail = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, -1)) & set(prevList) - set(success))
+           pending = list(set(prevList) - set(success) - set(fail))
+    
+           # count stuff
+           numSuccess = len(success)
+           numFail = len(fail)
+           numPending = len(pending)
+    
+           #if stage == 'loaded_to_datastore' and batchType == "P2":
+           #    for n in fail: print n
+    
+           str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m  " % (self.getIntAsString(numPending), self.getIntAsString(numSuccess), self.getIntAsString(numFail))
+           sys.stdout.write("|%-19s" % str)
+    
+           prevList = success
+           print >> DATFILE, stage, numSuccess, numFail, numPending
+    
+           self.czarDb.insertStats(stage, self.config.dvoLabel, batchType, numPending, numSuccess, numFail)
+    
+           if stage == 'processed': numPendingProcessed = numPending
+    
+        sys.stdout.write("|\n")
+        DATFILE.close()
+    
+        return numPendingProcessed
+    
+    '''
+    Writes table separator
+    '''
+    def writeTableSeparator(self, colCount):
+    
+        sys.stdout.write("+----+-------")
+        for i in range(0, colCount): sys.stdout.write("+-------------------")
+        sys.stdout.write("+\n")
+    
+    '''
+    Prints various timing stats for this batch type
+    '''
+    def printStats(self, batchType):
+    
+        rate = self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "1 HOUR") / 1.0 
+    
+        self.logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" % 
+                (batchType,
+                 rate,
+                 self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "24 HOUR"),
+                 self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "1 WEEK") / 7.0, 
+                 self.ippToPspsDb.getTotalDetectionsPublished(batchType, self.config.epoch, self.config.dvoLabel),
+                 self.ippToPspsDb.getTimeOfLastBatchPublished(batchType, self.config.epoch, self.config.dvoLabel)))
+    
+        return rate
+   
 
-'''
-def prepareStr(masterList, prevList, thisList, FILE, name):
+    '''
+    Overrides base-class version
+    '''
+    def printUsage(self):
+        super(Metrics, self).printUsage()
+        print " [hours]"
 
-    unsuccessful = list(set(prevList) - set(thisList))
 
-    countTotal = len(masterList)
-    countSuccessful = len(thisList)
-    countUnsuccessful = len(unsuccessful)
-
-    str = "%d" % countSuccessful
-    if countUnsuccessful: str = str + "(%d)" % countUnsuccessful
-
-    print >> FILE, name, countSuccessful, countUnsuccessful, (countTotal - countSuccessful - countUnsuccessful)
-    #;sys.stdout.softspace=0; 
-    return str
-
-'''
-Prints (to the log) one row of data (one row being one batch-type)
-'''
-def printTableRow(batchType):
-
-    allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType)
-    processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOLABEL)
-    datastoreIDs = ippToPspsDb.getloadedToDatastoreIDs(batchType, EPOCH, DVOLABEL)
-    loadedToODMIDs = ippToPspsDb.getLoadedToODMIDs(batchType, EPOCH, DVOLABEL)
-    mergeWorthyIDs = ippToPspsDb.getMergeWothyIDs(batchType, EPOCH, DVOLABEL)
-    mergedIDs = ippToPspsDb.getMergedIDs(batchType, EPOCH, DVOLABEL)
-
-    tempFilename = batchType + '.dat'
-    DATFILE = open(tempFilename,'w')
-    prepareStr(allIDs, allIDs, allIDs, DATFILE, "inDVO"),
-
-    logger.info("|  %2s  |  %-12d  |  %-12s  |  %-12s  |  %-12s  |  %-12s  |  %-12s  |  %-12s  |  %-19s |" % (
-                batchType, 
-                len(allIDs), 
-                prepareStr(allIDs, allIDs, processedIDs, DATFILE, "processed"),
-                prepareStr(allIDs, processedIDs, datastoreIDs, DATFILE, "onDatastore"),
-                prepareStr(allIDs, datastoreIDs, loadedToODMIDs, DATFILE, "loadedToODM"),
-                prepareStr(allIDs, loadedToODMIDs, mergeWorthyIDs, DATFILE, "MergeWorthy"),
-                prepareStr(allIDs, mergeWorthyIDs, mergedIDs, DATFILE, "Merged"),
-                len(list(set(allIDs) - set(mergedIDs))),
-                ippToPspsDb.getTimeOfLastBatchPublished(batchType, EPOCH, DVOLABEL)))
-
-    plotMe(batchType, tempFilename)
-    DATFILE.close()
-    #os.remove(tempFilename)
 
 '''
 Start of program.
 '''
-if len(sys.argv) > 1: CONFIG  = sys.argv[1]
-else:
-    print "** Usage: " + sys.argv[0] + " <configPath>"
-    sys.exit(1)
-
-# open config file
-configDoc = ElementTree(file=CONFIG)
-
-logging.setLoggerClass(PSLogger)
-logger = logging.getLogger("metrics")
-logger.setup(configDoc, "metrics")
-
-ippToPspsDb = IppToPspsDb(logger, configDoc)
-gpc1Db = Gpc1Db(logger, configDoc)
-
-DVOLABEL = configDoc.find("dvo/gpc1Label").text
-EPOCH = configDoc.find("options/epoch").text
-
-logger.infoTitle("ippToPsps loading summary")
-logger.infoPair("Loading epoch", EPOCH)
-logger.infoPair("DVO label", DVOLABEL)
-logger.info("")
-logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+")
-logger.info("| Type |  Total in DVO  |   Processed    |  On datastore  | Loaded to ODM  |  Merge-worthy  |     Merged     |  Total missing |    Last published    |")
-logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+")
-printTableRow("P2")
-printTableRow("ST")
-logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+")
+metrics = Metrics(sys.argv)
+metrics.run()
+metrics.exitProgram("finished")