Changeset 34041 for branches/meh_branches/ppstack_test/ippToPsps/jython/metrics.py

Timestamp:

Jun 19, 2012, 5:24:19 PM (14 years ago)

Author:

mhuber

Message:

merging latest r34040 trunk changes to branch

Location:

branches/meh_branches/ppstack_test

Files:

• . (modified) (1 prop)
• ippToPsps/jython/metrics.py (modified) (13 diffs)

branches/meh_branches/ppstack_test/ippToPsps/jython/metrics.py

@@ -9 +9 @@
 from ipptopsps import IppToPsps
 from czardb import CzarDb
-from gpc1db import Gpc1Db
+from plot import Plot
 
 '''
@@ -24 +24 @@
         if len(sys.argv) > 2: self.parsePollTimeArg(sys.argv[2])
     
-        # create database objects
         self.czarDb = CzarDb(self.logger, self.config)
-        self.gpc1Db = Gpc1Db(self.logger, self.config)
+        self.plot = Plot(self.logger, self.config, self.ippToPspsDb)
    
     '''
@@ -38 +37 @@
             now = datetime.datetime.now()
             self.logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
+            self.logger.infoPair("Config", self.config.name)
             self.logger.infoPair("Loading epoch", self.config.epoch)
             self.logger.infoPair("DVO label", self.config.dvoLabel)
@@ -43 +43 @@
         
             self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
-            self.logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |")
+            self.logger.info("|Type| batches per hour | last 24 hours | per day this week | detections/batch | last published |")
             self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
             rateP2 = self.printStats("P2")
             rateST = self.printStats("ST")
+            rateOB = self.printStats("OB")
             self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
         
@@ -54 +55 @@
             colCount = len(stages)
             self.writeTableSeparator(colCount)
-            sys.stdout.write("|Type|  DVO  ")
+            sys.stdout.write("|Type")
         
             for stage in stages: sys.stdout.write("|%19s" % stage)
             sys.stdout.write("|\n")
-            sys.stdout.write("|    |       ")
+            sys.stdout.write("|    ")
             for stage in stages: sys.stdout.write("| Pend  Succ  Fail  ")
             sys.stdout.write("|\n")
@@ -64 +65 @@
             pendP2Processed = self.printTableRow(stages, "P2")
             pendSTProcessed = self.printTableRow(stages, "ST")
+            pendOBProcessed = self.printTableRow(stages, "OB")
             self.writeTableSeparator(colCount)
 
@@ -71 +73 @@
             try: self.logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST)) 
             except: pass
+            try: self.logger.infoPair("Estimated time for OBs", "%.1f hours" % (pendOBProcessed / rateOB)) 
+            except: pass
        
             self.checkClientStatus()
+
+            # create a density plot of pending stuff for czartool
+            self.plot.createDensityPlot("P2", True)
+            self.plot.createDensityPlot("ST", True)
+            self.plot.createDensityPlot("OB", True)
+
+            # some clean-up before going to sleep
+            self.ippToPspsDb.purgeDeadClients()
             if not self.waitForPollTime(): break
 
@@ -88 +100 @@
     def printTableRow(self, stages, batchType):
     
-        # get a master list of IDs in DVO for this batch type
-        #allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType, self.config.minRa, self.config.maxRa, self.config.minDec, self.config.maxDec)
-        allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType)
-        prevList = allIDs
-        numEverything = len(allIDs)
-    
-        sys.stdout.write("| %2s |%6d " % (batchType, numEverything))
-    
-        tempFilename = batchType + '.dat'
-        DATFILE = open(tempFilename,'w')
-        print >> DATFILE, "DVO", numEverything, 0, 0
+        pending = self.ippToPspsDb.getPendingIds(batchType)
+
+        sys.stdout.write("| %2s " % (batchType))
+   
+        firstStage = True
         for stage in stages:
     
            # get lists. Use unions with prev list to make sure the right stuff is included
-           success = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, 1)) & set(prevList)) 
-           fail = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, -1)) & set(prevList) - set(success))
-           pending = list(set(prevList) - set(success) - set(fail))
+           if firstStage:
+               success = self.ippToPspsDb.getStageIDs(batchType, stage, 1)
+               fail = list(set(self.ippToPspsDb.getStageIDs(batchType, stage, -1)) - set(success))
+           else: 
+               success = list(set(self.ippToPspsDb.getStageIDs(batchType, stage, 1)) & set(prevList)) 
+               fail = list(set(self.ippToPspsDb.getStageIDs(batchType, stage, -1)) & set(prevList) - set(success))
+               pending = list(set(prevList) - set(success) - set(fail))
     
            # count stuff
@@ -114 +124 @@
            #    for n in fail: print n
     
-           str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m  " % (self.getIntAsString(numPending), self.getIntAsString(numSuccess), self.getIntAsString(numFail))
+           str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m  " % (
+                   self.getIntAsString(numPending), 
+                   self.getIntAsString(numSuccess), 
+                   self.getIntAsString(numFail))
            sys.stdout.write("|%-19s" % str)
     
            prevList = success
-           print >> DATFILE, stage, numSuccess, numFail, numPending
-    
-           self.czarDb.insertStats(stage, self.config.dvoLabel, batchType, numPending, numSuccess, numFail)
+    
+           self.czarDb.insertStats(stage, self.config.name, batchType, numPending, numSuccess, numFail)
     
            if stage == 'processed': numPendingProcessed = numPending
     
+           firstStage = False
+
         sys.stdout.write("|\n")
-        DATFILE.close()
     
         return numPendingProcessed
@@ -134 +147 @@
     def writeTableSeparator(self, colCount):
     
-        sys.stdout.write("+----+-------")
+        sys.stdout.write("+----")
         for i in range(0, colCount): sys.stdout.write("+-------------------")
         sys.stdout.write("+\n")
-    
+   
+    '''
+    Safe divide function
+    '''
+    def divide(self, num, denom):
+        if denom == 0: return 0
+        else: return num / denom
+
     '''
     Prints various timing stats for this batch type
@@ -143 +163 @@
     def printStats(self, batchType):
     
-        rate = self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "1 HOUR") / 1.0 
-    
+        rate = self.ippToPspsDb.countBatches(batchType, "1 HOUR") / 1.0 
+   
+
+        batchesInLast24Hours = self.ippToPspsDb.countBatches(batchType, "24 HOUR")
+        detectionsInLast24Hours = self.ippToPspsDb.getTotalDetectionsPublished(batchType, "24 HOUR")
+
         self.logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" % 
                 (batchType,
                  rate,
-                 self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "24 HOUR"),
-                 self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "1 WEEK") / 7.0, 
-                 self.ippToPspsDb.getTotalDetectionsPublished(batchType, self.config.epoch, self.config.dvoLabel),
-                 self.ippToPspsDb.getTimeOfLastBatchPublished(batchType, self.config.epoch, self.config.dvoLabel)))
+                 batchesInLast24Hours,
+                 self.ippToPspsDb.countBatches(batchType, "1 WEEK") / 7.0, 
+                 self.divide(detectionsInLast24Hours, batchesInLast24Hours),
+                 self.ippToPspsDb.getTimeOfLastBatchPublished(batchType)))
     
         return rate
@@ -160 +184 @@
     '''
     def printUsage(self):
-        super(Metrics, self).printUsage()
-        print " [hours]"
+        super(Metrics, self).printUsage("[<hours>]")
 
 
@@ -168 +191 @@
 Start of program.
 '''
-metrics = Metrics(sys.argv)
-metrics.run()
-metrics.exitProgram("finished")
-
-
-
+try:
+    metrics = Metrics(sys.argv)
+    metrics.run()
+    metrics.exitProgram("completed")
+except: pass
+
+
+