Index: trunk/ippToPsps/jython/metrics.py =================================================================== --- trunk/ippToPsps/jython/metrics.py (revision 33289) +++ trunk/ippToPsps/jython/metrics.py (revision 33351) @@ -7,183 +7,169 @@ import datetime -from xml.etree.ElementTree import ElementTree, Element, tostring - -from config import Config -from pslogger import PSLogger -from ipptopspsdb import IppToPspsDb +from ipptopsps import IppToPsps from czardb import CzarDb from gpc1db import Gpc1Db ''' -Makes a gnuplot of the data +Metrics class ''' -def plotMe(batchType, file): +class Metrics(IppToPsps): - OUTPUTFILE = "plots/" + config.dvoLabel + "_" + batchType + ".png" - f=os.popen('gnuplot', 'w') + ''' + Constructor + ''' + def __init__(self, argv): + super(Metrics, self).__init__(argv) - if 0: - TERM = "X11" - else: - TERM = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" + if len(sys.argv) > 2: self.parsePollTimeArg(sys.argv[2]) + + # create database objects + self.czarDb = CzarDb(self.logger, self.config) + self.gpc1Db = Gpc1Db(self.logger, self.config) + + ''' + Run... + ''' + def run(self): + + while True: + + self.logger.infoTitle("ippToPsps loading summary") + now = datetime.datetime.now() + self.logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S")) + self.logger.infoPair("Loading epoch", self.config.epoch) + self.logger.infoPair("DVO label", self.config.dvoLabel) + self.config.printBoxCoords() - print >> f, "set term " + TERM + "; \ - set output \"" + OUTPUTFILE + "\"; \ - set title \"ippToPsps : " + batchType + " loading for " + config.dvoLabel + "\"; \ - set boxwidth; \ - set xtic rotate by -90 scale 0; \ - set style data histogram; \ - set style histogram rowstacked; \ - set style fill solid border -1; \ - set ylabel \"Items processed per loading stage\"; \ - set boxwidth 0.75; \ - plot '" + file + "' using 2:xtic(1) notitle lt 2, '' using 3 notitle lt 1, '' using 4 notitle lt rgb \"#808080\"" + self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + self.logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |") + self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + rateP2 = self.printStats("P2") + rateST = self.printStats("ST") + self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + + self.logger.info("") + + stages = self.ippToPspsDb.getStages() + colCount = len(stages) + self.writeTableSeparator(colCount) + sys.stdout.write("|Type| DVO ") + + for stage in stages: sys.stdout.write("|%19s" % stage) + sys.stdout.write("|\n") + sys.stdout.write("| | ") + for stage in stages: sys.stdout.write("| Pend Succ Fail ") + sys.stdout.write("|\n") + self.writeTableSeparator(colCount) + pendP2Processed = self.printTableRow(stages, "P2") + pendSTProcessed = self.printTableRow(stages, "ST") + self.writeTableSeparator(colCount) - f.flush() + self.logger.info("") + try: self.logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2)) + except: pass + try: self.logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST)) + except: pass + + self.checkClientStatus() + if not self.waitForPollTime(): break -''' -Formats an int to a string -''' -def getIntAsString(value): - if value == 0: return "" - return str(value) -''' -Prints (to the log) one row of data (one row being one batch-type) -''' -def printTableRow(stages, batchType): + ''' + Formats an int to a string + ''' + def getIntAsString(self, value): + if value == 0: return "" + return str(value) + + ''' + Prints (to the log) one row of data (one row being one batch-type) + ''' + def printTableRow(self, stages, batchType): + + # get a master list of IDs in DVO for this batch type + #allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType, self.config.minRa, self.config.maxRa, self.config.minDec, self.config.maxDec) + allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType) + prevList = allIDs + numEverything = len(allIDs) + + sys.stdout.write("| %2s |%6d " % (batchType, numEverything)) + + tempFilename = batchType + '.dat' + DATFILE = open(tempFilename,'w') + print >> DATFILE, "DVO", numEverything, 0, 0 + for stage in stages: + + # get lists. Use unions with prev list to make sure the right stuff is included + success = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, 1)) & set(prevList)) + fail = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, -1)) & set(prevList) - set(success)) + pending = list(set(prevList) - set(success) - set(fail)) + + # count stuff + numSuccess = len(success) + numFail = len(fail) + numPending = len(pending) + + #if stage == 'loaded_to_datastore' and batchType == "P2": + # for n in fail: print n + + str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m " % (self.getIntAsString(numPending), self.getIntAsString(numSuccess), self.getIntAsString(numFail)) + sys.stdout.write("|%-19s" % str) + + prevList = success + print >> DATFILE, stage, numSuccess, numFail, numPending + + self.czarDb.insertStats(stage, self.config.dvoLabel, batchType, numPending, numSuccess, numFail) + + if stage == 'processed': numPendingProcessed = numPending + + sys.stdout.write("|\n") + DATFILE.close() + + return numPendingProcessed + + ''' + Writes table separator + ''' + def writeTableSeparator(self, colCount): + + sys.stdout.write("+----+-------") + for i in range(0, colCount): sys.stdout.write("+-------------------") + sys.stdout.write("+\n") + + ''' + Prints various timing stats for this batch type + ''' + def printStats(self, batchType): + + rate = self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "1 HOUR") / 1.0 + + self.logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" % + (batchType, + rate, + self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "24 HOUR"), + self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "1 WEEK") / 7.0, + self.ippToPspsDb.getTotalDetectionsPublished(batchType, self.config.epoch, self.config.dvoLabel), + self.ippToPspsDb.getTimeOfLastBatchPublished(batchType, self.config.epoch, self.config.dvoLabel))) + + return rate + - # get a master list of IDs in DVO for this batch type - #allIDs = gpc1Db.getIDsInThisDVODbForThisStage(config.dvoLabel, batchType, config.minRa, config.maxRa, config.minDec, config.maxDec) - allIDs = gpc1Db.getIDsInThisDVODbForThisStage(config.dvoLabel, batchType) - prevList = allIDs - numEverything = len(allIDs) + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Metrics, self).printUsage() + print " [hours]" - sys.stdout.write("| %2s |%6d " % (batchType, numEverything)) - tempFilename = batchType + '.dat' - DATFILE = open(tempFilename,'w') - print >> DATFILE, "DVO", numEverything, 0, 0 - for stage in stages: - - # get lists. Use unions with prev list to make sure the right stuff is included - success = list(set(ippToPspsDb.getStageIDs(batchType, config.epoch, config.dvoLabel, stage, 1)) & set(prevList)) - fail = list(set(ippToPspsDb.getStageIDs(batchType, config.epoch, config.dvoLabel, stage, -1)) & set(prevList) - set(success)) - pending = list(set(prevList) - set(success) - set(fail)) - - # count stuff - numSuccess = len(success) - numFail = len(fail) - numPending = len(pending) - - #if stage == 'loaded_to_datastore' and batchType == "P2": - # for n in fail: print n - - str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m " % (getIntAsString(numPending), getIntAsString(numSuccess), getIntAsString(numFail)) - sys.stdout.write("|%-19s" % str) - - prevList = success - print >> DATFILE, stage, numSuccess, numFail, numPending - - czarDb.insertStats(stage, config.dvoLabel, batchType, numPending, numSuccess, numFail) - - if stage == 'processed': numPendingProcessed = numPending - - sys.stdout.write("|\n") - #plotMe(batchType, tempFilename) - DATFILE.close() - - return numPendingProcessed - -''' -Writes table separator -''' -def writetableSeparator(colCount): - - sys.stdout.write("+----+-------") - for i in range(0, colCount): sys.stdout.write("+-------------------") - sys.stdout.write("+\n") - -''' -Prints various timing stats for this batch type -''' -def printStats(batchType): - - rate = ippToPspsDb.countBatchesInLastPeriod(batchType, config.epoch, config.dvoLabel, "1 HOUR") / 1.0 - - logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" % - (batchType, - rate, - ippToPspsDb.countBatchesInLastPeriod(batchType, config.epoch, config.dvoLabel, "24 HOUR"), - ippToPspsDb.countBatchesInLastPeriod(batchType, config.epoch, config.dvoLabel, "1 WEEK") / 7.0, - ippToPspsDb.getTotalDetectionsPublished(batchType, config.epoch, config.dvoLabel), - ippToPspsDb.getTimeOfLastBatchPublished(batchType, config.epoch, config.dvoLabel))) - - return rate ''' Start of program. ''' -if len(sys.argv) > 1: CONFIGPATH = sys.argv[1] -else: - print "** Usage: " + sys.argv[0] + " [hours]" - sys.exit(1) +metrics = Metrics(sys.argv) +metrics.run() +metrics.exitProgram("finished") -if len(sys.argv) > 2: - HOURS = float(sys.argv[2]) - SECONDS = HOURS * 60.0 * 60.0 -else: - SECONDS = None -# create objects -config = Config(CONFIGPATH) -logger = config.getLogger(sys.argv[0]) -ippToPspsDb = IppToPspsDb(logger, config) -czarDb = CzarDb(logger, config) -gpc1Db = Gpc1Db(logger, config) -while True: - - logger.infoTitle("ippToPsps loading summary") - now = datetime.datetime.now() - logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S")) - logger.infoPair("Loading epoch", config.epoch) - logger.infoPair("DVO label", config.dvoLabel) - config.printBoxCoords() - - logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") - logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |") - logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") - rateP2 = printStats("P2") - rateST = printStats("ST") - logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") - - logger.info("") - - stages = ippToPspsDb.getStages() - colCount = len(stages) - writetableSeparator(colCount) - sys.stdout.write("|Type| DVO ") - - for stage in stages: sys.stdout.write("|%19s" % stage) - sys.stdout.write("|\n") - sys.stdout.write("| | ") - for stage in stages: sys.stdout.write("| Pend Succ Fail ") - sys.stdout.write("|\n") - writetableSeparator(colCount) - pendP2Processed = printTableRow(stages, "P2") - pendSTProcessed = printTableRow(stages, "ST") - writetableSeparator(colCount) - - logger.info("") - try: logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2)) - except: pass - try: logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST)) - except: pass - - if SECONDS: - logger.infoPair("Waiting for", "%f hours" % HOURS) - time.sleep(SECONDS) - else: break - config.refresh() -