IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Feb 23, 2012, 4:14:32 PM (14 years ago)
Author:
rhenders
Message:

now a subclass of IppToPsps

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/jython/metrics.py

    r33289 r33351  
    77import datetime
    88
    9 from xml.etree.ElementTree import ElementTree, Element, tostring
    10 
    11 from config import Config
    12 from pslogger import PSLogger
    13 from ipptopspsdb import IppToPspsDb
     9from ipptopsps import IppToPsps
    1410from czardb import CzarDb
    1511from gpc1db import Gpc1Db
    1612
    1713'''
    18 Makes a gnuplot of the data
     14Metrics class
    1915'''
    20 def plotMe(batchType, file):
     16class Metrics(IppToPsps):
    2117
    22     OUTPUTFILE = "plots/" + config.dvoLabel + "_" + batchType + ".png"
    23     f=os.popen('gnuplot', 'w')
     18    '''
     19    Constructor
     20    '''
     21    def __init__(self, argv):
     22        super(Metrics, self).__init__(argv)
    2423
    25     if 0:
    26         TERM = "X11"
    27     else:
    28         TERM = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8"
     24        if len(sys.argv) > 2: self.parsePollTimeArg(sys.argv[2])
     25   
     26        # create database objects
     27        self.czarDb = CzarDb(self.logger, self.config)
     28        self.gpc1Db = Gpc1Db(self.logger, self.config)
     29   
     30    '''
     31    Run...
     32    '''
     33    def run(self):
     34
     35        while True:
     36   
     37            self.logger.infoTitle("ippToPsps loading summary")
     38            now = datetime.datetime.now()
     39            self.logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
     40            self.logger.infoPair("Loading epoch", self.config.epoch)
     41            self.logger.infoPair("DVO label", self.config.dvoLabel)
     42            self.config.printBoxCoords()
    2943       
    30     print >> f, "set term " + TERM + "; \
    31               set output \"" + OUTPUTFILE + "\"; \
    32               set title \"ippToPsps : " + batchType + " loading for " + config.dvoLabel + "\"; \
    33               set boxwidth; \
    34               set xtic rotate by -90 scale 0; \
    35               set style data histogram; \
    36               set style histogram rowstacked; \
    37               set style fill solid border -1; \
    38               set ylabel \"Items processed per loading stage\"; \
    39               set boxwidth 0.75; \
    40               plot '" + file + "' using 2:xtic(1) notitle lt 2, '' using 3 notitle lt 1, '' using 4 notitle lt rgb \"#808080\""
     44            self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
     45            self.logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |")
     46            self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
     47            rateP2 = self.printStats("P2")
     48            rateST = self.printStats("ST")
     49            self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
     50       
     51            self.logger.info("")
     52       
     53            stages = self.ippToPspsDb.getStages()
     54            colCount = len(stages)
     55            self.writeTableSeparator(colCount)
     56            sys.stdout.write("|Type|  DVO  ")
     57       
     58            for stage in stages: sys.stdout.write("|%19s" % stage)
     59            sys.stdout.write("|\n")
     60            sys.stdout.write("|    |       ")
     61            for stage in stages: sys.stdout.write("| Pend  Succ  Fail  ")
     62            sys.stdout.write("|\n")
     63            self.writeTableSeparator(colCount)
     64            pendP2Processed = self.printTableRow(stages, "P2")
     65            pendSTProcessed = self.printTableRow(stages, "ST")
     66            self.writeTableSeparator(colCount)
    4167
    42     f.flush()
     68            self.logger.info("")
     69            try: self.logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2))
     70            except: pass
     71            try: self.logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST))
     72            except: pass
     73       
     74            self.checkClientStatus()
     75            if not self.waitForPollTime(): break
    4376
    44 '''
    45 Formats an int to a string
    46 '''
    47 def getIntAsString(value):
    48     if value == 0: return ""
    49     return str(value)
    5077
    51 '''
    52 Prints (to the log) one row of data (one row being one batch-type)
    53 '''
    54 def printTableRow(stages, batchType):
     78    '''
     79    Formats an int to a string
     80    '''
     81    def getIntAsString(self, value):
     82        if value == 0: return ""
     83        return str(value)
     84   
     85    '''
     86    Prints (to the log) one row of data (one row being one batch-type)
     87    '''
     88    def printTableRow(self, stages, batchType):
     89   
     90        # get a master list of IDs in DVO for this batch type
     91        #allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType, self.config.minRa, self.config.maxRa, self.config.minDec, self.config.maxDec)
     92        allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType)
     93        prevList = allIDs
     94        numEverything = len(allIDs)
     95   
     96        sys.stdout.write("| %2s |%6d " % (batchType, numEverything))
     97   
     98        tempFilename = batchType + '.dat'
     99        DATFILE = open(tempFilename,'w')
     100        print >> DATFILE, "DVO", numEverything, 0, 0
     101        for stage in stages:
     102   
     103           # get lists. Use unions with prev list to make sure the right stuff is included
     104           success = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, 1)) & set(prevList))
     105           fail = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, -1)) & set(prevList) - set(success))
     106           pending = list(set(prevList) - set(success) - set(fail))
     107   
     108           # count stuff
     109           numSuccess = len(success)
     110           numFail = len(fail)
     111           numPending = len(pending)
     112   
     113           #if stage == 'loaded_to_datastore' and batchType == "P2":
     114           #    for n in fail: print n
     115   
     116           str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m  " % (self.getIntAsString(numPending), self.getIntAsString(numSuccess), self.getIntAsString(numFail))
     117           sys.stdout.write("|%-19s" % str)
     118   
     119           prevList = success
     120           print >> DATFILE, stage, numSuccess, numFail, numPending
     121   
     122           self.czarDb.insertStats(stage, self.config.dvoLabel, batchType, numPending, numSuccess, numFail)
     123   
     124           if stage == 'processed': numPendingProcessed = numPending
     125   
     126        sys.stdout.write("|\n")
     127        DATFILE.close()
     128   
     129        return numPendingProcessed
     130   
     131    '''
     132    Writes table separator
     133    '''
     134    def writeTableSeparator(self, colCount):
     135   
     136        sys.stdout.write("+----+-------")
     137        for i in range(0, colCount): sys.stdout.write("+-------------------")
     138        sys.stdout.write("+\n")
     139   
     140    '''
     141    Prints various timing stats for this batch type
     142    '''
     143    def printStats(self, batchType):
     144   
     145        rate = self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "1 HOUR") / 1.0
     146   
     147        self.logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" %
     148                (batchType,
     149                 rate,
     150                 self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "24 HOUR"),
     151                 self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "1 WEEK") / 7.0,
     152                 self.ippToPspsDb.getTotalDetectionsPublished(batchType, self.config.epoch, self.config.dvoLabel),
     153                 self.ippToPspsDb.getTimeOfLastBatchPublished(batchType, self.config.epoch, self.config.dvoLabel)))
     154   
     155        return rate
     156   
    55157
    56     # get a master list of IDs in DVO for this batch type
    57     #allIDs = gpc1Db.getIDsInThisDVODbForThisStage(config.dvoLabel, batchType, config.minRa, config.maxRa, config.minDec, config.maxDec)
    58     allIDs = gpc1Db.getIDsInThisDVODbForThisStage(config.dvoLabel, batchType)
    59     prevList = allIDs
    60     numEverything = len(allIDs)
     158    '''
     159    Overrides base-class version
     160    '''
     161    def printUsage(self):
     162        super(Metrics, self).printUsage()
     163        print " [hours]"
    61164
    62     sys.stdout.write("| %2s |%6d " % (batchType, numEverything))
    63165
    64     tempFilename = batchType + '.dat'
    65     DATFILE = open(tempFilename,'w')
    66     print >> DATFILE, "DVO", numEverything, 0, 0
    67     for stage in stages:
    68 
    69        # get lists. Use unions with prev list to make sure the right stuff is included
    70        success = list(set(ippToPspsDb.getStageIDs(batchType, config.epoch, config.dvoLabel, stage, 1)) & set(prevList))
    71        fail = list(set(ippToPspsDb.getStageIDs(batchType, config.epoch, config.dvoLabel, stage, -1)) & set(prevList) - set(success))
    72        pending = list(set(prevList) - set(success) - set(fail))
    73 
    74        # count stuff
    75        numSuccess = len(success)
    76        numFail = len(fail)
    77        numPending = len(pending)
    78 
    79        #if stage == 'loaded_to_datastore' and batchType == "P2":
    80        #    for n in fail: print n
    81 
    82        str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m  " % (getIntAsString(numPending), getIntAsString(numSuccess), getIntAsString(numFail))
    83        sys.stdout.write("|%-19s" % str)
    84 
    85        prevList = success
    86        print >> DATFILE, stage, numSuccess, numFail, numPending
    87 
    88        czarDb.insertStats(stage, config.dvoLabel, batchType, numPending, numSuccess, numFail)
    89 
    90        if stage == 'processed': numPendingProcessed = numPending
    91 
    92     sys.stdout.write("|\n")
    93     #plotMe(batchType, tempFilename)
    94     DATFILE.close()
    95 
    96     return numPendingProcessed
    97 
    98 '''
    99 Writes table separator
    100 '''
    101 def writetableSeparator(colCount):
    102 
    103     sys.stdout.write("+----+-------")
    104     for i in range(0, colCount): sys.stdout.write("+-------------------")
    105     sys.stdout.write("+\n")
    106 
    107 '''
    108 Prints various timing stats for this batch type
    109 '''
    110 def printStats(batchType):
    111 
    112     rate = ippToPspsDb.countBatchesInLastPeriod(batchType, config.epoch, config.dvoLabel, "1 HOUR") / 1.0
    113 
    114     logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" %
    115             (batchType,
    116              rate,
    117              ippToPspsDb.countBatchesInLastPeriod(batchType, config.epoch, config.dvoLabel, "24 HOUR"),
    118              ippToPspsDb.countBatchesInLastPeriod(batchType, config.epoch, config.dvoLabel, "1 WEEK") / 7.0,
    119              ippToPspsDb.getTotalDetectionsPublished(batchType, config.epoch, config.dvoLabel),
    120              ippToPspsDb.getTimeOfLastBatchPublished(batchType, config.epoch, config.dvoLabel)))
    121 
    122     return rate
    123166
    124167'''
    125168Start of program.
    126169'''
    127 if len(sys.argv) > 1: CONFIGPATH  = sys.argv[1]
    128 else:
    129     print "** Usage: " + sys.argv[0] + " <configPath> [hours]"
    130     sys.exit(1)
     170metrics = Metrics(sys.argv)
     171metrics.run()
     172metrics.exitProgram("finished")
    131173
    132 if len(sys.argv) > 2:
    133     HOURS = float(sys.argv[2])
    134     SECONDS = HOURS * 60.0 * 60.0
    135 else:
    136     SECONDS = None
    137174
    138 # create objects
    139 config = Config(CONFIGPATH)
    140 logger = config.getLogger(sys.argv[0])
    141 ippToPspsDb = IppToPspsDb(logger, config)
    142 czarDb = CzarDb(logger, config)
    143 gpc1Db = Gpc1Db(logger, config)
    144175
    145 while True:
    146 
    147     logger.infoTitle("ippToPsps loading summary")
    148     now = datetime.datetime.now()
    149     logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
    150     logger.infoPair("Loading epoch", config.epoch)
    151     logger.infoPair("DVO label", config.dvoLabel)
    152     config.printBoxCoords()
    153 
    154     logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
    155     logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |")
    156     logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
    157     rateP2 = printStats("P2")
    158     rateST = printStats("ST")
    159     logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
    160 
    161     logger.info("")
    162 
    163     stages = ippToPspsDb.getStages()
    164     colCount = len(stages)
    165     writetableSeparator(colCount)
    166     sys.stdout.write("|Type|  DVO  ")
    167 
    168     for stage in stages: sys.stdout.write("|%19s" % stage)
    169     sys.stdout.write("|\n")
    170     sys.stdout.write("|    |       ")
    171     for stage in stages: sys.stdout.write("| Pend  Succ  Fail  ")
    172     sys.stdout.write("|\n")
    173     writetableSeparator(colCount)
    174     pendP2Processed = printTableRow(stages, "P2")
    175     pendSTProcessed = printTableRow(stages, "ST")
    176     writetableSeparator(colCount)
    177 
    178     logger.info("")
    179     try: logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2))
    180     except: pass
    181     try: logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST))
    182     except: pass
    183 
    184     if SECONDS:
    185         logger.infoPair("Waiting for", "%f hours" % HOURS)
    186         time.sleep(SECONDS)
    187     else: break
    188     config.refresh()
    189 
Note: See TracChangeset for help on using the changeset viewer.