IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Mar 30, 2012, 2:49:37 PM (14 years ago)
Author:
eugene
Message:

merge changes from trunk

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20111122/ippToPsps/jython/metrics.py

    r32852 r33638  
    77import datetime
    88
    9 from xml.etree.ElementTree import ElementTree, Element, tostring
    10 
    11 from pslogger import PSLogger
    12 from ipptopspsdb import IppToPspsDb
     9from ipptopsps import IppToPsps
    1310from czardb import CzarDb
    1411from gpc1db import Gpc1Db
    15 
    16 '''
    17 Makes a gnuplot of the data
    18 '''
    19 def plotMe(batchType, file):
    20 
    21     OUTPUTFILE = "plots/" + DVOLABEL + "_" + batchType + ".png"
    22     f=os.popen('gnuplot', 'w')
    23 
    24     if 0:
    25         TERM = "X11"
    26     else:
    27         TERM = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8"
    28        
    29     print >> f, "set term " + TERM + "; \
    30               set output \"" + OUTPUTFILE + "\"; \
    31               set title \"ippToPsps : " + batchType + " loading for " + DVOLABEL + "\"; \
    32               set boxwidth; \
    33               set xtic rotate by -90 scale 0; \
    34               set style data histogram; \
    35               set style histogram rowstacked; \
    36               set style fill solid border -1; \
    37               set ylabel \"Items processed per loading stage\"; \
    38               set boxwidth 0.75; \
    39               plot '" + file + "' using 2:xtic(1) notitle lt 2, '' using 3 notitle lt 1, '' using 4 notitle lt rgb \"#808080\""
    40 
    41     f.flush()
    42 
    43 '''
    44 Formats an int to a string
    45 '''
    46 def getIntAsString(value):
    47     if value == 0: return ""
    48     return str(value)
    49 
    50 '''
    51 Prints (to the log) one row of data (one row being one batch-type)
    52 '''
    53 def printTableRow(stages, batchType):
    54 
    55     # get a master list of IDs in DVO for this batch type
    56     if MINRA and MAXRA and MINDEC and MAXDEC:
    57         allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType, MINRA, MAXRA, MINDEC, MAXDEC)
    58     else:
    59         allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType)
    60     prevList = allIDs
    61     numEverything = len(allIDs)
    62 
    63     sys.stdout.write("| %2s |%6d " % (batchType, numEverything))
    64 
    65     tempFilename = batchType + '.dat'
    66     DATFILE = open(tempFilename,'w')
    67     print >> DATFILE, "DVO", numEverything, 0, 0
    68     for stage in stages:
    69 
    70        # get lists. Use unions with prev list to make sure the right stuff is included
    71        success = list(set(ippToPspsDb.getStageIDs(batchType, EPOCH, DVOLABEL, stage, 1)) & set(prevList))
    72        fail = list(set(ippToPspsDb.getStageIDs(batchType, EPOCH, DVOLABEL, stage, -1)) & set(prevList) - set(success))
    73        pending = list(set(prevList) - set(success) - set(fail))
    74 
    75        # count stuff
    76        numSuccess = len(success)
    77        numFail = len(fail)
    78        numPending = len(pending)
    79 
    80        #if stage == 'loaded_to_datastore' and batchType == "P2":
    81        #    for n in fail: print n
    82 
    83        str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m  " % (getIntAsString(numPending), getIntAsString(numSuccess), getIntAsString(numFail))
    84        sys.stdout.write("|%-19s" % str)
    85 
    86        prevList = success
    87        print >> DATFILE, stage, numSuccess, numFail, numPending
    88 
    89        czarDb.insertStats(stage, DVOLABEL, batchType, numPending, numSuccess, numFail)
    90 
    91        if stage == 'processed': numPendingProcessed = numPending
    92 
    93     sys.stdout.write("|\n")
    94     plotMe(batchType, tempFilename)
    95     DATFILE.close()
    96 
    97     return numPendingProcessed
    98 
    99 '''
    100 Writes table separator
    101 '''
    102 def writetableSeparator(colCount):
    103 
    104     sys.stdout.write("+----+-------")
    105     for i in range(0, colCount): sys.stdout.write("+-------------------")
    106     sys.stdout.write("+\n")
    107 
    108 '''
    109 Prints various timing stats for this batch type
    110 '''
    111 def printStats(batchType):
    112 
    113     rate = ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "1 HOUR") / 1.0
    114 
    115     logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" %
    116             (batchType,
    117              rate,
    118              ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "24 HOUR"),
    119              ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "1 WEEK") / 7.0,
    120              ippToPspsDb.getTotalDetectionsPublished(batchType, EPOCH, DVOLABEL),
    121              ippToPspsDb.getTimeOfLastBatchPublished(batchType, EPOCH, DVOLABEL)))
    122 
    123     return rate
     12from plot import Plot
     13
     14'''
     15Metrics class
     16'''
     17class Metrics(IppToPsps):
     18
     19    '''
     20    Constructor
     21    '''
     22    def __init__(self, argv):
     23        super(Metrics, self).__init__(argv)
     24
     25        if len(sys.argv) > 2: self.parsePollTimeArg(sys.argv[2])
     26   
     27        # create database objects
     28        self.czarDb = CzarDb(self.logger, self.config)
     29        self.gpc1Db = Gpc1Db(self.logger, self.config)
     30        self.plot = Plot(self.logger, self.config, self.ippToPspsDb)
     31   
     32    '''
     33    Run...
     34    '''
     35    def run(self):
     36
     37        while True:
     38   
     39            self.logger.infoTitle("ippToPsps loading summary")
     40            now = datetime.datetime.now()
     41            self.logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
     42            self.logger.infoPair("Loading epoch", self.config.epoch)
     43            self.logger.infoPair("DVO label", self.config.dvoLabel)
     44            self.config.printBoxCoords()
     45       
     46            self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
     47            self.logger.info("|Type| batches per hour | last 24 hours | per day this week | detections/batch | last published |")
     48            self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
     49            rateP2 = self.printStats("P2")
     50            rateST = self.printStats("ST")
     51            self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
     52       
     53            self.logger.info("")
     54       
     55            stages = self.ippToPspsDb.getStages()
     56            colCount = len(stages)
     57            self.writeTableSeparator(colCount)
     58            sys.stdout.write("|Type|  DVO  ")
     59       
     60            for stage in stages: sys.stdout.write("|%19s" % stage)
     61            sys.stdout.write("|\n")
     62            sys.stdout.write("|    |       ")
     63            for stage in stages: sys.stdout.write("| Pend  Succ  Fail  ")
     64            sys.stdout.write("|\n")
     65            self.writeTableSeparator(colCount)
     66            pendP2Processed = self.printTableRow(stages, "P2")
     67            pendSTProcessed = self.printTableRow(stages, "ST")
     68            self.writeTableSeparator(colCount)
     69
     70            self.logger.info("")
     71            try: self.logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2))
     72            except: pass
     73            try: self.logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST))
     74            except: pass
     75       
     76            self.checkClientStatus()
     77
     78            # create a density plot of pending stuff for czartool
     79            self.plot.createDensityPlot("P2", True)
     80            self.plot.createDensityPlot("ST", True)
     81
     82            if not self.waitForPollTime(): break
     83
     84
     85    '''
     86    Formats an int to a string
     87    '''
     88    def getIntAsString(self, value):
     89        if value == 0: return ""
     90        return str(value)
     91   
     92    '''
     93    Prints (to the log) one row of data (one row being one batch-type)
     94    '''
     95    def printTableRow(self, stages, batchType):
     96   
     97        # get a master list of IDs in DVO for this batch type
     98        rows = self.gpc1Db.getItemsInThisDVODbForThisStage(self.config.dvoLabel,
     99                batchType,
     100                self.config.minRa,
     101                self.config.maxRa,
     102                self.config.minDec,
     103                self.config.maxDec)
     104
     105        # we only want to IDs
     106        allIDs = []
     107        for row in rows:
     108            try: allIDs.append(row[0])
     109            except: pass
     110
     111        #allIDs = self.ippToPspsDb.getPendingIdsForThisConfig(self.config.name)
     112
     113        prevList = allIDs
     114        numEverything = len(allIDs)
     115   
     116        sys.stdout.write("| %2s |%6d " % (batchType, numEverything))
     117   
     118        tempFilename = batchType + '.dat'
     119        for stage in stages:
     120   
     121           # get lists. Use unions with prev list to make sure the right stuff is included
     122           success = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, 1)) & set(prevList))
     123           fail = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, -1)) & set(prevList) - set(success))
     124           pending = list(set(prevList) - set(success) - set(fail))
     125   
     126           # count stuff
     127           numSuccess = len(success)
     128           numFail = len(fail)
     129           numPending = len(pending)
     130   
     131           #if stage == 'loaded_to_datastore' and batchType == "P2":
     132           #    for n in fail: print n
     133   
     134           str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m  " % (
     135                   self.getIntAsString(numPending),
     136                   self.getIntAsString(numSuccess),
     137                   self.getIntAsString(numFail))
     138           sys.stdout.write("|%-19s" % str)
     139   
     140           prevList = success
     141   
     142           self.czarDb.insertStats(stage, self.config.dvoLabel, batchType, numPending, numSuccess, numFail)
     143   
     144           if stage == 'processed': numPendingProcessed = numPending
     145   
     146        sys.stdout.write("|\n")
     147   
     148        return numPendingProcessed
     149   
     150    '''
     151    Writes table separator
     152    '''
     153    def writeTableSeparator(self, colCount):
     154   
     155        sys.stdout.write("+----+-------")
     156        for i in range(0, colCount): sys.stdout.write("+-------------------")
     157        sys.stdout.write("+\n")
     158   
     159    '''
     160    Safe divide function
     161    '''
     162    def divide(self, num, denom):
     163        if denom == 0: return 0
     164        else: return num / denom
     165
     166    '''
     167    Prints various timing stats for this batch type
     168    '''
     169    def printStats(self, batchType):
     170   
     171        rate = self.ippToPspsDb.countBatches(batchType, self.config.epoch, self.config.dvoLabel, "1 HOUR") / 1.0
     172   
     173
     174        batchesInLast24Hours = self.ippToPspsDb.countBatches(batchType, self.config.epoch, self.config.dvoLabel, "24 HOUR")
     175        detectionsInLast24Hours = self.ippToPspsDb.getTotalDetectionsPublished(batchType, self.config.epoch, self.config.dvoLabel, "24 HOUR")
     176
     177        self.logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" %
     178                (batchType,
     179                 rate,
     180                 batchesInLast24Hours,
     181                 self.ippToPspsDb.countBatches(batchType, self.config.epoch, self.config.dvoLabel, "1 WEEK") / 7.0,
     182                 self.divide(detectionsInLast24Hours, batchesInLast24Hours),
     183                 self.ippToPspsDb.getTimeOfLastBatchPublished(batchType, self.config.epoch, self.config.dvoLabel)))
     184   
     185        return rate
     186   
     187
     188    '''
     189    Overrides base-class version
     190    '''
     191    def printUsage(self):
     192        super(Metrics, self).printUsage("[<hours>]")
     193
     194
    124195
    125196'''
    126197Start of program.
    127198'''
    128 if len(sys.argv) > 1: CONFIG  = sys.argv[1]
    129 else:
    130     print "** Usage: " + sys.argv[0] + " <configPath> [hours]"
    131     sys.exit(1)
    132 
    133 if len(sys.argv) > 2:
    134     HOURS = float(sys.argv[2])
    135     SECONDS = HOURS * 60.0 * 60.0
    136 else:
    137     SECONDS = None
    138 
    139 
    140 
    141 # open config file
    142 configDoc = ElementTree(file=CONFIG)
    143 
    144 logging.setLoggerClass(PSLogger)
    145 logger = logging.getLogger("metrics")
    146 logger.setup(configDoc, "metrics")
    147 
    148 # create database objects
    149 ippToPspsDb = IppToPspsDb(logger, configDoc)
    150 czarDb = CzarDb(logger, configDoc)
    151 gpc1Db = Gpc1Db(logger, configDoc)
    152 
    153 DVOLABEL = configDoc.find("dvo/gpc1Label").text
    154 EPOCH = configDoc.find("options/epoch").text
    155 
    156 while True:
    157 
    158     logger.infoTitle("ippToPsps loading summary")
    159     now = datetime.datetime.now()
    160     logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
    161     logger.infoPair("Loading epoch", EPOCH)
    162     logger.infoPair("DVO label", DVOLABEL)
    163 
    164     # get equatorial coord constraints, if any
    165     MINRA = None
    166     MAXRA = None
    167     MINDEC = None
    168     MAXDEC = None
    169 
    170     try:
    171         MINRA = float(configDoc.find("dvo/minRA").text)
    172         MAXRA = float(configDoc.find("dvo/maxRA").text)
    173         MINDEC = float(configDoc.find("dvo/minDec").text)
    174         MAXDEC = float(configDoc.find("dvo/maxDec").text)
    175         logger.infoPair("Coordinates", "%.1f to %.1f RA, %.1f to %.1f Dec" % (MINRA, MAXRA, MINDEC, MAXDEC))
    176     except:
    177         pass
    178    
    179 
    180     logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
    181     logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |")
    182     logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
    183     rateP2 = printStats("P2")
    184     rateST = printStats("ST")
    185     logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
    186 
    187     logger.info("")
    188 
    189     stages = ippToPspsDb.getStages()
    190     colCount = len(stages)
    191     writetableSeparator(colCount)
    192     sys.stdout.write("|Type|  DVO  ")
    193 
    194     for stage in stages: sys.stdout.write("|%19s" % stage)
    195     sys.stdout.write("|\n")
    196     sys.stdout.write("|    |       ")
    197     for stage in stages: sys.stdout.write("| Pend  Succ  Fail  ")
    198     sys.stdout.write("|\n")
    199     writetableSeparator(colCount)
    200     pendP2Processed = printTableRow(stages, "P2")
    201     pendSTProcessed = printTableRow(stages, "ST")
    202     writetableSeparator(colCount)
    203 
    204     logger.info("")
    205     try: logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2))
    206     except: pass
    207     try: logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST))
    208     except: pass
    209 
    210     if SECONDS:
    211         logger.infoPair("Waiting for", "%f hours" % HOURS)
    212         time.sleep(SECONDS)
    213     else: break
    214 
    215 
     199metrics = Metrics(sys.argv)
     200metrics.run()
     201metrics.exitProgram("finished")
     202
     203
     204
Note: See TracChangeset for help on using the changeset viewer.