IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 33259


Ignore:
Timestamp:
Feb 14, 2012, 12:02:48 PM (14 years ago)
Author:
rhenders
Message:

Big changes to support a new Config class that encapuslates the xml config file

Location:
trunk/ippToPsps/jython
Files:
1 added
15 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/jython/batch.py

    r33235 r33259  
    3030    def __init__(self,
    3131                 logger,
    32                  configPath,
    33                  doc,
     32                 config,
    3433                 gpc1Db,
    3534                 ippToPspsDb,
    … …  
    4241        self.everythingOK = False
    4342        self.readHeader = False
    44         self.configPath = configPath
    45         self.doc = doc
     43        self.config = config
    4644        self.fits = fits
    4745        self.useFullTables = useFullTables
    … …  
    7573               return
    7674
    77         # get info from config
    78         self.survey = self.doc.find("options/survey").text
    79         self.pspsSurvey = self.doc.find("options/pspsSurvey").text
    80         self.dvoGpc1Label = self.doc.find("dvo/gpc1Label").text
    81         self.dvoLocation = self.doc.find("dvo/location").text
    82         self.scratchDb = ScratchDb(logger, self.doc, self.useFullTables)
     75        self.scratchDb = ScratchDb(logger, self.config, self.useFullTables)
    8376
    8477        if not self.scratchDb.everythingOK: return
    … …  
    8780        self.tablesToExport = []
    8881
    89         if self.survey != "":
    90             self.surveyID = self.scratchDb.getSurveyID(self.survey)
     82        if self.config.survey != "":
     83            self.surveyID = self.scratchDb.getSurveyID(self.config.survey)
    9184        else:
    9285            self.surveyID = -1;
    9386       
    94         # get some options from the config
    95         self.testMode = int(self.doc.find("options/testMode").text)
    96         self.dataRelease = int(self.doc.find("metadata/dataRelease").text)
    97 
    9887        # create datastore object
    99         self.datastore = Datastore(self.logger, self.doc, self.ippToPspsDb)
     88        self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb)
    10089
    10190        # get local storage location from config
    10291        self.batchName = Batch.getNameFromID(self.batchID)
    103         self.basePath = self.doc.find("localOutPath").text
    10492        self.subDir = Batch.getSubDir(
    105                 self.basePath,
     93                self.config.basePath,
    10694                self.batchType,
    107                 self.dvoGpc1Label)
     95                self.config.dvoLabel)
    10896
    10997        self.localOutPath = Batch.getOutputPath(
    110                 self.basePath,
     98                self.config.basePath,
    11199                self.batchType,
    112                 self.dvoGpc1Label,
     100                self.config.dvoLabel,
    113101                self.batchID)
    114102
    … …  
    125113        self.logger.infoTitle("New " + self.batchType + " batch")
    126114        self.logger.infoPair("Batch name", self.batchName)
    127         self.logger.infoPair("Survey", self.survey)
    128115        self.logger.infoPair("Survey ID", "%d" % self.surveyID)
    129         self.logger.infoPair("Publishing to PSPS as survey", self.pspsSurvey)
    130         self.logger.infoPair("DVO location", self.dvoLocation)
    131116        self.logger.infoBool("Use full DVO tables?", self.useFullTables)
    132117
    … …  
    227212         if value != "NULL": return value
    228213         else:
    229              if not self.testMode: return "NULL"
     214             if not self.config.test: return "NULL"
    230215             header[key] = default
    231216             self.logger.infoPair("Hardcoding " + key + " to", header[key])
    … …  
    256241        root.attrib['type'] = self.batchType
    257242        root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    258         if self.batchType != "IN": root.attrib['survey'] = self.pspsSurvey
     243        if self.batchType != "IN": root.attrib['survey'] = self.config.pspsSurvey
    259244        try: self.minObjID
    260245        except: pass
    … …  
    287272        # set up filenams and paths
    288273        tarFile = Batch.getTarFile(self.batchID)
    289         tarPath = Batch.getTarPath(self.basePath, self.batchType, self.dvoGpc1Label, self.batchID)
     274        tarPath = Batch.getTarPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID)
    290275        tarballFile = Batch.getTarballFile(self.batchID)
    291         tarballPath = Batch.getTarballPath(self.basePath, self.batchType, self.dvoGpc1Label, self.batchID)
     276        tarballPath = Batch.getTarballPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID)
    292277
    293278        # tar directory
    … …  
    498483
    499484        # TODO path to DVO prog hardcoded temporarily
    500         cmd = "../src/dvograbber " + self.configPath + " " + self.dvoLocation
     485        cmd = "../src/dvograbber " + self.config.path + " " + self.config.dvoLocation
    501486        self.logger.infoPair("Running DVO", cmd)
    502487        p = Popen(cmd, shell=True, stdout=PIPE)
    … …  
    537522            else:
    538523                self.writeBatchManifest()
    539                 if int(self.doc.find("options/publishToDatastore").text):
     524                if self.config.datastorePublishing:
    540525                    # tar and zip ready for publication to datastore
    541526                    if self.tarAndZip():
    … …  
    544529                        Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile)
    545530
    546                 if int(self.doc.find("options/reportNulls").text): self.reportNullsInAllPspsTables(False)
     531                if self.config.reportNulls: self.reportNullsInAllPspsTables(False)
    547532
    548533from datastore import Datastore

  • trunk/ippToPsps/jython/czardb.py

    r32831 r33259  
    1717    Constructor
    1818    '''
    19     def __init__(self, logger, doc):
    20         super(CzarDb, self).__init__(logger, doc, "czardatabase")
     19    def __init__(self, logger, config):
     20        super(CzarDb, self).__init__(logger, config, "czardatabase")
    2121
    2222

  • trunk/ippToPsps/jython/datastore.py

    r32835 r33259  
    1919
    2020    '''
    21     def __init__(self, logger, doc, ippToPspsDb):
     21    def __init__(self, logger, config, ippToPspsDb):
    2222   
    2323        # setup logger
    2424        self.logger = logger
    25         self.doc = doc
     25        self.config = config
    2626        self.ippToPspsDb = ippToPspsDb
    27         self.logger.debug("Datastore constructor")
    28 
    29         # open config
    30         self.product = doc.find("datastore/product").text
    31         self.type = doc.find("datastore/type").text
    32 
    33         self.logger.debug("Using product: '" + self.product + "' and type: '" + self.type + "'")
    3427
    3528    '''
    … …  
    4942        command  = "dsreg --add " + name + "\
    5043                    --link --datapath " + path + "\
    51                     --type " + self.type + "\
    52                     --product " + self.product + "\
     44                    --type " + self.config.datastoreType + "\
     45                    --product " + self.config.datastoreProduct + "\
    5346                    --list " + tempFile.name
    5447
    … …  
    8275                   --del " + name + " \
    8376                   --rm \
    84                    --product " + self.product
     77                   --product " + self.config.datastoreProduct
    8578
    8679        p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)

  • trunk/ippToPsps/jython/detectionbatch.py

    r33235 r33259  
    3333    def __init__(self,
    3434                 logger,
    35                  configPath,
    36                  configDoc,
     35                 config,
    3736                 gpc1Db,
    3837                 ippToPspsDb,
    … …  
    4342       super(DetectionBatch, self).__init__(
    4443               logger,
    45                configPath,
    46                configDoc,
     44               config,
    4745               gpc1Db,
    4846               ippToPspsDb,
    … …  
    7573
    7674       # if test mode
    77        if self.testMode:
     75       if self.config.test:
    7876           self.startX = 3
    7977           self.endX = 4
    … …  
    221219        self.scratchDb.updateFilterID("FrameMeta", self.filter)
    222220        self.scratchDb.updateAllRows("FrameMeta", "calibModNum", str(self.calibModNum))
    223         self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.dataRelease))
     221        self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.config.dataRelease))
    224222
    225223    '''
    … …  
    368366        self.scratchDb.updateFilterID(tableName, self.filter)
    369367        self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
    370         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
     368        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
    371369        if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO'])
    372370        self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0")
    … …  
    455453               , '" + self.dateStr + "' \
    456454               , 0 \
    457                , " + str(self.dataRelease) + "\
     455               , " + str(self.config.dataRelease) + "\
    458456               FROM " + ippTableName
    459457        self.scratchDb.execute(sql)
    … …  
    499497        self.scratchDb.execute(sql)
    500498
    501         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
     499        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
    502500
    503501    '''
    … …  
    528526
    529527        self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
    530         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
     528        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
    531529
    532530
    … …  
    752750    def importIppTables(self, filter=""):
    753751
    754        if self.testMode: regex = "XY33.psf"
     752       if self.config.test: regex = "XY33.psf"
    755753       else : regex = ".*.psf"
    756754 

  • trunk/ippToPsps/jython/dvo.py

    r33234 r33259  
    1616from java.lang import *
    1717from java.sql import *
    18 from xml.etree.ElementTree import ElementTree, Element, tostring
    1918
    2019
    … …  
    3534
    3635    '''
    37     def __init__(self, logger, doc):
     36    def __init__(self, logger, config):
    3837
    3938        # set up logging
    4039        self.logger = logger
    41         self.doc = doc
     40        self.config = config
    4241        self.logger.infoSeparator()
    43         self.dvoLocation = self.doc.find("dvo/location").text
    4442
    4543        # create database object
    46         self.scratchDb = ScratchDb(logger, self.doc, 1)
     44        self.scratchDb = ScratchDb(logger, self.config, 1)
    4745
    4846        # or decide if we are using the right DVO
    49         self.correctDvo = self.scratchDb.isCorrectDvo(self.dvoLocation)
     47        self.correctDvo = self.scratchDb.isCorrectDvo(self.config.dvoLocation)
    5048
    5149        # set up empty lists
    … …  
    5755
    5856        if not self.correctDvo:
    59             response = raw_input("* Wrong DVO is use. Do you want to reset and use '" + self.dvoLocation + "' instead (y/n)? ")
     57            print "*******************************************************************************"
     58            response = raw_input("**** Wrong DVO in use. Do you want to reset and use '" + self.config.dvoLocation + "' instead (y/n)? ")
    6059            if response == "y":
    61                 response = raw_input("* Are you ABSOLUTELY sure you want to do this? (y/n)? ")
     60                response = raw_input("**** Are you ABSOLUTELY sure you want to do this? (y/n)? ")
    6261                if response == "y":
    6362                    self.resetAllTables()
    … …  
    101100
    102101        # check if we have up-to-date version
    103         path = self.dvoLocation + "/Images.dat"
     102        path = self.config.dvoLocation + "/Images.dat"
    104103        if self.scratchDb.alreadyImportedThisDvoTable(path):
    105104            self.logger.infoPair("DVO Images.dat file", "up-to-date")
    … …  
    156155        if not self.correctDvo: return
    157156
    158         path =  self.dvoLocation + "/SkyTable.fits"
     157        path =  self.config.dvoLocation + "/SkyTable.fits"
    159158        if self.scratchDb.alreadyImportedThisDvoTable(path):
    160159            self.logger.infoPair("DVO SkyTable.fits file", "up-to-date")       
    … …  
    173172       
    174173        self.scratchDb.setImportedThisDvoTable(path)
    175         self.logger.infoPair("Finished importing SkyTable at", self.dvoLocation)
     174        self.logger.infoPair("Finished importing SkyTable at", self.config.dvoLocation)
    176175
    177176    '''
    … …  
    181180
    182181        halfSide = side/2.0
     182        print "%f %f %f" % (ra, dec, side)
    183183        self.setSkyArea(ra-halfSide, ra+halfSide, dec-halfSide, dec+halfSide)
    184184   
    … …  
    208208
    209209        # reset all lists
    210         allRegions = self.scratchDb.getDvoRegionsForThisBox(minRa, maxRa, minDec, maxDec)
     210        allRegions = self.scratchDb.getDvoRegions(minRa, maxRa, minDec, maxDec)
    211211        allIngestedRegions = self.scratchDb.getIngestedDvoRegions()
    212212        self.regionsToIngest = []
    … …  
    219219        for region in allRegions:
    220220
    221            cpmPath = self.dvoLocation + "/" + region + ".cpm"
    222            cptPath = self.dvoLocation + "/" + region + ".cpt"
     221           cpmPath = self.config.dvoLocation + "/" + region + ".cpm"
     222           cptPath = self.config.dvoLocation + "/" + region + ".cpt"
    223223
    224224           # check for existence of cpm and cpt files
    … …  
    262262        # go no further if we've already partly ingested a different DVO
    263263        if not self.correctDvo:
    264             self.logger.infoPair("Wrong DVO in use", self.dvoLocation)
     264            self.logger.infoPair("Wrong DVO in use", self.config.dvoLocation)
    265265            return
    266266
    … …  
    327327
    328328            # get combined size of cpm and cpt files
    329             size = size + self.getDiskSize(self.dvoLocation + "/" + region + ".cpm")
    330             size = size + self.getDiskSize(self.dvoLocation + "/" + region + ".cpt")
     329            size = size + self.getDiskSize(self.config.dvoLocation + "/" + region + ".cpm")
     330            size = size + self.getDiskSize(self.config.dvoLocation + "/" + region + ".cpt")
    331331
    332332        return size
    … …  
    357357        for region in self.regionsToIngest:
    358358
    359            cpmPath = self.dvoLocation + "/" + region + ".cpm"
    360            cptPath = self.dvoLocation + "/" + region + ".cpt"
     359           cpmPath = self.config.dvoLocation + "/" + region + ".cpm"
     360           cptPath = self.config.dvoLocation + "/" + region + ".cpt"
    361361
    362362           cpmTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpm")

  • trunk/ippToPsps/jython/fits.py

    r32455 r33259  
    1515
    1616    '''
    17     def __init__(self, logger, doc, originalPath):
     17    def __init__(self, logger, config, originalPath):
    1818
    1919       # set class variables
    2020       self.originalPath = originalPath
    2121       self.logger = logger
    22        self.doc = doc
     22       self.config = config
    2323       self.header = None
    24        self.localDir = self.doc.find("localOutPath").text
    2524
    2625       # does this file even exist?
    … …  
    3130       # ok, we have a file, now copy it locally to save on NFS overhead
    3231       self.logger.debugPair("FITS file", self.originalPath)
    33        #self.localCopyPath = self.localDir + "/temp.fits"
     32       #self.localCopyPath = self.config.basePath + "/temp.fits"
    3433       #shutil.copy2(self.originalPath, self.localCopyPath)
    3534

  • trunk/ippToPsps/jython/gpc1db.py

    r33186 r33259  
    2020    Constructor
    2121    '''
    22     def __init__(self, logger, doc):
    23         super(Gpc1Db, self).__init__(logger, doc, "gpc1database")
     22    def __init__(self, logger, config):
     23        super(Gpc1Db, self).__init__(logger, config, "gpc1database")
    2424
    2525    '''
    … …  
    227227        if len(files) < 1: return None
    228228
    229         return Fits(self.logger, self.doc, files[0]) # TODO just returning first file - check
     229        return Fits(self.logger, self.config, files[0]) # TODO just returning first file - check
    230230
    231231
    … …  
    304304
    305305                    # if we get here, then the cmf is readable, now check the stack_id
    306                     fits = Fits(self.logger, self.doc, path)
     306                    fits = Fits(self.logger, self.config, path)
    307307                    if fits.getPrimaryHeaderValue("STK_ID") == str(stackID):
    308308                        # we have the right file!

  • trunk/ippToPsps/jython/initbatch.py

    r33235 r33259  
    2727    def __init__(self,
    2828            logger,
    29             configPath,
    30             configDoc,
     29            config,
    3130            gpc1Db,
    3231            ippToPspsDb,
    3332            batchID):
    3433       super(InitBatch, self).__init__(logger,
    35                configPath,
    36                configDoc,
     34               config,
    3735               gpc1Db,
    3836               ippToPspsDb,

  • trunk/ippToPsps/jython/ipptopspsdb.py

    r32932 r33259  
    1717    Constructor
    1818    '''
    19     def __init__(self, logger, doc):
    20         super(IppToPspsDb, self).__init__(logger, doc, "ipptopspsdatabase")
     19    def __init__(self, logger, config):
     20        super(IppToPspsDb, self).__init__(logger, config, "ipptopspsdatabase")
    2121
    2222        self.MAX_FAILS = 5
    … …  
    590590    Creates a new batch. This is done with a locked table in order that no two clients can start work on the same item
    591591    '''
    592     def createNewBatch(self, batchType, stageID, survey, epoch, dvoDb, datastoreProduct, force):
     592    def createNewBatch(self, batchType, stageID, config):
    593593
    594594        batchID = -1;
    595595
    596         if force or \
    597             (not self.alreadyProcessed(batchType, stageID, epoch, dvoDb) \
    598             and not self.processingNow(batchType, stageID, epoch, dvoDb) \
    599             and not self.consistentlyFailed(batchType, stageID, epoch, dvoDb)):
     596        if config.force or \
     597            (not self.alreadyProcessed(batchType, stageID, config.epoch, config.dvoLabel) \
     598            and not self.processingNow(batchType, stageID, config.epoch, config.dvoLabel) \
     599            and not self.consistentlyFailed(batchType, stageID, config.epoch, config.dvoLabel)):
    600600
    601601            sql = "INSERT INTO batch ( \
    … …  
    608608                       '" + batchType + "', \
    609609                       " + str(stageID) + ", \
    610                        '" + survey + "', \
    611                        '" + dvoDb + "', \
    612                        '" + datastoreProduct + "' \
     610                       '" + config.survey + "', \
     611                       '" + config.dvoLabel + "', \
     612                       '" + config.datastoreProduct + "' \
    613613                       )"
    614614

  • trunk/ippToPsps/jython/load.py

    r33237 r33259  
    1010import socket
    1111import logging.config
    12 from xml.etree.ElementTree import ElementTree, Element, tostring
    13 
     12
     13from config import Config
    1414from pslogger import PSLogger
    1515from gpc1db import Gpc1Db
    … …  
    2828def publishAnyUnpublishedBatches(batchType):
    2929
    30     batchIDs = ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(EPOCH, DVOGPC1LABEL, batchType)
     30    batchIDs = ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(config.epoch, config.dvoLabel, batchType)
    3131    logger.infoPair("%s batches" % batchType, "%d" % len(batchIDs))
    3232
    … …  
    3434
    3535        batchName = Batch.getNameFromID(batchID)
    36         subDir = Batch.getSubDir(BASEPATH, batchType, DVOGPC1LABEL)
     36        subDir = Batch.getSubDir(config.basePath, batchType, config.dvoLabel)
    3737        tarballFile = Batch.getTarballFile(batchID)
    3838        logger.infoPair("Batch name", batchName)
    … …  
    4848    logger.infoPair("Box center and size", "%f/%f/%f" %(ra, dec, boxsize))
    4949
    50     allIDs = []
    51 
    52     # see if there are any IDs listed in the config
    53     try:
    54         for element in configDoc.findall('options/ids/id'):
    55             allIDs.append(int(element.text))
    56     except:
    57         logger.errorPair("Problem loading IDs from config under", "options/ids/id")
     50    allIDs = config.ids
    5851
    5952    if len(allIDs) > 0:
    … …  
    6356
    6457        allIDs = gpc1Db.getIDsInThisDVODbForThisStageInThisBox(
    65                 DVOGPC1LABEL,
     58                config.dvoLabel,
    6659                batchType,
    6760                ra,
    … …  
    7164        logger.infoPair("All %s items in DVO" % batchType, "%d" % len(allIDs))
    7265
    73     # if in FORCE mode, then queue full list
    74     if FORCE: ids = allIDs
    75 
    76     # if not in TEST mode, then only queue un-processed items
     66    # if in force mode, then queue full list
     67    if config.force: ids = allIDs
     68
     69    # if not in test mode, then only queue un-processed items
    7770    else:
    78         processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOGPC1LABEL)
    79         consistaentlyFailedIDs = ippToPspsDb.getConsistentlyFailedIDs(batchType, EPOCH, DVOGPC1LABEL)
     71        processedIDs = ippToPspsDb.getProcessedIDs(batchType, config.epoch, config.dvoLabel)
     72        consistaentlyFailedIDs = ippToPspsDb.getConsistentlyFailedIDs(batchType, config.epoch, config.dvoLabel)
    8073        ids = list(set(allIDs) - set(processedIDs) - set(consistaentlyFailedIDs))
    8174        logger.infoPair("Processed items", "%d" % len(processedIDs))
    … …  
    9790    for id in ids:
    9891
    99         batchID = ippToPspsDb.createNewBatch(
    100                 batchType,
    101                 id,
    102                 SURVEY,
    103                 EPOCH,
    104                 DVOGPC1LABEL,
    105                 datastore.product,
    106                 FORCE)
     92        batchID = ippToPspsDb.createNewBatch(batchType, id, config)
    10793       
    10894        if batchID < 0: continue;
    … …  
    11298        if batchType == "P2":
    11399            batch = DetectionBatch(logger,
    114                     CONFIG,
    115                     configDoc,
     100                    config,
    116101                    gpc1Db,
    117102                    ippToPspsDb,
    … …  
    121106        elif batchType == "ST":
    122107            batch = StackBatch(logger,
    123                     CONFIG,
    124                     configDoc,
     108                    config,
    125109                    gpc1Db,
    126110                    ippToPspsDb,
    … …  
    135119        logger.infoSeparator()
    136120
    137         # if in TEST mode, then quit after one batch
    138         if TEST: break
     121        # if in test mode, then quit after one batch
     122        if config.test: break
    139123
    140124    ippToPspsDb.unlockTables()
    … …  
    143127Start of program.
    144128'''
    145 if len(sys.argv) > 1: CONFIG = sys.argv[1]
     129if len(sys.argv) > 1: CONFIGPATH = sys.argv[1]
    146130else:
    147131    print "** Usage: " + sys.argv[0] + " <configPath> [init]"
    … …  
    154138
    155139# open config file
    156 configDoc = ElementTree(file=CONFIG)
    157 TEST = int(configDoc.find("options/testMode").text)
    158 
    159 # set up logging
    160 logging.setLoggerClass(PSLogger)
    161 logger = logging.getLogger("ippToPspsLog")
    162 
    163 # get hostnamee and PID for unique log naming
    164 HOST = socket.gethostname()
    165 PID = os.getpid()
    166 
    167 # if in test mode, print log to screen, otherwise, only to file
    168 #logger.setup(configDoc, "ippToPsps", TEST, not TEST, HOST, PID) TODO put back
    169 logger.setup(configDoc, "ippToPsps")
     140config = Config(CONFIGPATH)
     141#logger = config.getLogger("load", 0, 1)
     142logger = config.getLogger("load")
    170143
    171144# create various objects
    172 dvo = Dvo(logger, configDoc)
    173 gpc1Db = Gpc1Db(logger, configDoc)
    174 ippToPspsDb = IppToPspsDb(logger, configDoc)
    175 datastore = Datastore(logger, configDoc, ippToPspsDb)
     145dvo = Dvo(logger, config)
     146gpc1Db = Gpc1Db(logger, config)
     147ippToPspsDb = IppToPspsDb(logger, config)
     148datastore = Datastore(logger, config, ippToPspsDb)
    176149
    177150# check we connected ok
    … …  
    181154# get values from the configutaion file
    182155POLLPERIOD = 600
    183 DVOGPC1LABEL = configDoc.find("dvo/gpc1Label").text
    184 FORCE = int(configDoc.find("options/force").text)
    185 EPOCH = configDoc.find("options/epoch").text
    186 PUBLISH = int(configDoc.find("options/publishToDatastore").text)
    187 BASEPATH = configDoc.find("localOutPath").text
    188 SURVEY = configDoc.find("options/survey").text
    189 
    190 # get batch types from config
    191 batchTypes = []
    192 if int(configDoc.find("options/queueP2").text) == 1: batchTypes.append("P2")
    193 if int(configDoc.find("options/queueST").text) == 1: batchTypes.append("ST")
    194 
    195 # get equatorial coord limits, if any
    196 try: MINRA = float(configDoc.find("dvo/minRA").text)
    197 except: MINRA = 0.0
    198 try: MAXRA = float(configDoc.find("dvo/maxRA").text)
    199 except: MAXRA = 360.0
    200 try: MINDEC = float(configDoc.find("dvo/minDec").text)
    201 except: MINDEC = -30.0
    202 try: MAXDEC = float(configDoc.find("dvo/maxDec").text)
    203 except: MAXDEC = 90.0
    204 try: BOXSIZE = float(configDoc.find("dvo/boxSize").text)
    205 except: BOXSIZE = 4.0
    206 
    207 # prompt user: FORCE and PUBLISH is a dangerous combination
    208 if FORCE and PUBLISH and not QUEUE_IN:
     156
     157# prompt user: force and publishing is a dangerous combination
     158if config.force and config.datastorePublishing and not QUEUE_IN:
    209159   response = raw_input("\n*** Are you sure you want to publish data with the 'force' option enabled (y/n)? ")
    210160   if response != "y": sys.exit(1)
    211161
    212 # prompt user: TEST and PUBLISH is a dangerous combination
    213 if TEST and PUBLISH and not QUEUE_IN:
     162# prompt user: test and publishing is a dangerous combination
     163if config.test and config.datastorePublishing and not QUEUE_IN:
    214164   response = raw_input("\n*** Are you sure you want to publish data with the 'testMode' option enabled (y/n)? ")
    215165   if response != "y": sys.exit(1)
    … …  
    218168logger.infoSeparator()
    219169logger.infoTitle("ippToPsps loading started")
    220 logger.infoPair("Configuration file", CONFIG)
    221 logger.infoPair("Loading epoch", EPOCH)
    222 logger.infoPair("DVO gpc1 label", DVOGPC1LABEL)
    223 for batchType in batchTypes: logger.infoPair("Queuing batch type", batchType)
    224 logger.infoBool("Forcing?", FORCE)
    225 logger.infoBool("Test mode?", TEST)
    226 logger.infoBool("Publishing?", PUBLISH)
    227170
    228171# if an IN batch is requested, create and quit
    229172if QUEUE_IN:
    230    batchID = ippToPspsDb.createNewBatch(
    231              "IN",
    232              0,
    233              SURVEY,
    234              EPOCH,
    235              DVOGPC1LABEL,
    236              datastore.product,
    237              1)
     173   batchID = ippToPspsDb.createNewBatch("IN", 0, config)
    238174   if batchID > 0:
    239        batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb, batchID)
     175       batch = InitBatch(logger, config, gpc1Db, ippToPspsDb, batchID)
    240176       batch.run()
    241177
    … …  
    251187
    252188'''
    253 BORDER = 1.60
    254 HALFBOX = BOXSIZE/2.0
    255 BOXSIZEWITHBORDER = BOXSIZE + (BORDER * 2)
    256189
    257190# this outer while loop simply waits a few minutes then starts queuing againas more stuff may appear in DVO over time
    258191while True:
    259192
     193    config.refresh()
     194    config.printAll()
     195
    260196    # starting positions
    261     ra = MINRA + HALFBOX
    262     dec = MINDEC + HALFBOX
     197    ra = config.minRa + config.halfBox
     198    dec = config.minDec + config.halfBox
    263199
    264200    # queue up batches that are processed but not loaded to datastore
    265201    logger.infoTitle("Previous failed datastore loads")
    266     for batchType in batchTypes: publishAnyUnpublishedBatches(batchType)
    267 
    268     # loop through full range of RA/Dec queueing stuff in boxes of size BOXSIZE
    269     while ra <= MAXRA:
    270 
    271        while dec <= MAXDEC:
     202    for batchType in config.batchTypes: publishAnyUnpublishedBatches(batchType)
     203
     204    # loop through full range of RA/Dec queueing stuff in boxes of size config.boxSize
     205    while ra <= config.maxRa:
     206
     207       while dec <= config.maxDec:
    272208
    273209           # for each batch type
    … …  
    275211           # - check if we should pre-load DVO region
    276212           # - process the items
    277            for batchType in batchTypes:
    278                ids = queueItemsInBox(batchType, ra, dec, BOXSIZE)
     213           for batchType in config.batchTypes:
     214
     215               ids = queueItemsInBox(batchType, ra, dec, config.boxSize)
    279216
    280217               if len(ids) < 1:
    … …  
    282219                   continue
    283220
    284                dvo.setSkyAreaAsBox(ra, dec, BOXSIZEWITHBORDER)
     221               dvo.setSkyAreaAsBox(ra, dec, config.boxSizeWithBorder)
    285222               dvo.printSummary()
    286223
    … …  
    291228
    292229               # do we pre-ingest stuff from DVO?
    293                if smfsPerGB > 40:
     230               if smfsPerGB > 0:
    294231                   if not dvo.sync():
    295232                       logger.errorPair("Could not sync DVO with MySQL", "skipping")
    … …  
    301238               logger.infoBool("Using pre-ingested DVO data?", useFullTables)
    302239               processTheseItems(batchType, ids, useFullTables)
    303 
    304                # in the TEST mode, we quit after submitting one batch
    305                if TEST: break
    306 
    307            dec = dec + BOXSIZE
    308            if TEST: break
    309 
    310        dec = MINDEC + HALFBOX
    311        ra = ra + BOXSIZE
    312        if TEST: break
     240               sys.exit(1)
     241               
     242               # in the test mode, we quit after submitting one batch
     243               if config.test: break
     244
     245           dec = dec + config.boxSize
     246           if config.test: break
     247
     248       dec = config.minDec + config.halfBox
     249       ra = ra + config.boxSize
     250       if config.test: break
    313251
    314252    # wait for the POLLPERIOD before checking for new ids

  • trunk/ippToPsps/jython/metrics.py

    r33191 r33259  
    99from xml.etree.ElementTree import ElementTree, Element, tostring
    1010
     11from config import Config
    1112from pslogger import PSLogger
    1213from ipptopspsdb import IppToPspsDb
    … …  
    1920def plotMe(batchType, file):
    2021
    21     OUTPUTFILE = "plots/" + DVOLABEL + "_" + batchType + ".png"
     22    OUTPUTFILE = "plots/" + config.dvoLabel + "_" + batchType + ".png"
    2223    f=os.popen('gnuplot', 'w')
    2324
    … …  
    2930    print >> f, "set term " + TERM + "; \
    3031              set output \"" + OUTPUTFILE + "\"; \
    31               set title \"ippToPsps : " + batchType + " loading for " + DVOLABEL + "\"; \
     32              set title \"ippToPsps : " + batchType + " loading for " + config.dvoLabel + "\"; \
    3233              set boxwidth; \
    3334              set xtic rotate by -90 scale 0; \
    … …  
    5455
    5556    # get a master list of IDs in DVO for this batch type
    56     if MINRA and MAXRA and MINDEC and MAXDEC:
    57         allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType, MINRA, MAXRA, MINDEC, MAXDEC)
    58     else:
    59         allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType)
     57    allIDs = gpc1Db.getIDsInThisDVODbForThisStage(config.dvoLabel, batchType, config.minRa, config.maxRa, config.minDec, config.maxDec)
    6058    prevList = allIDs
    6159    numEverything = len(allIDs)
    … …  
    6967
    7068       # get lists. Use unions with prev list to make sure the right stuff is included
    71        success = list(set(ippToPspsDb.getStageIDs(batchType, EPOCH, DVOLABEL, stage, 1)) & set(prevList))
    72        fail = list(set(ippToPspsDb.getStageIDs(batchType, EPOCH, DVOLABEL, stage, -1)) & set(prevList) - set(success))
     69       success = list(set(ippToPspsDb.getStageIDs(batchType, config.epoch, config.dvoLabel, stage, 1)) & set(prevList))
     70       fail = list(set(ippToPspsDb.getStageIDs(batchType, config.epoch, config.dvoLabel, stage, -1)) & set(prevList) - set(success))
    7371       pending = list(set(prevList) - set(success) - set(fail))
    7472
    … …  
    8785       print >> DATFILE, stage, numSuccess, numFail, numPending
    8886
    89        czarDb.insertStats(stage, DVOLABEL, batchType, numPending, numSuccess, numFail)
     87       czarDb.insertStats(stage, config.dvoLabel, batchType, numPending, numSuccess, numFail)
    9088
    9189       if stage == 'processed': numPendingProcessed = numPending
    9290
    9391    sys.stdout.write("|\n")
    94     plotMe(batchType, tempFilename)
     92    #plotMe(batchType, tempFilename)
    9593    DATFILE.close()
    9694
    … …  
    111109def printStats(batchType):
    112110
    113     rate = ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "1 HOUR") / 1.0
     111    rate = ippToPspsDb.countBatchesInLastPeriod(batchType, config.epoch, config.dvoLabel, "1 HOUR") / 1.0
    114112
    115113    logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" %
    116114            (batchType,
    117115             rate,
    118              ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "24 HOUR"),
    119              ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "1 WEEK") / 7.0,
    120              ippToPspsDb.getTotalDetectionsPublished(batchType, EPOCH, DVOLABEL),
    121              ippToPspsDb.getTimeOfLastBatchPublished(batchType, EPOCH, DVOLABEL)))
     116             ippToPspsDb.countBatchesInLastPeriod(batchType, config.epoch, config.dvoLabel, "24 HOUR"),
     117             ippToPspsDb.countBatchesInLastPeriod(batchType, config.epoch, config.dvoLabel, "1 WEEK") / 7.0,
     118             ippToPspsDb.getTotalDetectionsPublished(batchType, config.epoch, config.dvoLabel),
     119             ippToPspsDb.getTimeOfLastBatchPublished(batchType, config.epoch, config.dvoLabel)))
    122120
    123121    return rate
    … …  
    126124Start of program.
    127125'''
    128 if len(sys.argv) > 1: CONFIG  = sys.argv[1]
     126if len(sys.argv) > 1: CONFIGPATH  = sys.argv[1]
    129127else:
    130128    print "** Usage: " + sys.argv[0] + " <configPath> [hours]"
    … …  
    137135    SECONDS = None
    138136
    139 
    140 
    141 # open config file
    142 configDoc = ElementTree(file=CONFIG)
    143 
    144 logging.setLoggerClass(PSLogger)
    145 logger = logging.getLogger("metrics")
    146 logger.setup(configDoc, "metrics")
    147 
    148 # create database objects
    149 ippToPspsDb = IppToPspsDb(logger, configDoc)
    150 czarDb = CzarDb(logger, configDoc)
    151 gpc1Db = Gpc1Db(logger, configDoc)
    152 
    153 DVOLABEL = configDoc.find("dvo/gpc1Label").text
    154 EPOCH = configDoc.find("options/epoch").text
     137# create objects
     138config = Config(CONFIGPATH)
     139logger = config.getLogger("metrics")
     140ippToPspsDb = IppToPspsDb(logger, config)
     141czarDb = CzarDb(logger, config)
     142gpc1Db = Gpc1Db(logger, config)
    155143
    156144while True:
    … …  
    159147    now = datetime.datetime.now()
    160148    logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
    161     logger.infoPair("Loading epoch", EPOCH)
    162     logger.infoPair("DVO label", DVOLABEL)
    163 
    164     # get equatorial coord constraints, if any
    165     try:
    166         MINRA = float(configDoc.find("dvo/minRA").text)
    167         MAXRA = float(configDoc.find("dvo/maxRA").text)
    168         MINDEC = float(configDoc.find("dvo/minDec").text)
    169         MAXDEC = float(configDoc.find("dvo/maxDec").text)
    170     except:
    171         MINRA = 0.0
    172         MAXRA = 360.0
    173         MINDEC = -30.0
    174         MAXDEC = 90.0
    175 
    176     logger.infoPair("RA limits", "%.1f to %.1f" % (MINRA, MAXRA))
    177     logger.infoPair("Dec limits", "%.1f to %.1f" % (MINDEC, MAXDEC))
     149    logger.infoPair("Loading epoch", config.epoch)
     150    logger.infoPair("DVO label", config.dvoLabel)
     151    config.printBoxCoords()
    178152
    179153    logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")

  • trunk/ippToPsps/jython/mysql.py

    r33181 r33259  
    2121
    2222    '''
    23     def __init__(self, logger, doc, dbType):
     23    def __init__(self, logger, config, dbType):
    2424
    2525        # set up logging
    2626        self.logger = logger
    27         self.doc = doc
     27        self.config = config
    2828        self.logger.debug("MySql class constructor")
    2929
    3030        # open config and grab database parameters
    31         self.dbName = self.doc.find(dbType +"/name").text
    32         self.dbHost = self.doc.find(dbType +"/host").text
    33         self.dbUser = self.doc.find(dbType +"/user").text
    34         self.dbPass = self.doc.find(dbType +"/password").text
     31        self.dbName = config.getDbName(dbType)
     32        self.dbHost = config.getDbHost(dbType)
     33        self.dbUser = config.getDbUser(dbType)
     34        self.dbPass = config.getDbPassword(dbType)
    3535
    3636        # set up JDBC connection
    … …  
    8080        connectionID = self.getLastConnectionID()
    8181        if connectionID == self.connectionID:
    82             self.logger.error("NOT going to kill THIS connection ID")
     82            self.logger.debug("NOT going to kill THIS connection ID")
    8383            return
    8484

  • trunk/ippToPsps/jython/pslogger.py

    r33192 r33259  
    77
    88   '''
    9    Sets up logging using values from the ippToPsps config file
     9   Sets up logging using provided values and this process hostname and PID if sending to file
     10   This will default to stout and no file output
    1011   '''
    11    def setup(self, doc, name, stdout=1, sendToFile=0, host="host", pid=1234):
     12   def setup(self, name, basePath, dvoLabel, stdout=1, sendToFile=0):
    1213
    1314       formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S')
    … …  
    1516
    1617       if sendToFile:
    17            PATH = doc.find("localOutPath").text + "/log"
     18
     19           HOST = socket.gethostname()
     20           PID = os.getpid()
     21
     22           # generate path
     23           PATH = basePath + "/log"
    1824           if not os.path.exists(PATH): os.makedirs(PATH)
    19            DVOLABEL = doc.find("dvo/gpc1Label").text
    20            FULLPATH = PATH + "/" + name + "_" + DVOLABEL + "_" + host + "_" + str(pid) + ".log"
     25           FULLPATH = PATH + "/" + name + "_" + dvoLabel + "_" + HOST + "_" + str(PID) + ".log"
    2126
    2227           # opens file to be appended

  • trunk/ippToPsps/jython/scratchdb.py

    r33236 r33259  
    1919    Constructor
    2020    '''
    21     def __init__(self, logger, doc, useFull=0):
    22         super(ScratchDb, self).__init__(logger, doc, "localdatabase")
     21    def __init__(self, logger, config, useFull=0):
     22        super(ScratchDb, self).__init__(logger, config, "localdatabase")
    2323
    2424        if useFull:
    … …  
    484484
    485485    '''
    486     Gets a list of PSPS image IDs for this stack ID
    487     '''
    488     def getDvoRegionsForThisBox(self, minRa, maxRa, minDec, maxDec):
     486    Gets a list of DVO regions that overlap with the defined box
     487    '''
     488    def getDvoRegions(self, minRa, maxRa, minDec, maxDec):
    489489
    490490        self.logger.debug("Querying DVO SkyTable for FITS files in this region")
     491
     492        regions = []
     493        self.getDvoRegionsForThisBox(regions, minRa, maxRa, minDec, maxDec)
     494
     495        # deal with 0/360 boundary
     496        if minRa < 0: self.getDvoRegionsForThisBox(regions, minRa + 360, 361 ,minDec, maxDec)
     497        if maxRa > 360: self.getDvoRegionsForThisBox(regions, 0, maxRa - 360 ,minDec, maxDec)
     498
     499        return regions
     500
     501    '''
     502    Gets a list of DVO regions that overlap with the defined box
     503    '''
     504    def getDvoRegionsForThisBox(self, regions, minRa, maxRa, minDec, maxDec):
    491505
    492506        sql = "SELECT name FROM " + self.dvoSkyTable + " \
    … …  
    498512        try:
    499513            rs = self.executeQuery(sql)
    500         except:
    501             self.logger.exception("Can't query for imageIDs")
    502 
    503         files = []
    504         while (rs.next()):
    505             files.append(rs.getString(1))
    506         rs.close()
    507 
    508         return files
    509 
    510 
     514            while (rs.next()): regions.append(rs.getString(1))
     515            rs.close()
     516        except:
     517            self.logger.exception("Can't query for DVO regions")
     518
     519

  • trunk/ippToPsps/jython/stackbatch.py

    r33235 r33259  
    3030    def __init__(self,
    3131                 logger,
    32                  configPath,
    33                  configDoc,
     32                 config,
    3433                 gpc1Db,
    3534                 ippToPspsDb,
    … …  
    4039       super(StackBatch, self).__init__(
    4140               logger,
    42                configPath,
    43                configDoc,
     41               config,
    4442               gpc1Db,
    4543               ippToPspsDb,
    … …  
    4745               batchID,
    4846               "ST",
    49                gpc1Db.getStackStageCmf(configDoc.find("dvo/gpc1Label").text, stackID),
     47               gpc1Db.getStackStageCmf(config.dvoLabel, stackID),
    5048               useFullTables)
    5149
    … …  
    307305        self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID))
    308306        self.scratchDb.updateFilterID("StackMeta", self.filter)
    309         self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.dataRelease))
     307        self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.config.dataRelease))
    310308        self.updateStackTypeID("StackMeta")
    311309
    … …  
    403401        self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID))
    404402        self.scratchDb.updateFilterID("StackDetection", self.filter)
    405         self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.dataRelease))
     403        self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.config.dataRelease))
    406404        self.scratchDb.updateAllRows("StackDetection", "primaryF", "0")
    407405        self.scratchDb.updateAllRows("StackDetection", "activeFlag", "0")
    … …  
    467465        self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID))
    468466        self.scratchDb.updateFilterID("StackApFlx", self.filter)
    469         self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.dataRelease))
     467        self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.config.dataRelease))
    470468        self.scratchDb.updateAllRows("StackApFlx", "primaryF", "0")
    471469        self.scratchDb.updateAllRows("StackApFlx", "activeFlag", "0")
Note: See TracChangeset for help on using the changeset viewer.