Index: branches/eam_branches/ipp-20121219/ippToPsps/config/ippToPspsDbSchema.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/config/ippToPspsDbSchema.sql (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/config/ippToPspsDbSchema.sql (revision 35076) @@ -58,11 +58,11 @@ `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, `id` bigint(20) unsigned NOT NULL auto_increment, - `config` varchar(30) default NULL, + `skychunk` varchar(30) default NULL, `ra_center` float default NULL, `dec_center` float default NULL, `box_side` float default NULL, PRIMARY KEY (`id`), - KEY `fk_config` (`config`), - CONSTRAINT `fk_config` FOREIGN KEY (`config`) REFERENCES `config` (`name`) ON DELETE CASCADE + KEY `fk_skychunk` (`skychunk`), + CONSTRAINT `fk_skychunk` FOREIGN KEY (`skychunk`) REFERENCES `skychunk` (`name`) ON DELETE CASCADE ) ENGINE=InnoDB AUTO_INCREMENT=44588 DEFAULT CHARSET=latin1; SET character_set_client = @saved_cs_client; @@ -80,5 +80,5 @@ `pid` bigint(20) unsigned NOT NULL, `host` varchar(20) NOT NULL, - `config` varchar(30) default NULL, + `skychunk` varchar(30) default NULL, `paused` tinyint(1) default '0', `killed` tinyint(1) default '0', @@ -91,11 +91,11 @@ -- --- Table structure for table `config` --- - -DROP TABLE IF EXISTS `config`; -SET @saved_cs_client = @@character_set_client; -SET character_set_client = utf8; -CREATE TABLE `config` ( +-- Table structure for table `skychunk` +-- + +DROP TABLE IF EXISTS `skychunk`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `skychunk` ( `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, `name` varchar(30) NOT NULL, @@ -200,5 +200,5 @@ CREATE TABLE `stripe` ( `client_id` bigint(20) unsigned default NULL, - `config` varchar(30) default NULL, + `skychunk` varchar(30) default NULL, `ra_center` float default NULL, UNIQUE KEY `unique_key` (`client_id`), Index: branches/eam_branches/ipp-20121219/ippToPsps/doc/upgrade.txt =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/doc/upgrade.txt (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/doc/upgrade.txt (revision 35076) @@ -18,4 +18,8 @@ encompases both the static configuration information and the information about a processing chunk. + + I propose a new split of config -- put only the global system + information in 'config' and define a new concept 'chunk', which + corresponds to the region of the sky currently being processed * client : an instance of an ipptopsps running proccess Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/Makefile.am =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/Makefile.am (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/Makefile.am (revision 35076) @@ -32,4 +32,5 @@ scratchdb.py \ setupScratchDb.py \ + skychunk.py \ stackbatch.py Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/batch.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/batch.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/batch.py (revision 35076) @@ -30,4 +30,5 @@ logger, config, + skychunk, gpc1Db, ippToPspsDb, @@ -46,4 +47,5 @@ self.readHeader = False self.config = config + self.skychunk = skychunk self.fits = fits self.useFullTables = useFullTables @@ -55,8 +57,4 @@ # self.testprint() - - self.configDir = os.getenv("IPPTOPSPS_DATA") - if self.configDir is not None: - self.configDir = self.configDir + "/" # set up class variables @@ -67,5 +65,5 @@ self.scratchDb = scratchDb self.batchType = batchType; - self.pspsVoTableFilePath = self.configDir + "tables." + batchType + ".vot" + self.pspsVoTableFilePath = self.config.configDir + "tables." + batchType + ".vot" # self.testprint() @@ -85,23 +83,23 @@ self.tablesToExport = [] - if self.config.survey != "": - self.surveyID = self.scratchDb.getSurveyID(self.config.survey) + if self.skychunk.survey != "": + self.surveyID = self.scratchDb.getSurveyID(self.skychunk.survey) else: self.surveyID = -1; # create datastore object - self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) - - # get local storage location from config + self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb) + + # get local storage location from skychunk self.batchName = Batch.getNameFromID(self.batchID) self.subDir = Batch.getSubDir( - self.config.basePath, + self.skychunk.basePath, self.batchType, - self.config.dvoLabel) + self.skychunk.dvoLabel) self.localOutPath = Batch.getOutputPath( - self.config.basePath, + self.skychunk.basePath, self.batchType, - self.config.dvoLabel, + self.skychunk.dvoLabel, self.batchID) @@ -253,5 +251,5 @@ root.attrib['type'] = self.batchType root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - root.attrib['survey'] = self.config.pspsSurvey + root.attrib['survey'] = self.skychunk.pspsSurvey # min/max object IDs @@ -294,7 +292,7 @@ # set up filenams and paths tarFile = Batch.getTarFile(self.batchID) - tarPath = Batch.getTarPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID) + tarPath = Batch.getTarPath(self.skychunk.basePath, self.batchType, self.skychunk.dvoLabel, self.batchID) tarballFile = Batch.getTarballFile(self.batchID) - tarballPath = Batch.getTarballPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID) + tarballPath = Batch.getTarballPath(self.skychunk.basePath, self.batchType, self.skychunk.dvoLabel, self.batchID) # tar directory @@ -516,5 +514,5 @@ # TODO path to DVO prog hardcoded temporarily - cmd = "../src/dvograbber " + self.config.settingsPath + " " + self.scratchDb.dbName + " " + self.config.dvoLocation + cmd = "../src/dvograbber " + self.config.settingsPath + " " + self.scratchDb.dbName + " " + self.skychunk.dvoLocation self.logger.infoPair("Running DVO", cmd) p = Popen(cmd, shell=True, stdout=PIPE) @@ -532,5 +530,5 @@ ''' Creates and publishes a batch - TODO all methods call below should throw exceptions on failure + TODO all method calls below should throw exceptions on failure ''' def run(self): @@ -561,5 +559,5 @@ if self.writeBatchManifest(): - if self.config.datastorePublishing: + if self.skychunk.datastorePublishing: # tar and zip ready for publication to datastore Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/batchRepublisher.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/batchRepublisher.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/batchRepublisher.py (revision 35076) @@ -30,5 +30,5 @@ self.exitProgram("incorrect args") - self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb) self.BATCHID = int(sys.argv[2]) @@ -40,5 +40,5 @@ batchName = Batch.getNameFromID(self.BATCHID) batchType = self.ippToPspsDb.getBatchType(self.BATCHID) - subDir = Batch.getSubDir(self.config.basePath, batchType, self.config.dvoLabel) + subDir = Batch.getSubDir(self.skychunk.basePath, batchType, self.skychunk.dvoLabel) tarballFile = Batch.getTarballFile(self.BATCHID) @@ -50,5 +50,5 @@ self.logger.infoPair("Reseting batch in database", "%d" % self.BATCHID) self.ippToPspsDb.resetBatch(self.BATCHID) - self.logger.infoPair("publishing to", self.config.datastoreProduct) + self.logger.infoPair("publishing to", self.skychunk.datastoreProduct) Batch.publishToDatastore(self.datastore, self.BATCHID, batchName, subDir, tarballFile) Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/cleanup.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/cleanup.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/cleanup.py (revision 35076) @@ -26,5 +26,5 @@ super(Cleanup, self).__init__(argv) - self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb) self.dxlayer = DXLayer(self.logger) if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) @@ -40,6 +40,6 @@ self.logger.infoSeparator() - self.logger.infoPair("Config", self.config.name) - self.config.printDeletionPolicy() + self.logger.infoPair("Skychunk", self.skychunk.name) + self.skychunk.printDeletionPolicy() self.clean("IN") self.clean("P2") @@ -89,9 +89,9 @@ # delete stuff from local disk - if self.config.deleteLocal: + if self.skychunk.deleteLocal: count = 0 for id in deleteFromLocalIDs: - if Batch.deleteFromDisk(self.logger, self.config.basePath, batchType, self.config.dvoLabel, id): + if Batch.deleteFromDisk(self.logger, self.skychunk.basePath, batchType, self.skychunk.dvoLabel, id): self.ippToPspsDb.updateDeletedLocal(id, 1) count = count + 1 @@ -100,5 +100,5 @@ # remove stuff from datastore - if self.config.deleteDatastore: + if self.skychunk.deleteDatastore: count = 0 for id in deleteFromDatastoreIDs: @@ -112,5 +112,5 @@ # remove stuff from DXLayer - if self.config.deleteDxLayer: + if self.skychunk.deleteDxLayer: count = 0 for id in deleteFromDxLayerIDs: Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/config.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/config.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/config.py (revision 35076) @@ -1,4 +1,6 @@ #!/usr/bin/env jython +import sys +import os import logging from xml.etree.ElementTree import ElementTree, Element, tostring @@ -6,8 +8,7 @@ from pslogger import PSLogger - ''' -A class encapsulating a ippToPsps configuration. This information is stored in the 'config' table -of the ipptopsps database, but there are some higher level config details in the 'settings.xml' file +A class encapsulating the globald ippToPsps configuration information. +This is stored in the 'settings.xml' file in the config directory ''' class Config(object): @@ -18,17 +19,37 @@ Basically reads the entire config and stores values to class variables ''' - def __init__(self, programName, name, configDir): + def __init__(self): - self.programName = programName - self.name = name - self.configDir = configDir + self.test = False + for arg in sys.argv: + if arg == "-test": + self.test = True + sys.argv.remove(arg) + if arg == "-t": + self.test = True + sys.argv.remove(arg) + + ## name of the top-level jython script + self.programName = os.path.basename(sys.argv[0]) + + ## this is somewhat crude: ipptopsps programs are called like this: + ## ippjython foo.py [chunk] [other options] + ## the logging code (called below by getLogger) wants to include 'name' in the + ## output file. we blindly set name == argv[1] + ## programs which are called without argv[1] will use 'none' in the log file + if len(sys.argv) >= 2: + self.name = sys.argv[1] + + # XXX this probably goes in 'config.py' + self.configDir = os.getenv("IPPTOPSPS_DATA") + if self.configDir is None: + self.configDir = "../config/" + else: + self.configDir = self.configDir + "/" # self.configdir is set in ipptopsps.jy on init based on env(IPPTOPSPS_DATA) # for test purposes, an uninstalled system may use the config information from # a relative path - if self.configDir is None: - self.settingsPath = "../config/settings.xml" # TODO - else: - self.settingsPath = self.configDir + "settings.xml" + self.settingsPath = self.configDir + "settings.xml" self.logger = None @@ -37,46 +58,9 @@ self.logPath = self.settingsDoc.find("logPath").text - # XXX this is poor -- the czartool stuff is not related to - # ipptopsps and should be moved elsewhere - self.czarPlotsPath = self.settingsDoc.find("czarPlotsPath").text - - # this is the border (in degrees) that we place around any loading box of - # PS1 pointings to ensure we pull a large enough area out of DVO - # XXX this is a poor place to put this information -- it is completely gpc-specific. - # probably should go elsewhere - self.BORDER = 1.65 - self.isLoaded = False - - ''' - Prints everything for this config - ''' - def printAll(self): - - self.logger.infoTitle("Config") - - try: - self.logger.infoSeparator() - self.logger.infoPair("Config name", self.name) - self.logger.infoPair("Survey", self.survey) - self.logger.infoPair("Publishing to PSPS as survey", self.pspsSurvey) - self.logger.infoPair("Loading epoch", self.epoch) - self.logger.infoPair("Data release", "%d" % self.dataRelease) - for batchType in self.batchTypes: self.logger.infoPair("Queuing batch type", batchType) - except: - pass - - self.printDvoInfo() - self.printDatastoreInfo() - self.printBoxCoords() - self.printDeletionPolicy() - try: self.logger.infoSeparator() - except: pass - - ''' - Queuing this batch type? - ''' - def queuingThisBatchType(self, batchType): - if batchType in self.batchTypes: return 1 - return 0 + print "config.programName: ", self.programName + print "config.configDir: ", self.configDir + print "config.settingsPath: ", self.settingsPath + print "config.logPath: ", self.logPath + print "config.test: ", self.test ''' @@ -85,58 +69,16 @@ def getLogger(self, host, pid, stdout=1, sendToFile=0): + print "get Logger: ", host, pid logging.setLoggerClass(PSLogger) + print "done Logger 1" self.logger = logging.getLogger(self.programName) - self.logger.setup(self.programName, self.logPath, self.name, host, pid, stdout, sendToFile) + print "done Logger 2" + if (self.name is None): + self.logger.setup(self.programName, self.logPath, "none", host, pid, stdout, sendToFile) + else: + self.logger.setup(self.programName, self.logPath, self.name, host, pid, stdout, sendToFile) + print "done Logger 3" return self.logger - - ''' - Prints the currently set DVO info - ''' - def printDvoInfo(self): - - try: - self.logger.infoPair("DVO label", self.dvoLabel) - self.logger.infoPair("DVO location", self.dvoLocation) - except: - pass - - ''' - Prints datastore info - ''' - def printDatastoreInfo(self): - - try: - self.logger.infoBool("Datastore publishing?", self.datastorePublishing) - if self.datastorePublishing: - self.logger.infoPair("Datastore type", self.datastoreProduct) - self.logger.infoPair("Datastore product", self.datastoreType) - except: - pass - - ''' - Prints the current deletion policy - ''' - def printDeletionPolicy(self): - - try: - self.logger.infoBool("Deleting local?", self.deleteLocal) - self.logger.infoBool("Deleting datastore?", self.deleteDatastore) - self.logger.infoBool("Deleting DXLayer?", self.deleteDxLayer) - except: - pass - - ''' - Prints the currently set RA/Dec bounding box - ''' - def printBoxCoords(self): - - try: - self.logger.infoPair("RA limits", "%.1f -> %.1f" % (self.minRa, self.maxRa)) - self.logger.infoPair("Dec limits", "%.1f -> %.1f" % (self.minDec, self.maxDec)) - self.logger.infoPair("Loading box size", "%.1f degrees" % self.boxSize) - except: - pass - ''' @@ -147,3 +89,2 @@ def getDbUser(self, dbType): return self.settingsDoc.find(dbType +"/user").text def getDbPassword(self, dbType): return self.settingsDoc.find(dbType +"/password").text - Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/console.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/console.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/console.py (revision 35076) @@ -37,5 +37,5 @@ self.frame = JFrame( - "ippToPsps console for config '" + self.config.name + "'", + "ippToPsps console for skychunk '" + self.skychunk.name + "'", layout=BorderLayout(), size=(1000, 500), @@ -74,5 +74,5 @@ button = JButton('Purge dead', actionPerformed=self.purgeDead) buttonPanel.add(button) - button = JButton('Change config', actionPerformed=self.changeConfig) + button = JButton('Change skychunk', actionPerformed=self.changeSkychunk) buttonPanel.add(button) @@ -136,17 +136,17 @@ self.ippToPspsDb.purgeDeadClients() self.refreshClientTable(None) - def changeConfig(self, event): + def changeSkychunk(self, event): ids = self.getSelectedIds() if len(ids) < 1: JOptionPane.showMessageDialog(None, "No clients selected", "Error", JOptionPane.ERROR_MESSAGE) return - comboBox = JComboBox(self.ippToPspsDb.getActiveConfigList()) + comboBox = JComboBox(self.ippToPspsDb.getActiveSkychunkList()) if JOptionPane.showConfirmDialog(None, comboBox, - "Choose a config", + "Choose a skychunk", JOptionPane.OK_CANCEL_OPTION, JOptionPane.PLAIN_MESSAGE) == JOptionPane.CANCEL_OPTION: return - self.ippToPspsDb.setConfigForLoaders(comboBox.getSelectedItem(), ids) + self.ippToPspsDb.setSkychunkForLoaders(comboBox.getSelectedItem(), ids) self.refreshClientTable(None) Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/datastore.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/datastore.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/datastore.py (revision 35076) @@ -1,3 +1,4 @@ #!/usr/bin/env jython +# EAM : config -> skychunk DONE from subprocess import call, PIPE, Popen @@ -19,9 +20,9 @@ ''' - def __init__(self, logger, config, ippToPspsDb): + def __init__(self, logger, skychunk, ippToPspsDb): # setup logger self.logger = logger - self.config = config + self.skychunk = skychunk self.ippToPspsDb = ippToPspsDb @@ -42,6 +43,6 @@ command = "dsreg --add " + name + "\ --link --datapath " + path + "\ - --type " + self.config.datastoreType + "\ - --product " + self.config.datastoreProduct + "\ + --type " + self.skychunk.datastoreType + "\ + --product " + self.skychunk.datastoreProduct + "\ --list " + tempFile.name @@ -75,5 +76,5 @@ --del " + name + " \ --rm \ - --product " + self.config.datastoreProduct + --product " + self.skychunk.datastoreProduct p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE) Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/datastoreRemover.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/datastoreRemover.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/datastoreRemover.py (revision 35076) @@ -24,5 +24,5 @@ self.exitProgram("incorrect args") - self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb) ''' Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/detectionbatch.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/detectionbatch.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/detectionbatch.py (revision 35076) @@ -36,4 +36,5 @@ logger, config, + skychunk, gpc1Db, ippToPspsDb, @@ -46,4 +47,5 @@ logger, config, + skychunk, gpc1Db, ippToPspsDb, @@ -224,5 +226,5 @@ self.scratchDb.updateFilterID("FrameMeta", self.filter) self.scratchDb.updateAllRows("FrameMeta", "calibModNum", str(self.calibModNum)) - self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.skychunk.dataRelease)) ''' @@ -374,5 +376,5 @@ self.scratchDb.updateFilterID(tableName, self.filter) self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO']) self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0") @@ -472,5 +474,5 @@ , '" + self.dateStr + "' \ , 0 \ - , " + str(self.config.dataRelease) + "\ + , " + str(self.skychunk.dataRelease) + "\ FROM " + ippTableName # self.logger.info(sql) @@ -526,5 +528,5 @@ self.scratchDb.execute(sql) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) ''' @@ -571,5 +573,5 @@ WHERE a.objID = b.objID AND a.detectID = b.detectID" self.scratchDb.execute(sql) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) ''' @@ -607,5 +609,5 @@ b.expTime, \ b.airMass, \ - " + str(self.config.dataRelease) + " \ + " + str(self.skychunk.dataRelease) + " \ FROM \ Detection_" + ota + " as a \ @@ -647,5 +649,5 @@ self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) ''' Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/dvo.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/dvo.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/dvo.py (revision 35076) @@ -33,9 +33,10 @@ ''' - def __init__(self, logger, config, scratchDbName=None): + def __init__(self, logger, config, skychunk, scratchDbName=None): # set up logging self.logger = logger self.config = config + self.skychunk = skychunk # set up empty lists @@ -57,10 +58,10 @@ # decide if we are using the right DVO for this scratchDb - self.correctDvo = self.scratchDb.isCorrectDvo(self.config.dvoLocation) + self.correctDvo = self.scratchDb.isCorrectDvo(self.skychunk.dvoLocation) # this scratch Db either has no DVO info ingested, or a different DVO ingested. Prompt user if not self.correctDvo: print "*******************************************************************************" - self.logger.warning("Wrong or no DVO in use. Do you want to reset and use '" + self.config.dvoLocation + "'?") + self.logger.warning("Wrong or no DVO in use. Do you want to reset and use '" + self.skychunk.dvoLocation + "'?") response = raw_input("(y/n) ") if response == "y": @@ -104,5 +105,5 @@ # check if we have up-to-date version - path = self.config.dvoLocation + "/Photcodes.dat" + path = self.skychunk.dvoLocation + "/Photcodes.dat" if self.scratchDb.alreadyImportedThisDvoTable(path): self.logger.debugPair("DVO Photcodes.dat file", "up-to-date") @@ -129,5 +130,5 @@ # check if we have up-to-date version - path = self.config.dvoLocation + "/Images.dat" + path = self.skychunk.dvoLocation + "/Images.dat" if self.scratchDb.alreadyImportedThisDvoTable(path): self.logger.debugPair("DVO Images.dat file", "up-to-date") @@ -154,5 +155,5 @@ if not self.correctDvo: return False - path = self.config.dvoLocation + "/SkyTable.fits" + path = self.skychunk.dvoLocation + "/SkyTable.fits" if self.scratchDb.alreadyImportedThisDvoTable(path): self.logger.debugPair("DVO SkyTable.fits file", "up-to-date") @@ -185,5 +186,5 @@ cmd += " -dbuser " + self.scratchDb.dbUser cmd += " -dbpass " + self.scratchDb.dbPass - cmd += " -D CATDIR " + self.config.dvoLocation + cmd += " -D CATDIR " + self.skychunk.dvoLocation self.logger.infoPair("Running dvopsps", cmd) @@ -196,5 +197,5 @@ self.scratchDb.setImportedThisDvoTable(path) - self.logger.infoPair("Finished importing SkyTable at", self.config.dvoLocation) + self.logger.infoPair("Finished importing SkyTable at", self.skychunk.dvoLocation) return True @@ -259,5 +260,5 @@ paths = [] for fileType in self.ingestFileTypes: - paths.append(self.config.dvoLocation + "/" + regionPath + "." + fileType) + paths.append(self.skychunk.dvoLocation + "/" + regionPath + "." + fileType) # check for the existence of all interested file types @@ -332,5 +333,5 @@ # go no further if we've already partly ingested a different DVO if not self.correctDvo: - self.logger.infoPair("Wrong DVO in use", self.config.dvoLocation) + self.logger.infoPair("Wrong DVO in use", self.skychunk.dvoLocation) return @@ -418,5 +419,5 @@ # get combined size of all interested files for fileType in self.ingestFileTypes: - size = size + self.getDiskSize(self.config.dvoLocation + "/" + region + "." + fileType) + size = size + self.getDiskSize(self.skychunk.dvoLocation + "/" + region + "." + fileType) # EAM TEST I/O self.logger.infoPair(fileType, size) @@ -570,5 +571,5 @@ def nativeIngestDetections(self, raCenter, decCenter, boxSize): - # XXX put the chunk below in a separate method + # XXX put the skychunk below in a separate method # drop detections table @@ -610,5 +611,5 @@ cmd += " -dbuser " + self.scratchDb.dbUser cmd += " -dbpass " + self.scratchDb.dbPass - cmd += " -D CATDIR " + self.config.dvoLocation + cmd += " -D CATDIR " + self.skychunk.dvoLocation cmd += " -region " cmd += " " + str(raCenter-halfSize) @@ -617,5 +618,5 @@ cmd += " " + str(decCenter+halfSize) - if self.config.parallel: + if self.skychunk.parallel: cmd += " -parallel" Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/dvodetections.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/dvodetections.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/dvodetections.py (revision 35076) @@ -20,7 +20,7 @@ Constructor ''' - def __init__(self, logger, config, scratchDbName): + def __init__(self, logger, config, skychunk, scratchDbName): - super(DvoDetections, self).__init__(logger, config, scratchDbName) + super(DvoDetections, self).__init__(logger, config, skychunk, scratchDbName) # declare DVO file types of interest @@ -32,6 +32,6 @@ def ingestRegion(self, region): - cpmPath = self.config.dvoLocation + "/" + region + ".cpm" - cptPath = self.config.dvoLocation + "/" + region + ".cpt" + cpmPath = self.skychunk.dvoLocation + "/" + region + ".cpm" + cptPath = self.skychunk.dvoLocation + "/" + region + ".cpt" cpmTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpm") Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/dvoobjects.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/dvoobjects.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/dvoobjects.py (revision 35076) @@ -12,5 +12,4 @@ from dvo import Dvo - ''' A class for ingesting DVO detections into MySQL @@ -21,7 +20,7 @@ Constructor ''' - def __init__(self, logger, config, scratchDbName=None): + def __init__(self, logger, config, skychunk, scratchDbName=None): - super(DvoObjects, self).__init__(logger, config, scratchDbName) + super(DvoObjects, self).__init__(logger, config, skychunk, scratchDbName) # declare DVO file types of interest @@ -38,5 +37,5 @@ for fileType in self.ingestFileTypes: - path = self.config.dvoLocation + "/" + region + "." + fileType + path = self.skychunk.dvoLocation + "/" + region + "." + fileType tableName = self.scratchDb.getDbFriendlyTableName(region + "." + fileType) @@ -64,8 +63,8 @@ cmd += " -dbuser " + self.scratchDb.dbUser cmd += " -dbpass " + self.scratchDb.dbPass - cmd += " -D CATDIR " + self.config.dvoLocation + cmd += " -D CATDIR " + self.skychunk.dvoLocation cmd += " -cpt " + region - if self.config.parallel: + if self.skychunk.parallel: cmd += " -parallel" Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/initbatch.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/initbatch.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/initbatch.py (revision 35076) @@ -28,4 +28,5 @@ logger, config, + skychunk, gpc1Db, ippToPspsDb, @@ -34,4 +35,5 @@ super(InitBatch, self).__init__(logger, config, + skychunk, gpc1Db, ippToPspsDb, Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopsps.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopsps.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopsps.py (revision 35076) @@ -1,3 +1,4 @@ #!/usr/bin/env jython +# EAM : config -> skychunk DONE import signal @@ -9,4 +10,5 @@ from config import Config +from skychunk import Skychunk from pslogger import PSLogger from ipptopspsdb import IppToPspsDb @@ -23,14 +25,8 @@ def __init__(self, argv, logToStdout=1, logToFile=0): - testmode = False - for arg in sys.argv: - if arg == "-test": - testmode = True - sys.argv.remove(arg) - if arg == "-t": - testmode = True - sys.argv.remove(arg) - - # are the arsg ok? all programs require a config name + # set up config object (global config information, also parse command-line options, eg -test / -t) + self.config = Config() + + # are the args ok? all programs require a config name if len(sys.argv) < 2: self.printUsage() @@ -38,6 +34,4 @@ # some class constants - self.PROGNAME = os.path.basename(sys.argv[0]) - CONFIGNAME = sys.argv[1] self.HOST = socket.gethostname() self.PID = os.getpid() @@ -47,42 +41,33 @@ self.SECONDS = None - self.configDir = os.getenv("IPPTOPSPS_DATA") - if self.configDir is not None: - self.configDir = self.configDir + "/" - - self.createNewConfig = False - self.rotateConfigs = False - # a new config? - if CONFIGNAME == "edit": self.createNewConfig = True - # should we rotate configs for this program? - elif CONFIGNAME == "all": self.rotateConfigs = True - - # set up config object - self.config = Config(self.PROGNAME, CONFIGNAME, self.configDir) - self.config.test = testmode - - if self.config.test: print "using test mode" - else: print "not using test mode" - self.logger = self.config.getLogger(self.HOST, self.PID, logToStdout, logToFile) + # argv[1] -> self.skychunk.name + self.skychunk = Skychunk(self.logger) + print "here... " + # create connection to databases database try: - self.ippToPspsDb = IppToPspsDb(self.logger, self.config) + self.ippToPspsDb = IppToPspsDb(self.logger, self.config, self.skychunk) except: self.exitProgram("Could not connect to ipptopsps Db") raise + print "here... " + self.checkClientStatus() + print "here... " # catch Ctrl-C signal signal.signal(signal.SIGINT, self.signal_handler) + print "here... " # title for log self.logger.infoSeparator() - self.logger.infoTitle("ippToPsps '" + self.PROGNAME + "' started") + self.logger.infoTitle("ippToPsps '" + self.config.programName + "' started") self.logger.infoPair("Host", self.HOST) self.logger.infoPair("PID", "%d" % self.PID) - self.logger.infoPair("Config", self.config.name) + self.logger.infoPair("Skychunk", self.skychunk.name) + print "here... " ''' @@ -122,5 +107,5 @@ # write message to log or stdout if no logger set up - msg = sys.argv[0] + " " + extra + msg = sys.argv[0] + " " + extra try: self.logger.errorPair("Usage:", msg) @@ -129,46 +114,58 @@ ''' - Refreshes the config then recreates objects with new settings - ''' - def refreshConfig(self): - - # new config? - if self.createNewConfig: - self.ippToPspsDb.editConfig() - self.ippToPspsDb.setConfigForThisClient(self.config.name, self.HOST, self.PID) - self.createNewConfig = False - - # if we are rotating configs, then look for next active one in list - if self.rotateConfigs: - self.ippToPspsDb.getConfigForThisClient(self.HOST, self.PID) - configs = self.ippToPspsDb.getActiveConfigList() + Refreshes the skychunk then recreates objects with new settings + ''' + def refreshSkychunk(self): + + print "refresh... " + print "self.skychunk.createNewSkychunk: ", self.skychunk.createNewSkychunk + + # new skychunk? + if self.skychunk.createNewSkychunk: + print "self.skychunk.createNewSkychunk: ", self.skychunk.createNewSkychunk + self.ippToPspsDb.editSkychunk() + print "self.skychunk.createNewSkychunk: ", self.skychunk.createNewSkychunk + self.ippToPspsDb.setSkychunkForThisClient(self.skychunk.name, self.HOST, self.PID) + print "self.skychunk.createNewSkychunk: ", self.skychunk.createNewSkychunk + self.skychunk.createNewSkychunk = False + + print "refresh... " + # if we are rotating skychunks, then look for next active one in list + if self.skychunk.rotateSkychunks: + self.ippToPspsDb.getSkychunkForThisClient(self.HOST, self.PID) + skychunks = self.ippToPspsDb.getActiveSkychunkList() i = 1 - for config in configs: - if config == self.config.name: break + for skychunk in skychunks: + if skychunk == self.skychunk.name: break i += 1 - if i >= len(configs): newConfig = configs[0] - else: newConfig = configs[i] - - self.ippToPspsDb.setConfigForThisClient(newConfig, self.HOST, self.PID) - - return self.ippToPspsDb.readConfig(self.HOST, self.PID) - - ''' - All implementing subclasses (programs) should call this method once in a while to see if it should be paused or killed, otherwise this updates the clients table with current time and reloads the config data + if i >= len(skychunks): newSkychunk = skychunks[0] + else: newSkychunk = skychunks[i] + + self.ippToPspsDb.setSkychunkForThisClient(newSkychunk, self.HOST, self.PID) + + print "refresh... " + return self.ippToPspsDb.readSkychunk(self.HOST, self.PID) + + ''' + All implementing subclasses (programs) should call this method once in a while to see if it should be paused or killed, otherwise this updates the clients table with current time and reloads the skychunk data ''' def checkClientStatus(self): - self.ippToPspsDb.updateClient(self.PROGNAME, self.HOST, self.PID) - + print "stat... " + self.ippToPspsDb.updateClient(self.config.programName, self.HOST, self.PID) + + print "stat... " if self.ippToPspsDb.isKilled(self.HOST, self.PID): self.exitProgram("killed via Db") - # this loop pauses the process if we have no config or we have set it to pause + # this loop pauses the process if we have no skychunk or we have set it to pause firstTimeIn = True - while not self.refreshConfig() or self.ippToPspsDb.isPaused(self.HOST, self.PID): - - self.ippToPspsDb.updateClient(self.PROGNAME, self.HOST, self.PID) + print "stat... " + while not self.refreshSkychunk() or self.ippToPspsDb.isPaused(self.HOST, self.PID): + + print "stat... " + self.ippToPspsDb.updateClient(self.config.programName, self.HOST, self.PID) if self.ippToPspsDb.isKilled(self.HOST, self.PID): self.exitProgram("killed while paused") @@ -177,4 +174,6 @@ time.sleep(self.PAUSEPERIOD) + print "stat done... " + ''' @@ -193,5 +192,5 @@ # write message to log or stdout if no logger set up try: - self.logger.infoPair(self.PROGNAME + " exited", exitReason) + self.logger.infoPair(self.config.programName + " exited", exitReason) except: print "*** Program exited: " + exitReason @@ -204,4 +203,2 @@ sys.exit(0) - - Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopspsdb.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopspsdb.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopspsdb.py (revision 35076) @@ -17,5 +17,5 @@ Constructor ''' - def __init__(self, logger, config): + def __init__(self, logger, config, skychunk): if (config.test): dbType = "ipptopspsdatabase_test" @@ -24,4 +24,6 @@ super(IppToPspsDb, self).__init__(logger, config, dbType) + self.skychunk = skychunk + self.MAX_FAILS = 5 @@ -33,7 +35,7 @@ sql = "SELECT DISTINCT batch_id \ FROM batch \ - WHERE timestamp > '" + self.config.epoch + "' \ + WHERE timestamp > '" + self.skychunk.epoch + "' \ AND batch_type = '" + batchType + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND merged = 1 \ AND " + column + " = 0" @@ -58,7 +60,7 @@ sql = "SELECT DISTINCT batch_id \ FROM batch \ - WHERE timestamp > '" + self.config.epoch + "' \ + WHERE timestamp > '" + self.skychunk.epoch + "' \ AND batch_type = '" + batchType + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND purged = 1 \ AND " + column + " = 0" @@ -84,7 +86,7 @@ sql = "SELECT DISTINCT batch_id \ FROM batch \ - WHERE timestamp > '" + self.config.epoch + "' \ + WHERE timestamp > '" + self.skychunk.epoch + "' \ AND batch_type = '" + batchType + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND (loaded_to_ODM = -1 OR merge_worthy = 1) \ AND " + column + " = 0" @@ -152,7 +154,7 @@ sql = "SELECT DISTINCT batch_id \ FROM batch \ - WHERE timestamp > '" + self.config.epoch + "' \ + WHERE timestamp > '" + self.skychunk.epoch + "' \ AND batch_type = '" + batchType + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND processed = 1\ AND loaded_to_datastore != 1 \ @@ -181,6 +183,6 @@ sql = "SELECT batch_id \ FROM batch \ - WHERE timestamp > '" + self.config.epoch + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + WHERE timestamp > '" + self.skychunk.epoch + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND loaded_to_datastore = 1 \ AND batch_type = '" + batchType + "' \ @@ -210,6 +212,6 @@ sql = "SELECT batch_id \ FROM batch \ - WHERE timestamp > '" + self.config.epoch + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + WHERE timestamp > '" + self.skychunk.epoch + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND loaded_to_datastore = 1 \ AND batch_type = '" + batchType + "' \ @@ -239,6 +241,6 @@ sql = "SELECT batch_id \ FROM batch \ - WHERE timestamp > '" + self.config.epoch + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + WHERE timestamp > '" + self.skychunk.epoch + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND merge_worthy = 1 \ AND batch_type = '" + batchType + "' \ @@ -266,6 +268,6 @@ sql = "SELECT DISTINCT batch_id \ FROM batch \ - WHERE timestamp > '" + self.config.epoch + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + WHERE timestamp > '" + self.skychunk.epoch + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND " + column + " = " + str(value) @@ -292,6 +294,6 @@ FROM batch \ WHERE batch_type = '" + batchType + "' \ - AND timestamp > '" + self.config.epoch + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND timestamp > '" + self.skychunk.epoch + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND processed = 1 \ AND loaded_to_datastore = 1 \ @@ -319,6 +321,6 @@ FROM batch \ WHERE batch_type = '" + batchType + "' \ - AND timestamp > '" + self.config.epoch + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND timestamp > '" + self.skychunk.epoch + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND " + column + " = " + str(value) @@ -391,6 +393,6 @@ WHERE batch_type = '" + batchType + "' \ AND processed = 1 \ - AND timestamp > '" + self.config.epoch + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND timestamp > '" + self.skychunk.epoch + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND timestamp BETWEEN now() - INTERVAL " + interval + " AND now()" @@ -416,6 +418,6 @@ WHERE batch_type = '" + batchType + "' \ AND loaded_to_datastore = 1 \ - AND timestamp > '" + self.config.epoch + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND timestamp > '" + self.skychunk.epoch + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ ORDER BY timestamp DESC LIMIT 1" @@ -515,5 +517,5 @@ WHERE stage_id = " + str(stage_id) + " \ AND batch_type = '" + batchType + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND batch_type != 'IN' \ AND timestamp BETWEEN now() - INTERVAL 12 HOUR AND now()" @@ -558,6 +560,6 @@ WHERE stage_id = " + str(stage_id) + " \ AND batch_type = '" + batchType + "' \ - AND timestamp > '" + self.config.epoch + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND timestamp > '" + self.skychunk.epoch + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND batch_type != 'IN' \ AND processed = 1" @@ -583,6 +585,6 @@ WHERE stage_id = " + str(stage_id) + " \ AND batch_type = '" + batchType + "' \ - AND timestamp > '" + self.config.epoch + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND timestamp > '" + self.skychunk.epoch + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND processed = -1" @@ -606,6 +608,6 @@ FROM batch \ WHERE batch_type = '" + batchType + "' \ - AND timestamp > '" + self.config.epoch + "' \ - AND dvo_db = '" + self.config.dvoLabel + "' \ + AND timestamp > '" + self.skychunk.epoch + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND processed = -1 \ GROUP BY stage_id HAVING numoccur >= " + str(self.MAX_FAILS) @@ -638,5 +640,5 @@ batchID = -1; - if self.config.force or \ + if self.skychunk.force or \ (not self.alreadyProcessed(batchType, stageID) \ and not self.processingNow(batchType, stageID) \ @@ -654,7 +656,7 @@ '" + batchType + "', \ " + str(stageID) + ", \ - '" + self.config.survey + "', \ - '" + self.config.dvoLabel + "', \ - '" + self.config.datastoreProduct + "' \ + '" + self.skychunk.survey + "', \ + '" + self.skychunk.dvoLabel + "', \ + '" + self.skychunk.datastoreProduct + "' \ )" self.logger.infoPair("heather:","sql") @@ -753,19 +755,19 @@ ''' - Sets the config field for this client - ''' - def setConfigForThisClient(self, config, host, pid): - - self.execute("UPDATE clients SET config = '" + config + "' \ + Sets the skychunk field for this client + ''' + def setSkychunkForThisClient(self, skychunk, host, pid): + + self.execute("UPDATE clients SET skychunk = '" + skychunk + "' \ WHERE host = '" + host + "' \ AND pid = " + str(pid)) ''' - Sets the config field for a set of loader clients - ''' - def setConfigForLoaders(self, config, ids): + Sets the skychunk field for a set of loader clients + ''' + def setSkychunkForLoaders(self, skychunk, ids): for id in ids: - self.execute("UPDATE clients SET config = '" + config + "' \ + self.execute("UPDATE clients SET skychunk = '" + skychunk + "' \ WHERE type = 'loader.py' \ AND id = " + str(id)) @@ -801,5 +803,5 @@ ''' def insertClient(self, type, host, pid): - self.execute("INSERT INTO clients (timestamp, type, host, pid, config) VALUES (now(), '" + type + "', '" + host + "', " + str(pid) + ", '" + self.config.name + "')") + self.execute("INSERT INTO clients (timestamp, type, host, pid, skychunk) VALUES (now(), '" + type + "', '" + host + "', " + str(pid) + ", '" + self.skychunk.name + "')") ''' @@ -845,41 +847,41 @@ ''' - Returns a list of available configs - ''' - def getActiveConfigList(self): + Returns a list of available skychunks + ''' + def getActiveSkychunkList(self): - sql = "SELECT DISTINCT name FROM config WHERE active = 1 ORDER BY name" - - configs = [] - try: - rs = self.executeQuery(sql) - while (rs.next()): configs.append(rs.getString(1)) - except: - self.logger.errorPair("Can't get config names", sql) + sql = "SELECT DISTINCT name FROM skychunk WHERE active = 1 ORDER BY name" + + skychunks = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): skychunks.append(rs.getString(1)) + except: + self.logger.errorPair("Can't get skychunk names", sql) finally: rs.close() - return configs - - ''' - Prompts user for info about a new or existing config and lets them neter the various details - ''' - def editConfig(self): + return skychunks + + ''' + Prompts user for info about a new or existing skychunk and lets them neter the various details + ''' + def editSkychunk(self): print " ********************************************************************" - print " * Config editor" + print " * Skychunk editor" print " *" - response = raw_input(" * Name for new config, or existing config to edit? ") + response = raw_input(" * Name for new skychunk, or existing skychunk to edit? ") if response == "": return - self.config.name = response - - # attempt to insert new config, if it already exists then carry on with update - sql = "INSERT INTO config (name) VALUES ('" + self.config.name + "')" + self.skychunk.name = response + + # attempt to insert new skychunk, if it already exists then carry on with update + sql = "INSERT INTO skychunk (name) VALUES ('" + self.skychunk.name + "')" try: self.execute(sql) except: pass - # for each column in the config table - sql = "SHOW COLUMNS FROM config" + # for each column in the skychunk table + sql = "SHOW COLUMNS FROM skychunk" rs = self.executeQuery(sql) while (rs.next()): @@ -895,5 +897,5 @@ # get current value for this field, if any try: - rs2 = self.executeQuery("SELECT " + field + " FROM config WHERE name = '" + self.config.name + "'") + rs2 = self.executeQuery("SELECT " + field + " FROM skychunk WHERE name = '" + self.skychunk.name + "'") rs2.first() default = rs2.getString(1) @@ -912,5 +914,5 @@ if type.find("varchar") != -1 or type.find("timestamp") != -1: quotes = "'" else: quotes = "" - self.execute("UPDATE config SET " + field + " = " + quotes + response + quotes + " WHERE name = '" + self.config.name + "'") + self.execute("UPDATE skychunk SET " + field + " = " + quotes + response + quotes + " WHERE name = '" + self.skychunk.name + "'") print " *" @@ -918,9 +920,9 @@ ''' - Returns config for this client - ''' - def getConfigForThisClient(self, host, pid): - - sql = "SELECT config \ + Returns skychunk for this client + ''' + def getSkychunkForThisClient(self, host, pid): + + sql = "SELECT skychunk \ FROM clients \ WHERE host = '" + host + "' \ @@ -930,8 +932,8 @@ rs = self.executeQuery(sql) rs.first() - self.config.name = rs.getString(1) - except: - self.logger.errorPair("No config set for", "%s (pid=%d)" % (host, pid)) - self.config.isLoaded = False + self.skychunk.name = rs.getString(1) + except: + self.logger.errorPair("No skychunk set for", "%s (pid=%d)" % (host, pid)) + self.skychunk.isLoaded = False return False @@ -939,12 +941,12 @@ ''' - Reads config from the database and populates Config object - ''' - def readConfig(self, host, pid): - - # first get config defined for this client - if not self.getConfigForThisClient(host, pid): return False - - # now load that config + Reads skychunk from the database and populates Skychunk object + ''' + def readSkychunk(self, host, pid): + + # first get skychunk defined for this client + if not self.getSkychunkForThisClient(host, pid): return False + + # now load that skychunk sql = "SELECT \ datastore_product \ @@ -970,58 +972,58 @@ ,queue_OB \ ,parallel \ - FROM config \ - WHERE name = '" + self.config.name + "'" - - self.config.batchTypes = [] + FROM skychunk \ + WHERE name = '" + self.skychunk.name + "'" + + self.skychunk.batchTypes = [] try: rs = self.executeQuery(sql) rs.first() - self.config.datastoreProduct = rs.getString(1) - self.config.datastoreType = rs.getString(2) - if rs.getInt(3) == 1: self.config.datastorePublishing = True - else: self.config.datastorePublishing = False - self.config.dvoLabel = rs.getString(4) + self.skychunk.datastoreProduct = rs.getString(1) + self.skychunk.datastoreType = rs.getString(2) + if rs.getInt(3) == 1: self.skychunk.datastorePublishing = True + else: self.skychunk.datastorePublishing = False + self.skychunk.dvoLabel = rs.getString(4) # if dvoLabel is null is can break queries, so set to something - if not self.config.dvoLabel: self.config.dvoLabel = "none" - self.config.dvoLocation = rs.getString(5) - self.config.minRa = rs.getDouble(6) - self.config.maxRa = rs.getDouble(7) - self.config.minDec = rs.getDouble(8) - self.config.maxDec = rs.getDouble(9) - - self.config.boxSize = rs.getDouble(10) - self.config.halfBox = self.config.boxSize/2.0 - self.config.boxSizeWithBorder = self.config.boxSize + (self.config.BORDER * 2) - - self.config.basePath = rs.getString(11) - self.config.dataRelease = rs.getInt(12) - - self.config.deleteLocal = rs.getInt(13) - self.config.deleteDatastore = rs.getInt(14) - self.config.deleteDxLayer = rs.getInt(15) - - self.config.epoch = rs.getString(16) - - self.config.survey = rs.getString(17) - self.config.pspsSurvey = rs.getString(18) - - if rs.getInt(19) == 1: self.config.batchTypes.append("P2") - if rs.getInt(20) == 1: self.config.batchTypes.append("ST") - if rs.getInt(21) == 1: self.config.batchTypes.append("OB") - - self.config.force = True # TODO - self.config.parallel = False # TODO - - if rs.getInt(22) == 1: self.config.parallel = True - - if self.config.parallel: print "USING parallel" - self.config.isLoaded = True - except: - self.logger.errorPair("Could not read config with name", self.config.name) - self.config.isLoaded = False - - return self.config.isLoaded + if not self.skychunk.dvoLabel: self.skychunk.dvoLabel = "none" + self.skychunk.dvoLocation = rs.getString(5) + self.skychunk.minRa = rs.getDouble(6) + self.skychunk.maxRa = rs.getDouble(7) + self.skychunk.minDec = rs.getDouble(8) + self.skychunk.maxDec = rs.getDouble(9) + + self.skychunk.boxSize = rs.getDouble(10) + self.skychunk.halfBox = self.skychunk.boxSize/2.0 + self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2) + + self.skychunk.basePath = rs.getString(11) + self.skychunk.dataRelease = rs.getInt(12) + + self.skychunk.deleteLocal = rs.getInt(13) + self.skychunk.deleteDatastore = rs.getInt(14) + self.skychunk.deleteDxLayer = rs.getInt(15) + + self.skychunk.epoch = rs.getString(16) + + self.skychunk.survey = rs.getString(17) + self.skychunk.pspsSurvey = rs.getString(18) + + if rs.getInt(19) == 1: self.skychunk.batchTypes.append("P2") + if rs.getInt(20) == 1: self.skychunk.batchTypes.append("ST") + if rs.getInt(21) == 1: self.skychunk.batchTypes.append("OB") + + self.skychunk.force = True # TODO + self.skychunk.parallel = False # TODO + + if rs.getInt(22) == 1: self.skychunk.parallel = True + + if self.skychunk.parallel: print "USING parallel" + self.skychunk.isLoaded = True + except: + self.logger.errorPair("Could not read skychunk with name", self.skychunk.name) + self.skychunk.isLoaded = False + + return self.skychunk.isLoaded ''' @@ -1031,8 +1033,8 @@ sql = "SELECT id FROM box \ - WHERE config = '" + self.config.name + "' \ + WHERE skychunk = '" + self.skychunk.name + "' \ AND ra_center = " + str(ra) + " \ AND dec_center = " + str(dec) + " \ - AND box_side = " + str(self.config.boxSize) + AND box_side = " + str(self.skychunk.boxSize) id = -1 @@ -1050,5 +1052,5 @@ ''' def removeAllBoxes(self): - self.execute("DELETE FROM box WHERE config = '" + self.config.name + "'") + self.execute("DELETE FROM box WHERE skychunk = '" + self.skychunk.name + "'") ''' @@ -1063,7 +1065,7 @@ sql = "INSERT INTO box \ - (config, ra_center, dec_center, box_side) \ + (skychunk, ra_center, dec_center, box_side) \ VALUES \ - ('" + self.config.name + "', " + str(ra) + ", " + str(dec) + ", " + str(self.config.boxSize) + ")" + ('" + self.skychunk.name + "', " + str(ra) + ", " + str(dec) + ", " + str(self.skychunk.boxSize) + ")" self.execute(sql) @@ -1104,10 +1106,10 @@ FROM box \ JOIN pending ON (id = box_id) \ - WHERE config = '" + self.config.name + "' \ + WHERE skychunk = '" + self.skychunk.name + "' \ AND batch_type = '" + batchType + "' \ AND ra_center NOT IN \ (SELECT ra_center \ FROM stripe \ - WHERE config = '" + self.config.name + "') \ + WHERE skychunk = '" + self.skychunk.name + "') \ GROUP BY ra_center LIMIT 1" @@ -1124,5 +1126,5 @@ FROM box \ JOIN pending ON (id = box_id) \ - WHERE config = '" + self.config.name + "' \ + WHERE skychunk = '" + self.skychunk.name + "' \ AND batch_type = '" + batchType + "' \ GROUP BY ra_center \ @@ -1144,5 +1146,5 @@ WHERE ra_center = " + str(raCenter) + " \ AND batch_type = '" + batchType + "' \ - AND config = '" + self.config.name + "' \ + AND skychunk = '" + self.skychunk.name + "' \ ORDER BY dec_center DESC" @@ -1177,6 +1179,6 @@ # now insert new stripe entry sql = "INSERT INTO stripe \ - (client_id, config, ra_center) \ - SELECT clients.id, clients.config, " + str(raCenter) + " \ + (client_id, skychunk, ra_center) \ + SELECT clients.id, clients.skychunk, " + str(raCenter) + " \ FROM clients \ WHERE host = '" + host + "' \ @@ -1206,5 +1208,5 @@ ''' - Returns ids for pending items for this config + Returns ids for pending items for this skychunk ''' def getPendingIds(self, batchType): @@ -1213,5 +1215,5 @@ FROM pending \ JOIN box ON (box_id = id) \ - WHERE config = '" + self.config.name + "' \ + WHERE skychunk = '" + self.skychunk.name + "' \ AND batch_type = '" + batchType + "'" @@ -1222,5 +1224,5 @@ rs.close() except: - self.logger.errorPair("Can't get pending ids for this config", sql) + self.logger.errorPair("Can't get pending ids for this skychunk", sql) return ids @@ -1253,10 +1255,10 @@ WHERE stage_id = " + str(stageID) + " \ AND batch_type = '" + batchType + "' \ - AND config = '" + self.config.name + "'" + AND skychunk = '" + self.skychunk.name + "'" self.execute(sql) ''' - Writes density-plot data to file for pending stuff over sky for given config + Writes density-plot data to file for pending stuff over sky for given skychunk ''' def createPendingDensityPlotData(self, batchType, DATFILE): @@ -1264,5 +1266,5 @@ sql = "SELECT id, ra_center, dec_center \ FROM box \ - WHERE config = '" + self.config.name + "' \ + WHERE skychunk = '" + self.skychunk.name + "' \ ORDER BY ra_center, dec_center" @@ -1317,5 +1319,5 @@ def getItemsInThisThisBox(self, ra, dec): - halfSide = self.config.boxSize/2.0 + halfSide = self.skychunk.boxSize/2.0 minRa = ra-halfSide maxRa = ra+halfSide @@ -1346,5 +1348,5 @@ FROM stripe \ JOIN clients ON (client_id = id) \ - WHERE clients.config = '" + self.config.name + "' \ + WHERE clients.skychunk = '" + self.skychunk.name + "' \ ORDER BY host, started" Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/loader.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/loader.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/loader.py (revision 35076) @@ -34,5 +34,5 @@ super(Loader, self).__init__(argv, 1, 1) - if self.config.parallel: + if self.skychunk.parallel: print "PARALLEL dvo" else: @@ -57,5 +57,5 @@ # create Datastore objects - self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb) # if an IN batch is requested, create and quit @@ -65,4 +65,5 @@ batch = InitBatch(self.logger, self.config, + self.skychunk, self.gpc1Db, self.ippToPspsDb, @@ -77,5 +78,5 @@ - # if an IN batch is requested, create and quit + # if the 'once' option is passed, we do not loop multiple times in 'run' self.onePassOnly = 0 if len(sys.argv) > 2 and sys.argv[2] == "once": @@ -85,29 +86,29 @@ self.parsePollTimeArg("0.0166") - self.config.printAll() + self.skychunk.printAll() ''' Overrides base-class version so we can ensure our scratch Db is using the right DVO Db ''' - def refreshConfig(self): - - ret = super(Loader, self).refreshConfig() + def refreshSkychunk(self): + + ret = super(Loader, self).refreshSkychunk() try: self.scratchDb except: return ret - self.dvoDetections = DvoDetections(self.logger, self.config, self.scratchDb.dbName) + self.dvoDetections = DvoDetections(self.logger, self.config, self.skychunk, self.scratchDb.dbName) return ret ''' - Overrides base_class version so we can break out of processing loops if the config changes + Overrides base_class version so we can break out of processing loops if the skychunk changes ''' def checkClientStatus(self): - oldConfigName = self.config.name + oldSkychunkName = self.skychunk.name super(Loader, self).checkClientStatus() - if oldConfigName != self.config.name: - self.logger.infoPair("Config changed", "from '" + oldConfigName + "' to '" + self.config.name + "'") - self.config.printAll() + if oldSkychunkName != self.skychunk.name: + self.logger.infoPair("Skychunk changed", "from '" + oldSkychunkName + "' to '" + self.skychunk.name + "'") + self.skychunk.printAll() return False @@ -125,5 +126,5 @@ abort = False - for batchType in self.config.batchTypes: + for batchType in self.skychunk.batchTypes: # get a stripe's worth of box IDs @@ -161,5 +162,5 @@ self.logger.infoPair("Box dimensions", "%.1f / %.1f / %.1f" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE'])) self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids)) - boxSizeWithBorder = boxDim['SIDE'] + (self.config.BORDER * 2) + boxSizeWithBorder = boxDim['SIDE'] + (self.skychunk.BORDER * 2) boxSizeSansBorder = boxDim['SIDE'] self.logger.infoPair("got here", "ok") @@ -179,5 +180,5 @@ # should do we pre-ingest stuff from DVO? - # NOTE EAM : this chunk loads the dvo detections into the mysql db + # NOTE EAM : this skychunk loads the dvo detections into the mysql db # XXXX EAM : this should happen for both P2 and Stack detection # XXXX EAM : in parallel model, this should happen for all P2 & Stack batches @@ -250,4 +251,5 @@ batch = DetectionBatch(self.logger, self.config, + self.skychunk, self.gpc1Db, self.ippToPspsDb, @@ -259,4 +261,5 @@ batch = StackBatch(self.logger, self.config, + self.skychunk, self.gpc1Db, self.ippToPspsDb, @@ -269,4 +272,5 @@ batch = ObjectBatch(self.logger, self.config, + self.skychunk, self.gpc1Db, self.ippToPspsDb, Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/objectbatch.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/objectbatch.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/objectbatch.py (revision 35076) @@ -32,4 +32,5 @@ logger, config, + skychunk, gpc1Db, ippToPspsDb, @@ -42,4 +43,5 @@ logger, config, + skychunk, gpc1Db, ippToPspsDb, @@ -52,5 +54,5 @@ try: - self.dvoObjects = DvoObjects(self.logger, self.config, self.scratchDb.dbName) + self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.scratchDb.dbName) except: self.logger.errorPair("Unable to create instance of", "DvoObjects") @@ -264,5 +266,5 @@ ,PSF_QF_PERF \ ,STARGAL_SEP \ - , " + str(self.config.dataRelease) + "\ + , " + str(self.skychunk.dataRelease) + "\ , RAND() \ FROM " + cptTableName Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/plot.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/plot.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/plot.py (revision 35076) @@ -15,16 +15,16 @@ Constructor ''' - def __init__(self, logger, config, ippToPspsDb): + def __init__(self, logger, skychunk, ippToPspsDb): self.logger = logger - self.config = config + self.skychunk = skychunk self.ippToPspsDb = ippToPspsDb ''' - Creates a density plot of pending exposures for this config + Creates a density plot of pending exposures for this skychunk ''' def createDensityPlot(self, batchType, forCzartool=False): - tempFilename = "./" + self.config.name + "_" + batchType + "_plotData.dat" + tempFilename = "./" + self.skychunk.name + "_" + batchType + "_plotData.dat" DATFILE = open(tempFilename,'w') max = self.ippToPspsDb.createPendingDensityPlotData(batchType, DATFILE) @@ -36,7 +36,8 @@ if forCzartool: - OUTPUTFILE = self.config.czarPlotsPath + "/ippToPsps_density_" + self.config.name + "_" + batchType + ".png" + ## XXX from config?? + OUTPUTFILE = self.skychunk.czarPlotsPath + "/ippToPsps_density_" + self.skychunk.name + "_" + batchType + ".png" else: - OUTPUTFILE = self.config.name + "_" + batchType + "_" + timestamp + ".png" + OUTPUTFILE = self.skychunk.name + "_" + batchType + "_" + timestamp + ".png" f = os.popen('/home/panstarrs/ipp/local/bin/gnuplot', 'w') @@ -49,8 +50,8 @@ print >> f, "set term " + TERM + "; \ set output \"" + OUTPUTFILE + "\"; \ - set title \"Unprocessed " + batchType + " items for '" + self.config.name + "' as of " + timestamp + "\"; \ + set title \"Unprocessed " + batchType + " items for '" + self.skychunk.name + "' as of " + timestamp + "\"; \ set grid; \ - set xrange [" + str(self.config.minRa) + ":" + str(self.config.maxRa) + "] reverse; \ - set yrange [" + str(self.config.minDec) + ":" + str(self.config.maxDec) + "]; \ + set xrange [" + str(self.skychunk.minRa) + ":" + str(self.skychunk.maxRa) + "] reverse; \ + set yrange [" + str(self.skychunk.minDec) + ":" + str(self.skychunk.maxDec) + "]; \ unset key; \ set palette rgb 22,13,10; \ Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/plotter.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/plotter.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/plotter.py (revision 35076) @@ -18,5 +18,5 @@ super(Plotter, self).__init__(argv) - self.plot = Plot(self.logger, self.config, self.ippToPspsDb) + self.plot = Plot(self.logger, self.skychunk, self.ippToPspsDb) if len(argv) < 3: Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/pollOdm.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/pollOdm.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/pollOdm.py (revision 35076) @@ -9,5 +9,5 @@ from ipptopsps import IppToPsps -from config import Config +from skychunk import Skychunk from odm import Odm from batch import Batch @@ -56,5 +56,5 @@ self.logger.infoSeparator() - self.logger.infoPair("Config", self.config.name) + self.logger.infoPair("Skychunk", self.skychunk.name) for stage in self.stages: Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/pslogger.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/pslogger.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/pslogger.py (revision 35076) @@ -11,5 +11,5 @@ This will default to stout and no file output ''' - def setup(self, programName, basePath, configName, host, pid, stdout=1, sendToFile=0): + def setup(self, programName, basePath, skychunkName, host, pid, stdout=1, sendToFile=0): formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S') @@ -21,5 +21,5 @@ PATH = basePath + "/log" if not os.path.exists(PATH): os.makedirs(PATH) - FULLPATH = PATH + "/" + programName + "_" + configName + "_" + host + "_" + str(pid) + ".log" + FULLPATH = PATH + "/" + programName + "_" + skychunkName + "_" + host + "_" + str(pid) + ".log" # opens file to be appended hdlr = logging.FileHandler(FULLPATH, "a") Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/queue.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/queue.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/queue.py (revision 35076) @@ -29,13 +29,13 @@ # create various objects self.gpc1Db = Gpc1Db(self.logger, self.config) - self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb) try: - self.dvoObjects = DvoObjects(self.logger, self.config) + self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk) except: self.exitProgram("Unable to create instance of DvoObject") raise - self.config.printAll() + self.skychunk.printAll() if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) @@ -43,5 +43,5 @@ ''' Main processing loop. - Tiled area is defined in config and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing + Tiled area is defined in 'skychunk' and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing by one or more loading clients that may be running on any host ''' @@ -54,5 +54,5 @@ # queue up batches that are processed but not loaded to datastore - for batchType in self.config.batchTypes: + for batchType in self.skychunk.batchTypes: self.logger.infoTitle("Previous failed datastore loads") @@ -66,8 +66,8 @@ if batchType == "OB": self.dvoObjects.setSkyArea( - self.config.minRa, - self.config.maxRa, - self.config.minDec, - self.config.maxDec) + self.skychunk.minRa, + self.skychunk.maxRa, + self.skychunk.minDec, + self.skychunk.maxDec) rows = self.dvoObjects.allPopulatedRegionInfo self.dvoObjects.printSummary() @@ -76,10 +76,10 @@ else: rows = self.gpc1Db.getItemsInThisDVODbForThisStage( - self.config.dvoLabel, + self.skychunk.dvoLabel, batchType, - self.config.minRa, - self.config.maxRa, - self.config.minDec, - self.config.maxDec) + self.skychunk.minRa, + self.skychunk.maxRa, + self.skychunk.minDec, + self.skychunk.maxDec) # EAM TEST I/O @@ -107,5 +107,5 @@ self.ippToPspsDb.storeAllItems(pending) - # loop through full range of RA/Dec queueing stuff in boxes of size self.config.boxSize + # loop through full range of RA/Dec queueing stuff in boxes of size self.skychunk.boxSize if len(ids) > 0: @@ -115,9 +115,9 @@ # starting positions - ra = self.config.minRa + self.config.halfBox - dec = self.config.minDec + self.config.halfBox + ra = self.skychunk.minRa + self.skychunk.halfBox + dec = self.skychunk.minDec + self.skychunk.halfBox - while ra <= self.config.maxRa: - while dec <= self.config.maxDec: + while ra <= self.skychunk.maxRa: + while dec <= self.skychunk.maxDec: box_id = self.ippToPspsDb.insertBox(ra, dec) @@ -128,12 +128,12 @@ ra, dec, - self.config.boxSize, + self.skychunk.boxSize, len(ids))) if len(ids) > 0: self.ippToPspsDb.insertPending(box_id, batchType, ids) - dec = dec + self.config.boxSize - dec = self.config.minDec + self.config.halfBox - ra = ra + self.config.boxSize + dec = dec + self.skychunk.boxSize + dec = self.skychunk.minDec + self.skychunk.halfBox + ra = ra + self.skychunk.boxSize self.logger.info("+-------------+-------------+-------------+-------------+") @@ -153,5 +153,5 @@ batchName = Batch.getNameFromID(batchID) - subDir = Batch.getSubDir(self.config.basePath, batchType, self.config.dvoLabel) + subDir = Batch.getSubDir(self.skychunk.basePath, batchType, self.skychunk.dvoLabel) tarballFile = Batch.getTarballFile(batchID) self.logger.infoPair("Batch name", batchName) Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/setupScratchDb.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/setupScratchDb.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/setupScratchDb.py (revision 35076) @@ -2,16 +2,23 @@ import sys +import os +import socket import stilts from java.lang import * from java.sql import * - -from ipptopsps import IppToPsps +from config import Config from scratchdb import ScratchDb +from ipptopspsdb import IppToPspsDb ''' SetupScratchDb class ''' -class SetupScratchDb(IppToPsps): +class SetupScratchDb(object): + + ## XXX EAM : 2013.01.31 - i have changed the 'config.py' concept + ## to only include the global static information (from + ## settings.xml), and stripped out the dynamic information to a + ## new entity called 'skychunk' ## XXX note that setupScratchDb.py fails if there are no valid @@ -47,16 +54,24 @@ ''' def __init__(self, argv): - super(SetupScratchDb, self).__init__(argv) + # set up config object + self.config = Config() - if len(argv) < 3: + if len(argv) < 2: self.printUsage() self.exitProgram("incorrect args") + # some class constants + self.HOST = socket.gethostname() + self.PID = os.getpid() + + # XXX Shouldn't config set up logger? + # (not yet, since we set HOST & PID with program...) + self.logger = self.config.getLogger(self.HOST, self.PID, 1, 0) + try: - self.scratchDb = ScratchDb(self.logger, self.config, '0', argv[2]) + self.scratchDb = ScratchDb(self.logger, self.config, argv[1]) except: self.exitProgram("Could not connect to a scratch Db") raise - def run(self): @@ -70,9 +85,6 @@ # install IN tables self.logger.infoPair("Installing", "initialization tables") - if self.configDir is None: - tables = stilts.treads("../config/tables.IN.vot") - else: - tablepath = self.configDir + "tables.IN.vot" - tables = stilts.treads(tablepath) + tablepath = self.config.configDir + "tables.IN.vot" + tables = stilts.treads(tablepath) for table in tables: @@ -86,9 +98,11 @@ self.scratchDb.createDvoTables() - ''' - Overrides base-class version - ''' def printUsage(self): - super(SetupScratchDb, self).printUsage("") + # write message to log or stdout if no logger set up + msg = os.path.basename(sys.argv[0]) + " " + try: + self.logger.errorPair("Usage:", msg) + except: + print "*** Usage: " + msg ''' @@ -100,4 +114,2 @@ setupScratchDb.exitProgram("completed") except: pass - - Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/skychunk.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/skychunk.py (revision 35076) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/skychunk.py (revision 35076) @@ -0,0 +1,133 @@ +#!/usr/bin/env jython + +import sys + +''' +A class encapsulating a ippToPsps processing skychunk. This information +is stored in the 'skychunk' table of the ipptopsps database +''' +class Skychunk(object): + + ''' + Constructor + + Basically reads the entire skychunk and stores values to class variables + ''' + def __init__(self, logger): + + # name of skychunk to work on (edit / all have special meanings) + self.name = sys.argv[1] + + self.createNewSkychunk = False + self.rotateSkychunks = False + + # a new skychunk? + if self.name == "edit": self.createNewSkychunk = True + + # should we rotate skychunks for this program? + if self.name == "all": self.rotateSkychunks = True + + print "name: ", self.name + print "new: ", self.createNewSkychunk + print "all: ", self.rotateSkychunks + + self.logger = logger + + # this is the border (in degrees) that we place around any loading box of + # PS1 pointings to ensure we pull a large enough area out of DVO + + # XXX this is a poor place to put this information -- it is completely gpc-specific. + # probably should go elsewhere + self.BORDER = 1.65 + self.isLoaded = False + + ''' + Prints everything for this skychunk + ''' + def printAll(self): + + self.logger.infoTitle("Skychunk") + + try: + self.logger.infoSeparator() + self.logger.infoPair("Skychunk name", self.name) + self.logger.infoPair("Survey", self.survey) + self.logger.infoPair("Publishing to PSPS as survey", self.pspsSurvey) + self.logger.infoPair("Loading epoch", self.epoch) + self.logger.infoPair("Data release", "%d" % self.dataRelease) + for batchType in self.batchTypes: self.logger.infoPair("Queuing batch type", batchType) + except: + pass + + self.printDvoInfo() + self.printDatastoreInfo() + self.printBoxCoords() + self.printDeletionPolicy() + try: self.logger.infoSeparator() + except: pass + + ''' + Queuing this batch type? + ''' + def queuingThisBatchType(self, batchType): + if batchType in self.batchTypes: return 1 + return 0 + + ''' + Creates a logger object and returns it + ''' + def getLogger(self, host, pid, stdout=1, sendToFile=0): + + logging.setLoggerClass(PSLogger) + self.logger = logging.getLogger(self.programName) + self.logger.setup(self.programName, self.logPath, self.name, host, pid, stdout, sendToFile) + + return self.logger + + ''' + Prints the currently set DVO info + ''' + def printDvoInfo(self): + + try: + self.logger.infoPair("DVO label", self.dvoLabel) + self.logger.infoPair("DVO location", self.dvoLocation) + except: + pass + + ''' + Prints datastore info + ''' + def printDatastoreInfo(self): + + try: + self.logger.infoBool("Datastore publishing?", self.datastorePublishing) + if self.datastorePublishing: + self.logger.infoPair("Datastore type", self.datastoreProduct) + self.logger.infoPair("Datastore product", self.datastoreType) + except: + pass + + ''' + Prints the current deletion policy + ''' + def printDeletionPolicy(self): + + try: + self.logger.infoBool("Deleting local?", self.deleteLocal) + self.logger.infoBool("Deleting datastore?", self.deleteDatastore) + self.logger.infoBool("Deleting DXLayer?", self.deleteDxLayer) + except: + pass + + ''' + Prints the currently set RA/Dec bounding box + ''' + def printBoxCoords(self): + + try: + self.logger.infoPair("RA limits", "%.1f -> %.1f" % (self.minRa, self.maxRa)) + self.logger.infoPair("Dec limits", "%.1f -> %.1f" % (self.minDec, self.maxDec)) + self.logger.infoPair("Loading box size", "%.1f degrees" % self.boxSize) + except: + pass Index: branches/eam_branches/ipp-20121219/ippToPsps/jython/stackbatch.py =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/jython/stackbatch.py (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/jython/stackbatch.py (revision 35076) @@ -33,4 +33,5 @@ logger, config, + skychunk, gpc1Db, ippToPspsDb, @@ -43,4 +44,5 @@ logger, config, + skychunk, gpc1Db, ippToPspsDb, @@ -49,5 +51,5 @@ batchID, "ST", - gpc1Db.getStackStageCmf(config.dvoLabel, stackID), + gpc1Db.getStackStageCmf(skychunk.dvoLabel, stackID), useFullTables) @@ -421,5 +423,5 @@ self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackMeta", self.filter) - self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.skychunk.dataRelease)) self.updateStackTypeID("StackMeta") @@ -528,5 +530,5 @@ self.scratchDb.updateFilterID("StackDetection", self.filter) - self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.skychunk.dataRelease)) self.scratchDb.updateAllRows("StackDetection", "primaryF", "0") @@ -612,5 +614,5 @@ self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackApFlx", self.filter) - self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.skychunk.dataRelease)) self.scratchDb.updateAllRows("StackApFlx", "primaryF", "0") self.scratchDb.updateAllRows("StackApFlx", "activeFlag", "0") @@ -646,5 +648,5 @@ self.scratchDb.updateAllRows("StackModelFit", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackModelFit", self.filter) - self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.skychunk.dataRelease)) self.scratchDb.updateAllRows("StackModelFit", "primaryF", "0") self.scratchDb.updateAllRows("StackModelFit", "activeFlag", "0") @@ -711,5 +713,5 @@ self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID)) - self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.skychunk.dataRelease)) ''' @@ -756,5 +758,5 @@ AND b.imageID = " + str(imageID) self.scratchDb.execute(sql) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) def populateStackDetectionCalib(self): @@ -787,5 +789,5 @@ b.expTime, \ b.airMass, \ - " + str(self.config.dataRelease) + " \ + " + str(self.skychunk.dataRelease) + " \ FROM \ StackDetection as a \ @@ -801,5 +803,5 @@ ## XXX write this with the select/insert - # self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) + # self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) ''' @@ -821,5 +823,5 @@ self.scratchDb.updateFilterID("ObjectCalColor", self.filter) - self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.config.dataRelease)) + self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.skychunk.dataRelease)) Index: branches/eam_branches/ipp-20121219/ippToPsps/test/fulltest.sh =================================================================== --- branches/eam_branches/ipp-20121219/ippToPsps/test/fulltest.sh (revision 35075) +++ branches/eam_branches/ipp-20121219/ippToPsps/test/fulltest.sh (revision 35076) @@ -25,11 +25,11 @@ # mkgpc1.sh user localhost eam eam -set mkgpc1 = 1 -set initdb = 1 -set mkdummy = 1 -set initscratch = 1 -set initbatch = 1 -set camqueue = 1 -set cambatch = 1 +set mkgpc1 = 0 +set initdb = 0 +set mkdummy = 0 +set initscratch = 0 +set camqueue = 0 +set initbatch = 0 +set cambatch = 0 set stackqueue = 1 set stackbatch = 1 @@ -54,33 +54,34 @@ endif -# create a dummy config so setupScratchDb.py does not fail -if ($mkdummy) then - ippjython queue.py -test edit <