Index: trunk/ippToPsps/jython/batch.py =================================================================== --- trunk/ippToPsps/jython/batch.py (revision 36726) +++ trunk/ippToPsps/jython/batch.py (revision 37246) @@ -37,11 +37,5 @@ batchID, batchType, - fits, - useFullTables): - - # print "starting the batch" - - # self.printline = 0 - # self.testprint() + fits): self.readHeader = False @@ -49,5 +43,4 @@ self.skychunk = skychunk self.fits = fits - self.useFullTables = useFullTables # set up logging @@ -55,6 +48,4 @@ self.logger.infoSeparator() self.logger.debug("Batch class constructor") - - # self.testprint() # set up class variables @@ -68,6 +59,4 @@ self.pspsVoTableFilePath = self.config.configDir + "tables." + batchType + ".vot" - # self.testprint() - # check FITS file is ok if self.fits: @@ -79,5 +68,6 @@ raise - self.scratchDb.setUseFullTables(self.useFullTables) + # define the dvo table names + self.scratchDb.setDvoTableNames() # TODO @@ -107,12 +97,10 @@ if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath) - # self.testprint() - # store today's date now = datetime.datetime.now(); self.dateStr = now.strftime("%Y-%m-%d") - # create DVO tables if accessing DVO directly - if not self.useFullTables: self.scratchDb.createDvoTables() + # create DVO tables (scratch database) + self.scratchDb.createDvoTables() # dump stuff to the log @@ -120,11 +108,8 @@ self.logger.infoPair("Batch name", self.batchName) self.logger.infoPair("Survey ID", "%d" % self.surveyID) - self.logger.infoBool("Use full DVO tables?", self.useFullTables) if self.fits: self.logger.infoPair("Input FITS file", self.fits.getOriginalPath()) self.logger.infoPair("Input FITS primary header", "%s cards found" % self.fits.getPrimaryHeaderCardCount()) - - # self.testprint() self.logger.infoPair("Output path", self.localOutPath) @@ -237,4 +222,40 @@ self.logger.errorPair("Missing header field", key) return "NULL" + + ''' + Returns the string keyword value from this header or else "NULL" + ''' + def getKeyValue(self, header, key): + + if key in header: return header[key] + else: + self.logger.errorPair("Missing header field", key) + return "NULL" + + ''' + Returns the string keyword value from this header or else "NULL" + ''' + def getKeyFloat(self, header, format, key): + + if key in header: + value = format % float(header[key]) + else: + self.logger.errorPair("Missing header field", key) + value = format % -999.9 + + return value + + ''' + Returns the string keyword value from this header or else "NULL" + ''' + def getKeyInt(self, header, defValue, key): + + if key in header: + value = str(int(header[key])) + else: + self.logger.errorPair("Missing header field", key) + value = str(defValue) + + return value ''' @@ -373,5 +394,5 @@ self.tablesToExport.append(table.name) - return self.alterPspsTables(); + return True ''' @@ -383,5 +404,5 @@ ''' - Alter PSPS tables + Alter PSPS tables (Subclass implements this) ''' def alterPspsTables(self): @@ -417,4 +438,5 @@ # print "read smf table ", table + self.logger.infoPair("read smf table", table) # drop any previous tables before import @@ -440,94 +462,4 @@ return True - ''' - Imports IPP tables from FITS file - - Accepts a regular expression filter so not all tables need to be imported - ''' - def importIppSTTables(self, columns="*", filter=""): - - self.logger.infoPair("Importing ST tables with filter", filter) - - # print "trying to read ", self.fits.getPath() - - # ST has 5 cmf files - all with the same table names. To counter this we do it like this. - - filters = ["g","r","i","z","y"] - count = 0 - for f in filters: - skip = 0 - if (f == "g"): - if self.gstackID > 0: - fileName = self.gfits.getPath() - else: - skip = 1 - if (f == "r"): - if self.rstackID > 0: - fileName = self.rfits.getPath() - else: - skip = 1 - if (f == "i"): - if self.istackID > 0: - fileName = self.ifits.getPath() - else: - skip = 1 - if (f == "z"): - if self.zstackID > 0: - fileName = self.zfits.getPath() - else: - skip = 1 - if (f == "y"): - if self.ystackID > 0: - fileName = self.yfits.getPath() - else: - skip = 1 - - if skip == 1: - self.logger.infoPair("no files for filter" , f) - else: - self.logger.infoPair("using filename:",fileName) - - try: - tables = stilts.treads(fileName) - except: - self.logger.errorPair("STILTS could not import from", fileName) - return False - - #count = 0 - for table in tables: - - # print "import smf table ", table - match = re.match(filter, table.name) - - if not match: continue - self.logger.infoPair("Reading IPP table", f + table.name) - table = stilts.tpipe(table, cmd='addcol table_index $0') - - table = stilts.tpipe(table, cmd='explodeall') - - # print "read smf table ", table - - # drop any previous tables before import - self.scratchDb.dropTable(f + table.name) - - # IPP FITS files are littered with infinities, so remove these - self.logger.debug("Removing Infinity values from all columns") - table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') - table = stilts.tpipe(table, cmd='replaceval -Infinity null *') - table = stilts.tpipe(table, cmd='replaceval Infinity null *') - # print "cleaned up values ", table - - try: - table.write(self.scratchDb.url + '#' + f + table.name) - # XXX this one is not needed - # self.scratchDb.killLastConnectionID() - count = count + 1 - except: - self.logger.exception("Problem writing table '" + f + table.name + "' to the database") - - self.logger.infoPair("Done. Imported", "%d tables" % count) - self.indexIppTables() - - return True ''' @@ -548,5 +480,4 @@ # get everything from table try: - print "reading table from mysql ", table _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table) self.scratchDb.killLastConnectionID() @@ -633,6 +564,5 @@ def run(self): - # this is badly named : it creates the tables and then fills in - # the ImageMeta tables for each ota (alterPspsTables) + # creates the empty PSPS tables if not self.createEmptyPspsTables(): self.logger.errorPair("Aborting this batch", "could not create empty PSPS tables") @@ -640,23 +570,21 @@ raise + # fill in basic, not cmf/smf data (eg, ImageMeta for detectionbatch) + if not self.alterPspsTables(): + self.logger.errorPair("Aborting this batch", "could not alter PSPS tables for batch type") + self.ippToPspsDb.updateProcessed(self.batchID, -1) + raise + # for P2/ST, this reads the detection tables from the CMF/SMF file(s) # for OB, this imports object data from DVO - if self.batchType == "ST": - self.logger.infoPair("did we get here","?") - if not self.importIppSTTables(): - self.logger.errorPair("skipping this batch", "could not import ST IPP tables") - self.ippToPspsDb.updateProcessed(self.batchID, -1) - raise - self.logger.infoPair("we did the ST","yes") - else: - # if not ST we do this - if not self.importIppTables(): - if self.batchType == "OB": - self.logger.errorPair("skipping this batch", "no dvo tables for this region") - self.ippToPspsDb.updateProcessed(self.batchID, 1) - return True - self.logger.errorPair("Aborting this batch", "could not import IPP tables") - self.ippToPspsDb.updateProcessed(self.batchID, -1) - raise + if not self.importIppTables(): + if self.batchType == "OB": + self.logger.errorPair("skipping this batch", "no dvo tables for this region") + self.ippToPspsDb.updateProcessed(self.batchID, 1) + return True + self.logger.errorPair("Aborting this batch", "could not import IPP tables") + self.ippToPspsDb.updateProcessed(self.batchID, -1) + raise + self.logger.infoPair("populatePspsTables","ok") if not self.populatePspsTables(): @@ -692,4 +620,8 @@ tarballFile) + if self.config.onebatch: + self.logger.infoPair("onebatch requested", "exiting") + sys.exit(0) + +# XXX why is this down here?? from datastore import Datastore -