Index: /trunk/ippToPsps/jython/batch.py =================================================================== --- /trunk/ippToPsps/jython/batch.py (revision 38994) +++ /trunk/ippToPsps/jython/batch.py (revision 38995) @@ -453,5 +453,5 @@ ''' def importIppTables(self, columns="*", filter=""): - if self.useOhanaMYSQLtoFITS: + if self.config.useOhanaMYSQLtoFITS: if not self.importIppTablesOhana(): self.logger.errorPair("problem running", "importIppTablesOhana") @@ -480,5 +480,5 @@ return False - self.scratchDb.loadFITStoMYSQL(self.fits.getPath(), "none") + self.scratchDb.loadFITStoMYSQL(self.fits.getPath()) ## may need to do this with direct sql: @@ -515,7 +515,7 @@ def importIppTablesStilts(self, columns="*", filter=""): - self.logger.infoPair("Importing tables with filter", filter) + self.logger.infoPair("Importing tables (stilts) with filter", filter) - # print "trying to read ", self.fits.getPath() + print "trying to read ", self.fits.getPath() try: @@ -527,16 +527,18 @@ self.tablesLoaded = [] + print "read tables from ", self.fits.getPath() + count = 0 for table in tables: # example of listing all columns in a table: - ## print "-----table: " + table.name - ## columns = table.columns() - ## for column in columns: - ## print column - ## - ## print "-----" - - # print "import smf table ", table + print "-----table: " + table.name + myColumns = table.columns() + for column in myColumns: + print column + print "-----" + + print "import smf table ", table.name + match = re.match(filter, table.name) if not match: continue @@ -555,7 +557,9 @@ self.logger.debug("Removing Infinity values from all columns") table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') + print "-----" table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + print "-----" table = stilts.tpipe(table, cmd='replaceval Infinity null *') - # print "cleaned up values ", table + print "cleaned up values ", table.name try: Index: /trunk/ippToPsps/jython/config.py =================================================================== --- /trunk/ippToPsps/jython/config.py (revision 38994) +++ /trunk/ippToPsps/jython/config.py (revision 38995) @@ -21,5 +21,5 @@ def __init__(self): - self.useOhanaMYSQLtoFITS = False + self.useOhanaMYSQLtoFITS = True self.test = False self.resetDvo = False @@ -41,4 +41,8 @@ # by default, we use gpc1test in a test, and do not if not a test self.gpc1test = self.test + + if sys.argv.count("-stilts"): + self.useOhanaMYSQLtoFITS = False + sys.argv.remove("-stilts") if sys.argv.count("-realgpc"): Index: /trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- /trunk/ippToPsps/jython/detectionbatch.py (revision 38994) +++ /trunk/ippToPsps/jython/detectionbatch.py (revision 38995) @@ -844,4 +844,5 @@ self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname]) self.logger.info("updated dvoids") + results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID") self.logger.info("deleted nulls") @@ -1060,6 +1061,6 @@ def importIppTables(self, filter=""): - ## XXX does the regex work? - regex = "XY*" + ## regex only works for -stilts import + regex = ".*" ## for a quick test, just do the first chip: @@ -1076,9 +1077,4 @@ columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG POSANGLE PLTSCALE PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_FWHM_MAJ PSF_FWHM_MIN PSF_THETA PSF_CORE PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_FLUX AP_FLUX_SIG AP_NPIX AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2" - #### crap crap crap - - ## this is apparently over-riding the choices above - ## regex = ".*" - ## the list of columns above works fine for *.psf, but breaks the *.deteff tables ## just read all columns Index: /trunk/ippToPsps/jython/dvo.py =================================================================== --- /trunk/ippToPsps/jython/dvo.py (revision 38994) +++ /trunk/ippToPsps/jython/dvo.py (revision 38995) @@ -590,5 +590,5 @@ ''' def importPhotcodes(self, path, tableName): - if True: + if self.config.useOhanaMYSQLtoFITS: self.importPhotcodesOhana(path,tableName) else: @@ -607,5 +607,11 @@ # table = stilts.tpipe(table, cmd='addcol -after TYPE TYPE_AS_INT "TYPE + 0"') - self.scratchDb.loadFITStoMYSQL (path, tableName) + self.scratchDb.loadFITStoMYSQL (path, "-tablename " + tableName) + + sql = "ALTER TABLE " + tableName + " ADD COLUMN TYPE_AS_INT INT AFTER TYPE" + self.scratchDb.execute(sql) + + sql = "UPDATE " + tableName + " SET TYPE_AS_INT = ascii(TYPE)" + self.scratchDb.execute(sql) self.logger.debugPair("Finished importing", tableName) Index: /trunk/ippToPsps/jython/scratchdb.py =================================================================== --- /trunk/ippToPsps/jython/scratchdb.py (revision 38994) +++ /trunk/ippToPsps/jython/scratchdb.py (revision 38995) @@ -809,5 +809,5 @@ Use Ohana fits_to_mysql to load FITS tables into scratch db ''' - def loadFITStoMYSQL(self, filename, tablename): + def loadFITStoMYSQL(self, filename, tablename=""): self.logger.infoPair("starting loadFITStoMYSQL: ", filename) @@ -819,9 +819,7 @@ cmd += " -dbuser " + self.dbUser cmd += " -dbpass " + self.dbPass + cmd += " " + tablename self.logger.debugPair("preparing command: ", cmd) - - if tablename != "none": - cmd += " -tablename " + tablename (stdoutFile, stdoutName) = tempfile.mkstemp(prefix="fits_to_mysql.stdout.", dir="/tmp") Index: /trunk/ippToPsps/jython/stackbatch.py =================================================================== --- /trunk/ippToPsps/jython/stackbatch.py (revision 38994) +++ /trunk/ippToPsps/jython/stackbatch.py (revision 38995) @@ -1387,8 +1387,68 @@ ''' Imports IPP tables from FITS file + + Accepts a regular expression filter so not all tables need to be imported + ''' + def importIppTables(self, columns="*", filter=""): + if self.config.useOhanaMYSQLtoFITS: + if not self.importIppTablesOhana(): + self.logger.errorPair("problem running", "importIppTablesOhana") + return False + else: + if not self.importIppTablesStilts(columns, filter): + self.logger.errorPair("problem running", "importIppTablesStilts") + return False + + return True + + ''' + Imports IPP tables from FITS file + Accepts a regular expression to match the tables so not all tables need to be imported + ''' + def importIppTablesOhana(self, columns="*", tableRE=""): + + self.logger.infoPair("Importing ST tables using ", "Ohana") + + self.tablesLoaded = [] + + count = 0 + for filter in self.filters: + if self.stackIDs[filter] <= 0: + self.logger.infoPair("no files for filter" , filter) + continue + + fileName = self.fits[filter].getPath() + + self.logger.infoPair("using filename:",fileName) + + ## output table names need to have "filter" prepended. + ## add this as an option + self.scratchDb.loadFITStoMYSQL(fileName, "-tableprefix " + filter) + ## XXX table = stilts.tpipe(table, cmd='addcol table_index $0') + + # get the list of tables in this file: + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False + + for table in tables: + self.logger.debugPair("Read IPP table", table.name) + myTable = filter + table.name.replace(".", "_") + self.tablesLoaded.append(myTable) + count += 1 + + self.logger.infoPair("Done. Imported", "%d tables" % count) + self.indexIppTables() + + return True + + ''' + Imports IPP tables from FITS file Accepts a regular expression to match the tables so not all tables need to be imported (This regex feature is not currently used...) ''' - def importIppTables(self, columns="*", tableRE=""): + def importIppTablesStilts(self, columns="*", tableRE=""): if self.config.retry: return True