Index: /trunk/ippToPsps/jython/batch.py =================================================================== --- /trunk/ippToPsps/jython/batch.py (revision 38998) +++ /trunk/ippToPsps/jython/batch.py (revision 38999) @@ -466,6 +466,4 @@ ''' Imports IPP tables from FITS file - - Accepts a regular expression filter so not all tables need to be imported ''' def importIppTablesOhana(self): @@ -473,21 +471,24 @@ self.logger.infoPair("Importing IPP tables", "using fits_to_mysql") + count = 0 + self.tablesLoaded = [] + + fileName = self.fits.getPath() + + # this option is equivalent to the stilts "addcol table_index $0" call + fitsOptions = " -sequence-column table_index" + fitsOptions += " -sequence-column-start 1" + + ## *** actually write the FITS data to mysql + self.scratchDb.loadFITStoMYSQL(fileName, fitsOptions) + ## use stilts to get a list of the tables from the header try: - tables = stilts.treads(self.fits.getPath()) + tables = stilts.treads(fileName) except: - self.logger.errorPair("STILTS could not import from", self.fits.getPath()) - return False - - self.scratchDb.loadFITStoMYSQL(self.fits.getPath()) - - ## may need to do this with direct sql: - # table = stilts.tpipe(table, cmd='addcol table_index $0') - - self.tablesLoaded = [] - - count = 0 + self.logger.errorPair("STILTS could not import from", fileName) + return False + for table in tables: - # example of listing all columns in a table: ## print "-----table: " + table.name Index: /trunk/ippToPsps/jython/diffbatch.py =================================================================== --- /trunk/ippToPsps/jython/diffbatch.py (revision 38998) +++ /trunk/ippToPsps/jython/diffbatch.py (revision 38999) @@ -564,8 +564,80 @@ ''' Imports IPP tables from FITS file + + Accepts a regular expression filter so not all tables need to be imported + ''' + def importIppTables(self, columns="*", filter=""): + if self.config.useOhanaMYSQLtoFITS: + if not self.importIppTablesOhana(): + self.logger.errorPair("problem running", "importIppTablesOhana") + return False + else: + if not self.importIppTablesStilts(columns, filter): + self.logger.errorPair("problem running", "importIppTablesStilts") + return False + + return True + + ''' + Imports IPP tables from FITS file + ''' + def importIppTablesOhana(self): + + self.logger.infoPair("Importing DF tables", "using fits_to_mysql") + + count = 0 + self.tablesLoaded = [] + self.haveApNpix = {} + + for num in self.number: + diffSkyFileID = self.number[num] + fileName = self.fits[num].getPath() + + self.logger.infoPair("using filename:",fileName) + + ## output table names need to have "_num" appended. + fitsOptions = "-tablesuffix _" + str(num) + + # this option is equivalent to the stilts "addcol table_index $0" call + fitsOptions += " -sequence-column table_index" + fitsOptions += " -sequence-column-start 1" + + ## *** actually write the FITS data to mysql + self.scratchDb.loadFITStoMYSQL(fileName, fitsOptions) + + ## use stilts to get a list of the tables from the header + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False + + # this is an optional field, check for it in the table below + self.haveApNpix[num] = 0 + + for table in tables: + # check for AP_NPIX columns: missing for SAS39 + if table.name == "SkyChip_psf": + columns = table.columns() + if 'AP_NPIX' in columns: + self.haveApNpix[num] = 1 + + myTable = table.name + "_" + str(num) + count = count + 1 + + self.tablesLoaded.append(myTable) + + # after all tables are read for this 'num', then index the set + self.indexIppTables(num) + + self.logger.infoPair("Done. Imported", "%d tables" % count) + return True + + ''' + Imports IPP tables from FITS file Accepts a regular expression to match the tables so not all tables need to be imported (This regex feature is not currently used...) ''' - def importIppTables(self, columns="*", tableRE=""): + def importIppTablesStilts(self, columns="*", tableRE=""): if self.config.retry: return True @@ -632,9 +704,4 @@ return True - - - - - ''' Overriding this method. Use regex to trim off, eg _XY33 extension Index: /trunk/ippToPsps/jython/forcedwarpbatch.py =================================================================== --- /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 38998) +++ /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 38999) @@ -747,11 +747,77 @@ return True - - + + ''' + Imports IPP tables from FITS file + + Accepts a regular expression filter so not all tables need to be imported + ''' + def importIppTables(self, columns="*", filter=""): + if self.config.useOhanaMYSQLtoFITS: + if not self.importIppTablesOhana(): + self.logger.errorPair("problem running", "importIppTablesOhana") + return False + else: + if not self.importIppTablesStilts(columns, filter): + self.logger.errorPair("problem running", "importIppTablesStilts") + return False + + return True + + ''' + Imports IPP tables from FITS file + ''' + def importIppTablesOhana(self): + + self.logger.infoPair("Importing DF tables", "using fits_to_mysql") + + count = 0 + self.tablesLoaded = [] + self.haveApNpix = {} + + for num in self.number: + forcedWarpID = self.number[num] + fileName = self.fits[num].getPath() + + self.logger.infoPair("loading for FW id", forcedWarpID) + + self.logger.infoPair("using filename:",fileName) + + ## output table names need to have "_num" appended. + fitsOptions = "-tablesuffix _" + str(forcedWarpID) + + # this option is equivalent to the stilts "addcol table_index $0" call + fitsOptions += " -sequence-column table_index" + fitsOptions += " -sequence-column-start 1" + + ## *** actually write the FITS data to mysql + self.scratchDb.loadFITStoMYSQL(fileName, fitsOptions) + + ## use stilts to get a list of the tables from the header + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False + + # this is an optional field, check for it in the table below + self.haveApNpix[num] = 0 + + for table in tables: + myTable = table.name + "_" + str(forcedWarpID) + count = count + 1 + + self.tablesLoaded.append(myTable) + + self.indexIppTables(forcedWarpID) + + # after all tables are read for this forcedWarpID, then index the set + self.logger.infoPair("Done. Imported", "%d tables" % count) + return True ''' This function reads the cmf/smf file and loads it into the database as its own table ''' - def importIppTables(self, columns="*", tableRE=""): + def importIppTablesStilts(self, columns="*", tableRE=""): self.tablesLoaded = [] @@ -760,8 +826,8 @@ for num in self.number: forcedWarpID = self.number[num] - if self.config.retry: return True + fileName = self.fits[num].getPath() + self.logger.infoPair("loading for add id", forcedWarpID) self.logger.infoPair("Importing FW tables with table match expression: ", tableRE) - fileName = self.fits[num].getPath() self.logger.infoPair("using filename:",fileName) Index: /trunk/ippToPsps/jython/stackbatch.py =================================================================== --- /trunk/ippToPsps/jython/stackbatch.py (revision 38998) +++ /trunk/ippToPsps/jython/stackbatch.py (revision 38999) @@ -1423,7 +1423,12 @@ ## output table names need to have "filter" prepended. - ## add this as an option - self.scratchDb.loadFITStoMYSQL(fileName, "-tableprefix " + filter) - ## XXX table = stilts.tpipe(table, cmd='addcol table_index $0') + fitsOptions = "-tableprefix " + filter + + # this pair of options is equivalent to the stilts "addcol table_index $0" call + fitsOptions += " -sequence-column table_index" + fitsOptions += " -sequence-column-start 1" + + ## *** actually write the FITS data to mysql + self.scratchDb.loadFITStoMYSQL(fileName, fitsOptions) # get the list of tables in this file: