Index: /branches/eam_branches/ipp-20120405/ippToPsps/config/ippToPspsDbSchema.sql =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/config/ippToPspsDbSchema.sql (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/config/ippToPspsDbSchema.sql (revision 33948) @@ -45,5 +45,5 @@ `purged` tinyint(4) NOT NULL default '0', PRIMARY KEY (`batch_id`) -) ENGINE=InnoDB AUTO_INCREMENT=326479 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=328286 DEFAULT CHARSET=latin1; SET character_set_client = @saved_cs_client; @@ -65,5 +65,5 @@ KEY `fk_config` (`config`), CONSTRAINT `fk_config` FOREIGN KEY (`config`) REFERENCES `config` (`name`) ON DELETE CASCADE -) ENGINE=InnoDB AUTO_INCREMENT=25457 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=44588 DEFAULT CHARSET=latin1; SET character_set_client = @saved_cs_client; @@ -87,5 +87,5 @@ PRIMARY KEY (`id`), UNIQUE KEY `unique_key` (`host`,`pid`) -) ENGINE=InnoDB AUTO_INCREMENT=961 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=1227 DEFAULT CHARSET=latin1; SET character_set_client = @saved_cs_client; @@ -110,5 +110,5 @@ `max_dec` double default '90', `box_size` double default '4', - `base_path` varchar(1000) default NULL, + `base_path` varchar(1000) default '/data/ipp005.0/ipptopsps/', `data_release` smallint(6) default '0', `delete_local` tinyint(1) default '0', @@ -120,4 +120,6 @@ `queue_P2` tinyint(1) default '1', `queue_ST` tinyint(1) default '0', + `queue_OB` tinyint(1) default '0', + `active` tinyint(1) default '1', UNIQUE KEY `name` (`name`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; @@ -141,4 +143,19 @@ -- +-- Table structure for table `object` +-- + +DROP TABLE IF EXISTS `object`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `object` ( + `batch_id` bigint(20) unsigned NOT NULL, + `region` varchar(50) default NULL, + KEY `batch_id` (`batch_id`), + CONSTRAINT `object_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- -- Table structure for table `pending` -- @@ -151,7 +168,6 @@ `batch_type` varchar(10) default NULL, `stage_id` bigint(20) default NULL, - `ra_bore` float default NULL, - `dec_bore` float default NULL, UNIQUE KEY `a_key` (`box_id`,`stage_id`,`batch_type`), + UNIQUE KEY `stage_id_key` (`stage_id`), CONSTRAINT `box_key` FOREIGN KEY (`box_id`) REFERENCES `box` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=latin1; @@ -199,3 +215,3 @@ /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; --- Dump completed on 2012-03-22 7:59:34 +-- Dump completed on 2012-04-10 6:00:59 Index: /branches/eam_branches/ipp-20120405/ippToPsps/config/settings.xml =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/config/settings.xml (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/config/settings.xml (revision 33948) @@ -5,5 +5,5 @@ - /data/ipp005.0/rhenders + /data/ipp005.0/ipptopsps Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/batch.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/batch.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/batch.py (revision 33948) @@ -39,5 +39,4 @@ useFullTables): - self.everythingOK = False self.readHeader = False self.config = config @@ -66,5 +65,5 @@ if not self.header: logger.errorPair("Could not read FITS for id", "%d" % id) - return + raise self.scratchDb.setUseFullTables(self.useFullTables) @@ -115,5 +114,4 @@ self.logger.infoPair("Output path", self.localOutPath) - self.everythingOK = True ''' @@ -510,28 +508,23 @@ def run(self): - if not self.everythingOK: - self.logger.errorPair("Aborting this batch", "could not initialize") - self.ippToPspsDb.updateProcessed(self.batchID, -1) - return - if not self.createEmptyPspsTables(): self.logger.errorPair("Aborting this batch", "could not create empty PSPS tables") self.ippToPspsDb.updateProcessed(self.batchID, -1) - return + raise if not self.importIppTables(): self.logger.errorPair("Aborting this batch", "could not import IPP tables") self.ippToPspsDb.updateProcessed(self.batchID, -1) - return + raise if not self.populatePspsTables(): self.logger.errorPair("Aborting this batch", "unable to populate PSPS tables") self.ippToPspsDb.updateProcessed(self.batchID, -1) - return + raise if not self.exportPspsTablesToFits(): self.logger.errorPair("Aborting this batch", "unable to export tables to FITS file") self.ippToPspsDb.updateProcessed(self.batchID, -1) - return + raise if self.writeBatchManifest(): Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/batchRepublisher.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/batchRepublisher.py (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/batchRepublisher.py (revision 33948) @@ -0,0 +1,71 @@ +#!/usr/bin/env jython + +from datastore import Datastore +import logging +import sys +import getopt +import traceback + +from ipptopsps import IppToPsps +from datastore import Datastore +from batch import Batch + +''' +BatchRepublisher class + +A program that republishes a bath to the datastore. + +It first 'resets' the batch in the ippToPsps database, i.e. pretends it has never been published, then re-publishes it to the datastore +''' +class BatchRepublisher(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(BatchRepublisher, self).__init__(argv) + + if len(sys.argv) < 3: + self.printUsage() + self.exitProgram("incorrect args") + + self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + self.BATCHID = int(sys.argv[2]) + + ''' + Runs... + ''' + def run(self): + + batchName = Batch.getNameFromID(self.BATCHID) + batchType = self.ippToPspsDb.getBatchType(self.BATCHID) + subDir = Batch.getSubDir(self.config.basePath, batchType, self.config.dvoLabel) + tarballFile = Batch.getTarballFile(self.BATCHID) + + self.logger.infoPair("batch ID", "%d" % self.BATCHID) + self.logger.infoPair("batchname", batchName) + self.logger.infoPair("batch type", batchType) + self.logger.infoPair("subdir", subDir) + self.logger.infoPair("tarballfile", tarballFile) + self.logger.infoPair("Reseting batch in database", "%d" % self.BATCHID) + self.ippToPspsDb.resetBatch(self.BATCHID) + self.logger.infoPair("publishing to", self.config.datastoreProduct) + Batch.publishToDatastore(self.datastore, self.BATCHID, batchName, subDir, tarballFile) + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(BatchRepublisher, self).printUsage("") + +''' +Start of program +''' +try: + batchRepublisher = BatchRepublisher(sys.argv) + batchRepublisher.run() + batchRepublisher.exitProgram("completed") +except Exception, e: + print str(e) + traceback.print_exc() + Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/cleanup.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/cleanup.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/cleanup.py (revision 33948) @@ -7,4 +7,5 @@ import sys import time +import traceback import logging.config @@ -41,4 +42,5 @@ self.logger.infoPair("Config", self.config.name) self.config.printDeletionPolicy() + self.clean("IN") self.clean("P2") self.clean("ST") @@ -48,4 +50,14 @@ if not self.waitForPollTime(): break + ''' + Reports results of cleanup for this set of IDs + ''' + def reportResults(self, name, ids, success): + + if len(ids) > 0: + self.logger.infoSeparator() + self.logger.infoPair("Removed from " + name, "%d of %d" % (success, len(ids))) + + ''' Finds stuff to delete for this batch type, then: @@ -62,5 +74,5 @@ loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType) - loadedDxlayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType) + deleteFromDxLayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType) mergedLocalIDs = self.ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType) @@ -68,15 +80,16 @@ purgedLocalIDs = self.ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType) - self.logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(loadedDatastoreIDs)) - self.logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(loadedDxlayerIDs)) - self.logger.infoPair("Merged but still on local disk", "%d" % len(mergedLocalIDs)) - self.logger.infoPair("Purged but still on datastore", "%d" % len(purgedDatastoreIDs)) - self.logger.infoPair("Purged but still on local disk", "%d" % len(purgedLocalIDs)) + deleteFromDatastoreIDs = loadedDatastoreIDs + purgedDatastoreIDs + deleteFromLocalIDs = mergedLocalIDs + purgedLocalIDs + + # report to log what we are going to do + self.logger.infoPair("To delete from datastore", "%d" % len(deleteFromDatastoreIDs)) + self.logger.infoPair("To delete from DXLayer", "%d" % len(deleteFromDxLayerIDs)) + self.logger.infoPair("To delete from local disk", "%d" % len(deleteFromLocalIDs)) + # delete stuff from local disk if self.config.deleteLocal: - self.logger.infoSeparator() count = 0 - ids = mergedLocalIDs + purgedLocalIDs - for id in ids: + for id in deleteFromLocalIDs: if Batch.deleteFromDisk(self.logger, self.config.basePath, batchType, self.config.dvoLabel, id): @@ -84,14 +97,10 @@ count = count + 1 - self.logger.infoSeparator() - self.logger.infoPair("Removed from local disk", "%d" % count) - self.logger.infoPair("Remaining on local disk", "%d" % (len(ids) - count)) - + self.reportResults("local disk", deleteFromLocalIDs, count) + # remove stuff from datastore if self.config.deleteDatastore: - self.logger.infoSeparator() count = 0 - ids = loadedDatastoreIDs + purgedDatastoreIDs - for id in ids: + for id in deleteFromDatastoreIDs: batchName = Batch.getNameFromID(id) @@ -100,12 +109,10 @@ count = count + 1 - self.logger.infoSeparator() - self.logger.infoPair("Removed from datastore", "%d" % count) - self.logger.infoPair("Remaining on datastore", "%d" % (len(ids) - count)) + self.reportResults("datastore", deleteFromDatastoreIDs, count) + # remove stuff from DXLayer if self.config.deleteDxLayer: - self.logger.infoSeparator() count = 0 - for id in loadedDxlayerIDs: + for id in deleteFromDxLayerIDs: if self.dxlayer.deleteBatch(id): @@ -113,9 +120,6 @@ count = count + 1 - self.logger.infoSeparator() - self.logger.infoPair("Removed from DXLayer", "%d" % count) - self.logger.infoPair("Remaining on DXLayer", "%d" % (len(loadedDxlayerIDs) - count)) + self.reportResults("DXLayer", deleteFromDxLayerIDs, count) - ''' Overrides base-class version @@ -128,5 +132,9 @@ Start of program ''' -cleanup = Cleanup(sys.argv) -cleanup.run() -cleanup.exitProgram("finished") +try: + cleanup = Cleanup(sys.argv) + cleanup.run() + cleanup.exitProgram("completed") +except Exception, e: + print str(e) + traceback.print_exc() Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/config.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/config.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/config.py (revision 33948) @@ -39,4 +39,6 @@ ''' def printAll(self): + + self.logger.infoTitle("Config") try: @@ -133,3 +135,2 @@ def getDbPassword(self, dbType): return self.settingsDoc.find(dbType +"/password").text - Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/console.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/console.py (revision 33948) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/console.py (revision 33948) @@ -0,0 +1,166 @@ +#!/usr/bin/env jython + +import sys +import getopt +import time + +from java.awt import BorderLayout +from java.awt import GridLayout +from java.awt import Dimension + +from javax.swing import JFrame +from javax.swing import JButton +from javax.swing import JLabel +from javax.swing import JTextField +from javax.swing import JButton +from javax.swing import JPanel +from javax.swing import JScrollPane +from javax.swing import JTable +from javax.swing import JTabbedPane +from javax.swing import JComboBox +from javax.swing import JOptionPane + +from javax.swing.table import DefaultTableModel + +from ipptopsps import IppToPsps + +''' +Console class +''' +class Console(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(Console, self).__init__(argv, 0, 0) + + self.frame = JFrame( + "ippToPsps console for config '" + self.config.name + "'", + layout=BorderLayout(), + size=(1000, 500), + windowClosing=self.exitProgram) + + tabbedPane = JTabbedPane(JTabbedPane.TOP) + tabbedPane.addTab("Clients", self.createClientsPanel()) + + #self.label = JLabel("Some title", JLabel.CENTER) + #self.frame.add(self.label, BorderLayout.NORTH) + + self.frame.add(tabbedPane, BorderLayout.CENTER) + + ''' + Creates a panel for managing clients + ''' + def createClientsPanel(self): + + panel = JPanel(layout=BorderLayout()) + + self.clientTable = JTable() + scrollPane = JScrollPane() + scrollPane.setPreferredSize(Dimension(100,125)) + scrollPane.getViewport().setView((self.clientTable)) + + # side panel of buttons + buttonPanel = JPanel(layout=GridLayout(6, 1)) + button = JButton('Pause', actionPerformed=self.pause) + buttonPanel.add(button) + button = JButton('Resume', actionPerformed=self.resume) + buttonPanel.add(button) + button = JButton('Kill', actionPerformed=self.kill) + buttonPanel.add(button) + button = JButton('Cancel kill', actionPerformed=self.cancelKill) + buttonPanel.add(button) + button = JButton('Purge dead', actionPerformed=self.purgeDead) + buttonPanel.add(button) + button = JButton('Change config', actionPerformed=self.changeConfig) + buttonPanel.add(button) + + button = JButton('Refresh', actionPerformed=self.refreshClientTable) + + panel.add(scrollPane, BorderLayout.CENTER) + panel.add(buttonPanel, BorderLayout.EAST) + panel.add(button, BorderLayout.SOUTH) + + self.refreshClientTable(None) + + return panel + + ''' + Gets table selection + ''' + def getSelectedIds(self): + rows = self.clientTable.getSelectedRows() + ids = [] + for row in rows: ids.append(self.clientDataModel.getValueAt(row, 0)) + return ids + + ''' + Refreshes clients table + ''' + def refreshClientTable(self, event): + + clients = self.ippToPspsDb.getClientInfo() + colNames = ('ID','host', 'started', 'timestamp', 'ra_center', 'paused?', 'killed?') + self.clientDataModel = DefaultTableModel(clients, colNames) + self.clientTable.setModel(self.clientDataModel) + + ''' + Overriden + ''' + def exitProgram(self, exitReason="quit"): + + self.frame.dispose() + super(Console, self).exitProgram("window closed") + + + ''' + Pause/kill button callbacks + ''' + def pause(self, event): + self.ippToPspsDb.pauseLoaders(True, self.getSelectedIds()) + self.refreshClientTable(None) + def resume(self, event): + self.ippToPspsDb.pauseLoaders(False, self.getSelectedIds()) + self.refreshClientTable(None) + def kill(self, event): + self.ippToPspsDb.killLoaders(True, self.getSelectedIds()) + self.refreshClientTable(None) + def cancelKill(self, event): + self.ippToPspsDb.killLoaders(False, self.getSelectedIds()) + self.refreshClientTable(None) + def remove(self, event): + self.ippToPspsDb.removeClientsByIds(self.getSelectedIds()) + self.refreshClientTable(None) + def purgeDead(self, event): + self.ippToPspsDb.purgeDeadClients() + self.refreshClientTable(None) + def changeConfig(self, event): + ids = self.getSelectedIds() + if len(ids) < 1: + JOptionPane.showMessageDialog(None, "No clients selected", "Error", JOptionPane.ERROR_MESSAGE) + return + comboBox = JComboBox(self.ippToPspsDb.getActiveConfigList()) + if JOptionPane.showConfirmDialog(None, + comboBox, + "Choose a config", + JOptionPane.OK_CANCEL_OPTION, + JOptionPane.PLAIN_MESSAGE) == JOptionPane.CANCEL_OPTION: return + + self.ippToPspsDb.setConfigForLoaders(comboBox.getSelectedItem(), ids) + self.refreshClientTable(None) + + + + ''' + Runs... + ''' + def run(self): + self.frame.setVisible(True) + +''' +Start of program +''' +console = Console(sys.argv) +console.run() + Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/datastore.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/datastore.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/datastore.py (revision 33948) @@ -102,5 +102,5 @@ firstInt = int(first[1:]) - lastInt = int(last[1:]) + lastInt = int(last[1:]) + 1 for i in range(firstInt, lastInt): Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/datastoreRemover.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/datastoreRemover.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/datastoreRemover.py (revision 33948) @@ -46,6 +46,8 @@ Start of program ''' -datastoreRemover = DatastoreRemover(sys.argv) -datastoreRemover.run() -datastoreRemover.exitProgram("finished") +try: + datastoreRemover = DatastoreRemover(sys.argv) + datastoreRemover.run() + datastoreRemover.exitProgram("completed") +except: pass Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/detectionbatch.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/detectionbatch.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/detectionbatch.py (revision 33948) @@ -53,11 +53,9 @@ useFullTables) - if not self.everythingOK: return - # get camera meta data meta = self.gpc1Db.getCameraStageMeta(self.id) if not meta: - self.everythingOK = False - return + self.logger.errorPair("Could not get", "camera metadata") + raise self.expID = meta[0]; @@ -93,14 +91,14 @@ # get a fre primary header values. if in test mode, then use defaults if self.safeDictionaryAccessWithDefault(self.header, 'MJD-OBS', "1") == "NULL": - self.everythingOK = False - return + self.logger.errorPair("Could not get", "MJD-OBS") + raise if self.safeDictionaryAccessWithDefault(self.header, 'EXPTIME', "1") == "NULL": - self.everythingOK = False - return + self.logger.errorPair("Could not get", "EXPTIME") + raise if self.safeDictionaryAccessWithDefault(self.header, 'FILTERID', "g.0000") == "NULL": - self.everythingOK = False - return + self.logger.errorPair("Could not get", "FILTERID") + raise self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0) @@ -418,4 +416,6 @@ ,momentYY \ ,apMag \ + ,kronFlux \ + ,kronFluxErr \ ,infoFlag \ ,sky \ @@ -436,7 +436,7 @@ ,X_PSF_SIG \ ,Y_PSF_SIG \ - ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+" \ - ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+")) / 1.085736) \ - ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \ + ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + " \ + ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + ")) / 1.085736) \ + ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + self.header['EXPTIME'] + " \ ,PSF_MAJOR \ ,PSF_MINOR \ @@ -448,4 +448,6 @@ ,MOMENTS_YY \ ,AP_MAG \ + ,KRON_FLUX / " + self.header['EXPTIME'] + " \ + ,KRON_FLUX_ERR / " + self.header['EXPTIME'] + " \ ,FLAGS\ ,SKY \ @@ -755,5 +757,5 @@ else : regex = ".*.psf" - columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG FLAGS SKY SKY_SIGMA EXT_NSIGMA" + columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS SKY SKY_SIGMA EXT_NSIGMA" return super(DetectionBatch, self).importIppTables(columns, regex) Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/dvo.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/dvo.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/dvo.py (revision 33948) @@ -17,5 +17,4 @@ from java.sql import * - ''' Abstract base-class for DVO classes @@ -48,5 +47,8 @@ # connect to the specified scratch database - self.scratchDb = ScratchDb(self.logger, self.config, scratchDbName) + try: + self.scratchDb = ScratchDb(self.logger, self.config, scratchDbName) + except: raise + self.scratchDb.setUseFullTables(True) @@ -90,5 +92,4 @@ return - ''' Loads the DVO Photcodes.dat FITS file @@ -218,4 +219,5 @@ self.regionsAlreadyIngested = [] self.regionsIngestedButOutOfDate = [] + tablesToCleanup = [] # loop through all regions in DVO inside this bounding-box @@ -256,4 +258,11 @@ continue + # we have no record of ingesting this region, but does the table exist? + # May hev been left behind after a crash, and therefore needs to be cleaned-up + for fileType in self.ingestFileTypes: + tableName = self.scratchDb.getDbFriendlyTableName(regionPath + "." + fileType) + if self.scratchDb.tableExists(tableName): + tablesToCleanup.append(tableName) + # check if we have out-of-date versions of any interested files, if so, add region to the purge list outOfDate = False @@ -279,4 +288,10 @@ self.sizeOfRegionsToPurge = self.getDiskSizeOfRegions(self.regionsToPurge) self.sizeOfRegionsToIngest = self.getDiskSizeOfRegions(self.regionsToIngest) + + # any tables need cleaning up? + if len(tablesToCleanup) > 0: + self.logger.infoPair("Found tables to be cleaned up", "%d" % len(tablesToCleanup)) + self.scratchDb.dropTables(tablesToCleanup) + ''' Index: anches/eam_branches/ipp-20120405/ippToPsps/jython/dvoToMySQL.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/dvoToMySQL.py (revision 33947) +++ (revision ) @@ -1,242 +1,0 @@ -#!/usr/bin/env jython - -import stilts -import datetime -import re -import sys -import os -import logging -import glob -from subprocess import call, PIPE, Popen - -from pslogger import PSLogger -from scratchdb import ScratchDb - -from java.lang import * -from java.sql import * -from xml.etree.ElementTree import ElementTree, Element, tostring - - -''' -Class for pulling IDs from a DVO database and shoving in a MySQL database -''' -class DvoToMySql(object): - - ''' - Constructor - - ''' - def __init__(self, logger, doc, RESETTABLES): - - # set up logging - self.logger = logger - self.doc = doc - self.logger.infoSeparator() - self.dvoLocation = self.doc.find("dvo/location").text - - # create database object - self.scratchDb = ScratchDb(logger, self.doc, 1) - - # some log stuff - self.logger.infoPair("Started", "dvoToMySql") - self.logger.infoPair("Importing DVO database at", self.dvoLocation) - - # create DVO tables - if RESETTABLES: self.scratchDb.resetDvoToMysqlTables() - - # import Images.dat table - self.logger.infoPair("Deleting from table", self.scratchDb.dvoMetaTable) - sql = "DELETE FROM " + self.scratchDb.dvoMetaTable - self.scratchDb.execute(sql) - - imagesTableName = self.importFits(self.dvoLocation, - "", - "Images.dat", - "SOURCE_ID IMAGE_ID CCDNUM EXTERN_ID FLAGS PHOTCODE NSTAR") - self.scratchDb.createIndex(imagesTableName, "EXTERN_ID") - - # insert into dvoMetaFull - # NB what we and smf files call IMAGE_ID, DVO calls EXTERN_ID. We are sticking - # with the smf name - self.logger.infoPair("Populating", "all image meta data") - sql = "INSERT INTO " + self.scratchDb.dvoMetaTable + " ( \ - sourceID, \ - imageID, \ - externID, \ - flags, \ - photcode \ - ) SELECT \ - SOURCE_ID, \ - IMAGE_ID, \ - EXTERN_ID, \ - FLAGS, \ - PHOTCODE \ - FROM " + imagesTableName - self.scratchDb.execute(sql) - - subdirs = ['n0000'] - - for subdir in subdirs: - - files = glob.glob(self.dvoLocation + "/" + subdir + "/*.cpm") - - #files = ['0247.06', '0244.06', '0244.10'] - - for file in files: - - # get just filename, without extension - file = os.path.basename(os.path.splitext(file)[0]) - self.logger.infoSeparator() - - if self.scratchDb.alreadyImportedThisDvoTable(file): continue - - # import cpm table and index - cpmTableName = self.importFits(self.dvoLocation, - subdir, - file + ".cpm", - "IMAGE_ID DET_ID OBJ_ID CAT_ID EXT_ID DB_FLAGS") - self.scratchDb.createIndex(cpmTableName, "IMAGE_ID") - self.scratchDb.createIndex(cpmTableName, "CAT_ID") - self.scratchDb.createIndex(cpmTableName, "OBJ_ID") - - # import cpt table and index - cptTableName = self.importFits(self.dvoLocation, - subdir, - file + ".cpt", - "OBJ_ID CAT_ID EXT_ID") - self.scratchDb.createIndex(cptTableName, "IMAGE_ID") - self.scratchDb.createIndex(cptTableName, "CAT_ID") - self.scratchDb.createIndex(cptTableName, "OBJ_ID") - - # shove SOURCE_IDs into measurement table - self.logger.infoPair("Adding", "SOURCE_IDs") - sql = "ALTER TABLE " + cpmTableName + " ADD COLUMN (SOURCE_ID SMALLINT)" - self.scratchDb.execute(sql) - sql = "UPDATE " + cpmTableName + " AS a, " + imagesTableName + " AS b \ - SET a.SOURCE_ID = b.SOURCE_ID \ - WHERE a.IMAGE_ID = b.IMAGE_ID" - self.scratchDb.execute(sql) - - # shove PSPS objID in measurement table - self.logger.infoPair("Adding","PSPS objIDs") - sql = "ALTER TABLE "+cpmTableName+" ADD COLUMN (PSPS_OBJ_ID BIGINT)" - self.scratchDb.execute(sql) - sql = "UPDATE "+cpmTableName+" AS a, "+cptTableName+" AS b \ - SET a.PSPS_OBJ_ID = b.EXT_ID \ - WHERE a.CAT_ID = b.CAT_ID \ - AND a.OBJ_ID = b.OBJ_ID" - self.scratchDb.execute(sql) - - # NB we use an INSERT IGNORE here. This is because of a known issue where multiple DVO cpm - # files can include the same detection, with the same IMAGE_ID/DET_ID pairing, but different - # PSPS object IDs assigned in the corresponding cpt file. This is believed to be a chip-boundary - # issue within DVO. So, for now, we take the first IMAGE_ID/DET_ID detection we find, and ignore the rest - self.logger.infoPair("Populating", self.scratchDb.dvoDetectionTable) - sql = "INSERT IGNORE INTO " + self.scratchDb.dvoDetectionTable + " (\ - sourceID \ - ,imageID \ - ,ippDetectID \ - ,detectID \ - ,ippObjID \ - ,objID \ - ,flags \ - ) SELECT \ - SOURCE_ID \ - ,IMAGE_ID \ - ,DET_ID \ - ,EXT_ID \ - ,CAT_ID * 1000000000 + OBJ_ID \ - ,PSPS_OBJ_ID \ - ,DB_FLAGS \ - FROM " + cpmTableName - try: - self.scratchDb.execute(sql) - except: - self.logger.error("FAILED: " + sql) - return - - # now drop what we don't need - self.logger.infoPair("Dropping table", cpmTableName) - self.scratchDb.dropTable(cpmTableName) - self.logger.infoPair("Dropping table", cptTableName) - self.scratchDb.dropTable(cptTableName) - - self.scratchDb.setImportedThisDvoTable(file) - - self.scratchDb.dropTable(imagesTableName) - - - ''' - Destructor - ''' - def __del__(self): - - self.logger.debug("DvoToMySql destructor") - - ''' - Imports a FITS file. A regex filter lets you choose which tables to pull from the file - ''' - def importFits(self, path, subdir, file, columns): - - fullPath = path + "/" + subdir + "/" + file - - if len(subdir) < 1: tableName = file - else: tableName = subdir + "_" + file - - tableName = tableName.replace('.', '_') - - self.logger.infoPair("Importing tables from file", fullPath) - self.logger.infoPair("Writing to database table", tableName) - - tables = stilts.treads(fullPath) - - count = 0 - for table in tables: - - self.logger.infoPair("Reading IPP table", table.name) - table = stilts.tpipe(table, cmd='explodeall') - - # IPP FITS files are littered with infinities, so remove these - self.logger.infoPair("Removing", "infinity values") - table = stilts.tpipe(table, cmd='replaceval -Infinity null *') - table = stilts.tpipe(table, cmd='replaceval Infinity null *') - - #try: - self.logger.infoPair("Writing FITS table to", "database") - table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName) - #except: - # self.logger.exception(" Problem writing table '" + table.name + "' to the database") - count = count + 1 - - self.logger.infoPair("Finished importing", "%d tables" % count) - - return tableName - - -''' -Start of program. -''' - -if len(sys.argv) > 1: CONFIG = sys.argv[1] -else: - print "** Usage: " + sys.argv[0] + " [reset]" - sys.exit(1) - -# open config file -configDoc = ElementTree(file=CONFIG) - -logging.setLoggerClass(PSLogger) -logger = logging.getLogger("dvoToMySQL") -logger.setup(configDoc, "dvoToMySQL", 0, 1) - -RESETTABLES = 0 - -if len(sys.argv) > 2 and sys.argv[2] == "reset": - response = raw_input("* Are you ABSOLUTELY sure you want to recreate the DVO tables (y/n)? ") - if response == "y": RESETTABLES = 1 - else: sys.exit(1) - -dvoToMySql = DvoToMySql(logger, configDoc, RESETTABLES) - -logger.infoPair("Program...", "complete") - Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/gpc1db.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/gpc1db.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/gpc1db.py (revision 33948) @@ -72,10 +72,10 @@ elif batchType == "ST": - stage = "staticsky_multi" - sql = "SELECT DISTINCT stack_id, ra, decl FROM staticskyInput \ + stage = "staticsky" + sql = "SELECT DISTINCT stack_id, radeg, decdeg FROM staticskyInput \ JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \ JOIN minidvodbRun USING(minidvodb_name) \ JOIN minidvodbProcessed USING(minidvodb_id) \ - JOIN stackRun USING(stack_id) JOIN skycells.skycell USING(skycell_id) \ + JOIN stackRun USING(stack_id) JOIN skycell USING(skycell_id) \ WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \ AND minidvodbRun.state = 'merged' \ @@ -83,6 +83,6 @@ AND addRun.stage = '" + stage + "' \ AND addRun.state = 'full' \ - AND decl BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ - AND ra BETWEEN " + str(minRA) + " AND " + str(maxRA) + AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ + AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) try: @@ -266,5 +266,5 @@ AND minidvodbRun.minidvodb_group = '" + dvoDb + "' \ AND minidvodbRun.state = 'merged' \ - AND stage = 'staticsky_multi'" + AND stage = 'staticsky'" ''' @@ -282,5 +282,5 @@ AND mergedvodbRun.state = 'full' \ AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \ - AND stage = 'staticsky_multi'" + AND stage = 'staticsky'" Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/initbatch.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/initbatch.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/initbatch.py (revision 33948) @@ -50,5 +50,6 @@ ''' def importIppTables(self, filter=""): - self.logger.debug("importIppTables method here to satidfy base-class") + self.logger.debug("importIppTables method here to satisfy base-class") + return True ''' @@ -63,4 +64,4 @@ def populatePspsTables(self): self.logger.debug("No processing required for this batch type") - return 1 + return True Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/ipptopsps.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/ipptopsps.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/ipptopsps.py (revision 33948) @@ -38,7 +38,10 @@ self.SECONDS = None + self.createNewConfig = False + self.rotateConfigs = False + # a new config? + if CONFIGNAME == "edit": self.createNewConfig = True # should we rotate configs for this program? - if CONFIGNAME == "all": self.rotateConfigs = True - else: self.rotateConfigs = False + elif CONFIGNAME == "all": self.rotateConfigs = True # set up config object @@ -47,5 +50,9 @@ # create connection to databases database - self.ippToPspsDb = IppToPspsDb(self.logger, self.config) + try: + self.ippToPspsDb = IppToPspsDb(self.logger, self.config) + except: + self.exitProgram("Could not connect to ipptopsps Db") + raise self.checkClientStatus() @@ -59,5 +66,5 @@ self.logger.infoPair("Host", self.HOST) self.logger.infoPair("PID", "%d" % self.PID) - self.logger.infoPair("Config", CONFIGNAME) + self.logger.infoPair("Config", self.config.name) ''' @@ -97,5 +104,5 @@ # write message to log or stdout if no logger set up - msg = sys.argv[0] + " " + extra + msg = sys.argv[0] + " " + extra try: self.logger.errorPair("Usage:", msg) @@ -103,9 +110,14 @@ print "*** Usage: " + msg - ''' Refreshes the config then recreates objects with new settings ''' def refreshConfig(self): + + # new config? + if self.createNewConfig: + self.ippToPspsDb.editConfig() + self.ippToPspsDb.setConfigForThisClient(self.config.name, self.HOST, self.PID) + self.createNewConfig = False # if we are rotating configs, then look for next active one in list @@ -163,5 +175,5 @@ # write message to log or stdout if no logger set up try: - self.logger.infoPair("Program exited", exitReason) + self.logger.infoPair(self.PROGNAME + " exited", exitReason) except: print "*** Program exited: " + exitReason Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/ipptopspsdb.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/ipptopspsdb.py (revision 33948) @@ -854,36 +854,58 @@ return configs - - ''' - Writes current Config object to Db - ''' - def writeConfig(self): - - sql = "UPDATE config \ - SET \ - datastore_product = '" + self.config.datastoreProduct + "' \ - ,datastore_type = '" + self.config.datastoreType + "' \ - ,datastore_publish = " + self.config.datastorePublishing + " \ - ,dvo_label = '" + self.config.dvoLabel + "' \ - ,dvo_location = '" + self.config.dvoLocation + "' \ - ,min_ra = " + self.config.minRa + " \ - ,max_ra = " + self.config.maxRa + " \ - ,min_dec = " + self.config.minDec + " \ - ,max_dec = " + self.config.maxDec + " \ - ,box_size = " + self.config.boxSize + " \ - ,base_path = '" + self.config.basePath + "' \ - ,data_release = " + self.config.dataRelease + " \ - ,delete_local = " + self.config.deleteLocal + " \ - ,delete_datastore = " + self.config.deleteDatastore + " \ - ,delete_dxlayer = " + self.config.deleteDxLayer + " \ - ,epoch = '" + self.config.epoch + "' \ - ,survey = '" + self.config.survey + "' \ - ,psps_survey = '" + self.config.pspsSurvey + "' \ - ,queue_P2 = " + self.config.queuingThisBatchType("P2") + " \ - ,queue_ST = " + self.config.queuingThisBatchType("ST") + " \ - ,queue_OB = " + self.config.queuingThisBatchType("OB") + " \ - WHERE name = '" + self.config.name + "'" - - self.execute(sql) + ''' + Prompts user for info about a new or existing config and lets them neter the various details + ''' + def editConfig(self): + + print " ********************************************************************" + print " * Config editor" + print " *" + response = raw_input(" * Name for new config, or existing config to edit? ") + if response == "": return + self.config.name = response + + # attempt to insert new config, if it already exists then carry on with update + sql = "INSERT INTO config (name) VALUES ('" + self.config.name + "')" + try: + self.execute(sql) + except: pass + + # for each column in the config table + sql = "SHOW COLUMNS FROM config" + rs = self.executeQuery(sql) + while (rs.next()): + + field = rs.getString(1) + type = rs.getString(2) + null = rs.getString(3) + + # things we don;t want the user to edit + if field == 'timestamp': continue + if field == 'name': continue + + # get current value for this field, if any + try: + rs2 = self.executeQuery("SELECT " + field + " FROM config WHERE name = '" + self.config.name + "'") + rs2.first() + default = rs2.getString(1) + except: pass + + question = " * " + field + " (" + type + ")" + + # there may not be a default value + try: question = question + " hit return to accept default of: '" + default + "'" + except: pass + + # prompt the user for new value or to accept existing value + question = question + "? " + response = raw_input(question) + if response != "": + if type.find("varchar") != -1 or type.find("timestamp") != -1: quotes = "'" + else: quotes = "" + self.execute("UPDATE config SET " + field + " = " + quotes + response + quotes + " WHERE name = '" + self.config.name + "'") + + print " *" + print " ********************************************************************" ''' @@ -951,4 +973,7 @@ else: self.config.datastorePublishing = False self.config.dvoLabel = rs.getString(4) + + # if dvoLabel is null is can break queries, so set to something + if not self.config.dvoLabel: self.config.dvoLabel = "none" self.config.dvoLocation = rs.getString(5) self.config.minRa = rs.getDouble(6) @@ -1255,5 +1280,4 @@ self.execute("CREATE TEMPORARY TABLE all_pending (stage_id bigint(20), ra_bore float, dec_bore float)") - count = 0 for row in rows: @@ -1262,8 +1286,8 @@ VALUES (%d, %f, %f)" % (row[0], row[1], row[2]) self.execute(sql) - count += 1 except: continue - self.logger.infoPair("All items written to Db", "%d" % count) + count = self.getRowCount("all_pending") + self.logger.infoPair("Items written to Db", "%d" % count) ''' Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/loader.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/loader.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/loader.py (revision 33948) @@ -45,9 +45,9 @@ self.scratchDb.disconnect() continue - self.logger.infoPair("Using scratch Db", self.scratchDb.dbName) break - if not self.scratchDb.connected: - self.logger.errorPair("Cannot connect to a", "scratch database") + if not self.scratchDb.connected: + self.exitProgram("Cannot connect to a scratch database") + raise # create Datastore objects @@ -108,4 +108,6 @@ ''' def run(self): + + if not self.scratchDb.connected: return numAttempts = 0 @@ -152,6 +154,7 @@ if batchType != "OB": # look in DVO for this box (with extra border) - #self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder) - self.dvoDetections.setSkyArea() + self.logger.infoPair("Querying DVO for this sky area", "") + self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder) + #self.dvoDetections.setSkyArea() sizeToBeIngested = self.dvoDetections.getDiskSizeOfRegionsToBeIngested() if sizeToBeIngested == 0.0: smfsPerGB = 999999999 @@ -160,5 +163,5 @@ self.logger.infoPair("smfs-per-GB", "%.1f" % smfsPerGB) # should do we pre-ingest stuff from DVO? - if smfsPerGB > 30: + if batchType == 'P2' and smfsPerGB > 30: if not self.dvoDetections.sync(): self.logger.errorPair("Could not sync DVO with MySQL", "skipping") @@ -195,38 +198,44 @@ self.ippToPspsDb.unlockTables() + self.ippToPspsDb.deletePendingItem(batchType, id) + + # catch any raised exceptions in batch constructors + try: + if batchType == "P2": + batch = DetectionBatch(self.logger, + self.config, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID, + useFullTables) + elif batchType == "ST": + batch = StackBatch(self.logger, + self.config, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID, + useFullTables) + + elif batchType == "OB": + batch = ObjectBatch(self.logger, + self.config, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID, + useFullTables) + + batch.run() - self.ippToPspsDb.deletePendingItem(batchType, id) - - if batchType == "P2": - batch = DetectionBatch(self.logger, - self.config, - self.gpc1Db, - self.ippToPspsDb, - self.scratchDb, - id, - batchID, - useFullTables) - elif batchType == "ST": - batch = StackBatch(self.logger, - self.config, - self.gpc1Db, - self.ippToPspsDb, - self.scratchDb, - id, - batchID, - useFullTables) - - elif batchType == "OB": - batch = ObjectBatch(self.logger, - self.config, - self.gpc1Db, - self.ippToPspsDb, - self.scratchDb, - id, - batchID, - useFullTables) - - batch.run() - + # if batch fails, ignore and move on to the next one + except: + self.logger.errorPair("Problem with this %s batch (%d)" % (batchType, id), "skipping") + pass + if not self.checkClientStatus(): return False @@ -253,6 +262,7 @@ loader = Loader(sys.argv) loader.run() - loader.exitProgram("finished") + loader.exitProgram("completed") except Exception, e: print str(e) traceback.print_exc() + Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/metrics.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/metrics.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/metrics.py (revision 33948) @@ -191,8 +191,10 @@ Start of program. ''' -metrics = Metrics(sys.argv) -metrics.run() -metrics.exitProgram("finished") - - - +try: + metrics = Metrics(sys.argv) + metrics.run() + metrics.exitProgram("completed") +except: pass + + + Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/mysql.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/mysql.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/mysql.py (revision 33948) @@ -34,4 +34,7 @@ self.dbPass = config.getDbPassword(dbType) + # a user friendly connection sring + connectionStr = self.dbName + "@" + self.dbHost + # set up JDBC connection self.url = "jdbc:mysql://"+self.dbHost+"/"+self.dbName+"?autoReconnect=true&user="+self.dbUser+"&password="+self.dbPass @@ -39,12 +42,10 @@ self.con = DriverManager.getConnection(self.url) self.connectionID = self.getLastConnectionID() - self.logger.debug("MySQL connection to %s with ID %d" % (dbType, self.connectionID)) + self.logger.infoPair("Connected to MySQL Db", connectionStr) except: - self.logger.error("Unable to connect to " + self.url) - self.everythingOK = False # TODO need this? + self.logger.errorPair("Unable to connect to", connectionStr) self.connected = False - return - - self.everythingOK = True + raise + self.connected = True @@ -88,4 +89,25 @@ sql = "KILL %d" % connectionID self.execute(sql) + + ''' + Checks whether this table exists + ''' + def tableExists(self, tableName): + + sql = "SELECT COUNT(*) \ + FROM information_schema.tables \ + WHERE table_schema = '" + self.dbName + "' \ + AND table_name = '" + tableName + "'" + + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) > 0: return True + else: return False + except: + self.logger.errorPair("Could not check if table exists", sql) + pass + + return False ''' @@ -159,4 +181,10 @@ ''' + Drops a list of tables + ''' + def dropTables(self, tables): + for table in tables: self.dropTable(table) + + ''' Drops a table ''' @@ -224,8 +252,25 @@ except: pass #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'") + + ''' + Checks that we have a connection + ''' + def isConnected(self): + + try: + self.con + self.connected = True + except: + self.logger.errorPair("Not connected to", self.url) + self.connected = False + + return self.connected + ''' TODO ''' def execute(self, sql): + + if not self.isConnected(): raise stmt = self.con.createStatement() @@ -238,4 +283,6 @@ def executeQuery(self, sql): + if not self.isConnected(): raise + stmt = self.con.createStatement() rs = stmt.executeQuery(sql) Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/objectbatch.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/objectbatch.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/objectbatch.py (revision 33948) @@ -51,7 +51,9 @@ 1) - if not self.everythingOK: return - - self.dvoObjects = DvoObjects(self.logger, self.config, self.scratchDb.dbName) + try: + self.dvoObjects = DvoObjects(self.logger, self.config, self.scratchDb.dbName) + except: + self.logger.errorPair("Unable to create instance of", "DvoObjects") + raise # create an output filename, which is {dvoINDEX}.FITS @@ -78,5 +80,7 @@ def alterPspsTables(self): - self.logger.debug("Altering PSPS tables") + # dec is reserved in MySQL, so STILTS replaces if with dec_, which PSPS doesn't like. so, force it back again using `` + self.scratchDb.execute("ALTER TABLE Object CHANGE dec_ `dec` double") + return True @@ -95,24 +99,20 @@ def insertMags(self, cpsTable): - # list of all filters we are interested in - allFilters = ['g', 'r', 'i', 'z', 'y'] + # list of all filters PSPS is interested in + interestedFilters = ['g', 'r', 'i', 'z', 'y'] - # this loop looks into the DVO Photcodes tables and gets the 'CODE' for each the filters above that are listed - filters = [] - for filter in allFilters: - code = self.scratchDb.getCodeForThisFilter(filter) - if code < 0: continue - filters.append([code, filter]) + filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters) # get a count of the available filters filterCount = len(filters) - self.logger.infoPair("Available filters", filters) + self.logger.infoPair("Available filters in Photcodes", filters) # the 'code' now defines the order in the cps file that the mags are listed for a given filter + self.logger.infoPair("Adding magnitudes from", "cps table") for filter in filters: sql = "UPDATE Object \ - JOIN " + cpsTable + " AS cps ON (cps.row = (Object.row*" + str(filterCount) + ")-" + str(filter[0]) + ") \ + JOIN " + cpsTable + " AS cps ON (cps.row = (Object.row*" + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ SET \ n" + filter[1] + " = NCODE \ @@ -120,8 +120,18 @@ ," + filter[1] + "MeanFlux = 3631 * POW(10.0, (-0.4*MAG)) \ ," + filter[1] + "MeanMagErr = MAG_ERR \ - ," + filter[1] + "Min = MAG_20 \ - ," + filter[1] + "Max = MAG_80 \ + ," + filter[1] + "Min = MAG_20/1000 \ + ," + filter[1] + "Max = MAG_80/1000 \ " - + + self.scratchDb.execute(sql) + + # now set to null all MeanMagErr values > 0.5 (cut set by Gene, 2012-04-12) + cut = 0.5 + self.logger.infoPair("Setting to NULL all MeanMagErr value >", "%f" % cut) + for filter in filters: + + sql = "UPDATE Object \ + SET " + filter[1] + "MeanMagErr = null \ + WHERE " + filter[1] + "MeanMagErr > " + str(cut) self.scratchDb.execute(sql) @@ -131,4 +141,5 @@ def updateColors(self): + self.logger.infoPair("Calculating", "colors") sql = "UPDATE Object \ SET \ @@ -147,24 +158,28 @@ self.logger.errorPair("Couldn't calculate colors", sql) - - ''' - Does the processing, i.e. pulling stuff from IPP tables into PSPS tables - ''' - def populatePspsTables(self): + ''' + Populates the Object table + ''' + def populateObjectTable(self): cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps") - self.logger.infoPair("Inserting objects from", "cpt") - sql = "INSERT INTO Object (\ + self.logger.infoPair("Populating", "ObjectCalColor") + self.logger.infoPair("Inserting objects from", "cpt file") + + # note `` around dec here, as this is a reserved word in MySQL + sql = "INSERT IGNORE INTO Object (\ objID \ ,ippObjID \ ,objInfoFlag \ + ,varFlag \ ,surveyID \ ,ra \ - ,dec_ \ + ,`dec` \ ,raErr \ ,decErr \ ,nDetections \ + ,dataRelease \ ) \ SELECT \ @@ -172,4 +187,5 @@ ,CAT_ID*1000000000 + OBJ_ID \ ,FLAGS \ + ,0 \ ," + str(self.surveyID) + " \ ,RA \ @@ -178,4 +194,5 @@ ,DEC_ERR \ ,NMEASURE \ + , " + str(self.config.dataRelease) + "\ FROM " + cptTableName @@ -184,4 +201,5 @@ except: self.logger.errorPair("Couldn't populate Object table", sql) + return False # add row count columns so we can perform joins to get colors @@ -190,8 +208,6 @@ self.scratchDb.addRowCountColumn(cpsTableName, "row") - self.logger.infoPair("Adding magnitudes from", "cps table") self.insertMags(cpsTableName) - self.logger.infoPair("Calculating", "colors") self.updateColors() @@ -205,2 +221,44 @@ return True + ''' + Populates the ObjectCalColor table + ''' + def populateObjectCalColorTable(self): + + self.logger.infoPair("Populating", "ObjectCalColor") + + sql = "INSERT INTO ObjectCalColor ( \ + objID \ + ,ippObjID \ + ,dataRelease \ + ) \ + SELECT \ + objID \ + ,ippObjID \ + ,dataRelease \ + FROM Object" + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't populate ObjectCalColor table", sql) + return False + + return True + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): + + if not self.populateObjectTable(): return False + #if not self.populateObjectCalColorTable(): return False + + # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number + self.logger.infoPair("Forcing uniqueness on", "objID in Object table") + rowCountBefore = self.scratchDb.getRowCount("Object") + self.scratchDb.execute("ALTER IGNORE TABLE Object ADD UNIQUE INDEX(objID)") + rowCountAfter = self.scratchDb.getRowCount("Object") + self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + + return True + Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/odm.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/odm.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/odm.py (revision 33948) @@ -78,5 +78,5 @@ DETAILS = doc.find("{%s}OdmBatchState/{%s}Details" % (NAMESPACE, NAMESPACE)).text results['DETAILS'] = DETAILS - if re.search("processing", STATE): + elif re.search("processing", STATE): loadedToODM = 1 if re.search("MergeWorthy", MESSAGE): Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/plot.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/plot.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/plot.py (revision 33948) @@ -62,4 +62,5 @@ f.flush() + f.close() #os.remove(tempFilename) Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/plotter.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/plotter.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/plotter.py (revision 33948) @@ -39,4 +39,6 @@ Start of program ''' -plotter = Plotter(sys.argv) -plotter.exitProgram("finished") +try: + plotter = Plotter(sys.argv) + plotter.exitProgram("completed") +except: pass Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/pollOdm.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/pollOdm.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/pollOdm.py (revision 33948) @@ -42,5 +42,5 @@ else: self.stages = [self.STAGE] - if self.BATCHTYPE == "all": self.batchTypes = ['P2', 'ST', 'OB'] + if self.BATCHTYPE == "all": self.batchTypes = ['IN', 'P2', 'ST', 'OB'] else: self.batchTypes = [self.BATCHTYPE] @@ -126,5 +126,5 @@ ''' def printUsage(self): - super(PollOdm, self).printUsage(" [hours]") + super(PollOdm, self).printUsage(" [hours]") @@ -132,5 +132,7 @@ Program starts here ''' -pollodm = PollOdm(sys.argv) -pollodm.run() -pollodm.exitProgram("finished") +try: + pollodm = PollOdm(sys.argv) + pollodm.run() + pollodm.exitProgram("completed") +except: pass Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/queue.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/queue.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/queue.py (revision 33948) @@ -27,9 +27,12 @@ super(Queue, self).__init__(argv) - # create various objects self.gpc1Db = Gpc1Db(self.logger, self.config) self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) - self.dvoObjects = DvoObjects(self.logger, self.config) + try: + self.dvoObjects = DvoObjects(self.logger, self.config) + except: + self.exitProgram("Unable to create instance of DvoObject") + raise self.config.printAll() @@ -115,5 +118,4 @@ while dec <= self.config.maxDec: - self.checkClientStatus() box_id = self.ippToPspsDb.insertBox(ra, dec) @@ -134,4 +136,5 @@ self.logger.info("+-------------+-------------+-------------+-------------+") + self.checkClientStatus() if not self.waitForPollTime(): break @@ -158,5 +161,5 @@ ''' def printUsage(self): - super(Cleanup, self).printUsage("[]") + super(Queue, self).printUsage("[]") @@ -164,5 +167,8 @@ Start of program. ''' -queue = Queue(sys.argv) -queue.run() -queue.exitProgram("finished") +try: + queue = Queue(sys.argv) + queue.run() + queue.exitProgram("completed") +except: pass + Index: anches/eam_branches/ipp-20120405/ippToPsps/jython/regionTest.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/regionTest.py (revision 33947) +++ (revision ) @@ -1,96 +1,0 @@ -from pslogger import PSLogger -from ipptopspsdb import IppToPspsDb -from gpc1db import Gpc1Db -from dvo import Dvo - -import sys -from xml.etree.ElementTree import ElementTree, Element, tostring -import logging - -''' -Start of program. -''' - -if len(sys.argv) > 1: CONFIG = sys.argv[1] -else: - print "** Usage: " + sys.argv[0] + " " - sys.exit(1) - -# open config file -configDoc = ElementTree(file=CONFIG) - -logging.setLoggerClass(PSLogger) -logger = logging.getLogger("regionTest") -logger.setup(configDoc, "regionTest") - - -gpc1Db = Gpc1Db(logger, configDoc) -dvo = Dvo(logger, configDoc) -#dvo.resetAllTables() - -DVOLABEL = configDoc.find("dvo/gpc1Label").text - -# get equatorial coord constraints, if any -try: - MINRA = float(configDoc.find("dvo/minRA").text) - MAXRA = float(configDoc.find("dvo/maxRA").text) - MINDEC = float(configDoc.find("dvo/minDec").text) - MAXDEC = float(configDoc.find("dvo/maxDec").text) -except: - MINRA = 0.0 - MAXRA = 360.0 - MINDEC = -30.0 - MAXDEC = 90.0 - -BORDER = 1.60 -#BOXSIZE = 4 -BOXSIZE = 1.0 -HALFBOX = BOXSIZE/2.0 - - - -ra = MINRA + HALFBOX -dec = MINDEC + HALFBOX - -while ra <= MAXRA: - while dec <= MAXDEC: - - logger.infoSeparator() - logger.infoPair("BOXSIZE", "%d" % BOXSIZE) - allIDs = gpc1Db.getIDsInThisDVODbForThisStageInThisBox(DVOLABEL, "P2", ra, dec, BOXSIZE) - logger.infoPair("Found", "%d exposures" % len(allIDs)) - - dvo.loadRegionInThisBoxWithBorder(ra, dec, BOXSIZE, BORDER) - - ''' - regionsToIngest = [] - regionsToPurge = [] - - dvo.getRegionsInThisBoxWithBorder(regionsToIngest, regionsToPurge, ra, dec, BOXSIZE, BORDER) - - size = dvo.getDiskSizeOfDvoRegions(regionsToIngest)/1073741824.0 - if size == 0: smfsPerGB = 0 - else: smfsPerGB = len(allIDs)/size - logger.infoPair("smf files per GB", "%f" % smfsPerGB) - ''' - dec = dec + BOXSIZE - break - ra = ra + BOXSIZE - break - -#regionsToIngest = [] -#regionsToPurge = [] -#dvo.getRegions(regionsToIngest, regionsToPurge) -#dvo.getRegions(regionsToIngest, regionsToPurge, MINRA-BORDER, MAXRA+BORDER, MINDEC-BORDER, MAXDEC+BORDER) -#size = dvo.getSizeOfDvoRegionFromFiles(files) -#logger.infoPair("Size of region in DVO", dvo.getFileSizeStr(size)) -#logger.infoPair("Average region file size", dvo.getFileSizeStr(size/len(files))) - -#dvo.loadRegion(329, 331.1, 1.4, 2) -#dvo.loadRegion(MINRA-BORDER, MAXRA+BORDER, MINDEC-BORDER, MAXDEC+BORDER) -#dvo.loadRegion() - -logger.infoPair("Program...", "complete") - - - Index: anches/eam_branches/ipp-20120405/ippToPsps/jython/reportloadfailures.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/reportloadfailures.py (revision 33947) +++ (revision ) @@ -1,50 +1,0 @@ -#!/usr/bin/env jython - -from subprocess import call, PIPE, Popen -import tempfile -from xml.etree.ElementTree import ElementTree, Element, tostring -import re -import sys - -import logging.config -from pslogger import PSLogger -from ipptopspsdb import IppToPspsDb -from odm import Odm -from batch import Batch - - -if len(sys.argv) > 1: CONFIG = sys.argv[1] -else: - print "\n** Usage: " + sys.argv[0] + " \n" - sys.exit(1) - -# open config file -configDoc = ElementTree(file=CONFIG) - -# set up logging -logging.setLoggerClass(PSLogger) -logger = logging.getLogger(sys.argv[0]) -logger.setup(configDoc, sys.argv[0]) - -ippToPspsDb = IppToPspsDb(logger, configDoc) -odm = Odm(logger, configDoc) - -DVOLABEL = configDoc.find("dvo/gpc1Label").text -EPOCH = configDoc.find("options/epoch").text - -ids = ippToPspsDb.getFailedLoadedToODMBatchIDs(EPOCH, DVOLABEL) - - -logger.infoPair("Batches failed load to ODM", "%d" % len(ids)) - - -results = {} - -for id in ids: - - if odm.checkBatch(id, results): - logger.info(" %9s : %s" % (Batch.getNameFromID(id), results['DETAILS'])) - else: - logger.info(" %9s : failed to poll ODM" % Batch.getNameFromID(id)) - - Index: anches/eam_branches/ipp-20120405/ippToPsps/jython/republishbatch.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/republishbatch.py (revision 33947) +++ (revision ) @@ -1,60 +1,0 @@ -import sys - -from pslogger import PSLogger -import logging -from xml.etree.ElementTree import ElementTree, Element, tostring - -from ipptopspsdb import IppToPspsDb -from datastore import Datastore -from batch import Batch - - -''' -A program that republishes a bath to the datastore. - -It first 'resets' the batch in the ippToPsps database, i.e. pretends it has never been published, then re-publishes it to the datastore -''' -if len(sys.argv) > 2: - CONFIG = sys.argv[1] - BATCHID = int(sys.argv[2]) -else: - print "** Usage: " + sys.argv[0] + " " - sys.exit(1) - -# open config file -configDoc = ElementTree(file=CONFIG) - -logging.setLoggerClass(PSLogger) -logger = logging.getLogger(sys.argv[0]) -logger.setup(configDoc, sys.argv[0]) - -ippToPspsDb = IppToPspsDb(logger, configDoc) -datastore = Datastore(logger, configDoc, ippToPspsDb) - -DVOLABEL = configDoc.find("dvo/gpc1Label").text -EPOCH = configDoc.find("options/epoch").text -BASEPATH = configDoc.find("localOutPath").text -DATASTOREPRODUCT = configDoc.find("datastore/product").text - -logger.infoTitle("Batch republisher") -logger.infoPair("Loading epoch", EPOCH) -logger.infoPair("DVO label", DVOLABEL) - - -batchName = Batch.getNameFromID(BATCHID) -BATCHTYPE = ippToPspsDb.getBatchType(BATCHID) -subDir = Batch.getSubDir(BASEPATH, BATCHTYPE, DVOLABEL) -tarballFile = Batch.getTarballFile(BATCHID) - -logger.infoPair("batch ID", "%d" % BATCHID) -logger.infoPair("batchname", batchName) -logger.infoPair("batch type", BATCHTYPE) -logger.infoPair("subdir", subDir) -logger.infoPair("tarballfile", tarballFile) -logger.infoPair("Reseting batch in database", "%d" % BATCHID) -ippToPspsDb.resetBatch(BATCHID) -logger.infoPair("publishing to", DATASTOREPRODUCT) -Batch.publishToDatastore(datastore, BATCHID, batchName, subDir, tarballFile) - - - Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/run.sh =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/run.sh (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/run.sh (revision 33948) @@ -1,1 +1,1 @@ -~/jre1.7.0/bin/java -Xbootclasspath/a:../jars/stilts.jar:../jars/commons-math-2.2.jar:../jars/mysql-connector-java-5.1.14-bin.jar -jar ~/jython2.5.3b1/jython.jar $1 $2 $3 $4 $5 $6 $7 $8 $9 +~ipp/jre1.7.0_03/bin/java -Xbootclasspath/a:../jars/stilts.jar:../jars/commons-math-2.2.jar:../jars/mysql-connector-java-5.1.14-bin.jar -jar ~/jython2.5.3b1/jython.jar $1 $2 $3 $4 $5 $6 $7 $8 $9 Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/scratchdb.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/scratchdb.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/scratchdb.py (revision 33948) @@ -278,5 +278,5 @@ rs.close() except: - self.logger.errorPai("Can't query for ingested regions", sql) + self.logger.errorPair("Can't query for ingested regions", sql) return regions @@ -408,6 +408,10 @@ self.logger.errorPair("Unable to delete " + fileType + " file from", self.dvoDoneTable) - sql = "DROP TABLE " + self.getDbFriendlyTableName(region + "." + fileType) - self.execute(sql) + tableName = self.getDbFriendlyTableName(region + "." + fileType) + sql = "DROP TABLE " + tableName + try: self.execute(sql) + except: + self.logger.errorPair("Unable to drop table", tableName) + pass deletedDetectionCount = detectionCount - self.getRowCount(self.dvoDetectionTable) @@ -540,18 +544,29 @@ ''' - Gets the 'code' for the provided filter - ''' - def getCodeForThisFilter(self, filter): - - code = -1 - sql = "SELECT CODE FROM " + self.dvoPhotcodesTable + " WHERE NAME = '" + filter + "'" - try: - rs = self.executeQuery(sql) - rs.first() - code = rs.getInt(1) + Gets a list of filters ordered as thy are in the DVO Photcodes table, as this reflects the order they apper in the cps file + ''' + def getOrderedListOfFiltersFromPhotcodesTable(self, interestedFilters): + + sql = "SELECT NAME FROM " + self.dvoPhotcodesTable + " WHERE " + + count = 0 + for filter in interestedFilters: + if count != 0: sql = sql + " OR " + sql = sql + " name = '" + filter + "'" + count += 1 + + sql = sql + " ORDER BY CODE" + + orderedFilters = [] + try: + rs = self.executeQuery(sql) + row = 1 + while (rs.next()): + orderedFilters.append([row,rs.getString(1)]) + row += 1 rs.close() except: - self.logger.errorPair("Could not get filter code using", sql ) - - return code - + self.logger.errorPair("Could not get ordered filter list", sql ) + + return orderedFilters + Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/setupScratchDb.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/setupScratchDb.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/setupScratchDb.py (revision 33948) @@ -25,5 +25,9 @@ self.exitProgram("incorrect args") - self.scratchDb = ScratchDb(self.logger, self.config, argv[2]) + try: + self.scratchDb = ScratchDb(self.logger, self.config, argv[2]) + except: + self.exitProgram("Could not connect to a scratch Db") + raise @@ -52,7 +56,9 @@ Program starts here ''' -setupScratchDb = SetupScratchDb(sys.argv) -setupScratchDb.run() -setupScratchDb.exitProgram("finished") +try: + setupScratchDb = SetupScratchDb(sys.argv) + setupScratchDb.run() + setupScratchDb.exitProgram("completed") +except: pass Index: /branches/eam_branches/ipp-20120405/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/stackbatch.py (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/jython/stackbatch.py (revision 33948) @@ -50,6 +50,4 @@ useFullTables) - if not self.everythingOK: return - self.stackType = "DEEP_STACK" # TODO @@ -57,6 +55,6 @@ meta = self.gpc1Db.getStackStageMeta(self.id) if not meta: - self.everythingOK = False - return + self.logger.errorPair("Could not get stack", "metadata") + raise self.filter = meta[0]; @@ -366,7 +364,7 @@ ,X_PSF_SIG \ ,Y_PSF_SIG \ - ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+str(self.expTime)+" \ - ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+str(self.expTime)+")) / 1.085736) \ - ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+str(self.expTime)+" \ + ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + " \ + ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + ")) / 1.085736) \ + ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + str(self.expTime) + " \ ,SKY \ ,SKY_SIGMA \ @@ -389,6 +387,6 @@ ,AP_MAG \ , NULL \ - ,KRON_FLUX \ - ,KRON_FLUX_ERR \ + ,KRON_FLUX / " + str(self.expTime) + " \ + ,KRON_FLUX_ERR / " + str(self.expTime) + " \ , NULL \ , NULL \ Index: anches/eam_branches/ipp-20120405/ippToPsps/jython/stilts.py =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/jython/stilts.py (revision 33947) +++ (revision ) @@ -1,5850 +1,0 @@ -# This module auto-generated by java class uk.ac.starlink.ttools.build.JyStilts. - -'''Provides access to STILTS commands. - -See the manual, http://www.starlink.ac.uk/stilts/sun256/ -for tutorial and full usage information. -''' - -from __future__ import generators -__author__ = 'Mark Taylor' - - -import jarray.array -import java.io.ByteArrayInputStream as _ByteArrayInputStream -import java.io.OutputStream as _OutputStream -import java.lang.Class as _Class -import java.lang.System as _System -import java.lang.reflect.Array as _Array -import java.util.ArrayList as _ArrayList -import uk.ac.starlink.table.ColumnInfo as _ColumnInfo -import uk.ac.starlink.table.MultiStarTableWriter as _MultiStarTableWriter -import uk.ac.starlink.table.StarTable as _StarTable -import uk.ac.starlink.table.StarTableFactory as _StarTableFactory -import uk.ac.starlink.table.StarTableOutput as _StarTableOutput -import uk.ac.starlink.table.TableSequence as _TableSequence -import uk.ac.starlink.table.Tables as _Tables -import uk.ac.starlink.table.WrapperStarTable as _WrapperStarTable -import uk.ac.starlink.table.WrapperRowSequence as _WrapperRowSequence -import uk.ac.starlink.task.InvokeUtils as _InvokeUtils -import uk.ac.starlink.ttools.Stilts as _Stilts -import uk.ac.starlink.ttools.filter.StepFactory as _StepFactory -import uk.ac.starlink.ttools.task.MapEnvironment as _MapEnvironment -import uk.ac.starlink.util.DataSource as _DataSource -import uk.ac.starlink.ttools.func.Arithmetic as Arithmetic -import uk.ac.starlink.ttools.func.Arrays as Arrays -import uk.ac.starlink.ttools.func.Conversions as Conversions -import uk.ac.starlink.ttools.func.Coords as Coords -import uk.ac.starlink.ttools.func.Distances as Distances -import uk.ac.starlink.ttools.func.Fluxes as Fluxes -import uk.ac.starlink.ttools.func.Formats as Formats -import uk.ac.starlink.ttools.func.Maths as Maths -import uk.ac.starlink.ttools.func.Strings as Strings -import uk.ac.starlink.ttools.func.Tilings as Tilings -import uk.ac.starlink.ttools.func.Times as Times - -class JyStarTable(_WrapperStarTable): - '''StarTable wrapper class for use within Jython. - -Decorates a uk.ac.starlink.table.StarTable -java object with methods for use within jython. -These include special bound functions to make it an -iterable object (with items which are table rows), -arithmetic + and * overloads for concatenation, -a write method for table viewing or output, -and methods representing STILTS filter functionality, -namely cmd_* methods for filters and mode_* methods -for output modes. - -As a general rule, any StarTable object which is -intented for use by JyStilts program code should be -wrapped in an instance of this class. - ''' - def __init__(self, base_table): - _WrapperStarTable.__init__(self, base_table) - def __iter__(self): - rowseq = self.getRowSequence() - while rowseq.next(): - yield self._create_row(rowseq.getRow()) - def __str__(self): - return '%s (?x%d)' % (self.getName(), self.getColumnCount()) - def __add__(self, other): - return tcat([self, other]) - def __mul__(self, count): - return tcat([self] * count) - def __rmul__(self, count): - return tcat([self] * count) - def columns(self): - '''Returns a tuple of ColumnInfo objects describing the columns of this table.''' - if hasattr(self, '_columns'): - return self._columns - else: - col_list = [] - for i in xrange(self.getColumnCount()): - col_list.append(_JyColumnInfo(self.getColumnInfo(i))) - self._columns = tuple(col_list) - return self.columns() - def parameters(self): - ''' -Returns a mapping of table parameter names to values. - -This does not provide all the information about the parameters, -for instance units and UCDs are not included. -For more detail, use the relevant StarTable methods. -Currently, this is not a live list, in the sense that changing -the returned dictionary will not affect the table parameter values. - ''' - if hasattr(self, '_parameters'): - return self._parameters - else: - params = {} - for p in self.getParameters(): - params[p.getInfo().getName()] = p.getValue() - self._parameters = params - return self.parameters() - def coldata(self, key): - '''Returns a sequence of all the values in a given column.''' - icol = self._column_index(key) - rowseq = self.getRowSequence() - while rowseq.next(): - yield rowseq.getCell(icol) - def _create_row(self, array): - row = _JyRow(array) - row.table = self - return row - def _column_index(self, key): - if type(key) is type(1): - if key >= 0: - return key - else: - return key + self.getColumnCount() - if hasattr(self, '_colmap'): - return self._colmap[key] - else: - colmap = {} - for ic, col in enumerate(self.columns()): - if not col in colmap: - colmap[col] = ic - colname = col.getName() - if not colname in colmap: - colmap[colname] = ic - self._colmap = colmap - return self._column_index(key) - def write(self, location=None, fmt='(auto)'): - '''Writes table to a file. - - The location parameter may give a filename or a - python file object open for writing. - if it is not supplied, standard output is used. - - The fmt parameter specifies output format. - Known output formats: - (auto) - jdbc - fits - fits-plus - fits-basic - fits-var - colfits-plus - colfits-basic - votable-tabledata - votable-binary-inline - votable-fits-href - votable-binary-href - votable-fits-inline - text - ascii - csv - csv-noheader - tst - html - html-element - latex - latex-document - mirage - ''' - sto = _StarTableOutput() - if hasattr(location, 'write') and hasattr(location, 'flush'): - ostrm = _JyOutputStream(location) - name = getattr(location, 'name', None) - handler = sto.getHandler(fmt, name) - sto.writeStarTable(self, ostrm, handler) - else: - if location is None: - location = '-' - sto.writeStarTable(self, location, fmt) - def cmd_addcol(self, *args): - '''\ - Add a new column called defined by the algebraic - expression . By default the new column appears after the last - column of the table, but you can position it either before or after - a specified column using the -before or -after flags respectively. - The -units, -ucd and -desc flags can be used to define metadata - values for the new column. - - Syntax for the and arguments is described in the - manual. - - The filtered table is returned. - - args is a list with words as elements: - [-after | -before ] - [-units ] [-ucd ] [-desc ] - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("addcol") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_addresolve(self, *args): - '''\ - Performs name resolution on the string-valued column - >col-id-objname< and appends two new columns >col-name-ra< and - >col-name-dec< containing the resolved Right Ascension and - Declination in degrees. - - Syntax for the argument is described in SUN/256. - - UCDs are added to the new columns in a way which tries to be - consistent with any UCDs already existing in the table. - - Since this filter works by interrogating a remote service, it will - obviously be slow. The current implementation is experimental; it - may be replaced in a future release by some way of doing the same - thing (perhaps a new STILTS task) which is able to work more - efficiently by dispatching multiple concurrent requests. - - This software uses source code created at the Centre de Donnees - astronomiques de Strasbourg, France. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("addresolve") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_addskycoords(self, *args): - '''\ - Add new columns to the table representing position on the sky. The - values are determined by converting a sky position whose coordinates - are contained in existing columns. The arguments give - identifiers for the two input coordinate columns in the coordinate - system named by , and the arguments name the two - new columns, which will be in the coordinate system named by - . The and coordinate system specifiers are - one of - - * icrs: ICRS (Hipparcos) (Right Ascension, Declination) - * fk5: FK5 J2000.0 (Right Ascension, Declination) - * fk4: FK4 B1950.0 (Right Ascension, Declination) - * galactic: IAU 1958 Galactic (Longitude, Latitude) - * supergalactic: de Vaucouleurs Supergalactic (Longitude, Latitude) - * ecliptic: Ecliptic (Longitude, Latitude) - - The -inunit and -outunit flags may be used to indicate the units of - the existing coordinates and the units for the new coordinates - respectively; use one of degrees, radians or sexagesimal (may be - abbreviated), otherwise degrees will be assumed. For sexagesimal, - the two corresponding columns must be string-valued in forms like - hh:mm:ss.s and dd:mm:ss.s respectively. - - For certain conversions, the value specified by the -epoch flag is - of significance. Where significant its value defaults to 2000.0. - - Syntax for the , and arguments is - described in the manual. - - The filtered table is returned. - - args is a list with words as elements: - [-epoch ] [-inunit deg|rad|sex] [-outunit deg|rad|sex] - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("addskycoords") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_assert(self, *args): - '''\ - Check that a boolean expression is true for each row. If the - expression does not evaluate true for any row of the table, - execution terminates with an error. As long as no error occurs, the - output table is identical to the input one. - - The exception generated by an assertion violation is of class - uk.ac.starlink.ttools.filter.AssertException although that is not - usually obvious if you are running from the shell in the usual way. - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("assert") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_badval(self, *args): - '''\ - For each column specified in any occurrence of the - value is replaced by a blank entry. - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("badval") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_cache(self): - '''\ - Stores in memory or on disk a temporary copy of the table at this - point in the pipeline. This can provide improvements in efficiency - if there is an expensive step upstream and a step which requires - more than one read of the data downstream. If you see an error like - "Can't re-read data from stream" then adding this step near the - start of the filters might help. - - The result of this filter is guaranteed to be random-access. - - See also the random filter, which caches only when the input table - is not random-access. - - The filtered table is returned. - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("cache") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_check(self): - '''\ - Runs checks on the table at the indicated point in the processing - pipeline. This is strictly a debugging measure, and may be - time-consuming for large tables. - - The filtered table is returned. - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("check") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_clearparams(self, *args): - '''\ - Clears the value of one or more named parameters. Each of the - values supplied may be either a parameter name or a simple - wildcard expression matching parameter names. Currently the only - wildcarding is a "*" to match any sequence of characters. - clearparams * will clear all the parameters in the table. - - It is not an error to supply s which do not exist in the - table - these have no effect. - - The filtered table is returned. - - args is a list with words as elements: - ... - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("clearparams") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_colmeta(self, *args): - '''\ - Modifies the metadata of one or more columns. Some or all of the - name, units, ucd and description of the column(s), identified by - can be set by using some or all of the listed flags. - Typically, will simply be the name of a single column. - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - [-name ] [-units ] [-ucd ] [-desc ] - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("colmeta") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_delcols(self, *args): - '''\ - Delete the specified columns. The same column may harmlessly be - specified more than once. - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("delcols") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_every(self, *args): - '''\ - Include only every 'th row in the result, starting with the - first row. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("every") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_explodecols(self, *args): - '''\ - Takes a list of specified columns which represent N-element arrays - and replaces each one with N scalar columns. Each of the columns - specified by must have a fixed-length array type, - though not all the arrays need to have the same number of elements. - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("explodecols") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_explodeall(self, *args): - '''\ - Replaces any columns which is an N-element arrays with N scalar - columns. Only columns with fixed array sizes are affected. The - action can be restricted to only columns of a certain shape using - the flags. - - If the -ifndim flag is used, then only columns of dimensionality - will be exploded. may be 1, 2, .... - - If the -ifshape flag is used, then only columns with a specific - shape will be exploded; is a space- or comma-separated list - of dimension extents, with the most rapidly-varying first, e.g. '2 5 - ' to explode all 2 x 5 element array columns. - - The filtered table is returned. - - args is a list with words as elements: - [-ifndim ] [-ifshape ] - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("explodeall") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_fixcolnames(self): - '''\ - Renames all columns and parameters in the input table so that they - have names which have convenient syntax for STILTS. For the most - part this means replacing spaces and other non-alphanumeric - characters with underscores. This is a convenience which lets you - use column names in algebraic expressions and other STILTS syntax. - - The filtered table is returned. - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("fixcolnames") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_head(self, *args): - '''\ - Include only the first rows of the table. If the table has - fewer than rows then it will be unchanged. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("head") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_keepcols(self, *args): - '''\ - Select the columns from the input table which will be included in - the output table. The output table will include only those columns - listed in , in that order. The same column may be listed - more than once, in which case it will appear in the output table - more than once. - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("keepcols") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_meta(self, *args): - '''\ - Provides information about the metadata for each column. This filter - turns the table sideways, so that each row of the output corresponds - to a column of the input. The columns of the output table contain - metadata items such as column name, units, UCD etc corresponding to - each column of the input table. - - By default the output table contains columns for the following - items: - - * Index: Position of column in table - * Name: Column name - * Class: Data type of objects in column - * Shape: Shape of array values - * ElSize: Size of each element in column (mostly useful for - strings) - * Units: Unit string - * Description: Description of data in the column - * UCD: Unified Content Descriptor - - as well as any table-specific column metadata items that the table - contains. - - However, the output may be customised by supplying one or more - headings. These may be selected from the above as well as the - following: - - * UCD_desc: Textual description of UCD - - as well as any table-specific metadata. It is not an error to - specify an item for which no metadata exists in any of the columns - (such entries will result in empty columns). - - Any table parameters of the input table are propagated to the output - one. - - The filtered table is returned. - - args is a list with words as elements: - [ ...] - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("meta") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_progress(self): - '''\ - Monitors progress by displaying the number of rows processed so far - on the terminal (standard error). This number is updated every - second or thereabouts; if all the processing is done in under a - second you may not see any output. If the total number of rows in - the table is known, an ASCII-art progress bar is updated, otherwise - just the number of rows seen so far is written. - - The filtered table is returned. - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("progress") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_random(self): - '''\ - Ensures that random access is available on this table. If the table - currently has random access, it has no effect. If only sequential - access is available, the table is cached so that downstream steps - will see the cached, hence random-access, copy. - - The filtered table is returned. - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("random") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_randomview(self): - '''\ - Ensures that steps downstream only use random access methods for - table access. If the table is sequential only, this will result in - an error. Only useful for debugging. - - The filtered table is returned. - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("randomview") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_repeat(self, *args): - '''\ - Repeats the rows of a table multiple times to produce a longer - table. The output table will have times as many rows as the - input table. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("repeat") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_replacecol(self, *args): - '''\ - Replaces the content of a column with the value of an algebraic - expression. The old values are discarded in favour of the result of - evaluating . You can specify the metadata for the new column - using the -name, -units, -ucd and -desc flags; for any of these - items which you do not specify, they will take the values from the - column being replaced. - - It is legal to reference the replaced column in the expression, so - for example "replacecol pixsize pixsize*2" just multiplies the - values in column pixsize by 2. - - Syntax for the and arguments is described in the - manual. - - The filtered table is returned. - - args is a list with words as elements: - [-name ] [-units ] [-ucd ] [-desc ] - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("replacecol") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_replaceval(self, *args): - '''\ - For each column specified in any instance of - is replaced by . The value string 'null' can be used for - either or to indicate a blank value (but see - also the badval filter). - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("replaceval") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_rowrange(self, *args): - '''\ - Includes only rows in a given range. The range can either be - supplied as " ", where row indices are inclusive, or " - +". In either case, the first row is numbered 1. - - Thus, to get the first hundred rows, use either "rowrange 1 100" or - "rowrange 1 +100" and to get the second hundred, either "rowrange - 101 200" or "rowrange 101 +100" - - The filtered table is returned. - - args is a list with words as elements: - |+ - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("rowrange") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_select(self, *args): - '''\ - Include in the output table only rows for which the expression - evaluates to true. must be an expression which - evaluates to a boolean value (true/false). - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("select") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_seqview(self): - '''\ - Ensures that steps downstream see the table as sequential access. - Any attempts at random access will fail. Only useful for debugging. - - The filtered table is returned. - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("seqview") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_setparam(self, *args): - '''\ - Sets a named parameter in the table to a given value. The parameter - named is set to the value . By default the type of the - parameter is determined automatically (if it looks like an integer - it's an integer etc) but this can be overridden using the -type - flag. The parameter description may be set using the -desc flag. - - The filtered table is returned. - - args is a list with words as elements: - [-type byte|short|int|long|float|double|boolean|string] - [-desc ] [-unit ] [-ucd ] - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("setparam") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_sort(self, *args): - '''\ - Sorts the table according to the value of one or more algebraic - expressions. The sort key expressions appear, as separate - (space-separated) words, in ; sorting is done on the first - expression first, but if that results in a tie then the second one - is used, and so on. - - Each expression must evaluate to a type that it makes sense to sort, - for instance numeric. If the -down flag is used, the sort order is - descending rather than ascending. - - Blank entries are by default considered to come at the end of the - collation sequence, but if the -nullsfirst flag is given then they - are considered to come at the start instead. - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - [-down] [-nullsfirst] - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("sort") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_sorthead(self, *args): - '''\ - Performs a sort on the table according to the value of one or more - algebraic expressions, retaining only rows at the head of - the resulting sorted table. The sort key expressions appear, as - separate (space-separated) words, in ; sorting is done on - the first expression first, but if that results in a tie then the - second one is used, and so on. Each expression must evaluate to a - type that it makes sense to sort, for instance numeric. - - If the -tail flag is used, then the last rows rather than - the first ones are retained. - - If the -down flag is used the sort order is descending rather than - ascending. - - Blank entries are by default considered to come at the end of the - collation sequence, but if the -nullsfirst flag is given then they - are considered to come at the start instead. - - This filter is functionally equivalent to using sort followed by - head, but it can be done in one pass and is usually cheaper on - memory and faster, as long as is significantly lower than - the size of the table. - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - [-tail] [-down] [-nullsfirst] - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("sorthead") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_stats(self, *args): - '''\ - Calculates statistics on the data in the table. This filter turns - the table sideways, so that each row of the output corresponds to a - column of the input. The columns of the output table contain - statistical items such as mean, standard deviation etc corresponding - to each column of the input table. - - By default the output table contains columns for the following - items: - - * Name: Column name - * Mean: Average - * StDev: Population Standard deviation - * Minimum: Numeric minimum - * Maximum: Numeric maximum - * NGood: Number of non-blank cells - - However, the output may be customised by supplying one or more - headings. These may be selected from the above as well as the - following: - - * NBad: Number of blank cells - * Variance: Population Variance - * SampStDev: Sample Standard Deviation - * SampVariance: Sample Variance - * Skew: Gamma 1 skewness measure - * Kurtosis: Gamma 2 peakedness measure - * Sum: Sum of values - * MinPos: Row index of numeric minimum - * MaxPos: Row index of numeric maximum - * Cardinality: Number of distinct values in column; values >100 - ignored - * Median: Middle value in sequence - * Quartile1: First quartile - * Quartile2: Second quartile - * Quartile3: Third quartile - - Additionally, the form "Q.nn" may be used to represent the quantile - corresponding to the proportion 0.nn, e.g.: - - * Q.25: First quartile - * Q.625: Fifth octile - - Any parameters of the input table are propagated to the output one. - - Note that quantile calculations (including median and quartiles) can - be expensive on memory. If you want to calculate quantiles for large - tables, it may be wise to reduce the number of columns to only those - you need the quantiles for earlier in the pipeline. No interpolation - is performed when calculating quantiles. - - The filtered table is returned. - - args is a list with words as elements: - [ ...] - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("stats") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_tablename(self, *args): - '''\ - Sets the table's name attribute to the given string. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("tablename") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_tail(self, *args): - '''\ - Include only the last rows of the table. If the table has - fewer than rows then it will be unchanged. - - The filtered table is returned. - - args is a list with words as elements: - - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("tail") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_transpose(self, *args): - '''\ - Transposes the input table so that columns become rows and vice - versa. The -namecol flag can be used to specify a column in the - input table which will provide the column names for the output - table. The first column of the output table will contain the column - names of the input table. - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - [-namecol ] - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("transpose") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def cmd_uniq(self, *args): - '''\ - Eliminates adjacent rows which have the same values. If used with no - arguments, then any row which has identical values to its - predecessor is removed. - - If the parameter is given then only the values in the - specified columns must be equal in order for the row to be removed. - - If the -count flag is given, then an additional column with the name - DupCount will be prepended to the table giving a count of the number - of duplicated input rows represented by each output row. A unique - row has a DupCount value of 1. - - Syntax for the argument is described in SUN/256. - - The filtered table is returned. - - args is a list with words as elements: - [-count] [] - ''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("uniq") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(self)) - def mode_out(self, out='-', ofmt='(auto)'): - '''\ - Writes a new table. - - Parameters: - - out = - The location of the output table. This is usually a filename - to write to. If it is equal to the special value "-" (the - default) the output table will be written to standard output. - - [Default: -] - - ofmt = - Specifies the format in which the output table will be written - (one of the ones in SUN/256 - matching is case-insensitive and - you can use just the first few letters). If it has the special - value "(auto)" (the default), then the output filename will be - examined to try to guess what sort of file is required usually - by looking at the extension. If it's not obvious from the - filename what output format is intended, an error will result. - - [Default: (auto)] - ''' - env = _JyEnvironment() - env.setValue('out', _map_env_value(out)) - env.setValue('ofmt', _map_env_value(ofmt)) - mode = _stilts.getModeFactory().createObject('out') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(self) - def mode_meta(self): - '''\ - Prints the table metadata to standard output. The name and type etc - of each column is tabulated, and table parameters are also shown. - - See the meta filter for more flexible output of table metadata. - ''' - env = _JyEnvironment() - mode = _stilts.getModeFactory().createObject('meta') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(self) - def mode_stats(self): - '''\ - Calculates and displays univariate statistics for each of the - numeric columns in the table. The following entries are shown for - each column as appropriate: - - * mean - * population standard deviation - * minimum - * maximum - * number of non-null entries - - See the stats filter for more flexible statistical calculations. - ''' - env = _JyEnvironment() - mode = _stilts.getModeFactory().createObject('stats') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(self) - def mode_count(self): - '''\ - Counts the number of rows and columns and writes the result to - standard output. - ''' - env = _JyEnvironment() - mode = _stilts.getModeFactory().createObject('count') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(self) - def mode_cgi(self, ofmt='votable'): - '''\ - Writes a table to standard output in a way suitable for use as - output from a CGI (Common Gateway Interface) program. This is very - much like out mode but a short CGI header giving the MIME - Content-Type is prepended to the output - - Parameters: - - ofmt = - Specifies the format in which the output table will be written - (one of the ones in SUN/256 - matching is case-insensitive and - you can use just the first few letters). - - [Default: votable] - ''' - env = _JyEnvironment() - env.setValue('ofmt', _map_env_value(ofmt)) - mode = _stilts.getModeFactory().createObject('cgi') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(self) - def mode_discard(self): - '''\ - Reads all the data in the table in sequential mode and discards it. - May be useful in conjunction with the assert filter. - ''' - env = _JyEnvironment() - mode = _stilts.getModeFactory().createObject('discard') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(self) - def mode_topcat(self): - '''\ - Attempts to display the output table directly in TOPCAT. If a TOPCAT - instance is already running on the local host, an attempt will be - made to open the table in that. A variety of mechanisms are used to - attempt communication with an existing TOPCAT instance. In order: - * SAMP using existing hub (TOPCAT v3.4+ only, requires SAMP hub to - be running) - * PLASTIC using existing hub (requires PLASTIC hub to be running) - * SOAP (requires TOPCAT to run with somewhat deprecated -soap flag, - may be limitations on table size) - * SAMP using internal, short-lived hub (TOPCAT v3.4+ only, running - hub not required, but may be slow. It's better to start an - external hub, e.g. topcat -exthub) Failing that, an attempt - will be made to launch a new TOPCAT instance for display. This only - works if the TOPCAT classes are on the class path. - - If large tables are involved, starting TOPCAT with the -disk flag is - probably a good idea. - ''' - env = _JyEnvironment() - mode = _stilts.getModeFactory().createObject('topcat') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(self) - def mode_samp(self, format='votable fits', client=None): - '''\ - Sends the table to registered SAMP-aware applications subscribed to - a suitable table load MType. SAMP, the Simple Application Messaging - Protocol, is a tool interoperability protocol. A SAMP Hub must be - running for this to work. - - Parameters: - - format = - Gives one or more table format types for attempting the table - transmission over SAMP. If multiple values are supplied, they - should be separated by spaces. Each value supplied for this - parameter corresponds to a different MType which may be used - for the transmission. If a single value is used, a SAMP - broadcast will be used. If multiple values are used, each - registered client will be interrogated to see whether it - subscribes to the corresponding MTypes in order; the first one - to which it is subscribed will be used to send the table. The - standard options are - - * votable: use MType table.load.votable - * fits: use MType table.load.fits - - If any other string is used which corresponds to one of - STILTS's known table output formats, an attempt will be made - to use an ad-hoc MType of the form table.load.format. - - [Default: votable fits] - - client = - Identifies a registered SAMP client which is to receive the - table. Either the client ID or the (case-insensitive) - application name may be used. If a non-null value is given, - then the table will be sent to only the first client with the - given name or ID. If no value is supplied the table will be - sent to all suitably subscribed clients. - ''' - env = _JyEnvironment() - env.setValue('format', _map_env_value(format)) - env.setValue('client', _map_env_value(client)) - mode = _stilts.getModeFactory().createObject('samp') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(self) - def mode_plastic(self, transport=None, client=None): - '''\ - Broadcasts the table to any registered Plastic-aware applications. - PLASTIC, the PLatform for AStronomical Tool InterConnection, is a - tool interoperability protocol. A Plastic hub must be running in - order for this to work. - - Parameters: - - transport = string|file - Determines the method (PLASTIC message) used to perform the - PLASTIC communication. The choices are - - * string: VOTable serialized as a string and passed as a call - parameter (ivo://votech.org/votable/load). Not suitable - for very large files. - * file: VOTable written to a temporary file and the filename - passed as a call parameter ( - ivo://votech.org/votable/loadFromURL). The file ought to - be deleted once it has been loaded. Not suitable for - inter-machine communication. - - If no value is set (null) then a decision will be taken based - on the apparent size of the table. - - client = - Gives the name of a PLASTIC listener application which is to - receive the broadcast table. If a non-null value is given, - then only the first registered application which reports its - application name as that value will receive the message. If no - value is supplied, the broadcast will be to all listening - applications. - ''' - env = _JyEnvironment() - env.setValue('transport', _map_env_value(transport)) - env.setValue('client', _map_env_value(client)) - mode = _stilts.getModeFactory().createObject('plastic') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(self) - def mode_tosql(self, protocol, db, dbtable, host='localhost', write='create', user='mbt', password=None): - '''\ - Writes a new table to an SQL database. You need the appropriate JDBC - drivers and -Djdcb.drivers set as usual (see SUN/256). - - Parameters: - - protocol = - The driver-specific sub-protocol specifier for the JDBC - connection. For MySQL's Connector/J driver, this is mysql, and - for PostgreSQL's driver it is postgresql. For other drivers, - you may have to consult the driver documentation. - - host = - The host which is acting as a database server. - - [Default: localhost] - - db = - The name of the database on the server into which the new - table will be written. - - dbtable = - The name of the table which will be written to the database. - - write = create|dropcreate|append - Controls how the values are written to a table in the - database. The options are: - - * create: Creates a new table before writing. It is an error - if a table of the same name already exists. - * dropcreate: Creates a new database table before writing. If - a table of the same name already exists, it is dropped - first. - * append: Appends to an existing table. An error results if - the named table has the wrong structure (number or types - of columns) for the data being written. - - [Default: create] - - user = - User name for the SQL connection to the database. - - [Default: mbt] - - password = - Password for the SQL connection to the database. - ''' - env = _JyEnvironment() - env.setValue('protocol', _map_env_value(protocol)) - env.setValue('db', _map_env_value(db)) - env.setValue('dbtable', _map_env_value(dbtable)) - env.setValue('host', _map_env_value(host)) - env.setValue('write', _map_env_value(write)) - env.setValue('user', _map_env_value(user)) - env.setValue('password', _map_env_value(password)) - mode = _stilts.getModeFactory().createObject('tosql') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(self) - -class RandomJyStarTable(JyStarTable): - '''Extends the JyStarTable wrapper class for random access. - - Instances of this class can be subscripted. - ''' - def __init__(self, base_table): - JyStarTable.__init__(self, base_table) - def __len__(self): - return int(self.getRowCount()) - def __getitem__(self, key): - if type(key) is type(slice(0)): - return [self._create_row(self.getRow(irow)) - for irow in _slice_range(key, len(self))] - elif key < 0: - irow = self.getRowCount() + key - return self._create_row(self.getRow(irow)) - else: - return self._create_row(self.getRow(key)) - def __str__(self): - return str(self.getName()) + '(' + str(self.getRowCount()) + 'x' + str(self.getColumnCount()) + ')' - def coldata(self, key): - '''Returns a sequence of all the values in a given column.''' - icol = self._column_index(key) - return _Coldata(self, icol) - -class _Coldata(object): - def __init__(self, table, icol): - self.table = table - self.icol = icol - self.nrow = len(table) - def __iter__(self): - rowseq = self.table.getRowSequence() - while rowseq.next(): - yield rowseq.getCell(self.icol) - def __len__(self): - return self.nrow - def __getitem__(self, key): - if type(key) is type(slice(0)): - return [self.table.getCell(irow, self.icol) - for irow in _slice_range(key, self.nrow)] - elif key < 0: - irow = self.nrow + key - return self.table.getCell(irow, self.icol) - else: - return self.table.getCell(key, self.icol) - -class _JyColumnInfo(_ColumnInfo): - def __init__(self, base): - _ColumnInfo.__init__(self, base) - def __str__(self): - return self.getName() - -class _JyRow(tuple): - def __init__(self, array): - tuple.__init__(self, array) - def __getitem__(self, key): - icol = self.table._column_index(key) - return tuple.__getitem__(self, icol) - -class _JyEnvironment(_MapEnvironment): - def __init__(self, grab_output=False): - _MapEnvironment.__init__(self) - if grab_output: - self._out = _MapEnvironment.getOutputStream(self) - else: - self._out = _System.out - self._err = _System.err - self._used = {} - def getOutputStream(self): - return self._out - def getErrorStream(self): - return self._err - def acquireValue(self, param): - _MapEnvironment.acquireValue(self, param) - self._used[param.getName()] = True - def getUnusedArgs(self): - return filter(lambda n: n not in self._used, self.getNames()) - -def _check_unused_args(env): - names = env.getUnusedArgs() - if (names): - raise SyntaxError('Unused STILTS parameters %s' % str(tuple([str(n) for n in names]))) - -def _check_multi_handler(handler): - if not _Class.forName('uk.ac.starlink.table.MultiStarTableWriter').isInstance(handler): - raise TypeError('Handler %s cannot write multiple tables' % handler.getFormatName()) - -def _slice_range(slice, leng): - start = slice.start - stop = slice.stop - step = slice.step - if start is None: - start = 0 - elif start < 0: - start += leng - if stop is None: - stop = leng - elif stop < 0: - stop += leng - if step is None: - return xrange(start, stop) - else: - return xrange(start, stop, step) - -class _JyOutputStream(_OutputStream): - def __init__(self, file): - self._file = file - def write(self, *args): - narg = len(args) - if narg is 1: - arg0 = args[0] - if type(arg0) is type(1): - pyarg = chr(arg0) - else: - pyarg = arg0 - elif narg is 3: - buf, off, leng = args - pyarg = buf[off:off + leng].tostring() - else: - raise SyntaxError('%d args?' % narg) - self._file.write(pyarg) - def close(self): - self._file.close() - def flush(self): - self._file.flush() - -class _JyTableSequence(_TableSequence): - def __init__(self, seq): - self._iter = iter(seq) - def nextTable(self): - try: - return self._iter.next() - except StopIteration: - return None - -class _JyDataSource(_DataSource): - def __init__(self, file): - buf = file.read(-1) - self._buffer = jarray.array(buf, 'b') - if hasattr(file, 'name'): - self.setName(file.name) - def getRawInputStream(self): - return _ByteArrayInputStream(self._buffer) -def import_star_table(table): - '''Imports a StarTable instance for use with JyStilts. - - This factory function takes an instance of the Java class - uk.ac.starlink.table.StarTable - and returns an instance of a wrapper subclass which has some - decorations useful in a python environment. - This includes stilts cmd_* and mode_* methods, as well as - python-friendly standard methods to make it behave as an - iterable, and where possible a container, of data rows, - and overloaded addition and multiplication operators - with the semantics of concatenation. - ''' - if table.isRandom(): - return RandomJyStarTable(table) - else: - return JyStarTable(table) - -def _map_env_value(pval): - if pval is None: - return None - elif pval is True: - return 'true' - elif pval is False: - return 'false' - elif isinstance(pval, _StarTable): - return pval - elif _is_container(pval, _StarTable): - return jarray.array(pval, _StarTable) - else: - return str(pval) - -def _is_container(value, type): - try: - if len(value) > 0: - for item in value: - if not isinstance(item, type): - return False - return True - else: - return False - except TypeError: - return False - -_stilts = _Stilts() - -_InvokeUtils.configureLogging(0, False) - -_param_alias_dict = {} -_param_alias_dict['in']='in_' - - -_stilts_lib_version = _stilts.getVersion() -_stilts_script_version = '2.2-1' -if _stilts_lib_version != _stilts_script_version: - print('WARNING: STILTS script/class library version mismatch (' + _stilts_script_version + '/' + _stilts_lib_version + ').') - print(' This may or may not cause trouble.') - -def tread(location, fmt='(auto)', random=False): - '''Reads a table from a filename, URL or python file object. - - The random argument determines whether random access is required - for the table. - Setting it true may improve efficiency, but perhaps at the cost - of memory usage and load time for large tables. - - The fmt argument must be supplied if the table format cannot - be auto-detected. - - In general supplying a filename is preferred; the current implementation - may be much more expensive on memory if a python file object is used. - - Auto-detected in-formats: - fits-plus - colfits-plus - colfits-basic - fits - votable - - Known in-formats: - fits-plus - colfits-plus - colfits-basic - fits - votable - ascii - csv - tst - ipac - wdc - - The result of the function is a JyStilts table object. - ''' - fact = _StarTableFactory(random) - if hasattr(location, 'read'): - datsrc = _JyDataSource(location) - table = fact.makeStarTable(datsrc, fmt) - else: - table = fact.makeStarTable(location, fmt) - return import_star_table(table) - -def treads(location, fmt='(auto)', random=False): - '''Reads multiple tables from a filename, URL or python file object. - - It only makes sense to use this function rather than tread() if the - format is, or may be, one which can contain multiple tables. - Generally this means VOTable or FITS or one of their variants. - - The random argument determines whether random access is required - for the table. - Setting it true may improve efficiency, but perhaps at the cost - of memory usage and load time for large tables. - - The fmt argument must be supplied if the table format cannot - be auto-detected. - - In general supplying a filename is preferred; the current implementation - may be much more expensive on memory if a python file object is used. - - The result of the function is a list of JyStilts table objects. - ''' - fact = _StarTableFactory(random) - if hasattr(location, 'read'): - datsrc = _JyDataSource(location) - else: - datsrc = _DataSource.makeDataSource(location) - tseq = fact.makeStarTables(datsrc, fmt) - tables = _Tables.tableArray(tseq) - return map(import_star_table, tables) - -def twrite(table, location=None, fmt='(auto)'): - '''Writes table to a file. - - The location parameter may give a filename or a - python file object open for writing. - if it is not supplied, standard output is used. - - The fmt parameter specifies output format. - Known output formats: - (auto) - jdbc - fits - fits-plus - fits-basic - fits-var - colfits-plus - colfits-basic - votable-tabledata - votable-binary-inline - votable-fits-href - votable-binary-href - votable-fits-inline - text - ascii - csv - csv-noheader - tst - html - html-element - latex - latex-document - mirage - ''' - sto = _StarTableOutput() - if hasattr(location, 'write') and hasattr(location, 'flush'): - ostrm = _JyOutputStream(location) - name = getattr(location, 'name', None) - handler = sto.getHandler(fmt, name) - sto.writeStarTable(table, ostrm, handler) - else: - if location is None: - location = '-' - sto.writeStarTable(table, location, fmt) - -def twrites(tables, location=None, fmt='(auto)'): - '''Writes a sequence of tables to a single container file. - - The tables parameter gives an iterable over JyStilts table objects - The location parameter may give a filename or a python file object - open for writing. If it is not supplied, standard output is used. - - The fmt parameter specifies output format. - Note that not all formats can write multiple tables; - an error will result if an attempt is made to write - multiple tables to a single-table only format. - Known multi-table output formats: - (auto) - fits - fits-plus - fits-basic - fits-var - colfits-plus - colfits-basic - votable-tabledata - votable-binary-inline - votable-fits-href - votable-binary-href - votable-fits-inline - text - html - html-element - ''' - sto = _StarTableOutput() - tseq = _JyTableSequence(tables) - if hasattr(location, 'write') and hasattr(location, 'flush'): - ostrm = _JyOutputStream(location) - name = getattr(location, 'name', None) - handler = sto.getHandler(fmt, name) - _check_multi_handler(handler) - handler.writeStarTables(tseq, ostrm) - else: - if location is None: - location = '-' - handler = sto.getHandler(fmt, location) - _check_multi_handler(handler) - handler.writeStarTables(tseq, location, sto) - -def tfilter(table, cmd): - '''Applies a filter operation to a table and returns the result. - In most cases, it's better to use one of the cmd_* functions. - ''' - step = _StepFactory.getInstance().createStep(cmd) - return import_star_table(step.wrap(table)) - -def calc(expression, **kwargs): - '''\ -Evaluates expressions. - -The return value is the output string. - -Parameters: - - expression = - An expression to evaluate. The functions in SUN/256 can be - used. - - table = - Input table. -''' - task = _stilts.getTaskFactory().createObject('calc') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment(grab_output=True) - env.setValue('expression', _map_env_value(expression)) - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - txt = env.getOutputText() - return str(txt.strip()) - -def coneskymatch(in_, **kwargs): - '''\ -Crossmatches table on sky position against remote cone service. - -The return value is the resulting table. - -Parameters: - - in_ =
- Input table. - - ra = - Expression which evaluates to the right ascension in degrees - in the ICRS coordinate system for the request at each row of - the input table. This will usually be the name or ID of a - column in the input table, or a function involving one. - - dec = - Expression which evaluates to the declination in degrees in - the ICRS coordinate system for the request at each row of the - input table. This will usually be the name or ID of a column - in the input table, or a function involving one. - - sr = - Expression which evaluates to the search radius in degrees for - the request at each row of the input table. This will often be - a constant numerical value, but may be the name or ID of a - column in the input table, or a function involving one. - - find = best|all|each - Determines which matches are retained. - - * best: Only the matching query table row closest to the - input table row will be output. Input table rows with no - matches will be omitted. - * all: All query table rows which match the input table row - will be output. Input table rows with no matches will be - omitted. - * each: There will be one output table row for each input - table row. If matches are found, the closest one from - the query table will be output, and in the case of no - matches, the query table columns will be blank. - - [Default: all] - - copycols = - List of columns from the input table which are to be copied to - the output table. Each column identified here will be - prepended to the columns of the combined output table, and its - value for each row taken from the input table row which - provided the parameters of the query which produced it. See - SUN/256 for list syntax. The default setting is "*", which - means that all columns from the input table are included in - the output. - - [Default: *] - - scorecol = - Gives the name of a column in the output table to contain the - distance between the requested central position and the actual - position of the returned row. The distance returned is an - angular distance in degrees. If a null value is chosen, no - distance column will appear in the output table. - - [Default: Separation] - - parallel = - Allows multiple cone searches to be performed concurrently. If - set to the default value, 1, the cone query corresponding to - the first row of the input table will be dispatched, when that - is completed the query corresponding to the second row will be - dispatched, and so on. If set to , then queries will be - overlapped in such a way that up to approximately may be - running at any one time. Whether this is a good idea, and what - might be a sensible maximum value for , depends on the - characteristics of the service being queried. - - [Default: 1] - - erract = abort|ignore|retry|retry - Determines what will happen if any of the individual cone - search requests fails. By default the task aborts. That may be - the best thing to do, but for unreliable or poorly implemented - services you may find that some searches fail and others - succeed so it can be best to continue operation in the face of - a few failures. The options are: - - * abort: failure of any query terminates the task - * ignore: failure of a query is treated the same as a query - which returns no rows - * retry: failed queries are retried until they succeed; use - with care - if the failure is for some good, or at least - reproducible reason this could prevent the task from - ever completing - * retry: failed queries are retried at most a fixed number - of times If they still fail the task terminates. - - [Default: abort] - - ostream = true|false - If set true, this will cause the operation to stream on - output, so that the output table is built up as the results - are obtained from the cone search service. The disadvantage of - this is that some output modes and formats need multiple - passes through the data to work, so depending on the output - destination, the operation may fail if this is set. Use with - care (or be prepared for the operation to fail). - - [Default: false] - - fixcols = none|dups|all - Determines how input columns are renamed before use in the - output table. The choices are: - - * none: columns are not renamed - * dups: columns which would otherwise have duplicate names in - the output will be renamed to indicate which table they - came from - * all: all columns will be renamed to indicate which table - they came from - - If columns are renamed, the new ones are determined by - suffix* parameters. - - [Default: dups] - - suffix0 =
- Input table. - - xdataN = - Gives a column name or expression for the x axis data for - table N. The expression is a numeric algebraic expression - based on column names as described in SUN/256 - - ydataN = - Gives a column name or expression for the y axis data for - table N. The expression is a numeric algebraic expression - based on column names as described in SUN/256 - - auxdataN = - Gives a column name or expression for the aux axis data for - table N. The expression is a numeric algebraic expression - based on column names as described in SUN/256 - - xlo = - The lower limit for the plotted x axis. If not set, a value - will be chosen which is low enough to accommodate all the - data. - - ylo = - The lower limit for the plotted y axis. If not set, a value - will be chosen which is low enough to accommodate all the - data. - - auxlo = - The lower limit for the plotted aux axis. If not set, a value - will be chosen which is low enough to accommodate all the - data. - - xhi = - The upper limit for the plotted x axis. If not set, a value - will be chosen which is high enough to accommodate all the - data. - - yhi = - The upper limit for the plotted y axis. If not set, a value - will be chosen which is high enough to accommodate all the - data. - - auxhi = - The upper limit for the plotted aux axis. If not set, a value - will be chosen which is high enough to accommodate all the - data. - - xlog = true|false - If false (the default), the scale on the x axis is linear; if - true it is logarithmic. - - [Default: false] - - ylog = true|false - If false (the default), the scale on the y axis is linear; if - true it is logarithmic. - - [Default: false] - - auxlog = true|false - If false (the default), the scale on the aux axis is linear; - if true it is logarithmic. - - [Default: false] - - xflip = true|false - If set true, the scale on the x axis will increase in the - opposite sense from usual (e.g. right to left rather than left - to right). - - [Default: false] - - yflip = true|false - If set true, the scale on the y axis will increase in the - opposite sense from usual (e.g. right to left rather than left - to right). - - [Default: false] - - auxflip = true|false - If set true, the scale on the aux axis will increase in the - opposite sense from usual (e.g. right to left rather than left - to right). - - [Default: false] - - xlabel = - Specifies a label to be used for annotating axis x. A default - values based on the plotted data will be used if no value is - supplied for this parameter. - - ylabel = - Specifies a label to be used for annotating axis y. A default - values based on the plotted data will be used if no value is - supplied for this parameter. - - auxlabel = - Specifies a label to be used for annotating axis aux. A - default values based on the plotted data will be used if no - value is supplied for this parameter. - - xerrorN = |[],[] - Gives expressions for the errors on X coordinates for table N. - The following forms are permitted: - - * : symmetric error value - * ,:distinct lower and upper error values - * ,: lower error value only - * ,: upper error value only - * null: no errors - - The expression in each case is a numeric algebraic expression - based on column names as described in SUN/256. - - yerrorN = |[],[] - Gives expressions for the errors on Y coordinates for table N. - The following forms are permitted: - - * : symmetric error value - * ,:distinct lower and upper error values - * ,: lower error value only - * ,: upper error value only - * null: no errors - - The expression in each case is a numeric algebraic expression - based on column names as described in SUN/256. - - auxshader = rainbow|pastel|... - Determines how data from auxiliary axes will be displayed. - Generally this is some kind of colour ramp. These are the - available colour fixing options: - - * rainbow - * pastel - * standard - * heat - * colour - * hue - * greyscale - * red-blue - - and these are the available colour modifying options: - - * hsv_h - * hsv_s - * hsv_v - * intensity - * rgb_red - * rgb_green - * rgb_blue - * yuv_y - * yuv_u - * yuv_v - * transparency - - [Default: rainbow] - - txtlabelN = - Gives an expression which will label each plotted point. If - given, the text (or number) resulting from evaluating the - expression will be written near each point which is plotted. - - subsetNS = - Gives the selection criterion for the subset labelled "NS". - This is a boolean expression which may be the name of a - boolean-valued column or any other boolean-valued expression. - Rows for which the expression evaluates true will be included - in the subset, and those for which it evaluates false will - not. - - nameNS = - Provides a name to use for a subset with the symbolic label - NS. This name will be used for display in the legend, if one - is displayed. - - colourNS = |red|blue|... - Defines the colour of markers plotted. The value may be a - 6-digit hexadecimal number giving red, green and blue - intensities, e.g. "ff00ff" for magenta. Alternatively it may - be the name of one of the pre-defined colours. These are - currently red, blue, green, grey, magenta, cyan, orange, pink, - yellow, black and white. - - For most purposes, either the American or the British spelling - is accepted for this parameter name. - - shapeNS = filled_circle|open_circle|... - Defines the shapes for the markers that are plotted in data - set NS. The following shapes are available: - - * filled_circle - * open_circle - * cross - * x - * open_square - * open_diamond - * open_triangle_up - * open_triangle_down - * filled_square - * filled_diamond - * filled_triangle_up - * filled_triangle_down - - sizeNS = - Defines the marker size in pixels for markers plotted in data - set NS. If the value is negative, an attempt will be made to - use a suitable size according to how many points there are to - be plotted. - - [Default: -1] - - transparencyNS = - Determines the transparency of plotted markers for data set - NS. A value of means that opacity is only achieved (the - background is only blotted out) when pixels of this colour - have been plotted on top of each other. - - The minimum value is 1, which means opaque markers. - - lineNS = DotToDot|LinearRegression - Determines what line if any will be plotted along with the - data points. The options are: - - * null: No line is plotted. - * DotToDot: Each point is joined to the next one in sequence - by a straight line. - * LinearRegression: A linear regression line is plotted based - on all the points which are visible in the plot. Note - that the regression coefficients take no account of - points out of the visible range. - - hideNS = true|false - Indicates whether the actual markers plotted for each point - should be hidden. Normally this is false, but you may want to - set it to true if the point positions are being revealed in - some other way, for instance by error markers or lines drawn - between them. - - [Default: false] - - errstyleNS = lines|capped_lines|... - Defines the way in which error bars (or ellipses, or...) will - be represented for data set NS if errors are being displayed. - The following options are available: - - * none - * lines - * capped_lines - * caps - * arrows - * ellipse - * crosshair_ellipse - * rectangle - * crosshair_rectangle - * filled_ellipse - * filled_rectangle - - [Default: lines] - - grid = true|false - If true, grid lines are drawn on the plot. If false, they are - absent. - - [Default: true] - - antialias = true|false - Controls whether lines are drawn using antialiasing, where - applicable. If lines are drawn to a bitmapped-type graphics - output format setting this parameter to true smooths the lines - out by using gradations of colour for diagonal lines, and - setting it false simply sets each pixel in the line to on or - off. For vector-type graphics output formats, or for cases in - which no diagonal lines are drawn, the setting of this - parameter has no effect. Setting it true may slow the plot - down slightly. - - [Default: true] - - sequence = ,,... - Can be used to control the sequence in which different - datasets and subsets are plotted. This will affect which - symbols are plotted on top of, and so potentially obscure, - which other ones. The value of this parameter is a - comma-separated list of the "NS" suffixes which appear on the - parameters which apply to subsets. The sets which are named - will be plotted in order, so the first-named one will be at - the bottom (most likely to be obscured). Note that if this - parameter is supplied, then only those sets which are named - will be plotted, so this parameter may also be used to - restrict which plots appear (though it may not be the most - efficient way of doing this). If no explicit value is supplied - for this parameter, sets will be plotted in some sequence - decided by STILTS (probably alphabetic by suffix). -''' - task = _stilts.getTaskFactory().createObject('plot2d') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - -def plot3d(**kwargs): - '''\ -3D Scatter Plot. - -Parameters: - - xpix = - The width of the output graphic in pixels. - - [Default: 300] - - ypix = - The height of the output graphic in pixels. - - [Default: 300] - - font = dialog|serif|... - Determines the font that will be used for textual annotation - of the plot, including axes etc. The available names are: - - * bitstream_charter - * bitstream_vera_sans - * bitstream_vera_sans_mono - * bitstream_vera_serif - * century_schoolbook_l - * courier - * courier_10_pitch - * cursor - * dejavu_lgc_sans - * dejavu_lgc_sans_condensed - * dejavu_lgc_sans_light - * dejavu_lgc_sans_mono - * dejavu_lgc_serif - * dejavu_lgc_serif_condensed - * dialog - * dialoginput - * dingbats - * hershey - * liberation_mono - * liberation_sans - * liberation_serif - * lucida_bright - * lucida_sans - * lucida_sans_typewriter - * luxi_mono - * luxi_sans - * luxi_serif - * monospaced - * nimbus_mono_l - * nimbus_roman_no9_l - * nimbus_sans_l - * nimbus_sans_l_condensed - * sansserif - * serif - * standard_symbols_l - * urw_bookman_l - * urw_chancery_l - * urw_gothic_l - * urw_palladio_l - * utopia - - [Default: dialog] - - fontsize = - Sets the font size used for plot annotations. - - [Default: 12] - - fontstyle = plain|bold|italic|bold-italic - Gives a style in which the font is to be applied for plot - annotations. Options are plain, bold, italic and bold-italic. - - [Default: plain] - - legend = true|false - Determines whether a legend showing which plotting style is - used for each data set. Defaults to true if there is more than - one set, false otherwise. - - title = - A one-line title to display at the top of the plot. - - omode = swing|out|cgi|discard|auto - Determines how the drawn plot will be output. - - * swing: Plot will be displayed in a window on the screen. - * out: Plot will be written to a file given by out using the - graphics format given by ofmt. - * cgi: Plot will be written in a way suitable for CGI use - direct from a web server. The output is in the graphics - format given by ofmt, preceded by a suitable - "Content-type" declaration. - * discard: Plot is drawn, but discarded. There is no output. - * auto: Behaves as swing or out mode depending on presence of - out parameter - - [Default: auto] - - out = - The location of the output file. This is usually a filename to - write to. If it is equal to the special value "-" the output - will be written to standard output. - - ofmt = png|gif|jpeg|pdf|eps|eps-gzip - Graphics format in which the plot is written to the output - file. One of: - - * png: image/png format - * gif: image/gif format - * jpeg: image/jpeg format - * pdf: application/pdf format - * eps: application/postscript format - * eps-gzip: application/postscript (gzip) format - - May default to a sensible value depending on the filename - given by out. - - inN =
- Input table. - - xdataN = - Gives a column name or expression for the x axis data for - table N. The expression is a numeric algebraic expression - based on column names as described in SUN/256 - - ydataN = - Gives a column name or expression for the y axis data for - table N. The expression is a numeric algebraic expression - based on column names as described in SUN/256 - - zdataN = - Gives a column name or expression for the z axis data for - table N. The expression is a numeric algebraic expression - based on column names as described in SUN/256 - - auxdataN = - Gives a column name or expression for the aux axis data for - table N. The expression is a numeric algebraic expression - based on column names as described in SUN/256 - - xlo = - The lower limit for the plotted x axis. If not set, a value - will be chosen which is low enough to accommodate all the - data. - - ylo = - The lower limit for the plotted y axis. If not set, a value - will be chosen which is low enough to accommodate all the - data. - - zlo = - The lower limit for the plotted z axis. If not set, a value - will be chosen which is low enough to accommodate all the - data. - - auxlo = - The lower limit for the plotted aux axis. If not set, a value - will be chosen which is low enough to accommodate all the - data. - - xhi = - The upper limit for the plotted x axis. If not set, a value - will be chosen which is high enough to accommodate all the - data. - - yhi = - The upper limit for the plotted y axis. If not set, a value - will be chosen which is high enough to accommodate all the - data. - - zhi = - The upper limit for the plotted z axis. If not set, a value - will be chosen which is high enough to accommodate all the - data. - - auxhi = - The upper limit for the plotted aux axis. If not set, a value - will be chosen which is high enough to accommodate all the - data. - - xlog = true|false - If false (the default), the scale on the x axis is linear; if - true it is logarithmic. - - [Default: false] - - ylog = true|false - If false (the default), the scale on the y axis is linear; if - true it is logarithmic. - - [Default: false] - - zlog = true|false - If false (the default), the scale on the z axis is linear; if - true it is logarithmic. - - [Default: false] - - auxlog = true|false - If false (the default), the scale on the aux axis is linear; - if true it is logarithmic. - - [Default: false] - - xflip = true|false - If set true, the scale on the x axis will increase in the - opposite sense from usual (e.g. right to left rather than left - to right). - - [Default: false] - - yflip = true|false - If set true, the scale on the y axis will increase in the - opposite sense from usual (e.g. right to left rather than left - to right). - - [Default: false] - - zflip = true|false - If set true, the scale on the z axis will increase in the - opposite sense from usual (e.g. right to left rather than left - to right). - - [Default: false] - - auxflip = true|false - If set true, the scale on the aux axis will increase in the - opposite sense from usual (e.g. right to left rather than left - to right). - - [Default: false] - - xlabel = - Specifies a label to be used for annotating axis x. A default - values based on the plotted data will be used if no value is - supplied for this parameter. - - ylabel = - Specifies a label to be used for annotating axis y. A default - values based on the plotted data will be used if no value is - supplied for this parameter. - - zlabel = - Specifies a label to be used for annotating axis z. A default - values based on the plotted data will be used if no value is - supplied for this parameter. - - auxlabel = - Specifies a label to be used for annotating axis aux. A - default values based on the plotted data will be used if no - value is supplied for this parameter. - - xerrorN = |[],[] - Gives expressions for the errors on X coordinates for table N. - The following forms are permitted: - - * : symmetric error value - * ,:distinct lower and upper error values - * ,: lower error value only - * ,: upper error value only - * null: no errors - - The expression in each case is a numeric algebraic expression - based on column names as described in SUN/256. - - yerrorN = |[],[] - Gives expressions for the errors on Y coordinates for table N. - The following forms are permitted: - - * : symmetric error value - * ,:distinct lower and upper error values - * ,: lower error value only - * ,: upper error value only - * null: no errors - - The expression in each case is a numeric algebraic expression - based on column names as described in SUN/256. - - zerrorN = |[],[] - Gives expressions for the errors on Z coordinates for table N. - The following forms are permitted: - - * : symmetric error value - * ,:distinct lower and upper error values - * ,: lower error value only - * ,: upper error value only - * null: no errors - - The expression in each case is a numeric algebraic expression - based on column names as described in SUN/256. - - auxshader = rainbow|pastel|... - Determines how data from auxiliary axes will be displayed. - Generally this is some kind of colour ramp. These are the - available colour fixing options: - - * rainbow - * pastel - * standard - * heat - * colour - * hue - * greyscale - * red-blue - - and these are the available colour modifying options: - - * hsv_h - * hsv_s - * hsv_v - * intensity - * rgb_red - * rgb_green - * rgb_blue - * yuv_y - * yuv_u - * yuv_v - * transparency - - [Default: rainbow] - - txtlabelN = - Gives an expression which will label each plotted point. If - given, the text (or number) resulting from evaluating the - expression will be written near each point which is plotted. - - subsetNS = - Gives the selection criterion for the subset labelled "NS". - This is a boolean expression which may be the name of a - boolean-valued column or any other boolean-valued expression. - Rows for which the expression evaluates true will be included - in the subset, and those for which it evaluates false will - not. - - nameNS = - Provides a name to use for a subset with the symbolic label - NS. This name will be used for display in the legend, if one - is displayed. - - colourNS = |red|blue|... - Defines the colour of markers plotted. The value may be a - 6-digit hexadecimal number giving red, green and blue - intensities, e.g. "ff00ff" for magenta. Alternatively it may - be the name of one of the pre-defined colours. These are - currently red, blue, green, grey, magenta, cyan, orange, pink, - yellow, black and white. - - For most purposes, either the American or the British spelling - is accepted for this parameter name. - - shapeNS = filled_circle|open_circle|... - Defines the shapes for the markers that are plotted in data - set NS. The following shapes are available: - - * filled_circle - * open_circle - * cross - * x - * open_square - * open_diamond - * open_triangle_up - * open_triangle_down - * filled_square - * filled_diamond - * filled_triangle_up - * filled_triangle_down - - sizeNS = - Defines the marker size in pixels for markers plotted in data - set NS. If the value is negative, an attempt will be made to - use a suitable size according to how many points there are to - be plotted. - - [Default: -1] - - transparencyNS = - Determines the transparency of plotted markers for data set - NS. A value of means that opacity is only achieved (the - background is only blotted out) when pixels of this colour - have been plotted on top of each other. - - The minimum value is 1, which means opaque markers. - - lineNS = DotToDot|LinearRegression - Determines what line if any will be plotted along with the - data points. The options are: - - * null: No line is plotted. - * DotToDot: Each point is joined to the next one in sequence - by a straight line. - * LinearRegression: A linear regression line is plotted based - on all the points which are visible in the plot. Note - that the regression coefficients take no account of - points out of the visible range. - - hideNS = true|false - Indicates whether the actual markers plotted for each point - should be hidden. Normally this is false, but you may want to - set it to true if the point positions are being revealed in - some other way, for instance by error markers or lines drawn - between them. - - [Default: false] - - errstyleNS = lines|capped_lines|... - Defines the way in which error bars (or ellipses, or...) will - be represented for data set NS if errors are being displayed. - The following options are available: - - * none - * lines - * capped_lines - * caps - * arrows - * cuboid - * ellipse - * crosshair_ellipse - * rectangle - * crosshair_rectangle - * filled_ellipse - * filled_rectangle - - [Default: lines] - - grid = true|false - If true, grid lines are drawn on the plot. If false, they are - absent. - - [Default: true] - - antialias = true|false - Controls whether lines are drawn using antialiasing, where - applicable. If lines are drawn to a bitmapped-type graphics - output format setting this parameter to true smooths the lines - out by using gradations of colour for diagonal lines, and - setting it false simply sets each pixel in the line to on or - off. For vector-type graphics output formats, or for cases in - which no diagonal lines are drawn, the setting of this - parameter has no effect. Setting it true may slow the plot - down slightly. - - [Default: true] - - sequence = ,,... - Can be used to control the sequence in which different - datasets and subsets are plotted. This will affect which - symbols are plotted on top of, and so potentially obscure, - which other ones. The value of this parameter is a - comma-separated list of the "NS" suffixes which appear on the - parameters which apply to subsets. The sets which are named - will be plotted in order, so the first-named one will be at - the bottom (most likely to be obscured). Note that if this - parameter is supplied, then only those sets which are named - will be plotted, so this parameter may also be used to - restrict which plots appear (though it may not be the most - efficient way of doing this). If no explicit value is supplied - for this parameter, sets will be plotted in some sequence - decided by STILTS (probably alphabetic by suffix). - - fog = - Sets the level of fogging used to provide a visual indication - of depth. Object plotted further away from the viewer appear - more washed-out by a white fog. The default value gives a bit - of fogging; increase it to make the fog thicker, or set to - zero if no fogging is required. - - [Default: 1.0] - - phi = - Angle in degrees through which the 3D plot is rotated abound - the Z axis prior to drawing. - - [Default: 30.0] - - theta = - Angle in degrees through which the 3D plot is rotated towards - the viewer (i.e. about the horizontal axis of the viewing - plane) prior to drawing. - - [Default: 15.0] -''' - task = _stilts.getTaskFactory().createObject('plot3d') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - -def plothist(**kwargs): - '''\ -Histogram. - -Parameters: - - xpix = - The width of the output graphic in pixels. - - [Default: 400] - - ypix = - The height of the output graphic in pixels. - - [Default: 300] - - font = dialog|serif|... - Determines the font that will be used for textual annotation - of the plot, including axes etc. The available names are: - - * bitstream_charter - * bitstream_vera_sans - * bitstream_vera_sans_mono - * bitstream_vera_serif - * century_schoolbook_l - * courier - * courier_10_pitch - * cursor - * dejavu_lgc_sans - * dejavu_lgc_sans_condensed - * dejavu_lgc_sans_light - * dejavu_lgc_sans_mono - * dejavu_lgc_serif - * dejavu_lgc_serif_condensed - * dialog - * dialoginput - * dingbats - * hershey - * liberation_mono - * liberation_sans - * liberation_serif - * lucida_bright - * lucida_sans - * lucida_sans_typewriter - * luxi_mono - * luxi_sans - * luxi_serif - * monospaced - * nimbus_mono_l - * nimbus_roman_no9_l - * nimbus_sans_l - * nimbus_sans_l_condensed - * sansserif - * serif - * standard_symbols_l - * urw_bookman_l - * urw_chancery_l - * urw_gothic_l - * urw_palladio_l - * utopia - - [Default: dialog] - - fontsize = - Sets the font size used for plot annotations. - - [Default: 12] - - fontstyle = plain|bold|italic|bold-italic - Gives a style in which the font is to be applied for plot - annotations. Options are plain, bold, italic and bold-italic. - - [Default: plain] - - legend = true|false - Determines whether a legend showing which plotting style is - used for each data set. Defaults to true if there is more than - one set, false otherwise. - - title = - A one-line title to display at the top of the plot. - - omode = swing|out|cgi|discard|auto - Determines how the drawn plot will be output. - - * swing: Plot will be displayed in a window on the screen. - * out: Plot will be written to a file given by out using the - graphics format given by ofmt. - * cgi: Plot will be written in a way suitable for CGI use - direct from a web server. The output is in the graphics - format given by ofmt, preceded by a suitable - "Content-type" declaration. - * discard: Plot is drawn, but discarded. There is no output. - * auto: Behaves as swing or out mode depending on presence of - out parameter - - [Default: auto] - - out = - The location of the output file. This is usually a filename to - write to. If it is equal to the special value "-" the output - will be written to standard output. - - ofmt = png|gif|jpeg|pdf|eps|eps-gzip - Graphics format in which the plot is written to the output - file. One of: - - * png: image/png format - * gif: image/gif format - * jpeg: image/jpeg format - * pdf: application/pdf format - * eps: application/postscript format - * eps-gzip: application/postscript (gzip) format - - May default to a sensible value depending on the filename - given by out. - - inN =
- Input table. - - xdataN = - Gives a column name or expression for the x axis data for - table N. The expression is a numeric algebraic expression - based on column names as described in SUN/256 - - xlo = - The lower limit for the plotted x axis. If not set, a value - will be chosen which is low enough to accommodate all the - data. - - xhi = - The upper limit for the plotted x axis. If not set, a value - will be chosen which is high enough to accommodate all the - data. - - xlog = true|false - If false (the default), the scale on the x axis is linear; if - true it is logarithmic. - - [Default: false] - - xflip = true|false - If set true, the scale on the x axis will increase in the - opposite sense from usual (e.g. right to left rather than left - to right). - - [Default: false] - - xlabel = - Specifies a label to be used for annotating axis x. A default - values based on the plotted data will be used if no value is - supplied for this parameter. - - subsetNS = - Gives the selection criterion for the subset labelled "NS". - This is a boolean expression which may be the name of a - boolean-valued column or any other boolean-valued expression. - Rows for which the expression evaluates true will be included - in the subset, and those for which it evaluates false will - not. - - nameNS = - Provides a name to use for a subset with the symbolic label - NS. This name will be used for display in the legend, if one - is displayed. - - colourNS = |red|blue|... - Defines the colour of bars plotted for data set NS. The value - may be a 6-digit hexadecimal number giving red, green and blue - intensities, e.g. "ff00ff" for magenta. Alternatively it may - be the name of one of the pre-defined colours. These are - currently red, blue, green, grey, magenta, cyan, orange, pink, - yellow, black and white. - - For most purposes, either the American or the British spelling - is accepted for this parameter name. - - barstyleNS = fill|open|... - Defines how histogram bars will be drawn for dataset NS. The - options are: - - * fill - * open - * tops - * spikes - * fillover - * openover - - [Default: fill] - - linewidthNS = - Defines the line width for lines drawn as part of the bars for - dataset NS. Only certain bar styles are affected by the line - width. - - [Default: 2] - - dashNS = dot|dash|...| - Defines the dashing pattern for lines drawn for dataset NS. To - generate a dashed line the value may be one of the named dash - types: - - * dot - * dash - * longdash - * dotdash - - or may be a comma-separated string of on/off length values - such as "4,2,8,2". A null value indicates a solid line. Only - certain bar styles are affected by the dash pattern. - - grid = true|false - If true, grid lines are drawn on the plot. If false, they are - absent. - - [Default: true] - - antialias = true|false - Controls whether lines are drawn using antialiasing, where - applicable. If lines are drawn to a bitmapped-type graphics - output format setting this parameter to true smooths the lines - out by using gradations of colour for diagonal lines, and - setting it false simply sets each pixel in the line to on or - off. For vector-type graphics output formats, or for cases in - which no diagonal lines are drawn, the setting of this - parameter has no effect. Setting it true may slow the plot - down slightly. - - [Default: true] - - sequence = ,,... - Can be used to control the sequence in which different - datasets and subsets are plotted. This will affect which - symbols are plotted on top of, and so potentially obscure, - which other ones. The value of this parameter is a - comma-separated list of the "NS" suffixes which appear on the - parameters which apply to subsets. The sets which are named - will be plotted in order, so the first-named one will be at - the bottom (most likely to be obscured). Note that if this - parameter is supplied, then only those sets which are named - will be plotted, so this parameter may also be used to - restrict which plots appear (though it may not be the most - efficient way of doing this). If no explicit value is supplied - for this parameter, sets will be plotted in some sequence - decided by STILTS (probably alphabetic by suffix). - - ylo = - Lower bound for Y axis. - - [Default: 0] - - yhi = - Upper bound for Y axis. Autogenerated from the data if not - supplied. - - ylog = true|false - Whether to use a logarithmic scale for the Y axis. - - [Default: false] - - ylabel = - Specifies a label for annotating the vertical axis. A default - value based on the type of histogram will be used if no value - is supplied for this parameter. - - [Default: Count] - - weightN = - Defines a weighting for each point accumulated to determine - the height of plotted bars. If this parameter has a value - other than 1 (the default) then instead of simply accumulating - the number of points per bin to determine bar height, the bar - height will be the sum over the weighting expression for the - points in each bin. Note that with weighting, the figure drawn - is no longer strictly speaking a histogram. - - When weighted, bars can be of negative height. An anomaly of - the plot as currently implemented is that the Y axis never - descends below zero, so any such bars are currently invisible. - This may be amended in a future release (contact the author to - lobby for such an amendment). - - [Default: 1] - - binwidth = - Defines the width on the X axis of histogram bins. If the X - axis is logarithmic, then this is a multiplicative value. - - norm = true|false - Determines whether bin counts are normalised. If true, - histogram bars are scaled such that summed height of all bars - over the whole dataset is equal to one. Otherwise (the - default), no scaling is done. - - [Default: false] - - cumulative = true|false - Determines whether historams are cumulative. When false (the - default), the height of each bar is determined by counting the - number of points which fall into the range on the X axis that - it covers. When true, the height is determined by counting all - the points between negative infinity and the upper bound of - the range on the X axis that it covers. - - [Default: false] - - binbase = - Adjusts the offset of the bins. By default zero (or one for - logarithmic X axis) is a boundary between bins; other - boundaries are defined by this and the bin width. If this - value is adjusted, the lower bound of one of the bins will be - set to this value, so all the bins move along by the - corresponding distance. - - [Default: 0] -''' - task = _stilts.getTaskFactory().createObject('plothist') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - -def server(**kwargs): - '''\ -Runs an HTTP server to perform STILTS commands. - -Parameters: - - port = - Port number on which the server should run. - - [Default: 2112] - - basepath = - Base path on the server at which request URLs are rooted. The - default is /stilts, which means that for instance requests to - execute task plot2d should be directed to the URL - http://host:portnum/stilts/task/plot2d?name=value&name=value... - - [Default: /stilts] - - tasks = ... - Gives a space-separated list of tasks which will be provided - by the running server. If the value is null then all tasks - will be available. However, some tasks don't make a lot of - sense to run from the server, so the default value is a - somewhat restricted list. If the server is being exposed to - external users, you might also want to reduce the list for - security reasons. - - [Default: calc coneskymatch regquery plot2d plot3d plothist - sqlclient sqlskymatch sqlupdate tcat tcatn tcopy tcube tjoin - tmatch1 tmatch2 tmatchn tmulti tmultin tpipe tskymatch2 - votcopy votlint] - - tablefactory = file|dirs:...|locator:... - This parameter determines how input table names (typically the - in parameter of table processing commands) are used to acquire - references to actual table data. The default behaviour is for - input table names to be treated as filenames, in conjunction - with some file type parameter. While this is usually sensible - for local use, in server situations it may be inappropriate, - since you don't want external users to have read access to - your entire filesystem. - - This parameter gives options for alternative ways of mapping - table names to table data items. The currently available - options are: - - * file: default behaviour - names are treated as filenames - * dirs:::...: following the "dirs:" prefix a list - of directories is specified which will be searched for - the file named. Note that the directory separator - character differs between operating systems; it is a - colon (":") for Unix-like OSs and a semi-colon (";") for - MS Windows. If a given name is identical to the - path-less filename in one of the directories, that - file is used as the referenced table. File type - information is ignored in this case, so the files must - be one of the types which STILTS can autodetect, - currently FITS or VOTable (FITS is more efficient). By - using this option, clients can be restricted to using a - fixed set of tables in a restricted part of the server's - file system. - * locator:: the must be the name of - a Java class on the classpath which implements the - interface uk.ac.starlink.ttools.task.TableLocator and - which has a no-arg constructor. An instance of this - class will be used to resolve names to tables. - - The usage and functionality of this parameter is experimental, - and may change significantly in future releases. - - [Default: file] -''' - task = _stilts.getTaskFactory().createObject('server') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - -def sqlclient(**kwargs): - '''\ -Executes SQL statements. - -Parameters: - - db = - URL which defines a connection to a database. This has the - form jdbc:: - the details are database- - and driver-dependent. Consult Sun's JDBC documentation and - that for the particular JDBC driver you are using for details. - Note that the relevant driver class will need to be on your - classpath and referenced in the jdbc.drivers system property - as well for the connection to be made. - - user = - User name for logging in to SQL database. Defaults to the - current username. - - [Default: mbt] - - password = - Password for logging in to SQL database. - - sql = - Text of an SQL statement for execution. This parameter may be - repeated, or statements may be separated by semicolon (";") - characters. -''' - task = _stilts.getTaskFactory().createObject('sqlclient') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - -def sqlskymatch(in_, **kwargs): - '''\ -Crossmatches table on sky position against SQL table. - -The return value is the resulting table. - -Parameters: - - in_ =
- Input table. - - ra = - Expression which evaluates to the right ascension in degrees - for the request at each row of the input table. This will - usually be the name or ID of a column in the input table, or a - function involving one. - - dec = - Expression which evaluates to the declination in degrees for - the request at each row of the input table. This will usually - be the name or ID of a column in the input table, or a - function involving one. - - sr = - Expression which evaluates to the search radius in degrees for - the request at each row of the input table. This will often be - a constant numerical value, but may be the name or ID of a - column in the input table, or a function involving one. - - find = best|all|each - Determines which matches are retained. - - * best: Only the matching query table row closest to the - input table row will be output. Input table rows with no - matches will be omitted. - * all: All query table rows which match the input table row - will be output. Input table rows with no matches will be - omitted. - * each: There will be one output table row for each input - table row. If matches are found, the closest one from - the query table will be output, and in the case of no - matches, the query table columns will be blank. - - [Default: all] - - copycols = - List of columns from the input table which are to be copied to - the output table. Each column identified here will be - prepended to the columns of the combined output table, and its - value for each row taken from the input table row which - provided the parameters of the query which produced it. See - SUN/256 for list syntax. The default setting is "*", which - means that all columns from the input table are included in - the output. - - [Default: *] - - scorecol = - Gives the name of a column in the output table to contain the - distance between the requested central position and the actual - position of the returned row. The distance returned is an - angular distance in degrees. If a null value is chosen, no - distance column will appear in the output table. - - [Default: Separation] - - erract = abort|ignore|retry|retry - Determines what will happen if any of the individual cone - search requests fails. By default the task aborts. That may be - the best thing to do, but for unreliable or poorly implemented - services you may find that some searches fail and others - succeed so it can be best to continue operation in the face of - a few failures. The options are: - - * abort: failure of any query terminates the task - * ignore: failure of a query is treated the same as a query - which returns no rows - * retry: failed queries are retried until they succeed; use - with care - if the failure is for some good, or at least - reproducible reason this could prevent the task from - ever completing - * retry: failed queries are retried at most a fixed number - of times If they still fail the task terminates. - - [Default: abort] - - ostream = true|false - If set true, this will cause the operation to stream on - output, so that the output table is built up as the results - are obtained from the cone search service. The disadvantage of - this is that some output modes and formats need multiple - passes through the data to work, so depending on the output - destination, the operation may fail if this is set. Use with - care (or be prepared for the operation to fail). - - [Default: false] - - fixcols = none|dups|all - Determines how input columns are renamed before use in the - output table. The choices are: - - * none: columns are not renamed - * dups: columns which would otherwise have duplicate names in - the output will be renamed to indicate which table they - came from - * all: all columns will be renamed to indicate which table - they came from - - If columns are renamed, the new ones are determined by - suffix* parameters. - - [Default: dups] - - suffix0 = = - Assigns new values for a given column. The assignment is made - in the form = where is the name of a - column in the SQL table and is the text of an - expression using STILTS's expression language, as described in - SUN/256. SQL table column names or $ID identifiers may be used - as variables in the usual way. - - This parameter may be supplied more than once to effect - multiple assignments, or multiple assignments may be made by - separating them with semicolons in the value of this - parameter. - - progress = true|false - If true, a spinner will be drawn on standard error which shows - how many rows have been updated so far. - - [Default: true] -''' - task = _stilts.getTaskFactory().createObject('sqlupdate') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - -def tcat(in_, **kwargs): - '''\ -Concatenates multiple similar tables. - -The return value is the resulting table. - -Parameters: - - in_ =
[
...] - Array of input tables. - - multi = true|false - Determines whether all tables, or just the first one, from - input table files will be used. If set false, then just the - first table from each file named by in will be used. If true, - then all tables present in those input files will be used. - This only has an effect for file formats which are capable of - containing more than one table, which effectively means FITS - and VOTable and their variants. - - [Default: false] - - seqcol = - Name of a column to be added to the output table which will - contain the sequence number of the input table from which each - row originated. This column will contain 1 for the rows from - the first concatenated table, 2 for the second, and so on. - - loccol = - Name of a column to be added to the output table which will - contain the location (as specified in the input parameter(s)) - of the input table from which each row originated. - - uloccol = - Name of a column to be added to the output table which will - contain the unique part of the location (as specified in the - input parameter(s)) of the input table from which each row - originated. If not null, parameters will also be added to the - output table giving the pre- and post-fix string common to all - the locations. For example, if the input tables are - "/data/cat_a1.fits" and "/data/cat_b2.fits" then the output - table will contain a new column which takes the - value "a1" for rows from the first table and "b2" for rows - from the second, and new parameters "_prefix" and - "_postfix" with the values "/data/cat_" and ".fits" - respectively. - - lazy = true|false - Whether to perform table resolution lazily. If true, each - table is only accessed when the time comes to add its rows to - the output; if false, then all the tables are accessed up - front. This is mostly a tuning parameter, and on the whole it - doesn't matter much how it is set, but for joining an enormous - number of tables setting it true may avoid running out of - resources. - - [Default: false] - - countrows = true|false - Whether to count the rows in the table before starting the - output. This is essentially a tuning parameter - if writing to - an output format which requires the number of rows up front - (such as normal FITS) it may result in skipping the number of - passes through the input files required for processing. Unless - you have a good understanding of the internals of the - software, your best bet for working out whether to set this - true or false is to try it both ways - - [Default: false] -''' - import uk.ac.starlink.ttools.task.TableCat - class _tcat_task(uk.ac.starlink.ttools.task.TableCat): - def __init__(self): - uk.ac.starlink.ttools.task.TableCat.__init__(self) - task = _tcat_task() - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - env.setValue('in_', _map_env_value(in_)) - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - table = task.createProducer(env).getTable() - _check_unused_args(env) - return import_star_table(table) - -def tcatn(**kwargs): - '''\ -Concatenates multiple tables. - -The return value is the resulting table. - -Parameters: - - nin = - The number of input tables for this task. For each of the - input tables N there will be associated parameters ifmtN, inN - and icmdN. - - inN = - Input table. - - seqcol = - Name of a column to be added to the output table which will - contain the sequence number of the input table from which each - row originated. This column will contain 1 for the rows from - the first concatenated table, 2 for the second, and so on. - - loccol = - Name of a column to be added to the output table which will - contain the location (as specified in the input parameter(s)) - of the input table from which each row originated. - - uloccol = - Name of a column to be added to the output table which will - contain the unique part of the location (as specified in the - input parameter(s)) of the input table from which each row - originated. If not null, parameters will also be added to the - output table giving the pre- and post-fix string common to all - the locations. For example, if the input tables are - "/data/cat_a1.fits" and "/data/cat_b2.fits" then the output - table will contain a new column which takes the - value "a1" for rows from the first table and "b2" for rows - from the second, and new parameters "_prefix" and - "_postfix" with the values "/data/cat_" and ".fits" - respectively. - - countrows = true|false - Whether to count the rows in the table before starting the - output. This is essentially a tuning parameter - if writing to - an output format which requires the number of rows up front - (such as normal FITS) it may result in skipping the number of - passes through the input files required for processing. Unless - you have a good understanding of the internals of the - software, your best bet for working out whether to set this - true or false is to try it both ways - - [Default: false] -''' - import uk.ac.starlink.ttools.task.TableCatN - class _tcatn_task(uk.ac.starlink.ttools.task.TableCatN): - def __init__(self): - uk.ac.starlink.ttools.task.TableCatN.__init__(self) - task = _tcatn_task() - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - table = task.createProducer(env).getTable() - _check_unused_args(env) - return import_star_table(table) - -def tcopy(in_, **kwargs): - '''\ -Converts between table formats. - -Parameters: - - in_ =
- Input table. -''' - task = _stilts.getTaskFactory().createObject('tcopy') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - env.setValue('in_', _map_env_value(in_)) - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - -def tcube(in_, **kwargs): - '''\ -Calculates N-dimensional histograms. - -The return value is the resulting table. - -Parameters: - - cols = ... - Columns to use for this task. One or more elements, - separated by spaces, should be given. Each one represents a - column in the table, using either its name or index. - - The number of columns listed in the value of this parameter - defines the dimensionality of the output data cube. - - in_ =
- Input table. - - bounds = []:[] ... - Gives the bounds for each dimension of the cube in data - coordinates. The form of the value is a space-separated list - of words, each giving an optional lower bound, then a colon, - then an optional upper bound, for instance "1:100 0:20" to - represent a range for two-dimensional output between 1 and 100 - of the first coordinate (table column) and between 0 and 20 - for the second. Either or both numbers may be omitted to - indicate that the bounds should be determined automatically by - assessing the range of the data in the table. A null value for - the parameter indicates that all bounds should be determined - automatically for all the dimensions. - - If any of the bounds need to be determined automatically in - this way, two passes through the data will be required, the - first to determine bounds and the second to populate the cube. - - binsizes = ... - Gives the extent of of the data bins (cube pixels) in each - dimension in data coordinates. The form of the value is a - space-separated list of values, giving a list of extents for - the first, second, ... dimension. Either this parameter or the - nbins parameter must be supplied. - - nbins = ... - Gives the number of bins (cube pixels) in each dimension. The - form of the value is a space-separated list of integers, - giving the number of pixels for the output cube in the first, - second, ... dimension. Either this parameter or the binsizes - parameter must be supplied. - - out = - The location of the output file. This is usually a filename to - write to. If it is equal to the special value "-" the output - will be written to standard output. - - The output cube is currently written as a single-HDU FITS - file. - - [Default: -] - - otype = byte|short|int|long|float|double - The type of numeric value which will fill the output array. If - no selection is made, the output type will be determined - automatically as the shortest type required to hold all the - values in the array. Currently, integers are always signed (no - BSCALE/BZERO), so for instance the largest value that can be - recorded in 8 bits is 127. - - scale = - Optionally gives a value by which the count in each bin is - scaled. If this value is null (the default) then for each row - that falls within the bounds of a pixel, the pixel value will - be incremented by 1. If a column ID is given, then instead of - 1 being added, the value of that column for the row in - question is added. The effect of this is that the output image - contains the mean of the given column for the rows - corresponding to each pixel rather than just a count of them. -''' - import uk.ac.starlink.ttools.task.TableCube - class _tcube_task(uk.ac.starlink.ttools.task.TableCube): - def __init__(self): - uk.ac.starlink.ttools.task.TableCube.__init__(self) - task = _tcube_task() - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - env.setValue('in_', _map_env_value(in_)) - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - table = task.createProducer(env).getTable() - _check_unused_args(env) - return import_star_table(table) - -def tjoin(**kwargs): - '''\ -Joins multiple tables side-to-side. - -The return value is the resulting table. - -Parameters: - - nin = - The number of input tables for this task. For each of the - input tables N there will be associated parameters ifmtN, inN - and icmdN. - - inN = - Input table. - - fixcols = none|dups|all - Determines how input columns are renamed before use in the - output table. The choices are: - - * none: columns are not renamed - * dups: columns which would otherwise have duplicate names in - the output will be renamed to indicate which table they - came from - * all: all columns will be renamed to indicate which table - they came from - - If columns are renamed, the new ones are determined by - suffix* parameters. - - [Default: dups] - - suffixN =
- Input table. -''' - import uk.ac.starlink.ttools.task.TableMatch1 - class _tmatch1_task(uk.ac.starlink.ttools.task.TableMatch1): - def __init__(self): - uk.ac.starlink.ttools.task.TableMatch1.__init__(self) - task = _tmatch1_task() - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - env.setValue('in_', _map_env_value(in_)) - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - table = task.createProducer(env).getTable() - _check_unused_args(env) - return import_star_table(table) - -def tmatch2(in1, in2, **kwargs): - '''\ -Crossmatches 2 tables using flexible criteria. - -The return value is the resulting table. - -Parameters: - - in1 = - Input table. - - in2 = - Input table. - - matcher = - Defines the nature of the matching that will be performed. - Depending on the name supplied, this may be positional - matching using celestial or Cartesian coordinates, exact - matching on the value of a string column, or other things. A - list and explanation of the available matching algorithms is - given in SUN/256. The value supplied for this parameter - determines the meanings of the values required by the params, - values* and tuning parameter(s). - - [Default: sky] - - values1 = - Defines the values from table 1 which are used to determine - whether a match has occurred. These will typically be - coordinate values such as RA and Dec and perhaps some per-row - error values as well, though exactly what values are required - is determined by the kind of match as determined by matcher. - Depending on the kind of match, the number and type of the - values required will be different. Multiple values should be - separated by whitespace; if whitespace occurs within a single - value it must be 'quoted' or "quoted". Elements of the - expression list are commonly just column names, but may be - algebraic expressions calculated from zero or more columns as - explained in SUN/256. - - values2 = - Defines the values from table 2 which are used to determine - whether a match has occurred. These will typically be - coordinate values such as RA and Dec and perhaps some per-row - error values as well, though exactly what values are required - is determined by the kind of match as determined by matcher. - Depending on the kind of match, the number and type of the - values required will be different. Multiple values should be - separated by whitespace; if whitespace occurs within a single - value it must be 'quoted' or "quoted". Elements of the - expression list are commonly just column names, but may be - algebraic expressions calculated from zero or more columns as - explained in SUN/256. - - params = - Determines the parameters of this match. This is typically one - or more tolerances such as error radii. It may contain zero or - more values; the values that are required depend on the match - type selected by the matcher parameter. If it contains - multiple values, they must be separated by spaces; values - which contain a space can be 'quoted' or "quoted". - - tuning = - Tuning values for the matching process, if appropriate. It may - contain zero or more values; the values that are permitted - depend on the match type selected by the matcher parameter. If - it contains multiple values, they must be separated by spaces; - values which contain a space can be 'quoted' or "quoted". If - this optional parameter is not supplied, sensible defaults - will be chosen. - - join = 1and2|1or2|all1|all2|1not2|2not1|1xor2 - Determines which rows are included in the output table. The - matching algorithm determines which of the rows from the first - table correspond to which rows from the second. This parameter - determines what to do with that information. Perhaps the most - obvious thing is to write out a table containing only rows - which correspond to a row in both of the two input tables. - However, you may also want to see the unmatched rows from one - or both input tables, or rows present in one table but - unmatched in the other, or other possibilities. The options - are: - - * 1and2: An output row for each row represented in both input - tables - * 1or2: An output row for each row represented in either or - both of the input tables - * all1: An output row for each matched or unmatched row in - table 1 - * all2: An output row for each matched or unmatched row in - table 2 - * 1not2: An output row only for rows which appear in the - first table but are not matched in the second table - * 2not1: An output row only for rows which appear in the - second table but are not matched in the first table - * 1xor2: An output row only for rows represented in one of - the input tables but not the other one - - [Default: 1and2] - - find = best|all - Determines which matches are retained. If best is selected, - then only the best match between the two tables will be - retained; in this case the data from a row of either input - table will appear in at most one row of the output table. If - all is selected, then all pairs of rows from the two input - tables which match the input criteria will be represented in - the output table. - - [Default: best] - - fixcols = none|dups|all - Determines how input columns are renamed before use in the - output table. The choices are: - - * none: columns are not renamed - * dups: columns which would otherwise have duplicate names in - the output will be renamed to indicate which table they - came from - * all: all columns will be renamed to indicate which table - they came from - - If columns are renamed, the new ones are determined by - suffix* parameters. - - [Default: dups] - - suffix1 =
[
...] - Array of input tables. - - multi = true|false - Determines whether all tables, or just the first one, from - input table files will be used. If set false, then just the - first table from each file named by in will be used. If true, - then all tables present in those input files will be used. - This only has an effect for file formats which are capable of - containing more than one table, which effectively means FITS - and VOTable and their variants. - - [Default: false] - - out = - The location of the output file. This is usually a filename to - write to. If it is equal to the special value "-" the output - will be written to standard output. - - [Default: -] -''' - task = _stilts.getTaskFactory().createObject('tmulti') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - env.setValue('in_', _map_env_value(in_)) - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - -def tmultin(**kwargs): - '''\ -Writes multiple processed tables to single container file. - -Parameters: - - nin = - The number of input tables for this task. For each of the - input tables N there will be associated parameters ifmtN, inN - and icmdN. - - inN = - Input table. - - out = - The location of the output file. This is usually a filename to - write to. If it is equal to the special value "-" the output - will be written to standard output. - - [Default: -] -''' - task = _stilts.getTaskFactory().createObject('tmultin') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - -def tpipe(in_, **kwargs): - '''\ -Performs pipeline processing on a table. - -The return value is the resulting table. - -Parameters: - - in_ =
- Input table. -''' - import uk.ac.starlink.ttools.task.TablePipe - class _tpipe_task(uk.ac.starlink.ttools.task.TablePipe): - def __init__(self): - uk.ac.starlink.ttools.task.TablePipe.__init__(self) - task = _tpipe_task() - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - env.setValue('in_', _map_env_value(in_)) - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - table = task.createProducer(env).getTable() - _check_unused_args(env) - return import_star_table(table) - -def tskymatch2(in1, in2, **kwargs): - '''\ -Crossmatches 2 tables on sky position. - -The return value is the resulting table. - -Parameters: - - in1 = - Input table. - - in2 = - Input table. - - ra1 = - Value in degrees for the right ascension of positions in table - 1 to be matched. This may simply be a column name, or it may - be an algebraic expression calculated from columns as - explained in SUN/256. If left blank, an attempt is made to - guess from UCDs, column names and unit annotations what - expression to use. - - dec1 = - Value in degrees for the declination of positions in table 1 - to be matched. This may simply be a column name, or it may be - an algebraic expression calculated from columns as explained - in SUN/256. If left blank, an attempt is made to guess from - UCDs, column names and unit annotations what expression to - use. - - ra2 = - Value in degrees for the right ascension of positions in table - 2 to be matched. This may simply be a column name, or it may - be an algebraic expression calculated from columns as - explained in SUN/256. If left blank, an attempt is made to - guess from UCDs, column names and unit annotations what - expression to use. - - dec2 = - Value in degrees for the declination of positions in table 2 - to be matched. This may simply be a column name, or it may be - an algebraic expression calculated from columns as explained - in SUN/256. If left blank, an attempt is made to guess from - UCDs, column names and unit annotations what expression to - use. - - error = - The maximum separation permitted between two objects for them - to count as a match. Units are arc seconds. - - tuning = - Tuning parameter that controls the pixel size used when - binning the rows. The legal range is from 0 (corresponding to - pixel size of about 60 degrees) to 20 (about 0.2 arcsec). The - value of this parameter will not affect the result but may - affect the performance in terms of CPU and memory resources - required. A default value will be chosen based on the size of - the error parameter, but it may be possible to improve - performance by adjusting the default value. The value used can - be seen by examining the progress output. If your match is - taking a long time or is failing from lack of memory it may be - worth trying different values for this parameter. - - join = 1and2|1or2|all1|all2|1not2|2not1|1xor2 - Determines which rows are included in the output table. The - matching algorithm determines which of the rows from the first - table correspond to which rows from the second. This parameter - determines what to do with that information. Perhaps the most - obvious thing is to write out a table containing only rows - which correspond to a row in both of the two input tables. - However, you may also want to see the unmatched rows from one - or both input tables, or rows present in one table but - unmatched in the other, or other possibilities. The options - are: - - * 1and2: An output row for each row represented in both input - tables - * 1or2: An output row for each row represented in either or - both of the input tables - * all1: An output row for each matched or unmatched row in - table 1 - * all2: An output row for each matched or unmatched row in - table 2 - * 1not2: An output row only for rows which appear in the - first table but are not matched in the second table - * 2not1: An output row only for rows which appear in the - second table but are not matched in the first table - * 1xor2: An output row only for rows represented in one of - the input tables but not the other one - - [Default: 1and2] - - find = best|all - Determines which matches are retained. If best is selected, - then only the best match between the two tables will be - retained; in this case the data from a row of either input - table will appear in at most one row of the output table. If - all is selected, then all pairs of rows from the two input - tables which match the input criteria will be represented in - the output table. - - [Default: best] -''' - import uk.ac.starlink.ttools.task.SkyMatch2 - class _tskymatch2_task(uk.ac.starlink.ttools.task.SkyMatch2): - def __init__(self): - uk.ac.starlink.ttools.task.SkyMatch2.__init__(self) - task = _tskymatch2_task() - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - env.setValue('in1', _map_env_value(in1)) - env.setValue('in2', _map_env_value(in2)) - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - table = task.createProducer(env).getTable() - _check_unused_args(env) - return import_star_table(table) - -def votcopy(in_='-', out='-', format='tabledata', **kwargs): - '''\ -Transforms between VOTable encodings. - -Parameters: - - in_ = - Location of the input VOTable. May be a URL, filename, or "-" - to indicate standard input. The input table may be compressed - using one of the known compression formats (Unix compress, - gzip or bzip2). - - [Default: -] - - out = - Location of the output VOTable. May be a filename or "-" to - indicate standard output. - - [Default: -] - - format = TABLEDATA|BINARY|FITS - Determines the encoding format of the table data in the output - document. If null is selected, then the tables will be - data-less (will contain no DATA element), leaving only the - document structure. Data-less tables are legal VOTable - elements. - - [Default: tabledata] - - charset = - Selects the Unicode encoding used for the output XML. The - available options and default are dependent on your JVM, but - the default probably corresponds to UTF-8. Use help=charset - for a full listing. - - cache = true|false - Determines whether the input tables are read into a cache - prior to being written out. The default is selected - automatically depending on the input table; so you should - normally leave this flag alone. - - href = true|false - In the case of BINARY or FITS encoding, this determines - whether the STREAM elements output will contain their data - inline or externally. If set false, the output document will - be self-contained, with STREAM data inline as base64-encoded - characters. If true, then for each TABLE in the document the - binary data will be written to a separate file and referenced - by an href attribute on the corresponding STREAM element. The - name of these files is usually determined by the name of the - main output file; but see also the base flag. - - base = - Determines the name of external output files written when the - href flag is true. Normally these are given names based on the - name of the output file. But if this flag is given, the names - will be based on the string. This flag is - compulsory if href is true and out=- (output is to standard - out), since in this case there is no default base name to use. -''' - task = _stilts.getTaskFactory().createObject('votcopy') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - -def votlint(votable='-', **kwargs): - '''\ -Validates VOTable documents. - -Parameters: - - votable = - Location of the VOTable to be checked. This may be a filename, - URL or "-" (the default), to indicate standard input. The - input may be compressed using one of the known compression - formats (Unix compress, gzip or bzip2). - - [Default: -] - - validate = true|false - Whether to validate the input document aganist the VOTable - DTD. If true (the default), then as well as votlint's own - checks, it is validated against an appropriate version of the - VOTable DTD which picks up such things as the presence of - unknown elements and attributes, elements in the wrong place, - and so on. Sometimes however, particularly when XML namespaces - are involved, the validator can get confused and may produce a - lot of spurious errors. Setting this flag false prevents this - validation step so that only votlint's own checks are - performed. In this case many violations of the VOTable - standard concerning document structure will go unnoticed. - - [Default: true] - - version = 1.0|1.1|1.2 - Selects the version of the VOTable standard which the input - table is supposed to exemplify. Currently the version can be - 1.0, 1.1 or 1.2. The version may also be specified within the - document using the "version" attribute of the document's - VOTABLE element; if it is and it conflicts with the value - specified by this flag, a warning is issued. - - out = - Destination file for output messages. May be a filename or "-" - to indicate standard output. - - [Default: -] -''' - task = _stilts.getTaskFactory().createObject('votlint') - for param in task.getParameters(): - pname = param.getName() - if pname in _param_alias_dict: - param.setName(_param_alias_dict[pname]) - env = _JyEnvironment() - for kw in kwargs.iteritems(): - key = kw[0] - value = kw[1] - env.setValue(key, _map_env_value(value)) - exe = task.createExecutable(env) - _check_unused_args(env) - exe.execute() - -def cmd_addcol(table, *args): - '''\ -Add a new column called defined by the algebraic -expression . By default the new column appears after the last -column of the table, but you can position it either before or after -a specified column using the -before or -after flags respectively. -The -units, -ucd and -desc flags can be used to define metadata -values for the new column. - -Syntax for the and arguments is described in the -manual. - -The filtered table is returned. - -args is a list with words as elements: - [-after | -before ] - [-units ] [-ucd ] [-desc ] - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("addcol") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_addresolve(table, *args): - '''\ -Performs name resolution on the string-valued column ->col-id-objname< and appends two new columns >col-name-ra< and ->col-name-dec< containing the resolved Right Ascension and -Declination in degrees. - -Syntax for the argument is described in SUN/256. - -UCDs are added to the new columns in a way which tries to be -consistent with any UCDs already existing in the table. - -Since this filter works by interrogating a remote service, it will -obviously be slow. The current implementation is experimental; it -may be replaced in a future release by some way of doing the same -thing (perhaps a new STILTS task) which is able to work more -efficiently by dispatching multiple concurrent requests. - -This software uses source code created at the Centre de Donnees -astronomiques de Strasbourg, France. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("addresolve") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_addskycoords(table, *args): - '''\ -Add new columns to the table representing position on the sky. The -values are determined by converting a sky position whose coordinates -are contained in existing columns. The arguments give -identifiers for the two input coordinate columns in the coordinate -system named by , and the arguments name the two -new columns, which will be in the coordinate system named by -. The and coordinate system specifiers are -one of - - * icrs: ICRS (Hipparcos) (Right Ascension, Declination) - * fk5: FK5 J2000.0 (Right Ascension, Declination) - * fk4: FK4 B1950.0 (Right Ascension, Declination) - * galactic: IAU 1958 Galactic (Longitude, Latitude) - * supergalactic: de Vaucouleurs Supergalactic (Longitude, Latitude) - * ecliptic: Ecliptic (Longitude, Latitude) - -The -inunit and -outunit flags may be used to indicate the units of -the existing coordinates and the units for the new coordinates -respectively; use one of degrees, radians or sexagesimal (may be -abbreviated), otherwise degrees will be assumed. For sexagesimal, -the two corresponding columns must be string-valued in forms like -hh:mm:ss.s and dd:mm:ss.s respectively. - -For certain conversions, the value specified by the -epoch flag is -of significance. Where significant its value defaults to 2000.0. - -Syntax for the , and arguments is -described in the manual. - -The filtered table is returned. - -args is a list with words as elements: - [-epoch ] [-inunit deg|rad|sex] [-outunit deg|rad|sex] - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("addskycoords") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_assert(table, *args): - '''\ -Check that a boolean expression is true for each row. If the -expression does not evaluate true for any row of the table, -execution terminates with an error. As long as no error occurs, the -output table is identical to the input one. - -The exception generated by an assertion violation is of class -uk.ac.starlink.ttools.filter.AssertException although that is not -usually obvious if you are running from the shell in the usual way. - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("assert") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_badval(table, *args): - '''\ -For each column specified in any occurrence of the -value is replaced by a blank entry. - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("badval") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_cache(table): - '''\ -Stores in memory or on disk a temporary copy of the table at this -point in the pipeline. This can provide improvements in efficiency -if there is an expensive step upstream and a step which requires -more than one read of the data downstream. If you see an error like -"Can't re-read data from stream" then adding this step near the -start of the filters might help. - -The result of this filter is guaranteed to be random-access. - -See also the random filter, which caches only when the input table -is not random-access. - -The filtered table is returned. -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("cache") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_check(table): - '''\ -Runs checks on the table at the indicated point in the processing -pipeline. This is strictly a debugging measure, and may be -time-consuming for large tables. - -The filtered table is returned. -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("check") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_clearparams(table, *args): - '''\ -Clears the value of one or more named parameters. Each of the - values supplied may be either a parameter name or a simple -wildcard expression matching parameter names. Currently the only -wildcarding is a "*" to match any sequence of characters. -clearparams * will clear all the parameters in the table. - -It is not an error to supply s which do not exist in the -table - these have no effect. - -The filtered table is returned. - -args is a list with words as elements: - ... -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("clearparams") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_colmeta(table, *args): - '''\ -Modifies the metadata of one or more columns. Some or all of the -name, units, ucd and description of the column(s), identified by - can be set by using some or all of the listed flags. -Typically, will simply be the name of a single column. - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - [-name ] [-units ] [-ucd ] [-desc ] - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("colmeta") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_delcols(table, *args): - '''\ -Delete the specified columns. The same column may harmlessly be -specified more than once. - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("delcols") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_every(table, *args): - '''\ -Include only every 'th row in the result, starting with the -first row. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("every") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_explodecols(table, *args): - '''\ -Takes a list of specified columns which represent N-element arrays -and replaces each one with N scalar columns. Each of the columns -specified by must have a fixed-length array type, -though not all the arrays need to have the same number of elements. - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("explodecols") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_explodeall(table, *args): - '''\ -Replaces any columns which is an N-element arrays with N scalar -columns. Only columns with fixed array sizes are affected. The -action can be restricted to only columns of a certain shape using -the flags. - -If the -ifndim flag is used, then only columns of dimensionality - will be exploded. may be 1, 2, .... - -If the -ifshape flag is used, then only columns with a specific -shape will be exploded; is a space- or comma-separated list -of dimension extents, with the most rapidly-varying first, e.g. '2 5 -' to explode all 2 x 5 element array columns. - -The filtered table is returned. - -args is a list with words as elements: - [-ifndim ] [-ifshape ] -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("explodeall") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_fixcolnames(table): - '''\ -Renames all columns and parameters in the input table so that they -have names which have convenient syntax for STILTS. For the most -part this means replacing spaces and other non-alphanumeric -characters with underscores. This is a convenience which lets you -use column names in algebraic expressions and other STILTS syntax. - -The filtered table is returned. -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("fixcolnames") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_head(table, *args): - '''\ -Include only the first rows of the table. If the table has -fewer than rows then it will be unchanged. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("head") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_keepcols(table, *args): - '''\ -Select the columns from the input table which will be included in -the output table. The output table will include only those columns -listed in , in that order. The same column may be listed -more than once, in which case it will appear in the output table -more than once. - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("keepcols") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_meta(table, *args): - '''\ -Provides information about the metadata for each column. This filter -turns the table sideways, so that each row of the output corresponds -to a column of the input. The columns of the output table contain -metadata items such as column name, units, UCD etc corresponding to -each column of the input table. - -By default the output table contains columns for the following -items: - - * Index: Position of column in table - * Name: Column name - * Class: Data type of objects in column - * Shape: Shape of array values - * ElSize: Size of each element in column (mostly useful for - strings) - * Units: Unit string - * Description: Description of data in the column - * UCD: Unified Content Descriptor - -as well as any table-specific column metadata items that the table -contains. - -However, the output may be customised by supplying one or more - headings. These may be selected from the above as well as the -following: - - * UCD_desc: Textual description of UCD - -as well as any table-specific metadata. It is not an error to -specify an item for which no metadata exists in any of the columns -(such entries will result in empty columns). - -Any table parameters of the input table are propagated to the output -one. - -The filtered table is returned. - -args is a list with words as elements: - [ ...] -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("meta") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_progress(table): - '''\ -Monitors progress by displaying the number of rows processed so far -on the terminal (standard error). This number is updated every -second or thereabouts; if all the processing is done in under a -second you may not see any output. If the total number of rows in -the table is known, an ASCII-art progress bar is updated, otherwise -just the number of rows seen so far is written. - -The filtered table is returned. -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("progress") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_random(table): - '''\ -Ensures that random access is available on this table. If the table -currently has random access, it has no effect. If only sequential -access is available, the table is cached so that downstream steps -will see the cached, hence random-access, copy. - -The filtered table is returned. -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("random") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_randomview(table): - '''\ -Ensures that steps downstream only use random access methods for -table access. If the table is sequential only, this will result in -an error. Only useful for debugging. - -The filtered table is returned. -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("randomview") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_repeat(table, *args): - '''\ -Repeats the rows of a table multiple times to produce a longer -table. The output table will have times as many rows as the -input table. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("repeat") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_replacecol(table, *args): - '''\ -Replaces the content of a column with the value of an algebraic -expression. The old values are discarded in favour of the result of -evaluating . You can specify the metadata for the new column -using the -name, -units, -ucd and -desc flags; for any of these -items which you do not specify, they will take the values from the -column being replaced. - -It is legal to reference the replaced column in the expression, so -for example "replacecol pixsize pixsize*2" just multiplies the -values in column pixsize by 2. - -Syntax for the and arguments is described in the -manual. - -The filtered table is returned. - -args is a list with words as elements: - [-name ] [-units ] [-ucd ] [-desc ] - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("replacecol") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_replaceval(table, *args): - '''\ -For each column specified in any instance of -is replaced by . The value string 'null' can be used for -either or to indicate a blank value (but see -also the badval filter). - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("replaceval") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_rowrange(table, *args): - '''\ -Includes only rows in a given range. The range can either be -supplied as " ", where row indices are inclusive, or " - +". In either case, the first row is numbered 1. - -Thus, to get the first hundred rows, use either "rowrange 1 100" or -"rowrange 1 +100" and to get the second hundred, either "rowrange -101 200" or "rowrange 101 +100" - -The filtered table is returned. - -args is a list with words as elements: - |+ -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("rowrange") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_select(table, *args): - '''\ -Include in the output table only rows for which the expression - evaluates to true. must be an expression which -evaluates to a boolean value (true/false). - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("select") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_seqview(table): - '''\ -Ensures that steps downstream see the table as sequential access. -Any attempts at random access will fail. Only useful for debugging. - -The filtered table is returned. -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("seqview") - sargs = [] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_setparam(table, *args): - '''\ -Sets a named parameter in the table to a given value. The parameter -named is set to the value . By default the type of the -parameter is determined automatically (if it looks like an integer -it's an integer etc) but this can be overridden using the -type -flag. The parameter description may be set using the -desc flag. - -The filtered table is returned. - -args is a list with words as elements: - [-type byte|short|int|long|float|double|boolean|string] - [-desc ] [-unit ] [-ucd ] - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("setparam") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_sort(table, *args): - '''\ -Sorts the table according to the value of one or more algebraic -expressions. The sort key expressions appear, as separate -(space-separated) words, in ; sorting is done on the first -expression first, but if that results in a tie then the second one -is used, and so on. - -Each expression must evaluate to a type that it makes sense to sort, -for instance numeric. If the -down flag is used, the sort order is -descending rather than ascending. - -Blank entries are by default considered to come at the end of the -collation sequence, but if the -nullsfirst flag is given then they -are considered to come at the start instead. - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - [-down] [-nullsfirst] -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("sort") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_sorthead(table, *args): - '''\ -Performs a sort on the table according to the value of one or more -algebraic expressions, retaining only rows at the head of -the resulting sorted table. The sort key expressions appear, as -separate (space-separated) words, in ; sorting is done on -the first expression first, but if that results in a tie then the -second one is used, and so on. Each expression must evaluate to a -type that it makes sense to sort, for instance numeric. - -If the -tail flag is used, then the last rows rather than -the first ones are retained. - -If the -down flag is used the sort order is descending rather than -ascending. - -Blank entries are by default considered to come at the end of the -collation sequence, but if the -nullsfirst flag is given then they -are considered to come at the start instead. - -This filter is functionally equivalent to using sort followed by -head, but it can be done in one pass and is usually cheaper on -memory and faster, as long as is significantly lower than -the size of the table. - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - [-tail] [-down] [-nullsfirst] -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("sorthead") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_stats(table, *args): - '''\ -Calculates statistics on the data in the table. This filter turns -the table sideways, so that each row of the output corresponds to a -column of the input. The columns of the output table contain -statistical items such as mean, standard deviation etc corresponding -to each column of the input table. - -By default the output table contains columns for the following -items: - - * Name: Column name - * Mean: Average - * StDev: Population Standard deviation - * Minimum: Numeric minimum - * Maximum: Numeric maximum - * NGood: Number of non-blank cells - -However, the output may be customised by supplying one or more - headings. These may be selected from the above as well as the -following: - - * NBad: Number of blank cells - * Variance: Population Variance - * SampStDev: Sample Standard Deviation - * SampVariance: Sample Variance - * Skew: Gamma 1 skewness measure - * Kurtosis: Gamma 2 peakedness measure - * Sum: Sum of values - * MinPos: Row index of numeric minimum - * MaxPos: Row index of numeric maximum - * Cardinality: Number of distinct values in column; values >100 - ignored - * Median: Middle value in sequence - * Quartile1: First quartile - * Quartile2: Second quartile - * Quartile3: Third quartile - -Additionally, the form "Q.nn" may be used to represent the quantile -corresponding to the proportion 0.nn, e.g.: - - * Q.25: First quartile - * Q.625: Fifth octile - -Any parameters of the input table are propagated to the output one. - -Note that quantile calculations (including median and quartiles) can -be expensive on memory. If you want to calculate quantiles for large -tables, it may be wise to reduce the number of columns to only those -you need the quantiles for earlier in the pipeline. No interpolation -is performed when calculating quantiles. - -The filtered table is returned. - -args is a list with words as elements: - [ ...] -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("stats") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_tablename(table, *args): - '''\ -Sets the table's name attribute to the given string. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("tablename") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_tail(table, *args): - '''\ -Include only the last rows of the table. If the table has -fewer than rows then it will be unchanged. - -The filtered table is returned. - -args is a list with words as elements: - -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("tail") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_transpose(table, *args): - '''\ -Transposes the input table so that columns become rows and vice -versa. The -namecol flag can be used to specify a column in the -input table which will provide the column names for the output -table. The first column of the output table will contain the column -names of the input table. - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - [-namecol ] -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("transpose") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def cmd_uniq(table, *args): - '''\ -Eliminates adjacent rows which have the same values. If used with no -arguments, then any row which has identical values to its -predecessor is removed. - -If the parameter is given then only the values in the -specified columns must be equal in order for the row to be removed. - -If the -count flag is given, then an additional column with the name -DupCount will be prepended to the table giving a count of the number -of duplicated input rows represented by each output row. A unique -row has a DupCount value of 1. - -Syntax for the argument is described in SUN/256. - -The filtered table is returned. - -args is a list with words as elements: - [-count] [] -''' - pfilt = _StepFactory.getInstance().getFilterFactory().createObject("uniq") - sargs = [str(a) for a in args] - argList = _ArrayList(sargs) - step = pfilt.createStep(argList.iterator()) - return import_star_table(step.wrap(table)) - -def mode_out(table, out='-', ofmt='(auto)'): - '''\ -Writes a new table. - -Parameters: - - out = - The location of the output table. This is usually a filename - to write to. If it is equal to the special value "-" (the - default) the output table will be written to standard output. - - [Default: -] - - ofmt = - Specifies the format in which the output table will be written - (one of the ones in SUN/256 - matching is case-insensitive and - you can use just the first few letters). If it has the special - value "(auto)" (the default), then the output filename will be - examined to try to guess what sort of file is required usually - by looking at the extension. If it's not obvious from the - filename what output format is intended, an error will result. - - [Default: (auto)] -''' - env = _JyEnvironment() - env.setValue('out', _map_env_value(out)) - env.setValue('ofmt', _map_env_value(ofmt)) - mode = _stilts.getModeFactory().createObject('out') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(table) - -def mode_meta(table): - '''\ -Prints the table metadata to standard output. The name and type etc -of each column is tabulated, and table parameters are also shown. - -See the meta filter for more flexible output of table metadata. -''' - env = _JyEnvironment() - mode = _stilts.getModeFactory().createObject('meta') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(table) - -def mode_stats(table): - '''\ -Calculates and displays univariate statistics for each of the -numeric columns in the table. The following entries are shown for -each column as appropriate: - - * mean - * population standard deviation - * minimum - * maximum - * number of non-null entries - -See the stats filter for more flexible statistical calculations. -''' - env = _JyEnvironment() - mode = _stilts.getModeFactory().createObject('stats') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(table) - -def mode_count(table): - '''\ -Counts the number of rows and columns and writes the result to -standard output. -''' - env = _JyEnvironment() - mode = _stilts.getModeFactory().createObject('count') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(table) - -def mode_cgi(table, ofmt='votable'): - '''\ -Writes a table to standard output in a way suitable for use as -output from a CGI (Common Gateway Interface) program. This is very -much like out mode but a short CGI header giving the MIME -Content-Type is prepended to the output - -Parameters: - - ofmt = - Specifies the format in which the output table will be written - (one of the ones in SUN/256 - matching is case-insensitive and - you can use just the first few letters). - - [Default: votable] -''' - env = _JyEnvironment() - env.setValue('ofmt', _map_env_value(ofmt)) - mode = _stilts.getModeFactory().createObject('cgi') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(table) - -def mode_discard(table): - '''\ -Reads all the data in the table in sequential mode and discards it. -May be useful in conjunction with the assert filter. -''' - env = _JyEnvironment() - mode = _stilts.getModeFactory().createObject('discard') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(table) - -def mode_topcat(table): - '''\ -Attempts to display the output table directly in TOPCAT. If a TOPCAT -instance is already running on the local host, an attempt will be -made to open the table in that. A variety of mechanisms are used to -attempt communication with an existing TOPCAT instance. In order: - * SAMP using existing hub (TOPCAT v3.4+ only, requires SAMP hub to - be running) - * PLASTIC using existing hub (requires PLASTIC hub to be running) - * SOAP (requires TOPCAT to run with somewhat deprecated -soap flag, - may be limitations on table size) - * SAMP using internal, short-lived hub (TOPCAT v3.4+ only, running - hub not required, but may be slow. It's better to start an - external hub, e.g. topcat -exthub) Failing that, an attempt -will be made to launch a new TOPCAT instance for display. This only -works if the TOPCAT classes are on the class path. - -If large tables are involved, starting TOPCAT with the -disk flag is -probably a good idea. -''' - env = _JyEnvironment() - mode = _stilts.getModeFactory().createObject('topcat') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(table) - -def mode_samp(table, format='votable fits', client=None): - '''\ -Sends the table to registered SAMP-aware applications subscribed to -a suitable table load MType. SAMP, the Simple Application Messaging -Protocol, is a tool interoperability protocol. A SAMP Hub must be -running for this to work. - -Parameters: - - format = - Gives one or more table format types for attempting the table - transmission over SAMP. If multiple values are supplied, they - should be separated by spaces. Each value supplied for this - parameter corresponds to a different MType which may be used - for the transmission. If a single value is used, a SAMP - broadcast will be used. If multiple values are used, each - registered client will be interrogated to see whether it - subscribes to the corresponding MTypes in order; the first one - to which it is subscribed will be used to send the table. The - standard options are - - * votable: use MType table.load.votable - * fits: use MType table.load.fits - - If any other string is used which corresponds to one of - STILTS's known table output formats, an attempt will be made - to use an ad-hoc MType of the form table.load.format. - - [Default: votable fits] - - client = - Identifies a registered SAMP client which is to receive the - table. Either the client ID or the (case-insensitive) - application name may be used. If a non-null value is given, - then the table will be sent to only the first client with the - given name or ID. If no value is supplied the table will be - sent to all suitably subscribed clients. -''' - env = _JyEnvironment() - env.setValue('format', _map_env_value(format)) - env.setValue('client', _map_env_value(client)) - mode = _stilts.getModeFactory().createObject('samp') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(table) - -def mode_plastic(table, transport=None, client=None): - '''\ -Broadcasts the table to any registered Plastic-aware applications. -PLASTIC, the PLatform for AStronomical Tool InterConnection, is a -tool interoperability protocol. A Plastic hub must be running in -order for this to work. - -Parameters: - - transport = string|file - Determines the method (PLASTIC message) used to perform the - PLASTIC communication. The choices are - - * string: VOTable serialized as a string and passed as a call - parameter (ivo://votech.org/votable/load). Not suitable - for very large files. - * file: VOTable written to a temporary file and the filename - passed as a call parameter ( - ivo://votech.org/votable/loadFromURL). The file ought to - be deleted once it has been loaded. Not suitable for - inter-machine communication. - - If no value is set (null) then a decision will be taken based - on the apparent size of the table. - - client = - Gives the name of a PLASTIC listener application which is to - receive the broadcast table. If a non-null value is given, - then only the first registered application which reports its - application name as that value will receive the message. If no - value is supplied, the broadcast will be to all listening - applications. -''' - env = _JyEnvironment() - env.setValue('transport', _map_env_value(transport)) - env.setValue('client', _map_env_value(client)) - mode = _stilts.getModeFactory().createObject('plastic') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(table) - -def mode_tosql(table, protocol, db, dbtable, host='localhost', write='create', user='mbt', password=None): - '''\ -Writes a new table to an SQL database. You need the appropriate JDBC -drivers and -Djdcb.drivers set as usual (see SUN/256). - -Parameters: - - protocol = - The driver-specific sub-protocol specifier for the JDBC - connection. For MySQL's Connector/J driver, this is mysql, and - for PostgreSQL's driver it is postgresql. For other drivers, - you may have to consult the driver documentation. - - host = - The host which is acting as a database server. - - [Default: localhost] - - db = - The name of the database on the server into which the new - table will be written. - - dbtable = - The name of the table which will be written to the database. - - write = create|dropcreate|append - Controls how the values are written to a table in the - database. The options are: - - * create: Creates a new table before writing. It is an error - if a table of the same name already exists. - * dropcreate: Creates a new database table before writing. If - a table of the same name already exists, it is dropped - first. - * append: Appends to an existing table. An error results if - the named table has the wrong structure (number or types - of columns) for the data being written. - - [Default: create] - - user = - User name for the SQL connection to the database. - - [Default: mbt] - - password = - Password for the SQL connection to the database. -''' - env = _JyEnvironment() - env.setValue('protocol', _map_env_value(protocol)) - env.setValue('db', _map_env_value(db)) - env.setValue('dbtable', _map_env_value(dbtable)) - env.setValue('host', _map_env_value(host)) - env.setValue('write', _map_env_value(write)) - env.setValue('user', _map_env_value(user)) - env.setValue('password', _map_env_value(password)) - mode = _stilts.getModeFactory().createObject('tosql') - consumer = mode.createConsumer(env) - _check_unused_args(env) - consumer.consume(table) - Index: /branches/eam_branches/ipp-20120405/ippToPsps/perl/pspsSchema2xml.pl =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/perl/pspsSchema2xml.pl (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/perl/pspsSchema2xml.pl (revision 33948) @@ -4,6 +4,6 @@ # # Script that searches a dir containing PSPS schema files, finds those that comtain the -# tables on interest then parses them into an XML format. Also generates C-header files -# containing enums that detail table column names and numbers. +# tables on interest then parses them into the VOTable format. Also generates trac-wiki +# formatted tables. # ####################################################################################### @@ -51,20 +51,4 @@ my $enumsHeader = ucfirst($type)."BatchEnums"; -open(OUT, ">".$enumsHeader.".h") or die("Error"); - -print OUT "#ifndef ".uc($enumsHeader)."_H\n"; -print OUT "#define ".uc($enumsHeader)."_H\n\n"; - -# tables file -my $tablesOutput = new IO::File(">tables.xml"); -my $tablesWriter = new XML::Writer(OUTPUT => $tablesOutput, DATA_MODE => 1, DATA_INDENT=>2); -$tablesWriter->xmlDecl('UTF-8'); -$tablesWriter->startTag('tableDescriptions', "type" => "$type"); - -# map file -my $mapOutput = new IO::File(">map.xml"); -my $mapWriter = new XML::Writer(OUTPUT => $mapOutput, DATA_MODE => 1, DATA_INDENT=>2); -$mapWriter->xmlDecl('UTF-8'); -$mapWriter->startTag('tabledata', "type" => "$type"); # VOTable file @@ -82,18 +66,8 @@ # finish up XML -$tablesWriter->endTag(); -$tablesWriter->end(); - -# finish up XML -$mapWriter->endTag(); -$mapWriter->end(); - -# finish up XML $votWriter->endTag(); # end RESOURCE tag $votWriter->endTag(); # end of TABLE tag $votWriter->end(); -print OUT "\n#endif"; -close OUT; close WIKI; @@ -208,4 +182,5 @@ parseTable("Object"); + #parseTable("ObjectCalColor"); } @@ -230,9 +205,4 @@ print WIKI "|| '''!".$tableNameOut."''' ||||||||||||||\n"; print WIKI "|| '''ODM attribute''' || '''units''' || '''Data type''' || '''Default''' || '''Description''' || '''IPP source''' || '''IPP variable/notes''' ||\n"; - print OUT "\ntypedef enum {\n"; - $tablesWriter->startTag('table', "name" => $tableNameOut); - $mapWriter->startTag('table', - "name" => $tableNameOut, - "ippfitsextension" => ""); $votWriter->startTag('TABLE', "name" => "$tableNameOut"); @@ -287,6 +257,4 @@ if (!$found) {print "Could not find table '$tableName'\n";} - $tablesWriter->endTag(); - $mapWriter->endTag(); $votWriter->startTag('DATA'); @@ -295,5 +263,4 @@ $votWriter->endTag(); # end DATA tag $votWriter->endTag(); # end TABLE tag - print OUT "} ".$tableNameOut.";\n"; close SCHEMA; @@ -384,13 +351,5 @@ $colNum++; - print OUT " ".uc($tableName)."_".uc($name)." = ".$colNum.",\n"; print WIKI "|| ".$name."||".$unit."||".$mstype."||".$default."||".$comment."|| || ||\n"; - - $tablesWriter->startTag('column', - "name" => $name, - "type" => $type, - "default" => $default, - "comment" => $fullComment); - $tablesWriter->endTag(); # get VOTable type @@ -413,8 +372,4 @@ $votWriter->endTag(); - $mapWriter->comment(" **MISSING** "); - -# $mapWriter->endTag(); - return $colNum; } Index: /branches/eam_branches/ipp-20120405/ippToPsps/src/Dvo.c =================================================================== --- /branches/eam_branches/ipp-20120405/ippToPsps/src/Dvo.c (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippToPsps/src/Dvo.c (revision 33948) @@ -188,6 +188,6 @@ this->dvoConfig = dvoConfigRead(&argc, argv); - for (int i=0; i= +END + +# How to read this data +FILE METADATA + PHU STR CHIP # The FITS file represents a single chip + EXTENSIONS STR CELL # The extensions represent cells + FPA.OBS STR FILENAME # A PHU keyword for unique identifier within the hierarchy level + CONTENT STR DETECTOR # How to determine content of FITS file + CONTENT.RULE STR {CHIP.ID} # How to derive the CONTENT when writing +END + +# What's in the FITS file? +CONTENTS METADATA + # CONTENT = chip name : type + CCID58-2-15a1 STR XY01:GPCChip # chip 00 + CCID58-2-18a1 STR XY02:GPCChip # chip 01 + CCID58-2-04a2 STR XY03:GPCChip # chip 02 + CCID58-1-01b1 STR XY04:GPCChip # chip 03 + CCID58-1-09b1 STR XY05:GPCChip # chip 04 + CCID58-2-09b1 STR XY06:GPCChip # chip 05 + CCID58-2-22a1 STR XY10:GPCChip # chip 06 + CCID58-2-01a1 STR XY11:GPCChip # chip 07 + CCID58-3-11a1 STR XY12:GPCChip # chip 08 + CCID58-3-06a1 STR XY13:GPCChip # chip 09 + CCID58-3-05b1 STR XY14:GPCChip # chip 10 + CCID58-3-08b1 STR XY15:GPCChip # chip 11 + CCID58-1-02b1 STR XY16:GPCChip # chip 12 + CCID58-2-07b2 STR XY17:GPCChip # chip 13 + CCID58-1-19a1 STR XY20:GPCChip # chip 14 + CCID58-1-04a1 STR XY21:GPCChip # chip 15 + CCID58-2-09a2 STR XY22:GPCChip # chip 16 + CCID58-1-02a1 STR XY23:GPCChip # chip 17 + CCID58-3-10b1 STR XY24:GPCChip # chip 18 + CCID58-3-05b2 STR XY25:GPCChip # chip 19 + CCID58-3-04b1 STR XY26:GPCChip # chip 20 + CCID58-1-18b1 STR XY27:GPCChip # chip 21 + CCID58-3-13a2 STR XY30:GPCChip # chip 22 + CCID58-3-05a1 STR XY31:GPCChip # chip 23 + CCID58-2-01a2 STR XY32:GPCChip # chip 24 + CCID58-1-02a2 STR XY33:GPCChip # chip 25 + CCID58-3-03b1 STR XY34:GPCChip # chip 26 + CCID58-1-01b2 STR XY35:GPCChip # chip 27 + CCID58-3-14b1 STR XY36:GPCChip # chip 28 + CCID58-2-23b2 STR XY37:GPCChip # chip 29 + CCID58-3-02b1 STR XY40:GPCChip # chip 30 + CCID58-1-03b2 STR XY41:GPCChip # chip 31 + CCID58-1-07b1 STR XY42:GPCChip # chip 32 + CCID58-2-11b2 STR XY43:GPCChip # chip 33 + CCID58-2-09a1 STR XY44:GPCChip # chip 34 + CCID58-3-04a2 STR XY45:GPCChip # chip 35 + CCID58-3-02a1 STR XY46:GPCChip # chip 36 + CCID58-3-14a1 STR XY47:GPCChip # chip 37 + CCID58-1-17b1 STR XY50:GPCChip # chip 38 + CCID58-2-09b2 STR XY51:GPCChip # chip 39 + CCID58-1-04b1 STR XY52:GPCChip # chip 40 + CCID58-1-05b2 STR XY53:GPCChip # chip 41 + CCID58-3-05a2 STR XY54:GPCChip # chip 42 + CCID58-3-03a2 STR XY55:GPCChip # chip 43 + CCID58-3-02a2 STR XY56:GPCChip # chip 44 + CCID58-3-11a2 STR XY57:GPCChip # chip 45 + CCID58-2-16b1 STR XY60:GPCChip # chip 46 + CCID58-1-03b1 STR XY61:GPCChip # chip 47 + CCID58-3-03b2 STR XY62:GPCChip # chip 48 + CCID58-1-02b2 STR XY63:GPCChip # chip 49 + CCID58-1-07a1 STR XY64:GPCChip # chip 50 + CCID58-2-04a1 STR XY65:GPCChip # chip 51 + CCID58-3-10a1 STR XY66:GPCChip # chip 52 + CCID58-1-09a1 STR XY67:GPCChip # chip 53 + CCID58-2-13b1 STR XY71:GPCChip # chip 54 + CCID58-1-14b1 STR XY72:GPCChip # chip 55 + CCID58-3-11b2 STR XY73:GPCChip # chip 56 + CCID58-1-05a2 STR XY74:GPCChip # chip 57 + CCID58-1-21a1 STR XY75:GPCChip # chip 58 + CCID58-2-05a1 STR XY76:GPCChip # chip 59 +END + +CHIPS METADATA + GPCChip METADATA + # Extension name, cellName:cellType + xy00 STR xy00:GPCCell + xy10 STR xy10:GPCCell + xy20 STR xy20:GPCCell + xy30 STR xy30:GPCCell + xy40 STR xy40:GPCCell + xy50 STR xy50:GPCCell + xy60 STR xy60:GPCCell + xy70 STR xy70:GPCCell + END +END + +# Specify the cell data +CELLS METADATA + GPCCell METADATA + CELL.TRIMSEC.SOURCE STR HEADER + CELL.TRIMSEC STR DATASEC + CELL.BIASSEC.SOURCE STR HEADER + CELL.BIASSEC STR BIASSEC + END +END + + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + FPA.FILTERID STR FILTERID + FPA.FILTER STR FILTERID + FPA.RA STR RA + FPA.DEC STR DEC + FPA.RADECSYS STR EQUINOX + FPA.OBSTYPE STR OBSTYPE + FPA.OBJECT STR OBJECT + FPA.OBS.MODE STR OBS_MODE + FPA.COMMENT STR CMMTOBS + FPA.AIRMASS STR AIRMASS + FPA.POSANGLE STR POSANGLE + FPA.ROTANGLE STR ROT + FPA.FOCUS STR M2Z + FPA.TIME STR SHUTOUTC + FPA.ALT STR ALT + FPA.AZ STR AZ + FPA.TEMP STR DETTEM + FPA.M1X STR M1X + FPA.M1Y STR M1Y + FPA.M1Z STR M1Z + FPA.M1TIP STR M1TIP + FPA.M1TILT STR M1TILT + FPA.M2X STR M2X + FPA.M2Y STR M2Y + FPA.M2Z STR M2Z + FPA.M2TIP STR M2TIP + FPA.M2TILT STR M2TILT + FPA.ENV.TEMP STR ENVTEM + FPA.ENV.HUMID STR ENVHUM + FPA.ENV.WIND STR ENVWIN + FPA.ENV.DIR STR ENVDIR + + FPA.TELTEMP.M1 STR TELTEMM1 # Primary mirror temps (C) + FPA.TELTEMP.M1CELL STR TELTEMMS # Primary mirror support temps (C) + FPA.TELTEMP.M2 STR TELTEMM2 # Secondary mirror temps (C + FPA.TELTEMP.SPIDER STR TELTEMSP # Spider temperatures (C) + FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C + FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP + CHIP.VIDEOCELL STR CELLMODE + + FPA.PON.TIME STR PONTIME + + CHIP.ID STR DETECTOR + CELL.XBIN STR CCDSUM + CELL.YBIN STR CCDSUM + CELL.X0 STR IMNPIX1 + CELL.Y0 STR IMNPIX2 + CELL.XPARITY STR ATM1_1 + CELL.YPARITY STR ATM2_2 + CHIP.TEMP STR CDETTEM + CHIP.TEMPERATURE STR DETTEM + FPA.EXPOSURE STR EXPREQ # Requested exposure time, presumably camera exposure time + CELL.EXPOSURE STR EXPTIME # Exposure time measured by shutter + CELL.DARKTIME STR DARKTIME # Exposure time for camera + CELL.TIME STR SHUTOUTC # Observation time + CELL.GAIN STR GAIN # Cell gain (e/ADU) + CELL.READNOISE STR RDNOISE # Cell read noise (ADU) + CELL.SATURATION STR MAXLIN # Saturation point (ADU) +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + FPA.TELESCOPE STR PS1 + FPA.INSTRUMENT STR GPC1 + FPA.LONGITUDE STR 10:25:01.417 # West Longitude in Hours 10.4170608521 + FPA.LATITUDE STR 20:42:25.558 # North Latitude in Degrees 20.7070999146 + FPA.ELEVATION F32 3048.0 # altitude in meters + FPA.DETECTOR STR GPC1 + FPA.TIMESYS STR UTC + CHIP.XSIZE S32 4846 + CHIP.YSIZE S32 4868 + CHIP.XPARITY S32 1 + CHIP.YPARITY S32 1 + CHIP.X0.DEPEND STR CHIP.NAME + CHIP.X0 METADATA + XY01 S32 4971 + XY02 S32 4971 + XY03 S32 4971 + XY04 S32 4971 + XY05 S32 4971 + XY06 S32 4971 + XY10 S32 9942 + XY11 S32 9942 + XY12 S32 9942 + XY13 S32 9942 + XY14 S32 9942 + XY15 S32 9942 + XY16 S32 9942 + XY17 S32 9942 + XY20 S32 14913 + XY21 S32 14913 + XY22 S32 14913 + XY23 S32 14913 + XY24 S32 14913 + XY25 S32 14913 + XY26 S32 14913 + XY27 S32 14913 + XY30 S32 19884 + XY31 S32 19884 + XY32 S32 19884 + XY33 S32 19884 # should be 19872 + XY34 S32 19884 + XY35 S32 19884 + XY36 S32 19884 + XY37 S32 19884 + XY40 S32 20041 + XY41 S32 20041 + XY42 S32 20041 + XY43 S32 20041 # should be 20029 + XY44 S32 20041 + XY45 S32 20041 + XY46 S32 20041 + XY47 S32 20041 + XY50 S32 25012 + XY51 S32 25012 + XY52 S32 25012 + XY53 S32 25012 + XY54 S32 25012 + XY55 S32 25012 + XY56 S32 25012 + XY57 S32 25012 + XY60 S32 29983 + XY61 S32 29983 + XY62 S32 29983 + XY63 S32 29983 + XY64 S32 29983 + XY65 S32 29983 + XY66 S32 29983 + XY67 S32 29983 + XY71 S32 34954 + XY72 S32 34954 + XY73 S32 34954 + XY74 S32 34954 + XY75 S32 34954 + XY76 S32 34954 + END + CHIP.Y0.DEPEND STR CHIP.NAME + CHIP.Y0 METADATA + XY01 S32 10286 + XY02 S32 15429 + XY03 S32 20572 + XY04 S32 25715 + XY05 S32 30858 + XY06 S32 36001 + XY10 S32 5143 + XY11 S32 10286 + XY12 S32 15429 + XY13 S32 20572 + XY14 S32 25715 + XY15 S32 30858 + XY16 S32 36001 + XY17 S32 41144 + XY20 S32 5143 + XY21 S32 10286 + XY22 S32 15429 + XY23 S32 20572 + XY24 S32 25715 + XY25 S32 30858 + XY26 S32 36001 + XY27 S32 41144 + XY30 S32 5143 + XY31 S32 10286 + XY32 S32 15429 + XY33 S32 20572 + XY34 S32 25715 # should be 25755 + XY35 S32 30858 + XY36 S32 36001 + XY37 S32 41144 + XY40 S32 306 + XY41 S32 5449 + XY42 S32 10592 + XY43 S32 15735 # should be 15707 + XY44 S32 20878 # should be 20890 + XY45 S32 26021 + XY46 S32 31164 + XY47 S32 36307 + XY50 S32 306 + XY51 S32 5449 + XY52 S32 10592 + XY53 S32 15735 + XY54 S32 20878 + XY55 S32 26021 + XY56 S32 31164 + XY57 S32 36307 + XY60 S32 306 + XY61 S32 5449 + XY62 S32 10592 + XY63 S32 15735 + XY64 S32 20878 + XY65 S32 26021 + XY66 S32 31164 + XY67 S32 36307 + XY71 S32 5449 + XY72 S32 10592 + XY73 S32 15735 + XY74 S32 20878 + XY75 S32 26021 + XY76 S32 31164 + END + CHIP.XPARITY.DEPEND STR CHIP.NAME + CHIP.XPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CHIP.YPARITY.DEPEND STR CHIP.NAME + CHIP.YPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CELL.READDIR S32 1 + CELL.BAD S32 0 + CELL.XSIZE S32 590 + CELL.YSIZE S32 598 + CELL.TIMESYS STR UTC +END + +# How to translation PS concepts into database lookups +DATABASE METADATA + # this rule is fragile : does not match the camera +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +END + + +# Where there might be some ambiguity, specify the format +FORMATS METADATA + FPA.RA STR DEGREES + FPA.DEC STR DEGREES + FPA.LONGITUDE STR HOURS + FPA.LATITUDE STR DEGREES + FPA.TIME STR YEAR.FIRST + CELL.TIME STR YEAR.FIRST + CELL.BINNING STR TOGETHER + # CELL.X0 STR FORTRAN + # CELL.Y0 STR FORTRAN + CELL.READNOISE STR ADU +END + +# Pattern correction information: + +# PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? +PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction + XY01 BOOL FALSE +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 + XY02 BOOL FALSE + XY03 BOOL FALSE + XY04 BOOL FALSE + XY05 BOOL FALSE + XY06 BOOL FALSE + XY10 BOOL FALSE + XY11 STR 0011100100000000000000000000000000000000000000000000000000000000 + XY12 BOOL FALSE + XY13 BOOL FALSE + XY14 STR 0000000000000000000100000000000000000000000000000000000000000000 # xy23 + XY15 STR 1111111100000000000000000000000000000000000000000000000000000000 # cols: 0 + XY16 BOOL FALSE + XY17 BOOL FALSE + XY20 BOOL FALSE + XY21 BOOL FALSE + XY22 BOOL FALSE + XY23 BOOL FALSE + XY24 BOOL FALSE + XY25 BOOL FALSE + XY26 BOOL FALSE + XY27 STR 1111111111111111111111111111111100000000000000000000000011111111 # cols: 0,1,2,3,7 + XY30 BOOL FALSE + XY31 STR 0000000000000000000000000000000000000000000000000000000011111111 # cols: 7 + XY32 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 + XY33 BOOL FALSE + XY34 BOOL FALSE + XY35 BOOL FALSE + XY36 BOOL FALSE + XY37 BOOL FALSE + XY40 BOOL FALSE + XY41 BOOL FALSE + XY42 BOOL FALSE + XY43 BOOL FALSE + XY44 BOOL FALSE + XY45 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 + XY46 BOOL FALSE + XY47 STR 1111111100000000000000001111111100000000111111110000000011111111 # cols: 0,3,5,7 + XY50 BOOL FALSE + XY51 BOOL FALSE + XY52 BOOL FALSE + XY53 BOOL FALSE + XY54 BOOL FALSE + XY55 BOOL FALSE + XY56 BOOL FALSE + XY57 STR 1111111111111111111111110000000000000000000000001111111111111111 # cols: 0,1,2,6,7 + XY60 STR 0000000000000000000000000000000000000000000001000000000000000000 # cell xy55 + XY61 BOOL FALSE + XY62 BOOL FALSE + XY63 BOOL FALSE + XY64 BOOL FALSE + XY65 BOOL FALSE +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 + XY66 BOOL FALSE + XY67 BOOL FALSE + XY71 BOOL FALSE + XY72 BOOL FALSE + XY73 BOOL FALSE + XY74 STR 0000000000000000111111110000000000000000000000000000000011111111 # cols: 2,7 + XY75 BOOL FALSE + XY76 BOOL FALSE +END + +PATTERN.CONTINUITY BOOL TRUE + PATTERN.CELL BOOL FALSE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# # XY06 STR 0101000000010000000110000100000000000000000000000000000000000000 +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000000000000000000000 +# XY15 STR 1111111110000000100000000000000000000000000000000000000000000000 +# # XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# # XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY24 BOOL FALSE +# # XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY25 BOOL FALSE +# # XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY26 BOOL FALSE +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# # XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# # XY36 STR 0001000000000000000000000000000000010000000000000000000000000000 +# XY36 BOOL FALSE +# # XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 ? +# XY37 BOOL FALSE +# # XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# # XY42 BOOL FALSE +# XY42 STR 0000000000000000000100000001000000000000000000000000000000000000 +# # XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY43 STR 0000000000000000000000000000000000000000000010000000000000000000 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# # XY62 STR 0000000000000000000000000000000000000000000000000000000000000010 +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# # XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY72 BOOL FALSE +# # XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY73 BOOL FALSE +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END Index: /branches/eam_branches/ipp-20120405/ippconfig/gpc1/ghost.model.mdc =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/gpc1/ghost.model.mdc (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/gpc1/ghost.model.mdc (revision 33948) @@ -1,15 +1,83 @@ +# Original center models +# GHOST.CENTER.X METADATA +# NORDER_X S32 3 +# NORDER_Y S32 3 +# VAL_X00_Y00 F64 -4.421024e+01 +# VAL_X01_Y00 F64 1.216270e-02 +# VAL_X02_Y00 F64 -9.721643e-08 +# VAL_X03_Y00 F64 9.976554e-11 +# VAL_X00_Y01 F64 -1.762476e-03 +# VAL_X01_Y01 F64 1.247212e-07 +# VAL_X02_Y01 F64 3.629557e-11 +# VAL_X00_Y02 F64 -1.040174e-07 +# VAL_X01_Y02 F64 1.074674e-10 +# VAL_X00_Y03 F64 1.564112e-11 +# NELEMENTS S32 10 +# END + +# GHOST.CENTER.Y METADATA +# NORDER_X S32 3 +# NORDER_Y S32 3 +# VAL_X00_Y00 F64 -2.189470e+00 +# VAL_X01_Y00 F64 -4.186514e-03 +# VAL_X02_Y00 F64 1.131554e-07 +# VAL_X03_Y00 F64 1.415192e-11 +# VAL_X00_Y01 F64 1.569104e-02 +# VAL_X01_Y01 F64 -1.782801e-07 +# VAL_X02_Y01 F64 8.179602e-11 +# VAL_X00_Y02 F64 2.577055e-07 +# VAL_X01_Y02 F64 8.879423e-11 +# VAL_X00_Y03 F64 5.767429e-11 +# NELEMENTS S32 10 +# END + +# Updated models of 2012-05-07 +# Corrected(?) parity +# GHOST.CENTER.X METADATA +# NORDER_X S32 3 +# NORDER_Y S32 3 +# VAL_X00_Y00 F64 3.314569e+01 +# VAL_X01_Y00 F64 1.110664e-02 +# VAL_X02_Y00 F64 -1.904158e-07 +# VAL_X03_Y00 F64 8.470224e-11 +# VAL_X00_Y01 F64 -1.616877e-03 +# VAL_X01_Y01 F64 1.646508e-07 +# VAL_X02_Y01 F64 5.623381e-11 +# VAL_X00_Y02 F64 -2.342888e-07 +# VAL_X01_Y02 F64 1.023761e-10 +# VAL_X00_Y03 F64 2.836980e-11 +# NELEMENTS S32 10 +# END + +# GHOST.CENTER.Y METADATA +# NORDER_X S32 3 +# NORDER_Y S32 3 +# VAL_X00_Y00 F64 -2.860068e+01 +# VAL_X01_Y00 F64 -8.790076e-03 +# VAL_X02_Y00 F64 2.456534e-07 +# VAL_X03_Y00 F64 2.947749e-11 +# VAL_X00_Y01 F64 1.948855e-02 +# VAL_X01_Y01 F64 -2.639927e-07 +# VAL_X02_Y01 F64 4.951474e-11 +# VAL_X00_Y02 F64 4.341716e-07 +# VAL_X01_Y02 F64 1.643046e-10 +# VAL_X00_Y03 F64 -1.737983e-12 +# NELEMENTS S32 10 +# END + +# Switched order GHOST.CENTER.X METADATA NORDER_X S32 3 NORDER_Y S32 3 - VAL_X00_Y00 F64 -4.421024e+01 - VAL_X01_Y00 F64 1.216270e-02 - VAL_X02_Y00 F64 -9.721643e-08 - VAL_X03_Y00 F64 9.976554e-11 - VAL_X00_Y01 F64 -1.762476e-03 - VAL_X01_Y01 F64 1.247212e-07 - VAL_X02_Y01 F64 3.629557e-11 - VAL_X00_Y02 F64 -1.040174e-07 - VAL_X01_Y02 F64 1.074674e-10 - VAL_X00_Y03 F64 1.564112e-11 + VAL_X00_Y00 F64 -1.215661e+02 + VAL_X01_Y00 F64 1.321875e-02 + VAL_X02_Y00 F64 -4.017026e-09 + VAL_X03_Y00 F64 1.148288e-10 + VAL_X00_Y01 F64 -1.908074e-03 + VAL_X01_Y01 F64 8.479150e-08 + VAL_X02_Y01 F64 1.635732e-11 + VAL_X00_Y02 F64 2.625405e-08 + VAL_X01_Y02 F64 1.125586e-10 + VAL_X00_Y03 F64 2.912432e-12 NELEMENTS S32 10 END @@ -18,17 +86,54 @@ NORDER_X S32 3 NORDER_Y S32 3 - VAL_X00_Y00 F64 -2.189470e+00 - VAL_X01_Y00 F64 -4.186514e-03 - VAL_X02_Y00 F64 1.131554e-07 - VAL_X03_Y00 F64 1.415192e-11 - VAL_X00_Y01 F64 1.569104e-02 - VAL_X01_Y01 F64 -1.782801e-07 - VAL_X02_Y01 F64 8.179602e-11 - VAL_X00_Y02 F64 2.577055e-07 - VAL_X01_Y02 F64 8.879423e-11 - VAL_X00_Y03 F64 5.767429e-11 + VAL_X00_Y00 F64 2.422174e+01 + VAL_X01_Y00 F64 4.170486e-04 + VAL_X02_Y00 F64 -1.934260e-08 + VAL_X03_Y00 F64 -1.173657e-12 + VAL_X00_Y01 F64 1.189352e-02 + VAL_X01_Y01 F64 -9.256748e-08 + VAL_X02_Y01 F64 1.140772e-10 + VAL_X00_Y02 F64 8.123932e-08 + VAL_X01_Y02 F64 1.328378e-11 + VAL_X00_Y03 F64 1.170865e-10 NELEMENTS S32 10 END + + +# These are semi-major/minor axes +# These are my (CZW) quadratic solutions +# GHOST.INNER.MAJOR METADATA +# NORDER_X S32 2 +# VAL_X00 F64 2.2622e+01 +# VAL_X01 F64 1.3822e-02 +# VAL_X02 F64 -3.7005e-07 +# NELEMENTS S32 3 +# END + +# GHOST.INNER.MINOR METADATA +# NORDER_X S32 2 +# VAL_X00 F64 4.9626e+01 +# VAL_X01 F64 -2.4257e-04 +# VAL_X02 F64 -1.2007e-07 +# NELEMENTS S32 3 +# END + +# GHOST.OUTER.MAJOR METADATA +# NORDER_X S32 2 +# VAL_X00 F64 9.6490e+01 +# VAL_X01 F64 9.4250e-03 +# VAL_X02 F64 6.5680e-07 +# NELEMENTS S32 3 +# END + +# GHOST.OUTER.MINOR METADATA +# NORDER_X S32 2 +# VAL_X00 F64 9.5450e+01 +# VAL_X01 F64 2.8775e-03 +# VAL_X02 F64 -3.1409e-07 +# NELEMENTS S32 3 +# END + +# These are the original linear solutions GHOST.INNER.MAJOR METADATA NORDER_X S32 1 Index: /branches/eam_branches/ipp-20120405/ippconfig/gpc1/ppImage.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/gpc1/ppImage.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/gpc1/ppImage.config (revision 33948) @@ -25,5 +25,8 @@ NOISEMAP BOOL FALSE # Apply read noise map - +# Final detrending options +HAS.VIDEO BOOL FALSE # Treat this OTA as if it has video data +USE.VIDEO.DARK BOOL FALSE # Use a video dark if we have video data? +USE.VIDEO.MASK BOOL TRUE # Use a video mask if we have video data? Index: /branches/eam_branches/ipp-20120405/ippconfig/gpc1/psastro.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/gpc1/psastro.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/gpc1/psastro.config (revision 33948) @@ -113,5 +113,7 @@ # *** make sure the choice of CATDIR (dvo database) contains the photcodes desired below -PSASTRO.CATDIR STR SYNTH.GRIZY +# PSASTRO.CATDIR STR SYNTH.GRIZY +PSASTRO.CATDIR STR PS1.REF.20120503 + DVO.GETSTAR.PHOTCODE STR i DVO.GETSTAR.MAX.RHO F32 3000.0 @@ -130,5 +132,5 @@ ZEROPT F32 24.563 PHOTCODE STR g - GHOST_MAX_MAG F32 -20.0 + GHOST_MAX_MAG F32 -16.5 END PHOTCODE.DATA METADATA @@ -346,9 +348,10 @@ PSASTRO.MATCH.FIT.NITER S32 5 - PSASTRO.CATDIR STR STS.PP5.REFCAT + # CATDIR with PS1 astrometry from PP5 pointing. Only works for that field + # PSASTRO.CATDIR STR STS.PP5.REFCAT END DEFAULT_RECIPE METADATA - PSASTRO.CATDIR STR PS1.REF.20120325 + PSASTRO.CATDIR STR PS1.REF.20120503 ZERO.POINT.USE.MEAN BOOL TRUE END @@ -356,4 +359,14 @@ LAP_ASTRO METADATA # PSASTRO.CATDIR STR 3PI.20110505.REFCAT - PSASTRO.CATDIR STR SYNTH.GRIZY -END +# PSASTRO.CATDIR STR SYNTH.GRIZY + PSASTRO.CATDIR STR PS1.REF.20120503 + ZERO.POINT.USE.MEAN BOOL TRUE +END + +SYNTH_CAT METADATA + PSASTRO.CATDIR STR SYNTH.GRIZY +END + +TEST_REFCAT METADATA + PSASTRO.CATDIR STR /data/ipp064.0/ipp/ippRefs/catdir.refcat.20120524.v0 +END Index: /branches/eam_branches/ipp-20120405/ippconfig/isp/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/isp/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/isp/camera.config (revision 33948) @@ -62,4 +62,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? CMF.DETEFF STR {CHIP.NAME}.deteff Index: /branches/eam_branches/ipp-20120405/ippconfig/lulin/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/lulin/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/lulin/camera.config (revision 33948) @@ -63,4 +63,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? CMF.DETEFF STR {CHIP.NAME}.deteff Index: /branches/eam_branches/ipp-20120405/ippconfig/megacam/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/megacam/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/megacam/camera.config (revision 33948) @@ -137,4 +137,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? CMF.DETEFF STR {CHIP.NAME}.deteff Index: /branches/eam_branches/ipp-20120405/ippconfig/mosaic2/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/mosaic2/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/mosaic2/camera.config (revision 33948) @@ -90,4 +90,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? CMF.DETEFF STR {CHIP.NAME}.deteff Index: /branches/eam_branches/ipp-20120405/ippconfig/recipes/filerules-mef.mdc =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/recipes/filerules-mef.mdc (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/recipes/filerules-mef.mdc (revision 33948) @@ -59,6 +59,8 @@ PPIMAGE.INPUT.SRC INPUT @FILES CHIP CMF PPIMAGE.MASK INPUT @DETDB CHIP MASK +PPIMAGE.VIDEOMASK INPUT @DETDB CHIP MASK PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.VIDEODARK INPUT @DETDB CHIP DARK PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE Index: /branches/eam_branches/ipp-20120405/ippconfig/recipes/filerules-simple.mdc =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/recipes/filerules-simple.mdc (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/recipes/filerules-simple.mdc (revision 33948) @@ -23,6 +23,8 @@ PPIMAGE.INPUT.SRC INPUT @FILES READOUT CMF PPIMAGE.MASK INPUT @DETDB CHIP MASK +PPIMAGE.VIDEOMASK INPUT @DETDB CHIP MASK PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.VIDEODARK INPUT @DETDB CHIP DARK PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE Index: /branches/eam_branches/ipp-20120405/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/recipes/filerules-split.mdc (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/recipes/filerules-split.mdc (revision 33948) @@ -36,4 +36,5 @@ PPIMAGE.INPUT INPUT @FILES CHIP IMAGE PPIMAGE.INPUT.MASK INPUT @FILES CHIP MASK +PPIMAGE.VIDEOMASK INPUT @DETDB CHIP MASK PPIMAGE.INPUT.VARIANCE INPUT @FILES CHIP VARIANCE PPIMAGE.INPUT.PSF INPUT @FILES CHIP PSF @@ -43,4 +44,5 @@ PPIMAGE.NOISEMAP INPUT @DETDB CHIP IMAGE PPIMAGE.DARK INPUT @DETDB CHIP DARK +PPIMAGE.VIDEODARK INPUT @DETDB CHIP DARK PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE PPIMAGE.FRINGE INPUT @DETDB CHIP FRINGE Index: /branches/eam_branches/ipp-20120405/ippconfig/recipes/ppImage.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/recipes/ppImage.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/recipes/ppImage.config (revision 33948) @@ -153,4 +153,9 @@ GAIN.OVERRIDE BOOL FALSE # Override a non-finite gain? + +# Final detrending options +HAS.VIDEO BOOL FALSE # Treat this OTA as if it has video data +USE.VIDEO.DARK BOOL FALSE # Use a video dark if we have video data? +USE.VIDEO.MASK BOOL FALSE # Use a video mask if we have video data? DETREND.CONSTRAINTS METADATA Index: /branches/eam_branches/ipp-20120405/ippconfig/recipes/psastro.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/recipes/psastro.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/recipes/psastro.config (revision 33948) @@ -254,6 +254,11 @@ END -LAP_ASTRO METADATA - +LAP_ASTRO METADATA +END + +SYNTH_CAT METADATA +END + +TEST_REFCAT METADATA END Index: /branches/eam_branches/ipp-20120405/ippconfig/recipes/psphot.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/recipes/psphot.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/recipes/psphot.config (revision 33948) @@ -38,4 +38,5 @@ SKY_CLIP_SIGMA F32 2.0 # statistic used to measure background SKY_SIG F32 1.0 # optional sky error for +SKY_SLOPE_MIN F32 3.0 # exit radial profile loop when abs(slope) is less than this # allowed values for SKY_STAT: @@ -57,4 +58,6 @@ PEAKS_POS2_NSIGMA_LIMIT F32 25.0 # peak signficance threshold for POS2 sources. (ppSub) # input pixels contribute less than this fraction of the flux +PEAKS_NMAX_TOTAL S32 0 # maximum allowed number of peaks 0 == unlimited + # parameters which adjust the footprint analysis USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging @@ -65,4 +68,5 @@ FOOTPRINT_CULL_NSIGMA_MIN F32 1 # Minimum height of colls in units of skyStdev FOOTPRINT_CULL_NSIGMA_PAD F32 0.01 # Fractional Padding for stdev +FOOTPRINT_USE_UNSUBTRACTED BOOL TRUE # find footprints without sources subtracted # parameter for the simple deblending @@ -186,4 +190,9 @@ KRON_ITERATIONS S32 2 +KRON_APPLY_WEIGHT BOOL FALSE +KRON_APPLY_WINDOW BOOL FALSE +KRON_SMOOTH BOOL FALSE +KRON_SMOOTH_SIGMA F32 1.7 +KRON_SMOOTH_NSIGMA S32 2 # Extended source fit parameters @@ -368,4 +377,5 @@ PSF_MODEL STR PS_MODEL_GAUSS EXT_MODEL STR PS_MODEL_QGAUSS + PEAKS_NMAX_TOTAL S32 50000 # maximum allowed number of peaks - To avoid memory explosion END Index: /branches/eam_branches/ipp-20120405/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/recipes/reductionClasses.mdc (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/recipes/reductionClasses.mdc (revision 33948) @@ -225,4 +225,29 @@ BACKGROUND_PSWARP STR BACKGROUND END + +DEFAULT_SYNTHCAT METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR SYNTH_CAT + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT + STATICSKY_CALIBRATION STR STATICSKY_CAL +END + # Sweetspot reduction class @@ -933,4 +958,28 @@ END +CNP_DATASET METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR CNP_DATASET + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT + STATICSKY_CALIBRATION STR STATICSKY_CAL +END + # random user tests (may not be stable or consistent) TEST1 METADATA @@ -952,2 +1001,26 @@ END +TEST_REFCAT METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR TEST_REFCAT + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT + STATICSKY_CALIBRATION STR STATICSKY_CAL +END + Index: /branches/eam_branches/ipp-20120405/ippconfig/sdssmosaic/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/sdssmosaic/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/sdssmosaic/camera.config (revision 33948) @@ -93,4 +93,5 @@ CMF.XSRC STR cmf.xsrc CMF.XFIT STR cmf.xfit + CMF.XRAD STR cmf.xrad # use .PSF and .EXT? CMF.DETEFF STR cmf.deteff Index: /branches/eam_branches/ipp-20120405/ippconfig/simmosaic/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/simmosaic/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/simmosaic/camera.config (revision 33948) @@ -72,4 +72,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? CMF.DETEFF STR {CHIP.NAME}.deteff Index: /branches/eam_branches/ipp-20120405/ippconfig/simple/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/simple/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/simple/camera.config (revision 33948) @@ -58,4 +58,5 @@ CMF.DATA STR psf CMF.XSRC STR xsrc + CMF.XRAD STR xrad CMF.XFIT STR xfit CMF.DETEFF STR deteff Index: /branches/eam_branches/ipp-20120405/ippconfig/ssp/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/ssp/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/ssp/camera.config (revision 33948) @@ -59,4 +59,5 @@ CMF.XSRC STR xsrc CMF.XFIT STR xfit + CMF.XRAD STR xrad CMF.DETEFF STR deteff Index: /branches/eam_branches/ipp-20120405/ippconfig/ssp/format.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/ssp/format.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/ssp/format.config (revision 33948) @@ -5,4 +5,5 @@ SIMPLE BOOL TRUE ### NAXIS S32 2 + DETECTOR STR rspiusb0 END Index: /branches/eam_branches/ipp-20120405/ippconfig/system.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/system.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/system.config (revision 33948) @@ -10,4 +10,5 @@ SDSS STR sdss/camera.config # Sloan Digital Sky Survey GPC1 STR gpc1/camera.config # Pan-STARRS GPC1 + GPC1-TDI STR gpc1-tdi/camera.config # Pan-STARRS GPC1 with TDI guiding ESOWFI STR esowfi/camera.config # ESO Wide-Field Imager LBCRED STR lbc_red/camera.config # Large Binocular Camera Red Index: /branches/eam_branches/ipp-20120405/ippconfig/tek/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/tek/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/tek/camera.config (revision 33948) @@ -60,4 +60,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? CMF.DETEFF STR {CHIP.NAME}.deteff Index: /branches/eam_branches/ipp-20120405/ippconfig/uh8k/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/uh8k/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/uh8k/camera.config (revision 33948) @@ -70,4 +70,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? CMF.DETEFF STR {CHIP.NAME}.deteff Index: /branches/eam_branches/ipp-20120405/ippconfig/vysos20/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/vysos20/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/vysos20/camera.config (revision 33948) @@ -63,4 +63,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? CMF.DETEFF STR {CHIP.NAME}.deteff Index: /branches/eam_branches/ipp-20120405/ippconfig/vysos5/camera.config =================================================================== --- /branches/eam_branches/ipp-20120405/ippconfig/vysos5/camera.config (revision 33947) +++ /branches/eam_branches/ipp-20120405/ippconfig/vysos5/camera.config (revision 33948) @@ -63,4 +63,5 @@ CMF.XSRC STR {CHIP.NAME}.xsrc # use .PSF and .EXT? CMF.XFIT STR {CHIP.NAME}.xfit # use .PSF and .EXT? + CMF.XRAD STR {CHIP.NAME}.xrad # use .PSF and .EXT? CMF.DETEFF STR {CHIP.NAME}.deteff Index: /branches/eam_branches/ipp-20120405/pstamp/scripts/pstamp_cleanup.pl =================================================================== --- /branches/eam_branches/ipp-20120405/pstamp/scripts/pstamp_cleanup.pl (revision 33947) +++ /branches/eam_branches/ipp-20120405/pstamp/scripts/pstamp_cleanup.pl (revision 33948) @@ -42,8 +42,7 @@ 'name=s' => \$name, 'outdir=s' => \$outdir, - 'reqType=s' => \$reqType, + 'reqType=s' => \$reqType, 'product=s' => \$product, 'uri=s' => \$uri, - 'outdir=s' => \$outdir, 'redirect-output' => \$redirect_output, 'verbose' => \$verbose, @@ -82,5 +81,5 @@ my_die("Cleanup not yet supported for reqType: $reqType", $req_id, $PS_EXIT_UNKNOWN_ERROR) - if ($reqType ne "pstamp") and ($reqType ne "NULL"); + if ($reqType ne "pstamp") and ($reqType ne "NULL") and ($reqType ne "dquery"); my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files