IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 29, 2012, 5:07:20 PM (14 years ago)
Author:
eugene
Message:

merging changes from trunk

Location:
branches/eam_branches/ipp-20120405/ippToPsps
Files:
6 deleted
28 edited
2 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20120405/ippToPsps/config/ippToPspsDbSchema.sql

    r33593 r33948  
    4545  `purged` tinyint(4) NOT NULL default '0',
    4646  PRIMARY KEY  (`batch_id`)
    47 ) ENGINE=InnoDB AUTO_INCREMENT=326479 DEFAULT CHARSET=latin1;
     47) ENGINE=InnoDB AUTO_INCREMENT=328286 DEFAULT CHARSET=latin1;
    4848SET character_set_client = @saved_cs_client;
    4949
    … …  
    6565  KEY `fk_config` (`config`),
    6666  CONSTRAINT `fk_config` FOREIGN KEY (`config`) REFERENCES `config` (`name`) ON DELETE CASCADE
    67 ) ENGINE=InnoDB AUTO_INCREMENT=25457 DEFAULT CHARSET=latin1;
     67) ENGINE=InnoDB AUTO_INCREMENT=44588 DEFAULT CHARSET=latin1;
    6868SET character_set_client = @saved_cs_client;
    6969
    … …  
    8787  PRIMARY KEY  (`id`),
    8888  UNIQUE KEY `unique_key` (`host`,`pid`)
    89 ) ENGINE=InnoDB AUTO_INCREMENT=961 DEFAULT CHARSET=latin1;
     89) ENGINE=InnoDB AUTO_INCREMENT=1227 DEFAULT CHARSET=latin1;
    9090SET character_set_client = @saved_cs_client;
    9191
    … …  
    110110  `max_dec` double default '90',
    111111  `box_size` double default '4',
    112   `base_path` varchar(1000) default NULL,
     112  `base_path` varchar(1000) default '/data/ipp005.0/ipptopsps/',
    113113  `data_release` smallint(6) default '0',
    114114  `delete_local` tinyint(1) default '0',
    … …  
    120120  `queue_P2` tinyint(1) default '1',
    121121  `queue_ST` tinyint(1) default '0',
     122  `queue_OB` tinyint(1) default '0',
     123  `active` tinyint(1) default '1',
    122124  UNIQUE KEY `name` (`name`)
    123125) ENGINE=InnoDB DEFAULT CHARSET=latin1;
    … …  
    141143
    142144--
     145-- Table structure for table `object`
     146--
     147
     148DROP TABLE IF EXISTS `object`;
     149SET @saved_cs_client     = @@character_set_client;
     150SET character_set_client = utf8;
     151CREATE TABLE `object` (
     152  `batch_id` bigint(20) unsigned NOT NULL,
     153  `region` varchar(50) default NULL,
     154  KEY `batch_id` (`batch_id`),
     155  CONSTRAINT `object_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`)
     156) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     157SET character_set_client = @saved_cs_client;
     158
     159--
    143160-- Table structure for table `pending`
    144161--
    … …  
    151168  `batch_type` varchar(10) default NULL,
    152169  `stage_id` bigint(20) default NULL,
    153   `ra_bore` float default NULL,
    154   `dec_bore` float default NULL,
    155170  UNIQUE KEY `a_key` (`box_id`,`stage_id`,`batch_type`),
     171  UNIQUE KEY `stage_id_key` (`stage_id`),
    156172  CONSTRAINT `box_key` FOREIGN KEY (`box_id`) REFERENCES `box` (`id`) ON DELETE CASCADE
    157173) ENGINE=InnoDB DEFAULT CHARSET=latin1;
    … …  
    199215/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
    200216
    201 -- Dump completed on 2012-03-22  7:59:34
     217-- Dump completed on 2012-04-10  6:00:59

  • branches/eam_branches/ipp-20120405/ippToPsps/config/settings.xml

    r33594 r33948  
    55
    66  <!-- path for storing logs -->
    7   <logPath>/data/ipp005.0/rhenders</logPath>
     7  <logPath>/data/ipp005.0/ipptopsps</logPath>
    88
    99  <!-- local scratch Db section -->

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/batch.py

    r33672 r33948  
    3939                 useFullTables):
    4040
    41         self.everythingOK = False
    4241        self.readHeader = False
    4342        self.config = config
    … …  
    6665            if not self.header:
    6766                logger.errorPair("Could not read FITS for id", "%d" % id)
    68                 return
     67                raise
    6968
    7069        self.scratchDb.setUseFullTables(self.useFullTables)
    … …  
    115114        self.logger.infoPair("Output path", self.localOutPath)
    116115
    117         self.everythingOK = True
    118116   
    119117    '''
    … …  
    510508    def run(self):
    511509
    512         if not self.everythingOK:
    513             self.logger.errorPair("Aborting this batch", "could not initialize")
    514             self.ippToPspsDb.updateProcessed(self.batchID, -1)
    515             return
    516 
    517510        if not self.createEmptyPspsTables():
    518511            self.logger.errorPair("Aborting this batch", "could not create empty PSPS tables")
    519512            self.ippToPspsDb.updateProcessed(self.batchID, -1)
    520             return
     513            raise
    521514
    522515        if not self.importIppTables():
    523516            self.logger.errorPair("Aborting this batch", "could not import IPP tables")
    524517            self.ippToPspsDb.updateProcessed(self.batchID, -1)
    525             return
     518            raise
    526519
    527520        if not self.populatePspsTables():
    528521            self.logger.errorPair("Aborting this batch", "unable to populate PSPS tables")
    529522            self.ippToPspsDb.updateProcessed(self.batchID, -1)
    530             return
     523            raise
    531524
    532525        if not self.exportPspsTablesToFits():
    533526            self.logger.errorPair("Aborting this batch", "unable to export tables to FITS file")
    534527            self.ippToPspsDb.updateProcessed(self.batchID, -1)
    535             return
     528            raise
    536529   
    537530        if self.writeBatchManifest():

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/cleanup.py

    r33708 r33948  
    77import sys
    88import time
     9import traceback
    910import logging.config
    1011
    … …  
    4142            self.logger.infoPair("Config", self.config.name)
    4243            self.config.printDeletionPolicy()
     44            self.clean("IN")
    4345            self.clean("P2")
    4446            self.clean("ST")
    … …  
    4850            if not self.waitForPollTime(): break
    4951   
     52    '''
     53    Reports results of cleanup for this set of IDs
     54    '''
     55    def reportResults(self, name, ids, success):
     56
     57        if len(ids) > 0:
     58            self.logger.infoSeparator()
     59            self.logger.infoPair("Removed from " + name, "%d of %d" % (success, len(ids)))
     60   
     61
    5062    '''
    5163    Finds stuff to delete for this batch type, then:
    … …  
    6274
    6375        loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType)
    64         loadedDxlayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType)
     76        deleteFromDxLayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType)
    6577        mergedLocalIDs = self.ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType)
    6678
    … …  
    6880        purgedLocalIDs = self.ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType)
    6981
    70         self.logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(loadedDatastoreIDs))
    71         self.logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(loadedDxlayerIDs))
    72         self.logger.infoPair("Merged but still on local disk", "%d" % len(mergedLocalIDs))
    73         self.logger.infoPair("Purged but still on datastore", "%d" % len(purgedDatastoreIDs))
    74         self.logger.infoPair("Purged but still on local disk", "%d" % len(purgedLocalIDs))
     82        deleteFromDatastoreIDs = loadedDatastoreIDs + purgedDatastoreIDs
     83        deleteFromLocalIDs = mergedLocalIDs + purgedLocalIDs
     84
     85        # report to log what we are going to do
     86        self.logger.infoPair("To delete from datastore", "%d" % len(deleteFromDatastoreIDs))
     87        self.logger.infoPair("To delete from DXLayer", "%d" % len(deleteFromDxLayerIDs))
     88        self.logger.infoPair("To delete from local disk", "%d" % len(deleteFromLocalIDs))
    7589   
     90        # delete stuff from local disk
    7691        if self.config.deleteLocal:
    77             self.logger.infoSeparator()
    7892            count = 0
    79             ids = mergedLocalIDs + purgedLocalIDs
    80             for id in ids:
     93            for id in deleteFromLocalIDs:
    8194       
    8295                if Batch.deleteFromDisk(self.logger, self.config.basePath, batchType, self.config.dvoLabel, id):
    … …  
    8497                    count = count + 1
    8598   
    86             self.logger.infoSeparator()
    87             self.logger.infoPair("Removed from local disk", "%d" % count)
    88             self.logger.infoPair("Remaining on local disk", "%d" % (len(ids) - count))
    89        
     99            self.reportResults("local disk", deleteFromLocalIDs, count)
    90100   
     101        # remove stuff from datastore
    91102        if self.config.deleteDatastore:
    92             self.logger.infoSeparator()
    93103            count = 0
    94             ids = loadedDatastoreIDs + purgedDatastoreIDs
    95             for id in ids:
     104            for id in deleteFromDatastoreIDs:
    96105       
    97106                batchName = Batch.getNameFromID(id)
    … …  
    100109                    count = count + 1
    101110   
    102             self.logger.infoSeparator()
    103             self.logger.infoPair("Removed from datastore", "%d" % count)
    104             self.logger.infoPair("Remaining on datastore", "%d" % (len(ids) - count))
     111            self.reportResults("datastore", deleteFromDatastoreIDs, count)
    105112       
     113        # remove stuff from DXLayer
    106114        if self.config.deleteDxLayer:
    107             self.logger.infoSeparator()
    108115            count = 0
    109             for id in loadedDxlayerIDs:
     116            for id in deleteFromDxLayerIDs:
    110117       
    111118                if self.dxlayer.deleteBatch(id):
    … …  
    113120                    count = count + 1
    114121   
    115             self.logger.infoSeparator()
    116             self.logger.infoPair("Removed from DXLayer", "%d" % count)
    117             self.logger.infoPair("Remaining on DXLayer", "%d" % (len(loadedDxlayerIDs) - count))
     122            self.reportResults("DXLayer", deleteFromDxLayerIDs, count)
    118123       
    119 
    120124    '''
    121125    Overrides base-class version
    … …  
    128132Start of program
    129133'''
    130 cleanup = Cleanup(sys.argv)
    131 cleanup.run()
    132 cleanup.exitProgram("finished")
     134try:
     135    cleanup = Cleanup(sys.argv)
     136    cleanup.run()
     137    cleanup.exitProgram("completed")
     138except Exception, e:
     139    print str(e)
     140    traceback.print_exc()

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/config.py

    r33558 r33948  
    3939    '''
    4040    def printAll(self):
     41
     42        self.logger.infoTitle("Config")
    4143
    4244        try:
    … …  
    133135    def getDbPassword(self, dbType): return self.settingsDoc.find(dbType +"/password").text
    134136
    135 

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/datastore.py

    r33259 r33948  
    102102
    103103        firstInt = int(first[1:])
    104         lastInt = int(last[1:])
     104        lastInt = int(last[1:]) + 1
    105105
    106106        for i in range(firstInt, lastInt):

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/datastoreRemover.py

    r33470 r33948  
    4646Start of program
    4747'''
    48 datastoreRemover = DatastoreRemover(sys.argv)
    49 datastoreRemover.run()
    50 datastoreRemover.exitProgram("finished")
     48try:
     49    datastoreRemover = DatastoreRemover(sys.argv)
     50    datastoreRemover.run()
     51    datastoreRemover.exitProgram("completed")
     52except: pass
    5153

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/detectionbatch.py

    r33353 r33948  
    5353               useFullTables)
    5454
    55        if not self.everythingOK: return
    56 
    5755       # get camera meta data
    5856       meta = self.gpc1Db.getCameraStageMeta(self.id)
    5957       if not meta:
    60            self.everythingOK = False
    61            return
     58           self.logger.errorPair("Could not get", "camera metadata")
     59           raise
    6260
    6361       self.expID = meta[0];
    … …  
    9391       # get a fre primary header values. if in test mode, then use defaults
    9492       if self.safeDictionaryAccessWithDefault(self.header, 'MJD-OBS', "1") == "NULL":
    95            self.everythingOK = False
    96            return
     93           self.logger.errorPair("Could not get", "MJD-OBS")
     94           raise
    9795
    9896       if self.safeDictionaryAccessWithDefault(self.header, 'EXPTIME', "1") == "NULL":
    99            self.everythingOK = False
    100            return
     97           self.logger.errorPair("Could not get", "EXPTIME")
     98           raise
    10199
    102100       if self.safeDictionaryAccessWithDefault(self.header, 'FILTERID', "g.0000") == "NULL":
    103            self.everythingOK = False
    104            return
     101           self.logger.errorPair("Could not get", "FILTERID")
     102           raise
    105103       
    106104       self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0)
    … …  
    418416               ,momentYY \
    419417               ,apMag \
     418               ,kronFlux \
     419               ,kronFluxErr \
    420420               ,infoFlag \
    421421               ,sky \
    … …  
    436436               ,X_PSF_SIG \
    437437               ,Y_PSF_SIG \
    438                ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+" \
    439                ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+")) / 1.085736) \
    440                ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \
     438               ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + " \
     439               ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + ")) / 1.085736) \
     440               ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + self.header['EXPTIME'] + " \
    441441               ,PSF_MAJOR \
    442442               ,PSF_MINOR \
    … …  
    448448               ,MOMENTS_YY \
    449449               ,AP_MAG \
     450               ,KRON_FLUX / " + self.header['EXPTIME'] + " \
     451               ,KRON_FLUX_ERR / " + self.header['EXPTIME'] + " \
    450452               ,FLAGS\
    451453               ,SKY \
    … …  
    755757       else : regex = ".*.psf"
    756758 
    757        columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG FLAGS SKY SKY_SIGMA EXT_NSIGMA"
     759       columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS SKY SKY_SIGMA EXT_NSIGMA"
    758760
    759761       return super(DetectionBatch, self).importIppTables(columns, regex)

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/dvo.py

    r33676 r33948  
    1717from java.sql import *
    1818
    19 
    2019'''
    2120Abstract base-class for DVO classes
    … …  
    4847       
    4948        # connect to the specified scratch database
    50         self.scratchDb = ScratchDb(self.logger, self.config, scratchDbName)
     49        try:
     50            self.scratchDb = ScratchDb(self.logger, self.config, scratchDbName)
     51        except: raise
     52
    5153        self.scratchDb.setUseFullTables(True)
    5254
    … …  
    9092            return
    9193
    92 
    9394    '''
    9495    Loads the DVO Photcodes.dat FITS file
    … …  
    218219        self.regionsAlreadyIngested = []
    219220        self.regionsIngestedButOutOfDate = []
     221        tablesToCleanup = []
    220222
    221223        # loop through all regions in DVO inside this bounding-box
    … …  
    256258               continue
    257259
     260           # we have no record of ingesting this region, but does the table exist?
     261           # May hev been left behind after a crash, and therefore needs to be cleaned-up
     262           for fileType in self.ingestFileTypes:
     263               tableName = self.scratchDb.getDbFriendlyTableName(regionPath + "." + fileType)
     264               if self.scratchDb.tableExists(tableName):
     265                   tablesToCleanup.append(tableName)
     266
    258267           # check if we have out-of-date versions of any interested files, if so, add region to the purge list
    259268           outOfDate = False
    … …  
    279288        self.sizeOfRegionsToPurge = self.getDiskSizeOfRegions(self.regionsToPurge)
    280289        self.sizeOfRegionsToIngest = self.getDiskSizeOfRegions(self.regionsToIngest)
     290
     291        # any tables need cleaning up?
     292        if len(tablesToCleanup) > 0:
     293            self.logger.infoPair("Found tables to be cleaned up", "%d" % len(tablesToCleanup))
     294            self.scratchDb.dropTables(tablesToCleanup)
     295           
    281296
    282297    '''

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/gpc1db.py

    r33527 r33948  
    7272        elif batchType == "ST":       
    7373
    74             stage = "staticsky_multi"
    75             sql = "SELECT DISTINCT stack_id, ra, decl FROM staticskyInput \
     74            stage = "staticsky"
     75            sql = "SELECT DISTINCT stack_id, radeg, decdeg FROM staticskyInput \
    7676                   JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \
    7777                   JOIN minidvodbRun USING(minidvodb_name) \
    7878                   JOIN minidvodbProcessed USING(minidvodb_id) \
    79                    JOIN  stackRun USING(stack_id) JOIN skycells.skycell USING(skycell_id) \
     79                   JOIN  stackRun USING(stack_id) JOIN skycell USING(skycell_id) \
    8080                   WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \
    8181                   AND minidvodbRun.state = 'merged' \
    … …  
    8383                   AND addRun.stage = '" + stage + "' \
    8484                   AND addRun.state = 'full' \
    85                    AND decl BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
    86                    AND ra BETWEEN " + str(minRA) + " AND " + str(maxRA)
     85                   AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
     86                   AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA)
    8787
    8888        try:
    … …  
    266266               AND minidvodbRun.minidvodb_group = '" + dvoDb + "' \
    267267               AND minidvodbRun.state = 'merged' \
    268                AND stage = 'staticsky_multi'"
     268               AND stage = 'staticsky'"
    269269        '''
    270270
    … …  
    282282               AND mergedvodbRun.state = 'full' \
    283283               AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \
    284                AND stage = 'staticsky_multi'"
     284               AND stage = 'staticsky'"
    285285
    286286

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/initbatch.py

    r33353 r33948  
    5050    '''
    5151    def importIppTables(self, filter=""):
    52         self.logger.debug("importIppTables method here to satidfy base-class")
     52        self.logger.debug("importIppTables method here to satisfy base-class")
     53        return True
    5354
    5455    '''
    … …  
    6364    def populatePspsTables(self):
    6465        self.logger.debug("No processing required for this batch type")
    65         return 1
     66        return True
    6667

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/ipptopsps.py

    r33711 r33948  
    3838        self.SECONDS = None
    3939
     40        self.createNewConfig = False
     41        self.rotateConfigs = False
     42        # a new config?
     43        if CONFIGNAME == "edit": self.createNewConfig = True
    4044        # should we rotate configs for this program?
    41         if CONFIGNAME == "all": self.rotateConfigs = True
    42         else:  self.rotateConfigs = False
     45        elif CONFIGNAME == "all": self.rotateConfigs = True
    4346
    4447        # set up config object
    … …  
    4750
    4851        # create connection to databases database
    49         self.ippToPspsDb = IppToPspsDb(self.logger, self.config)
     52        try:
     53            self.ippToPspsDb = IppToPspsDb(self.logger, self.config)
     54        except:
     55            self.exitProgram("Could not connect to ipptopsps Db")
     56            raise
    5057
    5158        self.checkClientStatus()
    … …  
    5966        self.logger.infoPair("Host", self.HOST)
    6067        self.logger.infoPair("PID", "%d" % self.PID)
    61         self.logger.infoPair("Config", CONFIGNAME)
     68        self.logger.infoPair("Config", self.config.name)
    6269
    6370    '''
    … …  
    97104
    98105        # write message to log or stdout if no logger set up
    99         msg = sys.argv[0] + " <configName|all> " + extra
     106        msg = sys.argv[0] + " <configName|all|edit> " + extra
    100107        try:
    101108            self.logger.errorPair("Usage:", msg)
    … …  
    103110            print "*** Usage: " + msg
    104111
    105 
    106112    '''
    107113    Refreshes the config then recreates objects with new settings
    108114    '''
    109115    def refreshConfig(self):
     116
     117        # new config?
     118        if self.createNewConfig:
     119            self.ippToPspsDb.editConfig()
     120            self.ippToPspsDb.setConfigForThisClient(self.config.name, self.HOST, self.PID)
     121            self.createNewConfig = False
    110122
    111123        # if we are rotating configs, then look for next active one in list
    … …  
    163175        # write message to log or stdout if no logger set up
    164176        try:
    165             self.logger.infoPair("Program exited", exitReason)
     177            self.logger.infoPair(self.PROGNAME + " exited", exitReason)
    166178        except:
    167179            print "*** Program exited: " + exitReason

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/ipptopspsdb.py

    r33710 r33948  
    854854        return configs
    855855
    856 
    857     '''
    858     Writes current Config object to Db
    859     '''
    860     def writeConfig(self):
    861 
    862         sql = "UPDATE config \
    863                SET \
    864                datastore_product = '" + self.config.datastoreProduct + "' \
    865                ,datastore_type = '" + self.config.datastoreType + "' \
    866                ,datastore_publish = " + self.config.datastorePublishing + " \
    867                ,dvo_label = '" + self.config.dvoLabel + "' \
    868                ,dvo_location = '" + self.config.dvoLocation + "' \
    869                ,min_ra = " + self.config.minRa + " \
    870                ,max_ra = " + self.config.maxRa + " \
    871                ,min_dec = " + self.config.minDec + " \
    872                ,max_dec = " + self.config.maxDec + " \
    873                ,box_size = " + self.config.boxSize + " \
    874                ,base_path = '" + self.config.basePath + "' \
    875                ,data_release = " + self.config.dataRelease + " \
    876                ,delete_local = " + self.config.deleteLocal + " \
    877                ,delete_datastore = " + self.config.deleteDatastore + " \
    878                ,delete_dxlayer = " + self.config.deleteDxLayer + " \
    879                ,epoch = '" + self.config.epoch + "' \
    880                ,survey = '" + self.config.survey + "' \
    881                ,psps_survey = '" + self.config.pspsSurvey + "' \
    882                ,queue_P2 = " + self.config.queuingThisBatchType("P2") + " \
    883                ,queue_ST = " + self.config.queuingThisBatchType("ST") + " \
    884                ,queue_OB = " + self.config.queuingThisBatchType("OB") + " \
    885                WHERE name = '" + self.config.name + "'"
    886 
    887         self.execute(sql)
     856    '''
     857    Prompts user for info about a new or existing config and lets them neter the various details
     858    '''
     859    def editConfig(self):
     860
     861       print " ********************************************************************"
     862       print " *              Config editor"
     863       print " *"
     864       response = raw_input(" * Name for new config, or existing config to edit? ")
     865       if response == "": return
     866       self.config.name = response
     867
     868       # attempt to insert new config, if it already exists then carry on with update
     869       sql = "INSERT INTO config (name) VALUES ('" + self.config.name + "')"
     870       try:       
     871           self.execute(sql)
     872       except: pass
     873
     874       # for each column in the config table
     875       sql = "SHOW COLUMNS FROM config"
     876       rs = self.executeQuery(sql)
     877       while (rs.next()):
     878
     879           field = rs.getString(1)
     880           type = rs.getString(2)
     881           null = rs.getString(3)
     882
     883           # things we don;t want the user to edit
     884           if field == 'timestamp': continue
     885           if field == 'name': continue
     886
     887           # get current value for this field, if any
     888           try:
     889               rs2 = self.executeQuery("SELECT " + field + " FROM config WHERE name = '" + self.config.name + "'")
     890               rs2.first()
     891               default = rs2.getString(1)
     892           except: pass
     893
     894           question = " * " + field + " (" + type + ")"
     895
     896           # there may not be a default value
     897           try: question = question +  " hit return to accept default of: '" + default + "'"
     898           except: pass
     899
     900           # prompt the user for new value or to accept existing value
     901           question = question +  "? "
     902           response = raw_input(question)
     903           if response != "":
     904               if type.find("varchar") != -1 or type.find("timestamp") != -1: quotes = "'"
     905               else: quotes = ""
     906               self.execute("UPDATE config SET " + field + " = " + quotes + response + quotes + " WHERE name = '" + self.config.name + "'")
     907       
     908       print " *"
     909       print " ********************************************************************"
    888910
    889911    '''
    … …  
    951973            else: self.config.datastorePublishing = False
    952974            self.config.dvoLabel = rs.getString(4)
     975
     976            # if dvoLabel is null is can break queries, so set to something
     977            if not self.config.dvoLabel: self.config.dvoLabel = "none"
    953978            self.config.dvoLocation = rs.getString(5)
    954979            self.config.minRa = rs.getDouble(6)
    … …  
    12551280
    12561281        self.execute("CREATE TEMPORARY TABLE all_pending (stage_id bigint(20), ra_bore float, dec_bore float)")
    1257         count = 0
    12581282        for row in rows:
    12591283
    … …  
    12621286                       VALUES (%d, %f, %f)" % (row[0], row[1], row[2])
    12631287                self.execute(sql)
    1264                 count += 1
    12651288            except: continue
    12661289
    1267         self.logger.infoPair("All items written to Db", "%d" % count)
     1290        count = self.getRowCount("all_pending")
     1291        self.logger.infoPair("Items written to Db", "%d" % count)
    12681292
    12691293    '''

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/loader.py

    r33677 r33948  
    4545                self.scratchDb.disconnect()
    4646                continue
    47             self.logger.infoPair("Using scratch Db", self.scratchDb.dbName)
    4847            break
    4948
    50         if not self.scratchDb.connected:
    51             self.logger.errorPair("Cannot connect to a", "scratch database")
     49        if not self.scratchDb.connected:
     50            self.exitProgram("Cannot connect to a scratch database")
     51            raise
    5252       
    5353        # create Datastore objects
    … …  
    108108    '''
    109109    def run(self):
     110
     111        if not self.scratchDb.connected: return
    110112
    111113        numAttempts = 0
    … …  
    152154                    if batchType != "OB":
    153155                        # look in DVO for this box (with extra border)
    154                         #self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
    155                         self.dvoDetections.setSkyArea()
     156                        self.logger.infoPair("Querying DVO for this sky area", "")
     157                        self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
     158                        #self.dvoDetections.setSkyArea()
    156159                        sizeToBeIngested = self.dvoDetections.getDiskSizeOfRegionsToBeIngested()
    157160                        if sizeToBeIngested == 0.0: smfsPerGB = 999999999
    … …  
    160163                        self.logger.infoPair("smfs-per-GB", "%.1f" % smfsPerGB)
    161164                        # should do we pre-ingest stuff from DVO?
    162                         if smfsPerGB > 30:
     165                        if batchType == 'P2' and smfsPerGB > 30:
    163166                            if not self.dvoDetections.sync():
    164167                                self.logger.errorPair("Could not sync DVO with MySQL", "skipping")
    … …  
    195198   
    196199            self.ippToPspsDb.unlockTables()
     200            self.ippToPspsDb.deletePendingItem(batchType, id)
     201
     202            # catch any raised exceptions in batch constructors
     203            try:
     204                if batchType == "P2":
     205                    batch = DetectionBatch(self.logger,
     206                            self.config,
     207                            self.gpc1Db,
     208                            self.ippToPspsDb,
     209                            self.scratchDb,
     210                            id,
     211                            batchID,
     212                            useFullTables)
     213                elif batchType == "ST":
     214                    batch = StackBatch(self.logger,
     215                            self.config,
     216                            self.gpc1Db,
     217                            self.ippToPspsDb,
     218                            self.scratchDb,
     219                            id,
     220                            batchID,
     221                            useFullTables)
     222   
     223                elif batchType == "OB":
     224                    batch = ObjectBatch(self.logger,
     225                            self.config,
     226                            self.gpc1Db,
     227                            self.ippToPspsDb,
     228                            self.scratchDb,
     229                            id,
     230                            batchID,
     231                            useFullTables)
     232   
     233                batch.run()
    197234   
    198             self.ippToPspsDb.deletePendingItem(batchType, id)
    199 
    200             if batchType == "P2":
    201                 batch = DetectionBatch(self.logger,
    202                         self.config,
    203                         self.gpc1Db,
    204                         self.ippToPspsDb,
    205                         self.scratchDb,
    206                         id,
    207                         batchID,
    208                         useFullTables)
    209             elif batchType == "ST":
    210                 batch = StackBatch(self.logger,
    211                         self.config,
    212                         self.gpc1Db,
    213                         self.ippToPspsDb,
    214                         self.scratchDb,
    215                         id,
    216                         batchID,
    217                         useFullTables)
    218    
    219             elif batchType == "OB":
    220                 batch = ObjectBatch(self.logger,
    221                         self.config,
    222                         self.gpc1Db,
    223                         self.ippToPspsDb,
    224                         self.scratchDb,
    225                         id,
    226                         batchID,
    227                         useFullTables)
    228    
    229             batch.run()
    230    
     235            # if batch fails, ignore and move on to the next one
     236            except:
     237                self.logger.errorPair("Problem with this %s batch (%d)" % (batchType, id), "skipping")
     238                pass
     239
    231240            if not self.checkClientStatus(): return False
    232241   
    … …  
    253262    loader = Loader(sys.argv)
    254263    loader.run()
    255     loader.exitProgram("finished")
     264    loader.exitProgram("completed")
    256265except Exception, e:
    257266    print str(e)
    258267    traceback.print_exc()
     268

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/metrics.py

    r33712 r33948  
    191191Start of program.
    192192'''
    193 metrics = Metrics(sys.argv)
    194 metrics.run()
    195 metrics.exitProgram("finished")
    196 
    197 
    198 
     193try:
     194    metrics = Metrics(sys.argv)
     195    metrics.run()
     196    metrics.exitProgram("completed")
     197except: pass
     198
     199
     200

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/mysql.py

    r33683 r33948  
    3434        self.dbPass = config.getDbPassword(dbType)
    3535
     36        # a user friendly connection sring
     37        connectionStr = self.dbName + "@" + self.dbHost
     38
    3639        # set up JDBC connection
    3740        self.url = "jdbc:mysql://"+self.dbHost+"/"+self.dbName+"?autoReconnect=true&user="+self.dbUser+"&password="+self.dbPass
    … …  
    3942            self.con = DriverManager.getConnection(self.url)
    4043            self.connectionID = self.getLastConnectionID()
    41             self.logger.debug("MySQL connection to %s with ID %d" % (dbType, self.connectionID))
     44            self.logger.infoPair("Connected to MySQL Db", connectionStr)
    4245        except:
    43             self.logger.error("Unable to connect to " + self.url)
    44             self.everythingOK = False # TODO need this?
     46            self.logger.errorPair("Unable to connect to", connectionStr)
    4547            self.connected = False
    46             return
    47 
    48         self.everythingOK = True
     48            raise
     49
    4950        self.connected = True
    5051
    … …  
    8889        sql = "KILL %d" % connectionID
    8990        self.execute(sql)
     91
     92    '''
     93    Checks whether this table exists
     94    '''
     95    def tableExists(self, tableName):
     96
     97        sql = "SELECT COUNT(*) \
     98               FROM information_schema.tables \
     99               WHERE table_schema = '" + self.dbName + "' \
     100               AND table_name = '" + tableName + "'"
     101
     102        try:
     103            rs = self.executeQuery(sql)
     104            rs.first()
     105            if rs.getInt(1) > 0: return True
     106            else: return False
     107        except:
     108            self.logger.errorPair("Could not check if table exists", sql)
     109            pass
     110
     111        return False
    90112
    91113    '''
    … …  
    159181
    160182    '''
     183    Drops a list of tables
     184    '''
     185    def dropTables(self, tables):
     186        for table in tables: self.dropTable(table)
     187
     188    '''
    161189    Drops a table
    162190    '''
    … …  
    224252        except: pass
    225253            #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'")
     254
     255    '''
     256    Checks that we have a connection
     257    '''
     258    def isConnected(self):
     259
     260        try:
     261            self.con
     262            self.connected = True
     263        except:
     264            self.logger.errorPair("Not connected to", self.url)
     265            self.connected = False
     266
     267        return self.connected
     268
    226269    '''
    227270    TODO
    228271    '''
    229272    def execute(self, sql):
     273
     274        if not self.isConnected(): raise
    230275
    231276        stmt = self.con.createStatement()
    … …  
    238283    def executeQuery(self, sql):
    239284
     285        if not self.isConnected(): raise
     286 
    240287        stmt = self.con.createStatement()
    241288        rs = stmt.executeQuery(sql)

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/objectbatch.py

    r33715 r33948  
    5151               1)
    5252
    53        if not self.everythingOK: return
    54 
    55        self.dvoObjects = DvoObjects(self.logger, self.config, self.scratchDb.dbName)
     53       try:
     54           self.dvoObjects = DvoObjects(self.logger, self.config, self.scratchDb.dbName)
     55       except:
     56           self.logger.errorPair("Unable to create instance of", "DvoObjects")
     57           raise
    5658
    5759       # create an output filename, which is {dvoINDEX}.FITS
    … …  
    7880    def alterPspsTables(self):
    7981
    80         self.logger.debug("Altering PSPS tables")
     82        # dec is reserved in MySQL, so STILTS replaces if with dec_, which PSPS doesn't like. so, force it back again using ``
     83        self.scratchDb.execute("ALTER TABLE Object CHANGE dec_ `dec` double")
     84
    8185        return True
    8286
    … …  
    9599    def insertMags(self, cpsTable):
    96100
    97         # list of all filters we are interested in
    98         allFilters = ['g', 'r', 'i', 'z', 'y']
     101        # list of all filters PSPS is interested in
     102        interestedFilters = ['g', 'r', 'i', 'z', 'y']
    99103       
    100         # this loop looks into the DVO Photcodes tables and gets the 'CODE' for each the filters above that are listed
    101         filters = []
    102         for filter in allFilters:
    103             code = self.scratchDb.getCodeForThisFilter(filter)
    104             if code < 0: continue
    105             filters.append([code, filter])
     104        filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters)
    106105   
    107106        # get a count of the available filters
    108107        filterCount = len(filters)
    109108
    110         self.logger.infoPair("Available filters", filters)
     109        self.logger.infoPair("Available filters in Photcodes", filters)
    111110
    112111        # the 'code' now defines the order in the cps file that the mags are listed for a given filter
     112        self.logger.infoPair("Adding magnitudes from", "cps table")
    113113        for filter in filters:
    114114
    115115            sql = "UPDATE Object \
    116                    JOIN " + cpsTable + " AS cps ON (cps.row = (Object.row*" + str(filterCount) + ")-" + str(filter[0]) + ") \
     116                   JOIN " + cpsTable + " AS cps ON (cps.row = (Object.row*" + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
    117117                   SET \
    118118                   n" + filter[1] + " = NCODE \
    … …  
    120120                   ," + filter[1] + "MeanFlux = 3631 * POW(10.0, (-0.4*MAG)) \
    121121                   ," + filter[1] + "MeanMagErr = MAG_ERR \
    122                    ," + filter[1] + "Min = MAG_20 \
    123                    ," + filter[1] + "Max = MAG_80 \
     122                   ," + filter[1] + "Min = MAG_20/1000 \
     123                   ," + filter[1] + "Max = MAG_80/1000 \
    124124                   "
    125 
     125           
     126            self.scratchDb.execute(sql)
     127
     128        # now set to null all MeanMagErr values > 0.5 (cut set by Gene, 2012-04-12)
     129        cut = 0.5
     130        self.logger.infoPair("Setting to NULL all MeanMagErr value >", "%f" % cut)
     131        for filter in filters:
     132
     133            sql = "UPDATE Object \
     134                   SET " + filter[1] + "MeanMagErr = null \
     135                   WHERE " + filter[1] + "MeanMagErr > " + str(cut)
    126136            self.scratchDb.execute(sql)
    127137
    … …  
    131141    def updateColors(self):
    132142
     143        self.logger.infoPair("Calculating", "colors")
    133144        sql = "UPDATE Object \
    134145               SET \
    … …  
    147158            self.logger.errorPair("Couldn't calculate colors", sql)
    148159
    149 
    150     '''
    151     Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
    152     '''
    153     def populatePspsTables(self):
     160    '''
     161    Populates the Object table
     162    '''
     163    def populateObjectTable(self):
    154164
    155165        cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
    156166        cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps")
    157167
    158         self.logger.infoPair("Inserting objects from", "cpt")
    159         sql = "INSERT INTO Object (\
     168        self.logger.infoPair("Populating", "ObjectCalColor")
     169        self.logger.infoPair("Inserting objects from", "cpt file")
     170
     171        # note `` around dec here, as this is a reserved word in MySQL
     172        sql = "INSERT IGNORE INTO Object (\
    160173               objID \
    161174               ,ippObjID \
    162175               ,objInfoFlag \
     176               ,varFlag \
    163177               ,surveyID \
    164178               ,ra \
    165                ,dec_ \
     179               ,`dec` \
    166180               ,raErr \
    167181               ,decErr \
    168182               ,nDetections \
     183               ,dataRelease \
    169184               ) \
    170185               SELECT \
    … …  
    172187               ,CAT_ID*1000000000 + OBJ_ID \
    173188               ,FLAGS \
     189               ,0 \
    174190               ," + str(self.surveyID) + " \
    175191               ,RA \
    … …  
    178194               ,DEC_ERR \
    179195               ,NMEASURE \
     196               , " + str(self.config.dataRelease) + "\
    180197               FROM " + cptTableName
    181198
    … …  
    184201        except:
    185202            self.logger.errorPair("Couldn't populate Object table", sql)
     203            return False
    186204
    187205        # add row count columns so we can perform joins to get colors
    … …  
    190208        self.scratchDb.addRowCountColumn(cpsTableName, "row")
    191209
    192         self.logger.infoPair("Adding magnitudes from", "cps table")
    193210        self.insertMags(cpsTableName)
    194211
    195         self.logger.infoPair("Calculating", "colors")
    196212        self.updateColors()
    197213
    … …  
    205221        return True
    206222
     223    '''
     224    Populates the ObjectCalColor table
     225    '''
     226    def populateObjectCalColorTable(self):
     227
     228        self.logger.infoPair("Populating", "ObjectCalColor")
     229
     230        sql = "INSERT INTO ObjectCalColor ( \
     231               objID \
     232               ,ippObjID \
     233               ,dataRelease \
     234               ) \
     235               SELECT \
     236               objID \
     237               ,ippObjID \
     238               ,dataRelease \
     239               FROM Object"
     240        try:
     241            self.scratchDb.execute(sql)
     242        except:
     243            self.logger.errorPair("Couldn't populate ObjectCalColor table", sql)
     244            return False
     245
     246        return True
     247
     248    '''
     249    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
     250    '''
     251    def populatePspsTables(self):
     252
     253        if not self.populateObjectTable(): return False
     254        #if not self.populateObjectCalColorTable(): return False
     255
     256        # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number
     257        self.logger.infoPair("Forcing uniqueness on", "objID in Object table")
     258        rowCountBefore = self.scratchDb.getRowCount("Object")
     259        self.scratchDb.execute("ALTER IGNORE TABLE Object ADD UNIQUE INDEX(objID)")
     260        rowCountAfter = self.scratchDb.getRowCount("Object")
     261        self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore))
     262
     263        return True
     264

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/odm.py

    r33559 r33948  
    7878                DETAILS = doc.find("{%s}OdmBatchState/{%s}Details" % (NAMESPACE, NAMESPACE)).text
    7979                results['DETAILS'] = DETAILS
    80             if re.search("processing", STATE):
     80            elif re.search("processing", STATE):
    8181                loadedToODM = 1
    8282            if re.search("MergeWorthy", MESSAGE):

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/plot.py

    r33685 r33948  
    6262
    6363        f.flush()
     64        f.close()
    6465        #os.remove(tempFilename)
    6566

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/plotter.py

    r33679 r33948  
    3939Start of program
    4040'''
    41 plotter = Plotter(sys.argv)
    42 plotter.exitProgram("finished")
     41try:
     42    plotter = Plotter(sys.argv)
     43    plotter.exitProgram("completed")
     44except: pass

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/pollOdm.py

    r33709 r33948  
    4242        else: self.stages = [self.STAGE]
    4343
    44         if self.BATCHTYPE == "all": self.batchTypes = ['P2', 'ST', 'OB']
     44        if self.BATCHTYPE == "all": self.batchTypes = ['IN', 'P2', 'ST', 'OB']
    4545        else: self.batchTypes = [self.BATCHTYPE]
    4646
    … …  
    126126    '''
    127127    def printUsage(self):
    128         super(PollOdm, self).printUsage("<P2|ST|OB|all> <unloaded|unmergeworthy|unmerged|all> [hours]")
     128        super(PollOdm, self).printUsage("<IN|P2|ST|OB|all> <unloaded|unmergeworthy|unmerged|all> [hours]")
    129129   
    130130   
    … …  
    132132Program starts here
    133133'''
    134 pollodm = PollOdm(sys.argv)
    135 pollodm.run()
    136 pollodm.exitProgram("finished")
     134try:
     135    pollodm = PollOdm(sys.argv)
     136    pollodm.run()
     137    pollodm.exitProgram("completed")
     138except: pass

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/queue.py

    r33674 r33948  
    2727        super(Queue, self).__init__(argv)
    2828
    29 
    3029        # create various objects
    3130        self.gpc1Db = Gpc1Db(self.logger, self.config)
    3231        self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb)
    33         self.dvoObjects = DvoObjects(self.logger, self.config)
     32        try:
     33            self.dvoObjects = DvoObjects(self.logger, self.config)
     34        except:
     35            self.exitProgram("Unable to create instance of DvoObject")
     36            raise
    3437
    3538        self.config.printAll()
    … …  
    115118                       while dec <= self.config.maxDec:
    116119           
    117                            self.checkClientStatus()
    118120                           box_id = self.ippToPspsDb.insertBox(ra, dec)
    119121           
    … …  
    134136                self.logger.info("+-------------+-------------+-------------+-------------+")
    135137
     138            self.checkClientStatus()
    136139            if not self.waitForPollTime(): break
    137140
    … …  
    158161    '''
    159162    def printUsage(self):
    160         super(Cleanup, self).printUsage("[<repeat time in hours>]")
     163        super(Queue, self).printUsage("[<repeat time in hours>]")
    161164
    162165
    … …  
    164167Start of program.
    165168'''
    166 queue = Queue(sys.argv)
    167 queue.run()
    168 queue.exitProgram("finished")                             
     169try:
     170    queue = Queue(sys.argv)
     171    queue.run()
     172    queue.exitProgram("completed")
     173except: pass
     174

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/run.sh

    r33717 r33948  
    1 ~/jre1.7.0/bin/java -Xbootclasspath/a:../jars/stilts.jar:../jars/commons-math-2.2.jar:../jars/mysql-connector-java-5.1.14-bin.jar -jar ~/jython2.5.3b1/jython.jar $1 $2 $3 $4 $5 $6 $7 $8 $9
     1~ipp/jre1.7.0_03/bin/java -Xbootclasspath/a:../jars/stilts.jar:../jars/commons-math-2.2.jar:../jars/mysql-connector-java-5.1.14-bin.jar -jar ~/jython2.5.3b1/jython.jar $1 $2 $3 $4 $5 $6 $7 $8 $9

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/scratchdb.py

    r33689 r33948  
    278278            rs.close()
    279279        except:
    280             self.logger.errorPai("Can't query for ingested regions", sql)
     280            self.logger.errorPair("Can't query for ingested regions", sql)
    281281
    282282        return regions
    … …  
    408408                    self.logger.errorPair("Unable to delete " + fileType + " file from", self.dvoDoneTable)
    409409
    410                sql = "DROP TABLE " + self.getDbFriendlyTableName(region + "." + fileType)
    411                self.execute(sql)
     410               tableName = self.getDbFriendlyTableName(region + "." + fileType)
     411               sql = "DROP TABLE " + tableName
     412               try: self.execute(sql)
     413               except:
     414                    self.logger.errorPair("Unable to drop table", tableName)
     415                    pass
    412416
    413417        deletedDetectionCount = detectionCount - self.getRowCount(self.dvoDetectionTable)
    … …  
    540544
    541545    '''
    542     Gets the 'code' for the provided filter
    543     '''
    544     def getCodeForThisFilter(self, filter):
    545 
    546         code = -1
    547         sql = "SELECT CODE FROM " + self.dvoPhotcodesTable + " WHERE NAME = '" + filter + "'"
    548         try:
    549             rs = self.executeQuery(sql)
    550             rs.first()
    551             code = rs.getInt(1)
     546    Gets a list of filters ordered as thy are in the DVO Photcodes table, as this reflects the order they apper in the cps file
     547    '''
     548    def getOrderedListOfFiltersFromPhotcodesTable(self, interestedFilters):
     549
     550        sql = "SELECT NAME FROM " + self.dvoPhotcodesTable + " WHERE "
     551
     552        count = 0
     553        for filter in interestedFilters:
     554           if count != 0: sql = sql + " OR "
     555           sql = sql + " name = '" + filter + "'"
     556           count += 1
     557
     558        sql = sql + " ORDER BY CODE"
     559
     560        orderedFilters = []
     561        try:
     562            rs = self.executeQuery(sql)
     563            row = 1
     564            while (rs.next()):
     565                orderedFilters.append([row,rs.getString(1)])
     566                row += 1
    552567            rs.close()
    553568        except:
    554             self.logger.errorPair("Could not get filter code using", sql )
    555 
    556         return code
    557 
     569            self.logger.errorPair("Could not get ordered filter list", sql )
     570
     571        return orderedFilters
     572

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/setupScratchDb.py

    r33589 r33948  
    2525            self.exitProgram("incorrect args")
    2626
    27         self.scratchDb = ScratchDb(self.logger, self.config, argv[2])
     27        try:
     28            self.scratchDb = ScratchDb(self.logger, self.config, argv[2])
     29        except:
     30            self.exitProgram("Could not connect to a scratch Db")
     31            raise
    2832
    2933
    … …  
    5256Program starts here
    5357'''
    54 setupScratchDb = SetupScratchDb(sys.argv)
    55 setupScratchDb.run()
    56 setupScratchDb.exitProgram("finished")
     58try:
     59    setupScratchDb = SetupScratchDb(sys.argv)
     60    setupScratchDb.run()
     61    setupScratchDb.exitProgram("completed")
     62except: pass
    5763
    5864

  • branches/eam_branches/ipp-20120405/ippToPsps/jython/stackbatch.py

    r33530 r33948  
    5050               useFullTables)
    5151
    52        if not self.everythingOK: return
    53 
    5452       self.stackType = "DEEP_STACK" # TODO
    5553
    … …  
    5755       meta = self.gpc1Db.getStackStageMeta(self.id)
    5856       if not meta:
    59            self.everythingOK = False
    60            return
     57           self.logger.errorPair("Could not get stack", "metadata")
     58           raise
    6159
    6260       self.filter = meta[0];
    … …  
    366364               ,X_PSF_SIG \
    367365               ,Y_PSF_SIG \
    368                ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+str(self.expTime)+" \
    369                ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+str(self.expTime)+")) / 1.085736) \
    370                ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+str(self.expTime)+" \
     366               ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + " \
     367               ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + ")) / 1.085736) \
     368               ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + str(self.expTime) + " \
    371369               ,SKY \
    372370               ,SKY_SIGMA \
    … …  
    389387               ,AP_MAG \
    390388               , NULL \
    391                ,KRON_FLUX \
    392                ,KRON_FLUX_ERR \
     389               ,KRON_FLUX / " + str(self.expTime) + " \
     390               ,KRON_FLUX_ERR / " + str(self.expTime) + " \
    393391               , NULL \
    394392               , NULL \

  • branches/eam_branches/ipp-20120405/ippToPsps/perl/pspsSchema2xml.pl

    r31759 r33948  
    44#
    55# Script that searches a dir containing PSPS schema files, finds those that comtain the
    6 # tables on interest then parses them into an XML format. Also generates C-header files
    7 # containing enums that detail table column names and numbers.
     6# tables on interest then parses them into the VOTable format. Also generates trac-wiki
     7# formatted tables.
    88#
    99#######################################################################################
    … …  
    5151
    5252my $enumsHeader = ucfirst($type)."BatchEnums";
    53 open(OUT, ">".$enumsHeader.".h") or die("Error");
    54 
    55 print OUT "#ifndef ".uc($enumsHeader)."_H\n";
    56 print OUT "#define ".uc($enumsHeader)."_H\n\n";
    57 
    58 # tables file
    59 my $tablesOutput = new IO::File(">tables.xml");
    60 my $tablesWriter = new XML::Writer(OUTPUT => $tablesOutput, DATA_MODE => 1, DATA_INDENT=>2);
    61 $tablesWriter->xmlDecl('UTF-8');
    62 $tablesWriter->startTag('tableDescriptions', "type" => "$type");
    63 
    64 # map file
    65 my $mapOutput = new IO::File(">map.xml");
    66 my $mapWriter = new XML::Writer(OUTPUT => $mapOutput, DATA_MODE => 1, DATA_INDENT=>2);
    67 $mapWriter->xmlDecl('UTF-8');
    68 $mapWriter->startTag('tabledata', "type" => "$type");
    6953
    7054# VOTable file
    … …  
    8266
    8367# finish up XML
    84 $tablesWriter->endTag();
    85 $tablesWriter->end();
    86 
    87 # finish up XML
    88 $mapWriter->endTag();
    89 $mapWriter->end();
    90 
    91 # finish up XML
    9268$votWriter->endTag(); # end RESOURCE tag
    9369$votWriter->endTag(); # end of TABLE tag
    9470$votWriter->end();
    9571
    96 print OUT "\n#endif";
    97 close OUT;
    9872close WIKI;
    9973
    … …  
    208182
    209183    parseTable("Object");
     184    #parseTable("ObjectCalColor");
    210185}
    211186
    … …  
    230205    print WIKI "|| '''!".$tableNameOut."''' ||||||||||||||\n";
    231206    print WIKI "|| '''ODM attribute''' || '''units''' || '''Data type''' || '''Default''' || '''Description''' || '''IPP source''' || '''IPP variable/notes''' ||\n";
    232     print OUT "\ntypedef enum {\n";
    233     $tablesWriter->startTag('table', "name" => $tableNameOut);
    234     $mapWriter->startTag('table',
    235             "name" => $tableNameOut,
    236             "ippfitsextension" => "");
    237207    $votWriter->startTag('TABLE',
    238208            "name" => "$tableNameOut");
    … …  
    287257
    288258    if (!$found) {print "Could not find table '$tableName'\n";}
    289     $tablesWriter->endTag();
    290     $mapWriter->endTag();
    291259
    292260    $votWriter->startTag('DATA');
    … …  
    295263        $votWriter->endTag(); # end DATA tag
    296264        $votWriter->endTag(); # end TABLE tag
    297         print OUT "} ".$tableNameOut.";\n";
    298265
    299266    close SCHEMA;
    … …  
    384351    $colNum++;
    385352
    386     print OUT "  ".uc($tableName)."_".uc($name)." = ".$colNum.",\n";
    387353    print WIKI "|| ".$name."||".$unit."||".$mstype."||".$default."||".$comment."|| || ||\n";
    388 
    389     $tablesWriter->startTag('column',
    390             "name" => $name,
    391             "type" => $type,
    392             "default" => $default,
    393             "comment" => $fullComment);
    394     $tablesWriter->endTag();
    395354
    396355    # get VOTable type
    … …  
    413372    $votWriter->endTag();
    414373
    415     $mapWriter->comment(" **MISSING** <map pspsName=\"$name\" ippType=\"$type\" ippName=\"\" comment=\"$fullComment\"/>");
    416 
    417 #    $mapWriter->endTag();
    418 
    419374    return $colNum;
    420375}

  • branches/eam_branches/ipp-20120405/ippToPsps/src/Dvo.c

    r33595 r33948  
    188188    this->dvoConfig = dvoConfigRead(&argc, argv);
    189189
    190     for (int i=0; i<argc; i++) free(argv[i]);
    191     free(argv);
     190    //for (int i=0; i<argc; i++) free(argv[i]);
     191    //free(argv);
    192192
    193193    // method pointers
Note: See TracChangeset for help on using the changeset viewer.