IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 35352


Ignore:
Timestamp:
Apr 3, 2013, 12:52:33 PM (13 years ago)
Author:
eugene
Message:

changed dvoDetectionFull mysql engine to 'MEMORY' to improve ingest speed

handle errrors in calls to dvopsps (only in detectionbatch.py)

removed a lot of excess verbosity used for debugging / testing

determine the mysql major version number via mysql query

added test option -realgpc to use test ipptopsps databases with a real gpc1 dataset

add version-dependent call to 'set session old_alter_table' to allow
the ALTER IGNORE statements to work on mysql v5.5 machines (like our
desktops).

raise an exception in case the load infile operations fail

remove unneeded 'killLastConnectionID'

extend logging time to milliseconds

allow an exposure to be skipped without aborting loader

check that a P2 image exists in the dvoImage table (otherwise skip the chip)

change the mysql floating point types for the dvoDetection table to use FLOAT/DOUBLE instead of REAL/FLOAT

remove the old test for valid astrometry of a chip (NASTRO >= 50)

this test was crude and not really the right test. it was also used
to verify that an image should / should not be included. now we check
against dvoImages to see if the chip should be loaded or not (DVO
rejects input images for bad astrometry)

skip an exposure if too many detections have been rejected. this is a sign of a problem and should require more attention

Location:
branches/eam_branches/ipp-20130307/ippToPsps
Files:
18 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20130307/ippToPsps/config/settings.xml

    r35217 r35352  
    4949
    5050  <!-- GPC1 Db section (REAL version) -->
     51  <!-- this db is used for real processing, or in a test if -realgpc
     52       is supplied on the cmd line -->
    5153  <gpc1database>
    5254    <name>gpc1</name>
    … …  
    8688  </localdatabase_test>
    8789
    88   <!-- GPC1 Db section (REAL VERSION) -->
    89   <gpc1database_test>
    90     <name>gpc1</name>
    91     <host>ippdb01</host>
    92     <user>ipp</user>
    93     <password>ipp</password>
    94   </gpc1database_test>
    95 
    96   <!-- GPC1 Db section (TEST VERSION)
     90  <!-- GPC1 Db section (TEST VERSION)-->
     91  <!-- this db is used for test processing UNLESS -realgpc is supplied
     92       on the cmd line -->
    9793  <gpc1database_test>
    9894    <name>gpc1test</name>
    … …  
    10096    <user>dvo</user>
    10197    <password>dvo</password>
    102   </gpc1database_test> -->
     98  </gpc1database_test>
    10399
    104100  <!-- ippToPsps Db section (TEST VERSION)  -->

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/batch.py

    r35174 r35352  
    415415
    416416          # drop any previous tables before import
    417           #self.scratchDb.dropTable(table.name)
     417          self.scratchDb.dropTable(table.name)
    418418
    419419          # IPP FITS files are littered with infinities, so remove these
    … …  
    426426          try:
    427427              table.write(self.scratchDb.url + '#' + table.name)
    428               # print "wrote to mysql ", table
    429               self.scratchDb.killLastConnectionID()
     428              # XXX this one is not needed
     429              # self.scratchDb.killLastConnectionID()
    430430              count = count + 1
    431431          except:
    … …  
    454454           # get everything from table
    455455           try:
     456               print "reading table from mysql ", table
    456457               _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table)
    457458               self.scratchDb.killLastConnectionID()
    … …  
    554555
    555556        if not self.populatePspsTables():
     557            if self.skipBatch:
     558                self.ippToPspsDb.updateProcessed(self.batchID, -1)
     559                return True
     560
    556561            self.logger.errorPair("Aborting this batch", "unable to populate PSPS tables")
    557562            self.ippToPspsDb.updateProcessed(self.batchID, -1)

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/config.py

    r35217 r35352  
    2828            if arg == "-t":
    2929                self.test = True
     30                sys.argv.remove(arg)
     31
     32        self.gpc1test = self.test
     33        for arg in sys.argv:
     34            if arg == "-realgpc":
     35                self.gpc1test = False
    3036                sys.argv.remove(arg)
    3137
    … …  
    6268
    6369
    64         print "config.programName: ", self.programName
    65         print "config.configDir: ", self.configDir
    66         print "config.settingsPath: ", self.settingsPath
    67         print "config.logPath: ", self.logPath
    68         print "config.test: ", self.test
    69         print "config.czarPlotsPath: ", self.czarPlotsPath
     70        # print "config.programName: ", self.programName
     71        # print "config.configDir: ", self.configDir
     72        # print "config.settingsPath: ", self.settingsPath
     73        # print "config.logPath: ", self.logPath
     74        # print "config.test: ", self.test
     75        # print "config.czarPlotsPath: ", self.czarPlotsPath
    7076
    7177        root = self.settingsDoc.getroot()
    … …  
    128134        for loaderhost in localdb.findall('loader'):
    129135          name = loaderhost.get('mach')
    130           print "name, myhost: ", name, myhost
     136          # print "name, myhost: ", name, myhost
    131137          if name == myhost:
    132138              self.scratchDbLeaf = loaderhost
    … …  
    135141        for loaderhost in localdb.findall('loader'):
    136142          name = loaderhost.get('mach')
    137           print "name, myhost: ", name, myhost
     143          # print "name, myhost: ", name, myhost
    138144          if name == 'generic':
    139145              self.scratchDbLeaf = loaderhost

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/detectionbatch.py

    r35171 r35352  
    212212        )"
    213213
    214         print "frame meta sql: ", sql
     214        # print "frame meta sql: ", sql
    215215
    216216        self.scratchDb.execute(sql)
    … …  
    607607         WHERE b.imageID = " + str(imageID) + \
    608608         " INTO OUTFILE '" + dumpFile + "'"
    609         print "sql: ", sql
    610         self.scratchDb.execute(sql)
     609
     610        try: self.scratchDb.execute(sql)
     611        except:
     612            self.logger.info("failed to select data for detectionCalib")
     613            self.logger.infoPair("sql: ", sql)
     614            raise
    611615
    612616        sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName
    613         print "sql: ", sql
    614         self.scratchDb.execute(sql)
     617        try: self.scratchDb.execute(sql)
     618        except:
     619            self.logger.info("failed to load data from infile for detectionCalib")
     620            self.logger.infoPair("sql: ", sql)
     621            raise
    615622       
    616623    '''
    … …  
    655662
    656663        # load corresponding header into memory
    657         header = self.fits.findAndReadHeader(extname)
     664        header = self.fits.findAndReadHeader(extname, self.config.test)
    658665        if not header:
    659             self.logger.errorPair("No header found for chip", chipname)
     666            # XXX There is no way to check if we SHOULD find a given
     667            # chip.  We have to accept the absence of the header as
     668            # information that the chip is not available.  This
     669            # message is too verbose, only print in test mode
     670            if self.config.test: self.logger.errorPair("No header found for chip", chipname)
    660671            return False
    661672       
    … …  
    773784        # this is a bit crude: if the chip is not present, this test will fail and the chip
    774785        # will be (correctly) skipped.  would be better to carry that information explicitly ("chip is missing")
    775         if not self.scratchDb.astrometricSolutionOK(chipname):
    776             self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
     786
     787        # XXX keep this in or not?
     788        # if not self.scratchDb.astrometricSolutionOK(chipname):
     789        #     self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
     790        #     return False
     791
     792        # does this chip exist in the DVO image table (if NOT, then skip it)
     793        if not self.scratchDb.haveThisChip(self.imageIDs[chipname]):
     794            self.logger.info("| %5s |            ------------- Chip not in DVO : rejecting ------------                    |" % chipname)
    777795            return False
     796
    778797        self.logger.info("populate stuff ");
    779798        # populate remainder of tables
    … …  
    803822        self.totalNullObjID    = self.totalNullObjID + results['NULLOBJID']
    804823        self.totalDetections   = self.totalDetections + rowCount
     824
     825        fractionKept = float(rowCount) / float(results['ORIGINALTOTAL'])
     826        # print "kept ", str(fractionKept), " detections from ", chipname
     827
     828        if fractionKept < 0.8:
     829            # print "Too many rejected detections (", str(fractionKept), ", fail on exposure for", chipname
     830            self.logger.infoPair("Too many rejected detections, fail on exposure for", chipname)
     831            self.skipBatch = True
     832            return False;
     833
    805834        #self.logger.info("updated totals")
    806835        # check we have something in this Detection table TODO add this to table above
    807836        if rowCount < 1:
    808             self.logger.infoPair("Skipping empty table for chipname", chipname)
     837            self.logger.infoPair("Empty table for chip, fail on exposure", chipname)
     838            self.skipBatch = True
    809839            return False;
    810840
    … …  
    849879
    850880        # loop through all OTAs again to update with DVO IDs
     881        self.skipBatch = False
    851882        self.validChips = []   
    852883        self.tablesToExport = []   
    … …  
    872903
    873904                if self.populatePspsTablesChip(ota, x, y, results, tables): otaCount = otaCount + 1
     905                if self.skipBatch:
     906                    self.logger.error("fatal problem for exposure, skipping")
     907                    return False
    874908
    875909        if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1
     910
     911        if self.skipBatch:
     912            self.logger.error("fatal problem for exposure, skipping")
     913            return False
    876914
    877915        # print totals

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/dvo.py

    r35223 r35352  
    591591    def nativeIngestDetections(self, boxId, boxDim):
    592592
    593         # XXX put the chunk below in a separate method
    594         # blow away existing dvoDetection table
    595 
    596593        # clear the 'ingested' field for all boxes owned by this host
    597594        self.ippToPspsDb.clearIngestedBoxes(self.scratchDb.dbHost)
    … …  
    607604        ##     raise
    608605
    609         # drop detections table
    610         self.logger.infoPair("Dropping table", self.scratchDb.dvoDetectionTable)
    611         self.scratchDb.dropTable(self.scratchDb.dvoDetectionTable)
    612        
    613         # create detections table
    614         self.logger.infoPair("Creating table", self.scratchDb.dvoDetectionTable)
    615         sql = "CREATE TABLE " + self.scratchDb.dvoDetectionTable + " LIKE dvoDetection"
    616         try: self.scratchDb.execute(sql)
    617         except:
    618             self.logger.errorPair("Unable to create table", self.scratchDb.dvoDetectionTable)
     606        # blow away existing dvoDetection table & re-crate
     607        if not self.scratchDb.resetDvoDetectionTable():
     608            self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
    619609            return False
    620         self.scratchDb.changeEngineToInnoDB(self.scratchDb.dvoDetectionTable)
    621        
    622         # add fileID column
    623         sql = "ALTER TABLE " + self.scratchDb.dvoDetectionTable + " ADD fileID INT NOT NULL"
    624         try: self.scratchDb.execute(sql)
    625         except:
    626             self.logger.errorPair("Unable to add fileID column to ", self.scratchDb.dvoDetectionTable)
    627             return False
    628 
    629         # add an index to the fileID column
    630         self.scratchDb.createIndex(self.scratchDb.dvoDetectionTable, "fileID")
    631610
    632611        # make sure we have an up-to-date Images table
    … …  
    665644        p = Popen(cmd, shell=True, stdout=PIPE)
    666645        p.wait()
     646        if p.returncode != 0:
     647            self.logger.errorPair("dvopsps failed on ", self.scratchDb.dvoDetectionTable)
     648            raise
    667649
    668650        ## dvopsps can take a very long time. try 2 or 3 times to re-establish communication with the db before proceeding?
    … …  
    670652        self.ippToPspsDb.setIngestedBox(boxId, self.scratchDb.dbHost)
    671653
    672         # add fileID column
     654        # run 'ANALYZE' to get indexes up-to-snuff
    673655        sql = "ANALYZE TABLE " + self.scratchDb.dvoDetectionTable
    674656        try: self.scratchDb.execute(sql)

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/fits.py

    r35097 r35352  
    8787
    8888    2880 is the magic number that the FITS format uses to break up data: extensions always begin on multiples of 2880 bytes
     89
     90    Note that we may not be able to distinguish the case where a header is expected from one where the absence of a header
     91    just tells us that the header is not available.  In some cases, we may not want to emit the error message ("not found")
     92
     93    XXX WOW! this block is reading the entire file to get the header.
     94    major inefficiency...  I thought the FITS interactions all went
     95    through STILTS
    8996    '''
    90     def findAndReadHeader(self, name):
     97    def findAndReadHeader(self, name, VERBOSE):
    9198
    9299        found = False
    … …  
    114121
    115122            self.file.seek(origIndex, 0)
    116             self.logger.errorPair("Could not read header", name)
     123            if VERBOSE: self.logger.errorPair("Could not read header", name)
    117124            return
    118125        else:

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/gpc1db.py

    r35185 r35352  
    2222    def __init__(self, logger, config):
    2323        # define database type
    24         if (config.test):
     24        if (config.gpc1test):
    2525            dbType = "gpc1database_test"
    2626        else:
    … …  
    104104        try:
    105105            # XXX EAM : test output
    106             self.logger.infoPair("sql for dvo items:", sql)
    107106            rs = self.executeQuery(sql)
    108107            while (rs.next()):
    … …  
    111110        except:
    112111            self.logger.exception("Can't query for ids in DVO")
     112            self.logger.infoPair("failed sql:", sql)
    113113            return rows
    114114

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/ipptopsps.py

    r35177 r35352  
    5555            raise
    5656
     57        print "check client status"
    5758        self.checkClientStatus()
    5859
    … …  
    142143    def checkClientStatus(self):
    143144
     145        print "update client"
    144146        self.ippToPspsDb.updateClient(self.config.programName, self.HOST, self.PID)
    145147
     148        print "check killed"
    146149        if self.ippToPspsDb.isKilled(self.HOST, self.PID):
    147150            self.exitProgram("killed via Db")
    … …  
    149152        # this loop pauses the process if we have no skychunk or we have set it to pause
    150153        firstTimeIn = True
     154        print "refresh skychunk"
    151155        while not self.refreshSkychunk() or self.ippToPspsDb.isPaused(self.HOST, self.PID):
    152156

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/loader.py

    r35226 r35352  
    3636
    3737        if self.skychunk.parallel:
    38             print "PARALLEL dvo"
     38            self.logger.info("using PARALLEL DVO")
    3939        else:
    40             print "SERIAL dvo"
     40            self.logger.info("using SERIAL DVO")
    4141
    4242        # connect to the gpc1 database

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/mysql.py

    r35222 r35352  
    4848            raise
    4949
     50        # determine the mysql major version number:
     51        sql = "SELECT @@version"
     52
     53        try:
     54            rs = self.executeQuery(sql)
     55            rs.first()
     56            fullVersion = rs.getString(1)
     57            self.version = fullVersion[0:3]
     58           
     59        except:
     60            print "failed to get version number"
     61            raise
     62
     63        # print "version number: ", self.version
    5064        self.connected = True
    5165

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/objectbatch.py

    r35223 r35352  
    372372        self.logger.infoPair("Forcing uniqueness on", "objID in Object table")
    373373        rowCountBefore = self.scratchDb.getRowCount("Object")
     374
     375        # XXX EAM : note that in mysql versions later than 5.1, this fails
     376        # unless the following is called first:
     377        # set session old_alter_table=1
     378        # follow the command with
     379        # set session old_alter_table=0
     380        # OF COURSE, this fails for mysql version < 5.5...
     381        if self.scratchDb.version > 5.1:
     382            self.scratchDb.execute("set session old_alter_table=1")
     383           
    374384        self.scratchDb.execute("ALTER IGNORE TABLE Object ADD UNIQUE INDEX(objID)")
     385        if self.scratchDb.version > 5.1:
     386            self.scratchDb.execute("set session old_alter_table=0")
     387
    375388        rowCountAfter = self.scratchDb.getRowCount("Object")
    376389        self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore))

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/pslogger.py

    r35097 r35352  
    1313   def setup(self, programName, basePath, skychunkName, host, pid, stdout=1, sendToFile=0):
    1414
    15        formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S')
     15       formatter = logging.Formatter('%(asctime)s.%(msecs)03d | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S')
    1616       self.setLevel(logging.INFO)
    1717

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/queue.py

    r35226 r35352  
    2929        super(Queue, self).__init__(argv)
    3030
    31         print "inited ipptopsps"
     31        # print "inited ipptopsps"
    3232
    3333        # create various objects
    3434        self.gpc1Db = Gpc1Db(self.logger, self.config)
    35         print "connect to gpc1"
     35        # print "connect to gpc1"
    3636
    3737        self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb)
    3838
    39         print "made datastore"
     39        # print "made datastore"
    4040
    4141        # connect to scratch database
    4242        self.scratchDb = ScratchDb(self.logger, self.config)
    4343
    44         print "connect to scratchdb"
     44        # print "connect to scratchdb"
    4545
    4646        try:
    … …  
    5050            raise
    5151
    52         print "got objects"
     52        # print "got objects"
    5353
    5454        self.skychunk.printAll()

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/scratchdb.py

    r35226 r35352  
    2929            dbType = "localdatabase"
    3030
    31         print "trying to connect: ", dbType
     31        # print "trying to connect: ", dbType
    3232
    3333        dbName = config.getDbName(dbType)
    … …  
    368368
    369369    '''
     370    Checks whether the given chipID is in the DVO Image table
     371    '''
     372    def haveThisChip(self, imageID):
     373
     374        sql = "SELECT IMAGE_ID FROM " + self.dvoImagesTable + " WHERE IMAGE_ID = " + str(imageID)
     375
     376        try:
     377            rs = self.executeQuery(sql)
     378            rs.first()
     379            if rs.getInt(1) == int(imageID):
     380                # print "DVO has imageID ", imageID
     381                return True
     382            else:
     383                print "DVO lacks imageID ", imageID
     384                print "result was ", rs.getInt(1)
     385                print "sql: ", sql
     386                return False
     387        except:
     388            print "DVO imageID query failed ", imageID
     389            print "sql: ", sql
     390            return False
     391
     392        return True
     393
     394    '''
    370395    Creates a table for for ID matching
    371396    '''
    … …  
    409434               objID BIGINT, \
    410435               flags INT, \
    411                zp REAL, \
    412                zpErr REAL, \
    413                airMass REAL, \
    414                expTime REAL, \
    415                ra FLOAT, \
    416                dec_ FLOAT, \
    417                raErr REAL, \
    418                decErr REAL, \
     436               zp FLOAT, \
     437               zpErr FLOAT, \
     438               airMass FLOAT, \
     439               expTime FLOAT, \
     440               ra DOUBLE, \
     441               dec_ DOUBLE, \
     442               raErr FLOAT, \
     443               decErr FLOAT, \
    419444               PRIMARY KEY (imageID, ippDetectID), \
    420445               KEY (objID, detectID) \
    … …  
    463488    Drops and recreates tables necessary for dvoToMySQL program. Be very careful before using this...
    464489    '''
     490    def resetDvoDetectionTable(self):
     491
     492       self.logger.infoPair("Dropping table", self.dvoDetectionTable)
     493       self.dropTable(self.dvoDetectionTable)
     494
     495       # create detections table
     496       self.logger.infoPair("Creating table", self.dvoDetectionTable)
     497       sql = "CREATE TABLE " + self.dvoDetectionTable + " LIKE dvoDetection"
     498       try: self.execute(sql)
     499       except:
     500           self.logger.errorPair("Unable to create table", self.dvoDetectionTable)
     501           return False
     502
     503       # XXX changing the dvoDetectionFull table to MEMORY
     504       # self.changeEngineToInnoDB(self.dvoDetectionTable)
     505
     506       # set engine to memory
     507       sql = "ALTER TABLE " + self.dvoDetectionTable + " ENGINE = MEMORY"
     508       try: self.execute(sql)
     509       except:
     510           self.logger.errorPair("Unable to set engine to MEMORY for ", self.dvoDetectionTable)
     511           return False
     512
     513       return True
     514
     515    '''
     516    Drops and recreates tables necessary for dvoToMySQL program. Be very careful before using this...
     517    '''
    465518    def resetAllDvoTables(self):
    466519
    … …  
    485538       self.changeEngineToInnoDB(self.dvoDoneTable)
    486539
    487        # create detections table
    488        self.logger.infoPair("Creating table", self.dvoDetectionTable)
    489        sql = "CREATE TABLE " + self.dvoDetectionTable + " LIKE dvoDetection"
    490        try: self.execute(sql)
    491        except:
    492            self.logger.errorPair("Unable to create table", self.dvoDetectionTable)
     540       # blow away existing dvoDetection table & re-crate
     541       if not self.resetDvoDetectionTable():
     542           self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
    493543           return False
    494        self.changeEngineToInnoDB(self.dvoDetectionTable)
    495 
    496        # add fileID column
    497        sql = "ALTER TABLE " + self.dvoDetectionTable + " ADD fileID INT NOT NULL"
    498        try: self.execute(sql)
    499        except:
    500            self.logger.errorPair("Unable to add fileID column to ", self.dvoDetectionTable)
    501            return False
    502 
    503        # add an index to the fileID column
    504        self.createIndex(self.dvoDetectionTable, "fileID")
    505 
    506        # XXX : this should be based on choice of useStilts (which is set in dvo.py and must be moved)
    507        if (False):
    508            # now add a delete cascading foreign key constraint on fileID
    509            sql = "ALTER TABLE dvoDetectionFull \
    510               ADD CONSTRAINT fk_fileID \
    511               FOREIGN KEY (fileID) \
    512               REFERENCES dvoDone(id) \
    513               ON DELETE CASCADE"
    514            try: self.execute(sql)
    515            except:
    516                self.logger.errorPair("Unable to create foreign key on", self.dvoDoneTable)
    517                return False
    518544
    519545       return True

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/skychunk.py

    r35179 r35352  
    2828        if self.name == "all": self.rotateSkychunks = True
    2929
    30         print "name: ", self.name
    31         print "new: ", self.createNewSkychunk
    32         print "all: ", self.rotateSkychunks
     30        # print "name: ", self.name
     31        # print "new: ", self.createNewSkychunk
     32        # print "all: ", self.rotateSkychunks
    3333
    3434        self.logger = logger

  • branches/eam_branches/ipp-20130307/ippToPsps/jython/stackbatch.py

    r35222 r35352  
    543543        # response = raw_input("updated dvo ")
    544544       
     545        # XXX EAM : note that in mysql versions later than 5.1, this fails
     546        # unless the following is called first:
     547        # set session old_alter_table=1
     548        # follow the command with
     549        # set session old_alter_table=0
     550        # OF COURSE, this fails for mysql version < 5.5...
     551        if self.scratchDb.version > 5.1:
     552            self.scratchDb.execute("set session old_alter_table=1")
     553
    545554        sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)"
    546555       
     556        if self.scratchDb.version > 5.1:
     557            self.scratchDb.execute("set session old_alter_table=0")
     558
    547559        self.scratchDb.execute(sql)
    548560        # response = raw_input("add primary key? ")
    … …  
    806818         ON (a.stackDetectID = b.detectID) where b.imageID = " + str(imageID) + \
    807819         " INTO OUTFILE '" + dumpFile + "'"
    808         print "sql: ", sql
    809         self.scratchDb.execute(sql)
     820
     821        try: self.scratchDb.execute(sql)
     822        except:
     823            self.logger.info("failed to select data for stackDetectionCalib")
     824            self.logger.infoPair("sql: ", sql)
     825            raise
    810826
    811827        sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName
    812         print "sql: ", sql
    813         self.scratchDb.execute(sql)
     828        try: self.scratchDb.execute(sql)
     829        except:
     830            self.logger.info("failed to load data from infile for stackDetectionCalib")
     831            self.logger.infoPair("sql: ", sql)
     832            raise
    814833
    815834        ## XXX write this with the select/insert

  • branches/eam_branches/ipp-20130307/ippToPsps/test/fulltest.sh

    r35225 r35352  
    11#!/bin/csh -f
    2 
    3 set stackqueuename = 0
    4 if ($#argv > 0) then
    5   set stackqueuename = $1
    6   echo "stackqueue: $stackqueuename"
    7 endif
    8 
    9 set objectqueuename = 0
    10 if ($#argv > 0) then
    11   set stackqueuename = $1
    12   echo "stackqueue: $stackqueuename"
    13 endif
    142
    153# a bit crude : get the datadir path by interpollation of the PATH element for psconfig
    … …  
    2412# before running this test, you must have a mysql user:
    2513# mkgpc1.sh user localhost eam eam
    26 
    2714set mkgpc1       = 1
    2815set initdb       = 1
    2916set initscratch  = 1
     17set initbatch    = 1
    3018set camqueue     = 1
    31 set initbatch    = 1
    3219set cambatch     = 1
    3320set stackqueue   = 1
    … …  
    3623set objectbatch  = 1
    3724
     25set args    = ""
     26
     27while ("$1" != "")
     28 switch ("$1")
     29  case -mkgpc1
     30    set mkgpc1 = 0
     31    breaksw;
     32  case -initdb
     33    set initdb = 0
     34    breaksw;
     35  case -initscratch
     36    set initscratch = 0
     37    breaksw;
     38  case -initbatch
     39    set initbatch = 0
     40    breaksw;
     41  case -camera
     42    set camqueue = 0
     43    set cambatch = 0
     44    breaksw;
     45  case -stack
     46    set stackqueue = 0
     47    set stackbatch = 0
     48    breaksw;
     49  case -object
     50    set objectqueue = 0
     51    set objectbatch = 0
     52    breaksw;
     53  default:
     54    set args=($args $1);
     55    breaksw;
     56  endsw
     57  shift
     58end
     59
     60set stackqueuename = 0
     61set objectqueuename = 0
     62
    3863set OUTDIR = "testdata"
    3964set queuename = "test"
    4065
    4166if ($mkgpc1) then
     67  echo ""
     68  echo " ---- mkgpc1data.dvo : creating the dvo & gpc1 database entries ----"
    4269  mkgpc1data.dvo
    4370endif
    … …  
    4572# create the ipptopsps database
    4673if ($initdb) then
     74  echo ""
     75  echo " ---- init ipptopsps_test ----"
    4776
    4877  # does the database already exist?
    … …  
    5584# re-create the ipptopsps database
    5685if ($initscratch) then
     86  echo ""
     87  echo " ---- init ipptopsps_test_scratch ----"
    5788
    5889  # create the database (ignore failure if it already exists)
    … …  
    6293endif
    6394
     95# this is poor : we need to queue something in order to generate an init batch
     96# this is because a skychunk carries the info about the datastore used to publish
     97# the batch
     98if ($initbatch) then
     99  echo ""
     100  echo " ---- make skychunk for IN batches ----"
     101
     102  ippjython queue.py -test edit <<EOF
     103initbatch
     104test_ds
     105IPP_PSPS
     1060
     107none
     108none
     1090
     1100
     1110
     1120
     1132
     114none
     1150
     1160
     1171
     1181
     1192012-12-10 00:00:00.0
     1203PI
     1213PI
     1220
     1230
     1240
     1251
     1260
     127y
     128y
     129EOF
     130endif
     131
     132  echo ""
     133  echo " ---- create init batch ----"
     134
     135  rm -rf $OUTDIR/IN
     136  ippjython loader.py -test initbatch init
     137endif
     138
    64139# this code is sensitive to a previously existing database?
    65140if ($camqueue) then
     141  echo ""
     142  echo " ---- queue P2 batches ----"
     143
    66144  ippjython queue.py -test edit <<EOF
    67145$queuename
    … …  
    94172endif
    95173
    96 if ($initbatch) then
    97   rm -rf $OUTDIR/IN
    98   ippjython loader.py -test $queuename init
    99 endif
    100 
    101174if ($cambatch) then
     175  echo ""
     176  echo " ---- load P2 batches ----"
     177
    102178  rm -rf $OUTDIR/P2
    103179  ippjython loader.py -test $queuename once
    … …  
    106182# this code is sensitive to a previously existing database?
    107183if ($stackqueue) then
     184  echo ""
     185  echo " ---- queue ST batches ----"
     186
    108187  if ($stackqueuename == 0) then
    109188    set stackqueuename = $queuename\_stk
    … …  
    140219
    141220if ($stackbatch) then
     221  echo ""
     222  echo " ---- load ST batches ----"
     223
    142224  if ($stackqueuename == 0) then
    143225    set stackqueuename = $queuename\_stk
    … …  
    149231# this code is sensitive to a previously existing database?
    150232if ($objectqueue) then
     233  echo ""
     234  echo " ---- queue OB batches ----"
     235
    151236  if ($objectqueuename == 0) then
    152237    set objectqueuename = $queuename\_obj
    … …  
    183268
    184269if ($objectbatch) then
     270  echo ""
     271  echo " ---- load OB batches ----"
     272
    185273  if ($objectqueuename == 0) then
    186274    set objectqueuename = $queuename\_obj

  • branches/eam_branches/ipp-20130307/ippToPsps/test/mkgpc1data.dvo

    r35188 r35352  
    196196    # PCA2X0Y2
    197197
     198    if ($i == 3)
     199      echo "NOTE: not adding image $i to DVO : this simulates images with poor astrometry"
     200      continue
     201    end
     202
    198203    echo addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
    199204    exec addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
Note: See TracChangeset for help on using the changeset viewer.