Index: /trunk/ippToPsps/config/settings.xml =================================================================== --- /trunk/ippToPsps/config/settings.xml (revision 35416) +++ /trunk/ippToPsps/config/settings.xml (revision 35417) @@ -15,4 +15,5 @@ dvo dvo + 16 @@ -21,4 +22,5 @@ dvo dvo + 16 @@ -27,4 +29,5 @@ dvo dvo + 16 @@ -33,4 +36,5 @@ dvo dvo + 16 @@ -39,4 +43,12 @@ dvo dvo + 16 + + + ipptopsps_scratch + ipp064 + dvo + dvo + 32 @@ -45,8 +57,11 @@ dvo dvo + 16 + gpc1 @@ -71,4 +86,12 @@ dvo dvo + 16 + + + ipptopsps_test_scratch + ipp064 + dvo + dvo + 32 @@ -77,4 +100,5 @@ dvo dvo + 2 @@ -83,16 +107,11 @@ dvo dvo + 2 - - - gpc1 - ippdb01 - ipp - ipp - - - + gpc1test @@ -100,5 +119,5 @@ dvo dvo - --> + Index: /trunk/ippToPsps/jython/batch.py =================================================================== --- /trunk/ippToPsps/jython/batch.py (revision 35416) +++ /trunk/ippToPsps/jython/batch.py (revision 35417) @@ -276,4 +276,7 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + if p.returncode != 0: + self.logger.errorPair("error running xmllint for ", tmpPath) + raise except: self.logger.errorPair("Popen failed", cmd) @@ -301,5 +304,4 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() - if p.returncode != 0: self.logger.errorPair("tar command", "failed") @@ -311,5 +313,4 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() - if p.returncode != 0: self.logger.errorPair("gzip command", "failed") @@ -415,5 +416,5 @@ # drop any previous tables before import - #self.scratchDb.dropTable(table.name) + self.scratchDb.dropTable(table.name) # IPP FITS files are littered with infinities, so remove these @@ -426,6 +427,6 @@ try: table.write(self.scratchDb.url + '#' + table.name) - # print "wrote to mysql ", table - self.scratchDb.killLastConnectionID() + # XXX this one is not needed + # self.scratchDb.killLastConnectionID() count = count + 1 except: @@ -454,4 +455,5 @@ # get everything from table try: + print "reading table from mysql ", table _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table) self.scratchDb.killLastConnectionID() @@ -518,4 +520,7 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + if p.returncode != 0: + self.logger.errorPair("dvograbber command", "failed") + return False rowCount = self.scratchDb.getRowCount("dvoDetection") @@ -554,4 +559,8 @@ if not self.populatePspsTables(): + if self.skipBatch: + self.ippToPspsDb.updateProcessed(self.batchID, -1) + return True + self.logger.errorPair("Aborting this batch", "unable to populate PSPS tables") self.ippToPspsDb.updateProcessed(self.batchID, -1) Index: /trunk/ippToPsps/jython/config.py =================================================================== --- /trunk/ippToPsps/jython/config.py (revision 35416) +++ /trunk/ippToPsps/jython/config.py (revision 35417) @@ -28,4 +28,10 @@ if arg == "-t": self.test = True + sys.argv.remove(arg) + + self.gpc1test = self.test + for arg in sys.argv: + if arg == "-realgpc": + self.gpc1test = False sys.argv.remove(arg) @@ -60,12 +66,10 @@ self.czarPlotsPath = self.settingsDoc.find("czarPlotsPath").text - - - print "config.programName: ", self.programName - print "config.configDir: ", self.configDir - print "config.settingsPath: ", self.settingsPath - print "config.logPath: ", self.logPath - print "config.test: ", self.test - print "config.czarPlotsPath: ", self.czarPlotsPath + # print "config.programName: ", self.programName + # print "config.configDir: ", self.configDir + # print "config.settingsPath: ", self.settingsPath + # print "config.logPath: ", self.logPath + # print "config.test: ", self.test + # print "config.czarPlotsPath: ", self.czarPlotsPath root = self.settingsDoc.getroot() @@ -119,4 +123,11 @@ return self.settingsDoc.find(dbType +"/password").text + def getDbMemory(self, dbType): + if dbType == 'localdatabase' or dbType == 'localdatabase_test': + return self.scratchDbLeaf.find('memory').text + else: + print "memory is not a valid option for non-scratch databases..." + raise + def setScratchDbLeaf(self, myhost): root = self.settingsDoc.getroot() @@ -128,5 +139,5 @@ for loaderhost in localdb.findall('loader'): name = loaderhost.get('mach') - print "name, myhost: ", name, myhost + # print "name, myhost: ", name, myhost if name == myhost: self.scratchDbLeaf = loaderhost @@ -135,5 +146,5 @@ for loaderhost in localdb.findall('loader'): name = loaderhost.get('mach') - print "name, myhost: ", name, myhost + # print "name, myhost: ", name, myhost if name == 'generic': self.scratchDbLeaf = loaderhost Index: /trunk/ippToPsps/jython/datastore.py =================================================================== --- /trunk/ippToPsps/jython/datastore.py (revision 35416) +++ /trunk/ippToPsps/jython/datastore.py (revision 35417) @@ -47,4 +47,5 @@ --list " + tempFile.name + print "dsreg cmd: ", command p = Popen(command, shell=True, stdout=PIPE) p.wait() Index: /trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- /trunk/ippToPsps/jython/detectionbatch.py (revision 35416) +++ /trunk/ippToPsps/jython/detectionbatch.py (revision 35417) @@ -212,5 +212,5 @@ )" - print "frame meta sql: ", sql + # print "frame meta sql: ", sql self.scratchDb.execute(sql) @@ -607,10 +607,17 @@ WHERE b.imageID = " + str(imageID) + \ " INTO OUTFILE '" + dumpFile + "'" - print "sql: ", sql - self.scratchDb.execute(sql) + + try: self.scratchDb.execute(sql) + except: + self.logger.info("failed to select data for detectionCalib") + self.logger.infoPair("sql: ", sql) + raise sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName - print "sql: ", sql - self.scratchDb.execute(sql) + try: self.scratchDb.execute(sql) + except: + self.logger.info("failed to load data from infile for detectionCalib") + self.logger.infoPair("sql: ", sql) + raise ''' @@ -655,7 +662,11 @@ # load corresponding header into memory - header = self.fits.findAndReadHeader(extname) + header = self.fits.findAndReadHeader(extname, self.config.test) if not header: - self.logger.errorPair("No header found for chip", chipname) + # XXX There is no way to check if we SHOULD find a given + # chip. We have to accept the absence of the header as + # information that the chip is not available. This + # message is too verbose, only print in test mode + if self.config.test: self.logger.errorPair("No header found for chip", chipname) return False @@ -670,4 +681,5 @@ # store these for later self.imageIDs[chipname] = header['IMAGEID'] + self.sourceIDs[chipname] = header['SOURCEID'] # populate ImageMeta @@ -691,4 +703,5 @@ # dictionary objects to hold imageIDs for later self.imageIDs = {} + self.sourceIDs = {} # loop through all OTAs and populate ImageMeta extensions @@ -773,7 +786,15 @@ # this is a bit crude: if the chip is not present, this test will fail and the chip # will be (correctly) skipped. would be better to carry that information explicitly ("chip is missing") - if not self.scratchDb.astrometricSolutionOK(chipname): - self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % chipname) + + # XXX keep this in or not? + # if not self.scratchDb.astrometricSolutionOK(chipname): + # self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % chipname) + # return False + + # does this chip exist in the DVO image table (if NOT, then skip it) + if not self.scratchDb.haveThisChip(self.imageIDs[chipname], self.sourceIDs[chipname]): + self.logger.info("| %5s | ------------- Chip not in DVO : rejecting ------------ |" % chipname) return False + self.logger.info("populate stuff "); # populate remainder of tables @@ -803,8 +824,19 @@ self.totalNullObjID = self.totalNullObjID + results['NULLOBJID'] self.totalDetections = self.totalDetections + rowCount + + fractionKept = float(rowCount) / float(results['ORIGINALTOTAL']) + # print "kept ", str(fractionKept), " detections from ", chipname + + if fractionKept < 0.8: + # print "Too many rejected detections (", str(fractionKept), ", fail on exposure for", chipname + self.logger.infoPair("Too many rejected detections, fail on exposure for", chipname) + self.skipBatch = True + return False; + #self.logger.info("updated totals") # check we have something in this Detection table TODO add this to table above if rowCount < 1: - self.logger.infoPair("Skipping empty table for chipname", chipname) + self.logger.infoPair("Empty table for chip, fail on exposure", chipname) + self.skipBatch = True return False; @@ -849,4 +881,5 @@ # loop through all OTAs again to update with DVO IDs + self.skipBatch = False self.validChips = [] self.tablesToExport = [] @@ -872,6 +905,13 @@ if self.populatePspsTablesChip(ota, x, y, results, tables): otaCount = otaCount + 1 - - if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1 + if self.skipBatch: + self.logger.error("fatal problem for exposure, skipping") + return False + + if "Chip" in self.imageIDs: + if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1 + if self.skipBatch: + self.logger.error("fatal problem for exposure, skipping") + return False # print totals @@ -886,7 +926,7 @@ self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") - # if we only have one table export, i.e. FrameMeta, then get out of here + # if we only have one table export, i.e. FrameMeta, then get out of here (skip batch, but do not abort) if len(self.tablesToExport) == 1: - + self.skipBatch = True self.logger.error("No tables to export") return False Index: /trunk/ippToPsps/jython/dvo.py =================================================================== --- /trunk/ippToPsps/jython/dvo.py (revision 35416) +++ /trunk/ippToPsps/jython/dvo.py (revision 35417) @@ -193,4 +193,7 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + if p.returncode != 0: + self.logger.errorPair("dvopsps failed on SkyTable.fits for ", self.skychunk.dvoLocation) + raise self.logger.infoPair("Adding index to", self.scratchDb.dvoSkyTable) @@ -591,7 +594,4 @@ def nativeIngestDetections(self, boxId, boxDim): - # XXX put the chunk below in a separate method - # blow away existing dvoDetection table - # clear the 'ingested' field for all boxes owned by this host self.ippToPspsDb.clearIngestedBoxes(self.scratchDb.dbHost) @@ -607,26 +607,8 @@ ## raise - # drop detections table - self.logger.infoPair("Dropping table", self.scratchDb.dvoDetectionTable) - self.scratchDb.dropTable(self.scratchDb.dvoDetectionTable) - - # create detections table - self.logger.infoPair("Creating table", self.scratchDb.dvoDetectionTable) - sql = "CREATE TABLE " + self.scratchDb.dvoDetectionTable + " LIKE dvoDetection" - try: self.scratchDb.execute(sql) - except: - self.logger.errorPair("Unable to create table", self.scratchDb.dvoDetectionTable) + # blow away existing dvoDetection table & re-crate + if not self.scratchDb.resetDvoDetectionTable(): + self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull") return False - self.scratchDb.changeEngineToInnoDB(self.scratchDb.dvoDetectionTable) - - # add fileID column - sql = "ALTER TABLE " + self.scratchDb.dvoDetectionTable + " ADD fileID INT NOT NULL" - try: self.scratchDb.execute(sql) - except: - self.logger.errorPair("Unable to add fileID column to ", self.scratchDb.dvoDetectionTable) - return False - - # add an index to the fileID column - self.scratchDb.createIndex(self.scratchDb.dvoDetectionTable, "fileID") # make sure we have an up-to-date Images table @@ -665,4 +647,7 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + if p.returncode != 0: + self.logger.errorPair("dvopsps failed on ", self.scratchDb.dvoDetectionTable) + raise ## dvopsps can take a very long time. try 2 or 3 times to re-establish communication with the db before proceeding? @@ -670,5 +655,5 @@ self.ippToPspsDb.setIngestedBox(boxId, self.scratchDb.dbHost) - # add fileID column + # run 'ANALYZE' to get indexes up-to-snuff sql = "ANALYZE TABLE " + self.scratchDb.dvoDetectionTable try: self.scratchDb.execute(sql) Index: /trunk/ippToPsps/jython/dvoobjects.py =================================================================== --- /trunk/ippToPsps/jython/dvoobjects.py (revision 35416) +++ /trunk/ippToPsps/jython/dvoobjects.py (revision 35417) @@ -67,4 +67,8 @@ if self.skychunk.parallel: + if self.scratchDb.dbHost == "localhost": + self.logger.info("dvopsps objects -parallel is incompatible with localhost for scratchdb") + raise + cmd += " -parallel" @@ -72,4 +76,7 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + if p.returncode != 0: + self.logger.errorPair("dvopsps failed on ", region) + raise # update lists after attempted sync Index: /trunk/ippToPsps/jython/fits.py =================================================================== --- /trunk/ippToPsps/jython/fits.py (revision 35416) +++ /trunk/ippToPsps/jython/fits.py (revision 35417) @@ -87,6 +87,13 @@ 2880 is the magic number that the FITS format uses to break up data: extensions always begin on multiples of 2880 bytes + + Note that we may not be able to distinguish the case where a header is expected from one where the absence of a header + just tells us that the header is not available. In some cases, we may not want to emit the error message ("not found") + + XXX WOW! this block is reading the entire file to get the header. + major inefficiency... I thought the FITS interactions all went + through STILTS ''' - def findAndReadHeader(self, name): + def findAndReadHeader(self, name, VERBOSE): found = False @@ -114,5 +121,5 @@ self.file.seek(origIndex, 0) - self.logger.errorPair("Could not read header", name) + if VERBOSE: self.logger.errorPair("Could not read header", name) return else: Index: /trunk/ippToPsps/jython/gpc1db.py =================================================================== --- /trunk/ippToPsps/jython/gpc1db.py (revision 35416) +++ /trunk/ippToPsps/jython/gpc1db.py (revision 35417) @@ -22,5 +22,5 @@ def __init__(self, logger, config): # define database type - if (config.test): + if (config.gpc1test): dbType = "gpc1database_test" else: @@ -104,5 +104,4 @@ try: # XXX EAM : test output - self.logger.infoPair("sql for dvo items:", sql) rs = self.executeQuery(sql) while (rs.next()): @@ -111,4 +110,5 @@ except: self.logger.exception("Can't query for ids in DVO") + self.logger.infoPair("failed sql:", sql) return rows Index: /trunk/ippToPsps/jython/ipptopsps.py =================================================================== --- /trunk/ippToPsps/jython/ipptopsps.py (revision 35416) +++ /trunk/ippToPsps/jython/ipptopsps.py (revision 35417) @@ -55,4 +55,5 @@ raise + print "check client status" self.checkClientStatus() @@ -142,6 +143,8 @@ def checkClientStatus(self): + print "update client" self.ippToPspsDb.updateClient(self.config.programName, self.HOST, self.PID) + print "check killed" if self.ippToPspsDb.isKilled(self.HOST, self.PID): self.exitProgram("killed via Db") @@ -149,4 +152,5 @@ # this loop pauses the process if we have no skychunk or we have set it to pause firstTimeIn = True + print "refresh skychunk" while not self.refreshSkychunk() or self.ippToPspsDb.isPaused(self.HOST, self.PID): Index: /trunk/ippToPsps/jython/loader.py =================================================================== --- /trunk/ippToPsps/jython/loader.py (revision 35416) +++ /trunk/ippToPsps/jython/loader.py (revision 35417) @@ -36,7 +36,7 @@ if self.skychunk.parallel: - print "PARALLEL dvo" + self.logger.info("using PARALLEL DVO") else: - print "SERIAL dvo" + self.logger.info("using SERIAL DVO") # connect to the gpc1 database @@ -198,5 +198,7 @@ # insertion if not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost): - self.dvoDetections.nativeIngestDetections(boxId, boxDim) + if not self.dvoDetections.nativeIngestDetections(boxId, boxDim): + self.logger.error("Unable to ingest detections with nativeIngest") + return False useFullTables = 1 Index: /trunk/ippToPsps/jython/mysql.py =================================================================== --- /trunk/ippToPsps/jython/mysql.py (revision 35416) +++ /trunk/ippToPsps/jython/mysql.py (revision 35417) @@ -46,4 +46,18 @@ self.logger.errorPair("Unable to connect to", connectionStr) self.connected = False + raise + + # determine the mysql major version number: + sql = "SELECT @@version" + + try: + rs = self.executeQuery(sql) + rs.first() + fullVersion = rs.getString(1) + majVersion = fullVersion[0:3] + self.version = float(majVersion) + + except: + print "failed to get version number" raise Index: /trunk/ippToPsps/jython/objectbatch.py =================================================================== --- /trunk/ippToPsps/jython/objectbatch.py (revision 35416) +++ /trunk/ippToPsps/jython/objectbatch.py (revision 35417) @@ -372,5 +372,18 @@ self.logger.infoPair("Forcing uniqueness on", "objID in Object table") rowCountBefore = self.scratchDb.getRowCount("Object") + + # XXX EAM : note that in mysql versions later than 5.1, this fails + # unless the following is called first: + # set session old_alter_table=1 + # follow the command with + # set session old_alter_table=0 + # OF COURSE, this fails for mysql version < 5.5... + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=1") + self.scratchDb.execute("ALTER IGNORE TABLE Object ADD UNIQUE INDEX(objID)") + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=0") + rowCountAfter = self.scratchDb.getRowCount("Object") self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) Index: /trunk/ippToPsps/jython/pslogger.py =================================================================== --- /trunk/ippToPsps/jython/pslogger.py (revision 35416) +++ /trunk/ippToPsps/jython/pslogger.py (revision 35417) @@ -13,5 +13,5 @@ def setup(self, programName, basePath, skychunkName, host, pid, stdout=1, sendToFile=0): - formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S') + formatter = logging.Formatter('%(asctime)s.%(msecs)03d | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S') self.setLevel(logging.INFO) Index: /trunk/ippToPsps/jython/queue.py =================================================================== --- /trunk/ippToPsps/jython/queue.py (revision 35416) +++ /trunk/ippToPsps/jython/queue.py (revision 35417) @@ -29,18 +29,18 @@ super(Queue, self).__init__(argv) - print "inited ipptopsps" + # print "inited ipptopsps" # create various objects self.gpc1Db = Gpc1Db(self.logger, self.config) - print "connect to gpc1" + # print "connect to gpc1" self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb) - print "made datastore" + # print "made datastore" # connect to scratch database self.scratchDb = ScratchDb(self.logger, self.config) - print "connect to scratchdb" + # print "connect to scratchdb" try: @@ -50,5 +50,5 @@ raise - print "got objects" + # print "got objects" self.skychunk.printAll() Index: /trunk/ippToPsps/jython/scratchdb.py =================================================================== --- /trunk/ippToPsps/jython/scratchdb.py (revision 35416) +++ /trunk/ippToPsps/jython/scratchdb.py (revision 35417) @@ -29,5 +29,5 @@ dbType = "localdatabase" - print "trying to connect: ", dbType + # print "trying to connect: ", dbType dbName = config.getDbName(dbType) @@ -38,4 +38,5 @@ self.dvoSkyTable = "dvoSkyTable" self.dvoPhotcodesTable = "dvoPhotcodes" + self.memoryTableSize = config.getDbMemory(dbType) ''' @@ -368,4 +369,45 @@ ''' + Checks whether the given chipID is in the DVO Image table + ''' + def haveThisChip(self, imageID, sourceID): + + sql = "SELECT EXTERN_ID FROM " \ + + self.dvoImagesTable + \ + " WHERE EXTERN_ID = " + str(imageID) + \ + " AND SOURCE_ID = " + str(sourceID) + + try: + rs = self.executeQuery(sql) + + if not rs: + print "missing result set for imageID query" + raise + + # if no returned rows, return an error + if not rs.next(): + print "DVO lacks imageID ", imageID + print "sql: ", sql + return False + + if rs.getInt(1) == int(imageID): + # print "DVO has imageID ", imageID + return True + + else: + print "DVO has wrong imageID? ", imageID + print "result was ", rs.getInt(1) + print "sql: ", sql + raise + + except: + print "DVO imageID query failed ", imageID + print "sql: ", sql + raise + + # how can I reach this? + return True + + ''' Creates a table for for ID matching ''' @@ -409,12 +451,12 @@ objID BIGINT, \ flags INT, \ - zp REAL, \ - zpErr REAL, \ - airMass REAL, \ - expTime REAL, \ - ra FLOAT, \ - dec_ FLOAT, \ - raErr REAL, \ - decErr REAL, \ + zp FLOAT, \ + zpErr FLOAT, \ + airMass FLOAT, \ + expTime FLOAT, \ + ra DOUBLE, \ + dec_ DOUBLE, \ + raErr FLOAT, \ + decErr FLOAT, \ PRIMARY KEY (imageID, ippDetectID), \ KEY (objID, detectID) \ @@ -463,4 +505,42 @@ Drops and recreates tables necessary for dvoToMySQL program. Be very careful before using this... ''' + def resetDvoDetectionTable(self): + + self.logger.infoPair("Dropping table", self.dvoDetectionTable) + self.dropTable(self.dvoDetectionTable) + + # we need a big enough memory table for the incoming detections + print "requested memory in gigs: ", self.memoryTableSize + tableSize = int(self.memoryTableSize) * 1024 * 1024 * 1024 + sql = "set @@max_heap_table_size = " + str(tableSize) + print "setting max heap size: ", sql + try: self.execute(sql) + except: + self.logger.errorPair("Unable to set max MEMORY table size for ", self.dvoDetectionTable) + return False + + # create detections table + self.logger.infoPair("Creating table", self.dvoDetectionTable) + sql = "CREATE TABLE " + self.dvoDetectionTable + " LIKE dvoDetection" + try: self.execute(sql) + except: + self.logger.errorPair("Unable to create table", self.dvoDetectionTable) + return False + + # XXX changing the dvoDetectionFull table to MEMORY + # self.changeEngineToInnoDB(self.dvoDetectionTable) + + # set engine to memory + sql = "ALTER TABLE " + self.dvoDetectionTable + " ENGINE = MEMORY" + try: self.execute(sql) + except: + self.logger.errorPair("Unable to set engine to MEMORY for ", self.dvoDetectionTable) + return False + + return True + + ''' + Drops and recreates tables necessary for dvoToMySQL program. Be very careful before using this... + ''' def resetAllDvoTables(self): @@ -485,35 +565,8 @@ self.changeEngineToInnoDB(self.dvoDoneTable) - # create detections table - self.logger.infoPair("Creating table", self.dvoDetectionTable) - sql = "CREATE TABLE " + self.dvoDetectionTable + " LIKE dvoDetection" - try: self.execute(sql) - except: - self.logger.errorPair("Unable to create table", self.dvoDetectionTable) + # blow away existing dvoDetection table & re-crate + if not self.resetDvoDetectionTable(): + self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull") return False - self.changeEngineToInnoDB(self.dvoDetectionTable) - - # add fileID column - sql = "ALTER TABLE " + self.dvoDetectionTable + " ADD fileID INT NOT NULL" - try: self.execute(sql) - except: - self.logger.errorPair("Unable to add fileID column to ", self.dvoDetectionTable) - return False - - # add an index to the fileID column - self.createIndex(self.dvoDetectionTable, "fileID") - - # XXX : this should be based on choice of useStilts (which is set in dvo.py and must be moved) - if (False): - # now add a delete cascading foreign key constraint on fileID - sql = "ALTER TABLE dvoDetectionFull \ - ADD CONSTRAINT fk_fileID \ - FOREIGN KEY (fileID) \ - REFERENCES dvoDone(id) \ - ON DELETE CASCADE" - try: self.execute(sql) - except: - self.logger.errorPair("Unable to create foreign key on", self.dvoDoneTable) - return False return True Index: /trunk/ippToPsps/jython/skychunk.py =================================================================== --- /trunk/ippToPsps/jython/skychunk.py (revision 35416) +++ /trunk/ippToPsps/jython/skychunk.py (revision 35417) @@ -28,7 +28,7 @@ if self.name == "all": self.rotateSkychunks = True - print "name: ", self.name - print "new: ", self.createNewSkychunk - print "all: ", self.rotateSkychunks + # print "name: ", self.name + # print "new: ", self.createNewSkychunk + # print "all: ", self.rotateSkychunks self.logger = logger Index: /trunk/ippToPsps/jython/stackbatch.py =================================================================== --- /trunk/ippToPsps/jython/stackbatch.py (revision 35416) +++ /trunk/ippToPsps/jython/stackbatch.py (revision 35417) @@ -543,6 +543,18 @@ # response = raw_input("updated dvo ") + # XXX EAM : note that in mysql versions later than 5.1, this fails + # unless the following is called first: + # set session old_alter_table=1 + # follow the command with + # set session old_alter_table=0 + # OF COURSE, this fails for mysql version < 5.5... + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=1") + sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)" + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=0") + self.scratchDb.execute(sql) # response = raw_input("add primary key? ") @@ -806,10 +818,17 @@ ON (a.stackDetectID = b.detectID) where b.imageID = " + str(imageID) + \ " INTO OUTFILE '" + dumpFile + "'" - print "sql: ", sql - self.scratchDb.execute(sql) + + try: self.scratchDb.execute(sql) + except: + self.logger.info("failed to select data for stackDetectionCalib") + self.logger.infoPair("sql: ", sql) + raise sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName - print "sql: ", sql - self.scratchDb.execute(sql) + try: self.scratchDb.execute(sql) + except: + self.logger.info("failed to load data from infile for stackDetectionCalib") + self.logger.infoPair("sql: ", sql) + raise ## XXX write this with the select/insert Index: /trunk/ippToPsps/test/fulltest.sh =================================================================== --- /trunk/ippToPsps/test/fulltest.sh (revision 35416) +++ /trunk/ippToPsps/test/fulltest.sh (revision 35417) @@ -1,15 +1,3 @@ #!/bin/csh -f - -set stackqueuename = 0 -if ($#argv > 0) then - set stackqueuename = $1 - echo "stackqueue: $stackqueuename" -endif - -set objectqueuename = 0 -if ($#argv > 0) then - set stackqueuename = $1 - echo "stackqueue: $stackqueuename" -endif # a bit crude : get the datadir path by interpollation of the PATH element for psconfig @@ -24,10 +12,9 @@ # before running this test, you must have a mysql user: # mkgpc1.sh user localhost eam eam - set mkgpc1 = 1 set initdb = 1 set initscratch = 1 +set initbatch = 1 set camqueue = 1 -set initbatch = 1 set cambatch = 1 set stackqueue = 1 @@ -36,8 +23,48 @@ set objectbatch = 1 +set args = "" + +while ("$1" != "") + switch ("$1") + case -mkgpc1 + set mkgpc1 = 0 + breaksw; + case -initdb + set initdb = 0 + breaksw; + case -initscratch + set initscratch = 0 + breaksw; + case -initbatch + set initbatch = 0 + breaksw; + case -camera + set camqueue = 0 + set cambatch = 0 + breaksw; + case -stack + set stackqueue = 0 + set stackbatch = 0 + breaksw; + case -object + set objectqueue = 0 + set objectbatch = 0 + breaksw; + default: + set args=($args $1); + breaksw; + endsw + shift +end + +set stackqueuename = 0 +set objectqueuename = 0 + set OUTDIR = "testdata" set queuename = "test" if ($mkgpc1) then + echo "" + echo " ---- mkgpc1data.dvo : creating the dvo & gpc1 database entries ----" mkgpc1data.dvo endif @@ -45,4 +72,6 @@ # create the ipptopsps database if ($initdb) then + echo "" + echo " ---- init ipptopsps_test ----" # does the database already exist? @@ -55,4 +84,6 @@ # re-create the ipptopsps database if ($initscratch) then + echo "" + echo " ---- init ipptopsps_test_scratch ----" # create the database (ignore failure if it already exists) @@ -62,6 +93,77 @@ endif +# `name` +# `datastore_product` +# `datastore_type` +# `datastore_publish` +# `dvo_label` +# `dvo_location` +# `min_ra` +# `max_ra` +# `min_dec` +# `max_dec` +# `box_size` +# `base_path` +# `data_release` +# `delete_local` +# `delete_datastore` +# `delete_dxlayer` +# `epoch` +# `survey` +# `psps_survey` +# `queue_P2` +# `queue_ST` +# `queue_OB` +# `active` +# `parallel` + +# this is poor : we need to queue something in order to generate an init batch +# this is because a skychunk carries the info about the datastore used to publish +# the batch +if ($initbatch) then + echo "" + echo " ---- make skychunk for IN batches ----" + + ippjython queue.py -test edit <