Index: /branches/eam_branches/ipp-20130307/ippToPsps/jython/batch.py =================================================================== --- /branches/eam_branches/ipp-20130307/ippToPsps/jython/batch.py (revision 35401) +++ /branches/eam_branches/ipp-20130307/ippToPsps/jython/batch.py (revision 35402) @@ -276,4 +276,7 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + if p.returncode != 0: + self.logger.errorPair("error running xmllint for ", tmpPath) + raise except: self.logger.errorPair("Popen failed", cmd) @@ -301,5 +304,4 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() - if p.returncode != 0: self.logger.errorPair("tar command", "failed") @@ -311,5 +313,4 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() - if p.returncode != 0: self.logger.errorPair("gzip command", "failed") @@ -519,4 +520,7 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + if p.returncode != 0: + self.logger.errorPair("dvograbber command", "failed") + return False rowCount = self.scratchDb.getRowCount("dvoDetection") Index: /branches/eam_branches/ipp-20130307/ippToPsps/jython/config.py =================================================================== --- /branches/eam_branches/ipp-20130307/ippToPsps/jython/config.py (revision 35401) +++ /branches/eam_branches/ipp-20130307/ippToPsps/jython/config.py (revision 35402) @@ -66,6 +66,4 @@ self.czarPlotsPath = self.settingsDoc.find("czarPlotsPath").text - - # print "config.programName: ", self.programName # print "config.configDir: ", self.configDir @@ -125,4 +123,11 @@ return self.settingsDoc.find(dbType +"/password").text + def getDbMemory(self, dbType): + if dbType == 'localdatabase' or dbType == 'localdatabase_test': + return self.scratchDbLeaf.find('memory').text + else: + print "memory is not a valid option for non-scratch databases..." + raise + def setScratchDbLeaf(self, myhost): root = self.settingsDoc.getroot() Index: /branches/eam_branches/ipp-20130307/ippToPsps/jython/datastore.py =================================================================== --- /branches/eam_branches/ipp-20130307/ippToPsps/jython/datastore.py (revision 35401) +++ /branches/eam_branches/ipp-20130307/ippToPsps/jython/datastore.py (revision 35402) @@ -47,4 +47,5 @@ --list " + tempFile.name + print "dsreg cmd: ", command p = Popen(command, shell=True, stdout=PIPE) p.wait() Index: /branches/eam_branches/ipp-20130307/ippToPsps/jython/detectionbatch.py =================================================================== --- /branches/eam_branches/ipp-20130307/ippToPsps/jython/detectionbatch.py (revision 35401) +++ /branches/eam_branches/ipp-20130307/ippToPsps/jython/detectionbatch.py (revision 35402) @@ -909,9 +909,9 @@ return False - if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1 - - if self.skipBatch: - self.logger.error("fatal problem for exposure, skipping") - return False + if "Chip" in self.imageIDs: + if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1 + if self.skipBatch: + self.logger.error("fatal problem for exposure, skipping") + return False # print totals @@ -926,7 +926,7 @@ self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") - # if we only have one table export, i.e. FrameMeta, then get out of here + # if we only have one table export, i.e. FrameMeta, then get out of here (skip batch, but do not abort) if len(self.tablesToExport) == 1: - + self.skipBatch = True self.logger.error("No tables to export") return False Index: /branches/eam_branches/ipp-20130307/ippToPsps/jython/dvo.py =================================================================== --- /branches/eam_branches/ipp-20130307/ippToPsps/jython/dvo.py (revision 35401) +++ /branches/eam_branches/ipp-20130307/ippToPsps/jython/dvo.py (revision 35402) @@ -193,4 +193,7 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + if p.returncode != 0: + self.logger.errorPair("dvopsps failed on SkyTable.fits for ", self.skychunk.dvoLocation) + raise self.logger.infoPair("Adding index to", self.scratchDb.dvoSkyTable) Index: /branches/eam_branches/ipp-20130307/ippToPsps/jython/dvoobjects.py =================================================================== --- /branches/eam_branches/ipp-20130307/ippToPsps/jython/dvoobjects.py (revision 35401) +++ /branches/eam_branches/ipp-20130307/ippToPsps/jython/dvoobjects.py (revision 35402) @@ -67,4 +67,8 @@ if self.skychunk.parallel: + if self.scratchDb.dbHost == "localhost": + self.logger.info("dvopsps objects -parallel is incompatible with localhost for scratchdb") + raise + cmd += " -parallel" @@ -72,4 +76,7 @@ p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + if p.returncode != 0: + self.logger.errorPair("dvopsps failed on ", region) + raise # update lists after attempted sync Index: /branches/eam_branches/ipp-20130307/ippToPsps/jython/loader.py =================================================================== --- /branches/eam_branches/ipp-20130307/ippToPsps/jython/loader.py (revision 35401) +++ /branches/eam_branches/ipp-20130307/ippToPsps/jython/loader.py (revision 35402) @@ -198,5 +198,7 @@ # insertion if not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost): - self.dvoDetections.nativeIngestDetections(boxId, boxDim) + if not self.dvoDetections.nativeIngestDetections(boxId, boxDim): + self.logger.error("Unable to ingest detections with nativeIngest") + return False useFullTables = 1 Index: /branches/eam_branches/ipp-20130307/ippToPsps/jython/mysql.py =================================================================== --- /branches/eam_branches/ipp-20130307/ippToPsps/jython/mysql.py (revision 35401) +++ /branches/eam_branches/ipp-20130307/ippToPsps/jython/mysql.py (revision 35402) @@ -55,5 +55,6 @@ rs.first() fullVersion = rs.getString(1) - self.version = fullVersion[0:3] + majVersion = fullVersion[0:3] + self.version = float(majVersion) except: @@ -61,5 +62,4 @@ raise - # print "version number: ", self.version self.connected = True Index: /branches/eam_branches/ipp-20130307/ippToPsps/jython/scratchdb.py =================================================================== --- /branches/eam_branches/ipp-20130307/ippToPsps/jython/scratchdb.py (revision 35401) +++ /branches/eam_branches/ipp-20130307/ippToPsps/jython/scratchdb.py (revision 35402) @@ -38,4 +38,5 @@ self.dvoSkyTable = "dvoSkyTable" self.dvoPhotcodesTable = "dvoPhotcodes" + self.memoryTableSize = config.getDbMemory(dbType) ''' @@ -509,4 +510,14 @@ self.dropTable(self.dvoDetectionTable) + # we need a big enough memory table for the incoming detections + print "requested memory in gigs: ", self.memoryTableSize + tableSize = int(self.memoryTableSize) * 1024 * 1024 * 1024 + sql = "set @@max_heap_table_size = " + str(tableSize) + print "setting max heap size: ", sql + try: self.execute(sql) + except: + self.logger.errorPair("Unable to set max MEMORY table size for ", self.dvoDetectionTable) + return False + # create detections table self.logger.infoPair("Creating table", self.dvoDetectionTable)