Index: /trunk/ippToPsps/jython/dvo.py =================================================================== --- /trunk/ippToPsps/jython/dvo.py (revision 39466) +++ /trunk/ippToPsps/jython/dvo.py (revision 39467) @@ -95,12 +95,12 @@ self.correctDvo = True if not self.loadSkyTable(): - self.correctDvo = False - return + os._exit(1) + if not self.loadPhotcodes(): - self.correctDvo = False - return + os._exit(1) + if not self.loadImages(): - self.correctDvo = False - return + os._exit(1) + ''' @@ -108,11 +108,8 @@ ''' def loadPhotcodes(self): - - # go no further if we've already partly ingested a different DVO - if not self.correctDvo: return if (self.skychunk.dvoLocation == 'none'): # print "do not load Photcodes for place-holder DVO " + self.skychunk.dvoLocation - return False + return True # check if we have up-to-date version @@ -120,5 +117,5 @@ if self.scratchDb.alreadyImportedThisDvoTable(path): self.logger.debugPair("DVO Photcodes.dat file", "up-to-date") - return + return True self.logger.infoSeparator() @@ -132,4 +129,5 @@ self.scratchDb.setImportedThisDvoTable(path) + return True ''' @@ -137,7 +135,4 @@ ''' def loadImages(self): - - # go no further if we've already partly ingested a different DVO - if not self.correctDvo: return # check if we have up-to-date version @@ -145,5 +140,5 @@ if self.scratchDb.alreadyImportedThisDvoTable(path): self.logger.debugPair("DVO Images.dat file", "up-to-date") - return + return True self.logger.infoSeparator() @@ -158,4 +153,5 @@ self.scratchDb.setImportedThisDvoTable(path) + return True ''' @@ -227,15 +223,12 @@ def loadSkyTable(self): - # go no further if we've already partly ingested a different DVO - if not self.correctDvo: return False - if (self.skychunk.dvoLocation == 'none'): # print "do not load SkyTable for place-holder DVO " + self.skychunk.dvoLocation - return False + return True path = self.skychunk.dvoLocation + "/SkyTable.fits" if self.scratchDb.alreadyImportedThisDvoTable(path): self.logger.debugPair("DVO SkyTable.fits file", "up-to-date") - return False + return True # self.logger.infoSeparator() @@ -296,7 +289,4 @@ def setSkyArea(self, minRa=-1., maxRa=361., minDec=-91., maxDec=91): - # go no further if we've already partly ingested a different DVO - if not self.correctDvo: return - if (maxRa - minRa <= 0) or (maxDec - minDec) <= 0: self.logger.errorPair("Zero range in either RA or Dec", Index: /trunk/ippToPsps/jython/forcedwarpbatch.py =================================================================== --- /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 39466) +++ /trunk/ippToPsps/jython/forcedwarpbatch.py (revision 39467) @@ -211,11 +211,21 @@ self.logger.infoPair("Processing table", tableName) + self.logger.infoPair("creating table", tableName) + sql = "CREATE TABLE " + tableName + " LIKE ForcedWarpMeta" - self.scratchDb.execute(sql) + try: self.scratchDb.execute(sql) + except: + self.logger.error("problem with SQL: " + sql) + raise header = self.header[num] self.logger.infoPair("found header","ok") + + photoCalID = str(self.scratchDb.getPhotoCalID(header['IMAGEID'])) + if photoCalID == -1: + self.logger.infoPair("image not found in DVO, skipping", str(num)) + return False # PHOTCODE and FILTER are not in the header; reconstruct here @@ -299,5 +309,6 @@ # note hardcoded pltscale - self.scratchDb.updateRecalStats(tableName, header['IMAGEID'], "0.25"); + if photoCalID != -1: + self.scratchDb.updateRecalStats(tableName, header['IMAGEID'], "0.25"); self.scratchDb.updateAllRows(tableName, "batchID", str(self.batchID)) @@ -307,4 +318,6 @@ self.scratchDb.updateAllRows(tableName, "processingVersion", str(self.skychunk.processingVersion)) # self.tablesToExport.append("ForcedWarpMeta") + #return false + return True ''' @@ -331,4 +344,5 @@ self.logger.infoPair("can't execute sql",sql) raise + return True ''' @@ -364,7 +378,10 @@ sqlLine.group("FpsfFlux", "PSF_INST_FLUX / " + expTimeString) sqlLine.group("FpsfFluxErr", "PSF_INST_FLUX_SIG / " + expTimeString) - sqlLine.group("xPosChip", "SRC_CHIP_X") - sqlLine.group("yPosChip", "SRC_CHIP_Y") - sqlLine.group("FccdID" , "SRC_CHIP_NUM") + + if self.haveSrcPos[num]: + sqlLine.group("xPosChip", "SRC_CHIP_X") + sqlLine.group("yPosChip", "SRC_CHIP_Y") + sqlLine.group("FccdID" , "SRC_CHIP_NUM") + sqlLine.group("FpsfMajorFWHM", "PSF_FWHM_MAJ * abs(PLTSCALE)") sqlLine.group("FpsfMinorFWHM", "PSF_FWHM_MIN * abs(PLTSCALE)") @@ -473,4 +490,5 @@ self.scratchDb.createIndex(maskTableName, "objID") self.scratchDb.createIndex(maskTableName, "ippDetectID") + return True ''' @@ -529,4 +547,5 @@ results = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "objID") self.scratchDb.createIndex(pspsTableName, "objID") + return True ''' @@ -556,16 +575,18 @@ sqlLine.group("lensObjShearE1", "LENS_E1_SH_OBJ") sqlLine.group("lensObjShearE2", "LENS_E2_SH_OBJ") - sqlLine.group("lensPSFSmearX11", "LENS_X11_SM_PSF") - sqlLine.group("lensPSFSmearX12", "LENS_X12_SM_PSF") - sqlLine.group("lensPSFSmearX22", "LENS_X22_SM_PSF") - sqlLine.group("lensPSFSmearE1", "LENS_E1_SM_PSF") - sqlLine.group("lensPSFSmearE2", "LENS_E2_SM_PSF") - sqlLine.group("lensPSFShearX11", "LENS_X11_SH_PSF") - sqlLine.group("lensPSFShearX12", "LENS_X12_SH_PSF") - sqlLine.group("lensPSFShearX22", "LENS_X22_SH_PSF") - sqlLine.group("lensPSFShearE1", "LENS_E1_SH_PSF") - sqlLine.group("lensPSFShearE2", "LENS_E2_SH_PSF") - sqlLine.group("psfE1", "LENS_E1_PSF") - sqlLine.group("psfE2", "LENS_E2_PSF") + + if self.haveLensPSF[num]: + sqlLine.group("lensPSFSmearX11", "LENS_X11_SM_PSF") + sqlLine.group("lensPSFSmearX12", "LENS_X12_SM_PSF") + sqlLine.group("lensPSFSmearX22", "LENS_X22_SM_PSF") + sqlLine.group("lensPSFSmearE1", "LENS_E1_SM_PSF") + sqlLine.group("lensPSFSmearE2", "LENS_E2_SM_PSF") + sqlLine.group("lensPSFShearX11", "LENS_X11_SH_PSF") + sqlLine.group("lensPSFShearX12", "LENS_X12_SH_PSF") + sqlLine.group("lensPSFShearX22", "LENS_X22_SH_PSF") + sqlLine.group("lensPSFShearE1", "LENS_E1_SH_PSF") + sqlLine.group("lensPSFShearE2", "LENS_E2_SH_PSF") + sqlLine.group("psfE1", "LENS_E1_PSF") + sqlLine.group("psfE2", "LENS_E2_PSF") sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) @@ -597,4 +618,5 @@ self.scratchDb.createIndex(pspsTableName, "objID") + return True ''' @@ -611,10 +633,14 @@ myTable = "ForcedWarpMeta_" + str(num) self.logger.infoPair("Populating", myTable) - self.populateForcedWarpMeta(num) + if not self.populateForcedWarpMeta(num): + self.logger.infoPair ("failure for ForcedWarpMeta, skipping ", str(num)) + continue self.tablesToExport.append(myTable) myTable = "ForcedWarpMeasurement_" + str(num) self.logger.infoPair("Populating", myTable) - self.populateForcedWarpMeasurement(num) + if not self.populateForcedWarpMeasurement(num): + self.logger.infoPair ("skippping ForcedWarpMeasurement", str(num)) + continue self.tablesToExport.append(myTable) @@ -629,9 +655,10 @@ self.populateForcedWarpExtended(num) self.tablesToExport.append(myTable) - - myTable = "ForcedWarpLensing_" + str(num) - self.logger.infoPair("Populating", myTable) - self.populateForcedWarpLensing(num) - self.tablesToExport.append(myTable) + + if self.haveLensing[num]: + myTable = "ForcedWarpLensing_" + str(num) + self.logger.infoPair("Populating", myTable) + self.populateForcedWarpLensing(num) + self.tablesToExport.append(myTable) myTable = "ForcedWarpToImage_" + str(num) @@ -667,9 +694,11 @@ def importIppTablesOhana(self): - self.logger.infoPair("Importing DF tables", "using fits_to_mysql") + self.logger.infoPair("Importing FW tables", "using fits_to_mysql") count = 0 self.tablesLoaded = [] - self.haveApNpix = {} + self.haveSrcPos = {} + self.haveLensPSF = {} + self.haveLensing = {} for num in self.number: @@ -699,7 +728,25 @@ # this is an optional field, check for it in the table below - self.haveApNpix[num] = 0 - + self.haveSrcPos[num] = 0 + self.haveLensPSF[num] = 0 + self.haveLensing[num] = 0 for table in tables: + # check for AP_NPIX columns: missing for SAS39 + self.logger.debugPair("checking table", table.name) + if table.name == "SkyChip.psf": + columns = table.columns() + for column in columns: + columnString = str(column) + if columnString == 'LENS_X11_SM_PSF': + self.haveLensPSF[num] = 1 + self.logger.debugPair("found LENS_X11_SM_PSF in file", fileName) + if columnString == 'LENS_X11_SM_OBJ': + self.haveLensing[num] = 1 + self.logger.debugPair("found LENS_X11_SM_OBJ in file", fileName) + if columnString == 'SRC_CHIP_X': + self.haveSrcPos[num] = 1 + self.logger.debugPair("found SRC_CHIP_X in file", fileName) + + myTable = table.name + "_" + str(num) count = count + 1 Index: /trunk/ippToPsps/jython/scratchdb.py =================================================================== --- /trunk/ippToPsps/jython/scratchdb.py (revision 39466) +++ /trunk/ippToPsps/jython/scratchdb.py (revision 39467) @@ -139,6 +139,6 @@ if name == 'NONE': - self.logger.errorPair("No valid PSF model ID: ", name) - return -1 + self.logger.infoPair("No valid PSF model ID: ", name) + return 0 sql = "SELECT fitModelID FROM FitModel WHERE name = '" + name + "'" @@ -252,5 +252,5 @@ if not rs.first(): - self.logger.infoPair("no zero point; image is missing, imageID: ", imageID) + self.logger.infoPair("no zero point; image is missing, externID: ", externID) return zeroPoint @@ -334,5 +334,5 @@ rs.first() if rs.getInt(1) > 0: - self.logger.debugPair("Already imported up-to-date version of", path) + self.logger.infoPair("Already imported up-to-date version of", path) return True else: Index: /trunk/ippToPsps/jython/sqlUtility.py =================================================================== --- /trunk/ippToPsps/jython/sqlUtility.py (revision 39466) +++ /trunk/ippToPsps/jython/sqlUtility.py (revision 39467) @@ -55,5 +55,8 @@ for i in range(len(self.values)): - output += "'" + self.values[i] + "'" + if self.values[i] == "NULL": + output += "NULL" + else: + output += "'" + self.values[i] + "'" if (i < len(self.values) - 1): output += ", "