Changeset 37246 for trunk/ippToPsps/jython/detectionbatch.py
- Timestamp:
- Aug 12, 2014, 4:10:18 PM (12 years ago)
- File:
-
- 1 edited
-
trunk/ippToPsps/jython/detectionbatch.py (modified) (19 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippToPsps/jython/detectionbatch.py
r37106 r37246 19 19 from ipptopspsdb import IppToPspsDb 20 20 from scratchdb import ScratchDb 21 from sqlUtility import sqlUtility 21 22 22 23 import logging.config … … 41 42 scratchDb, 42 43 camID, 43 batchID, 44 useFullTables): 44 batchID): 45 45 46 46 super(DetectionBatch, self).__init__( … … 54 54 batchID, 55 55 "P2", 56 gpc1Db.getCameraStageSmf(camID,skychunk.P2_smf_version), 57 useFullTables) 56 gpc1Db.getCameraStageSmf(camID,skychunk.P2_smf_version)) 58 57 59 58 # get camera meta data … … 99 98 raise 100 99 100 # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2 101 101 self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0) 102 102 … … 124 124 self.logger.infoPair("Proccesing table", "FrameMeta") 125 125 126 sql = "INSERT INTO FrameMeta (\ 127 frameID \ 128 ,frameName \ 129 ,cameraID \ 130 ,cameraConfigID \ 131 ,telescopeID \ 132 ,analysisVer \ 133 ,p1Recip \ 134 ,p2Recip \ 135 ,p3Recip \ 136 ,photoScat \ 137 ,expStart \ 138 ,expTime \ 139 ,airmass \ 140 ,raBore \ 141 ,decBore \ 142 ,ctype1 \ 143 ,ctype2 \ 144 ,crval1 \ 145 ,crval2 \ 146 ,crpix1 \ 147 ,crpix2 \ 148 ,cdelt1 \ 149 ,cdelt2 \ 150 ,pc001001 \ 151 ,pc001002 \ 152 ,pc002001 \ 153 ,pc002002 \ 154 ,polyOrder \ 155 ,pca1x3y0 \ 156 ,pca1x2y1 \ 157 ,pca1x1y2 \ 158 ,pca1x0y3 \ 159 ,pca1x2y0 \ 160 ,pca1x1y1 \ 161 ,pca1x0y2 \ 162 ,pca2x3y0 \ 163 ,pca2x2y1 \ 164 ,pca2x1y2 \ 165 ,pca2x0y3 \ 166 ,pca2x2y0 \ 167 ,pca2x1y1 \ 168 ,pca2x0y2 \ 169 ) VALUES ( \ 170 " + str(self.expID) + " \ 171 ,'" + self.expName + "' \ 172 ,1 \ 173 ,1 \ 174 ,1 \ 175 ,'" + str(self.analysisVer) + "' \ 176 ,' ' \ 177 ,' ' \ 178 ,' ' \ 179 ," + self.safeDictionaryAccess(self.header, 'ZPT_ERR') + " \ 180 ," + self.safeDictionaryAccess(self.header, 'MJD-OBS') + " \ 181 ," + self.safeDictionaryAccess(self.header, 'EXPREQ') + " \ 182 ," + self.safeDictionaryAccess(self.header, 'AIRMASS') + " \ 183 ," + self.safeDictionaryAccess(self.header, 'RA') + " \ 184 ," + self.safeDictionaryAccess(self.header, 'DEC') + " \ 185 ,'" + self.safeDictionaryAccess(self.header, 'CTYPE1') + "' \ 186 ,'" + self.safeDictionaryAccess(self.header, 'CTYPE2') + "' \ 187 ," + self.safeDictionaryAccess(self.header, 'CRVAL1') + " \ 188 ," + self.safeDictionaryAccess(self.header, 'CRVAL2') + " \ 189 ," + self.safeDictionaryAccess(self.header, 'CRPIX1') + " \ 190 ," + self.safeDictionaryAccess(self.header, 'CRPIX2') + " \ 191 ," + self.safeDictionaryAccess(self.header, 'CDELT1') + " \ 192 ," + self.safeDictionaryAccess(self.header, 'CDELT2') + " \ 193 ," + self.safeDictionaryAccess(self.header, 'PC001001') + " \ 194 ," + self.safeDictionaryAccess(self.header, 'PC001002') + " \ 195 ," + self.safeDictionaryAccess(self.header, 'PC002001') + " \ 196 ," + self.safeDictionaryAccess(self.header, 'PC002002') + " \ 197 ," + self.safeDictionaryAccess(self.header, 'NPLYTERM') + " \ 198 ," + self.safeDictionaryAccess(self.header, 'PCA1X3Y0') + " \ 199 ," + self.safeDictionaryAccess(self.header, 'PCA1X2Y1') + " \ 200 ," + self.safeDictionaryAccess(self.header, 'PCA1X1Y2') + " \ 201 ," + self.safeDictionaryAccess(self.header, 'PCA1X0Y3') + " \ 202 ," + self.safeDictionaryAccess(self.header, 'PCA1X2Y0') + " \ 203 ," + self.safeDictionaryAccess(self.header, 'PCA1X1Y1') + " \ 204 ," + self.safeDictionaryAccess(self.header, 'PCA1X0Y2') + " \ 205 ," + self.safeDictionaryAccess(self.header, 'PCA2X3Y0') + " \ 206 ," + self.safeDictionaryAccess(self.header, 'PCA2X2Y1') + " \ 207 ," + self.safeDictionaryAccess(self.header, 'PCA2X1Y2') + " \ 208 ," + self.safeDictionaryAccess(self.header, 'PCA2X0Y3') + " \ 209 ," + self.safeDictionaryAccess(self.header, 'PCA2X2Y0') + " \ 210 ," + self.safeDictionaryAccess(self.header, 'PCA2X1Y1') + " \ 211 ," + self.safeDictionaryAccess(self.header, 'PCA2X0Y2') + " \ 212 )" 213 214 # print "frame meta sql: ", sql 215 216 self.scratchDb.execute(sql) 126 sqlLine = sqlUtility("INSERT INTO FrameMeta (") 127 128 # sqlLine.group("p1Recip", ' '); 129 # sqlLine.group("p2Recip", ' '); 130 # sqlLine.group("p3Recip", ' '); 131 132 sqlLine.group("frameID", str(self.expID)); 133 sqlLine.group("frameName", self.expName); 134 sqlLine.group("cameraID", "1"); 135 sqlLine.group("cameraConfigID", "1"); 136 sqlLine.group("telescopeID", "1"); 137 sqlLine.group("analysisVer", self.analysisVer); 138 sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR')); 139 sqlLine.group("expStart", self.getKeyFloat(self.header, "%.10f", 'MJD-OBS')); 140 sqlLine.group("expTime", self.getKeyFloat(self.header, "%.8f", 'EXPREQ')); 141 sqlLine.group("airmass", self.getKeyFloat(self.header, "%.8f", 'AIRMASS')); 142 sqlLine.group("raBore", self.getKeyFloat(self.header, "%.10f", 'RA')); 143 sqlLine.group("decBore", self.getKeyFloat(self.header, "%.10f", 'DEC')); 144 sqlLine.group("ctype1", self.getKeyValue(self.header, 'CTYPE1')); 145 sqlLine.group("ctype2", self.getKeyValue(self.header, 'CTYPE2')); 146 sqlLine.group("crval1", self.getKeyFloat(self.header, "%.8f", 'CRVAL1')); 147 sqlLine.group("crval2", self.getKeyFloat(self.header, "%.8f", 'CRVAL2')); 148 sqlLine.group("crpix1", self.getKeyFloat(self.header, "%.8f", 'CRPIX1')); 149 sqlLine.group("crpix2", self.getKeyFloat(self.header, "%.8f", 'CRPIX2')); 150 sqlLine.group("cdelt1", self.getKeyFloat(self.header, "%.8e", 'CDELT1')); 151 sqlLine.group("cdelt2", self.getKeyFloat(self.header, "%.8e", 'CDELT2')); 152 sqlLine.group("pc001001", self.getKeyFloat(self.header, "%.8e", 'PC001001')); 153 sqlLine.group("pc001002", self.getKeyFloat(self.header, "%.8e", 'PC001002')); 154 sqlLine.group("pc002001", self.getKeyFloat(self.header, "%.8e", 'PC002001')); 155 sqlLine.group("pc002002", self.getKeyFloat(self.header, "%.8e", 'PC002002')); 156 sqlLine.group("polyOrder", self.getKeyValue(self.header, 'NPLYTERM')); 157 sqlLine.group("pca1x3y0", self.getKeyFloat(self.header, "%.8e", 'PCA1X3Y0')); 158 sqlLine.group("pca1x2y1", self.getKeyFloat(self.header, "%.8e", 'PCA1X2Y1')); 159 sqlLine.group("pca1x1y2", self.getKeyFloat(self.header, "%.8e", 'PCA1X1Y2')); 160 sqlLine.group("pca1x0y3", self.getKeyFloat(self.header, "%.8e", 'PCA1X0Y3')); 161 sqlLine.group("pca1x2y0", self.getKeyFloat(self.header, "%.8e", 'PCA1X2Y0')); 162 sqlLine.group("pca1x1y1", self.getKeyFloat(self.header, "%.8e", 'PCA1X1Y1')); 163 sqlLine.group("pca1x0y2", self.getKeyFloat(self.header, "%.8e", 'PCA1X0Y2')); 164 sqlLine.group("pca2x3y0", self.getKeyFloat(self.header, "%.8e", 'PCA2X3Y0')); 165 sqlLine.group("pca2x2y1", self.getKeyFloat(self.header, "%.8e", 'PCA2X2Y1')); 166 sqlLine.group("pca2x1y2", self.getKeyFloat(self.header, "%.8e", 'PCA2X1Y2')); 167 sqlLine.group("pca2x0y3", self.getKeyFloat(self.header, "%.8e", 'PCA2X0Y3')); 168 sqlLine.group("pca2x2y0", self.getKeyFloat(self.header, "%.8e", 'PCA2X2Y0')); 169 sqlLine.group("pca2x1y1", self.getKeyFloat(self.header, "%.8e", 'PCA2X1Y1')); 170 sqlLine.group("pca2x0y2", self.getKeyFloat(self.header, "%.8e", 'PCA2X0Y2')); 171 172 sql = sqlLine.make(") VALUES ( ", ")") 173 174 try: self.scratchDb.execute(sql) 175 except: 176 self.logger.errorPair('failed sql: ', sql) 177 raise 178 217 179 self.scratchDb.updateAllRows("FrameMeta", "batchID", str(self.batchID)) 218 180 self.scratchDb.updateAllRows("FrameMeta", "surveyID", str(self.surveyID)) … … 228 190 tableName = "ImageMeta_" + ota 229 191 230 # XXX we drop the table above so it is not left behind on failure 231 # drop then re-create table 232 # self.scratchDb.dropTable(tableName) 192 # we drop the table before calling this functoin so it is not left behind on failure 233 193 sql = "CREATE TABLE " + tableName + " LIKE ImageMeta" 234 194 try: self.scratchDb.execute(sql) 235 195 except: pass 196 236 197 if (ota[0:2] == "XY"): ccdID = ota[2:4] 237 198 else: ccdID = 0 238 199 200 psfFwhmMajor = self.getKeyFloat(header, "%.8f", 'FWHM_MAJ') 201 psfFwhmMinor = self.getKeyFloat(header, "%.8f", 'FWHM_MIN') 202 psfFwhm = 0.5*(float(psfFwhmMajor) + float(psfFwhmMinor)) 203 239 204 # insert image metadata into table 240 sql = "INSERT INTO " + tableName + " ( \ 241 frameID \ 242 ,ccdID \ 243 ,bias \ 244 ,biasScat \ 245 ,sky \ 246 ,skyScat \ 247 ,magSat \ 248 ,completMag \ 249 ,astroScat \ 250 ,photoScat \ 251 ,numAstroRef \ 252 ,numPhotoRef \ 253 ,nx \ 254 ,ny \ 255 ,psfFwhm \ 256 ,psfWidMajor \ 257 ,psfWidMinor \ 258 ,psfTheta \ 259 ,momentXX \ 260 ,momentXY \ 261 ,momentYY \ 262 ,momentM3C \ 263 ,momentM3S \ 264 ,momentM4C \ 265 ,momentM4S \ 266 ,momentR1 \ 267 ,momentRH \ 268 ,apResid \ 269 ,dapResid \ 270 ,detectorID \ 271 ,qaFlags \ 272 ,detrend1 \ 273 ,detrend2 \ 274 ,detrend3 \ 275 ,detrend4 \ 276 ,detrend5 \ 277 ,detrend6 \ 278 ,detrend7 \ 279 ,detrend8 \ 280 ,photoZero \ 281 ,ctype1 \ 282 ,ctype2 \ 283 ,crval1 \ 284 ,crval2 \ 285 ,crpix1 \ 286 ,crpix2 \ 287 ,cdelt1 \ 288 ,cdelt2 \ 289 ,pc001001 \ 290 ,pc001002 \ 291 ,pc002001 \ 292 ,pc002002 \ 293 ,polyOrder \ 294 ,pca1x3y0 \ 295 ,pca1x2y1 \ 296 ,pca1x1y2 \ 297 ,pca1x0y3 \ 298 ,pca1x2y0 \ 299 ,pca1x1y1 \ 300 ,pca1x0y2 \ 301 ,pca2x3y0 \ 302 ,pca2x2y1 \ 303 ,pca2x1y2 \ 304 ,pca2x0y3 \ 305 ,pca2x2y0 \ 306 ,pca2x1y1 \ 307 ,pca2x0y2 \ 308 ) VALUES ( \ 309 " + str(self.expID) + " \ 310 ," + str(ccdID) + " \ 311 ," + str(self.bias) + " \ 312 ," + str(self.biasScat) + " \ 313 ," + self.safeDictionaryAccess(header, 'MSKY_MN') + " \ 314 ," + self.safeDictionaryAccess(header, 'MSKY_SIG') + " \ 315 ," + self.safeDictionaryAccess(header, 'FSATUR') + " \ 316 ," + self.safeDictionaryAccess(header, 'FLIMIT') + " \ 317 ," + self.safeDictionaryAccess(header, 'CERROR') + " \ 318 ," + self.safeDictionaryAccess(self.header, 'ZPT_OBS') + " \ 319 ," + self.safeDictionaryAccess(header, 'NASTRO') + " \ 320 ," + self.safeDictionaryAccess(header, 'NASTRO') + " \ 321 ," + self.safeDictionaryAccess(header, 'CNAXIS1') + " \ 322 ," + self.safeDictionaryAccess(header, 'CNAXIS2') + " \ 323 , ( " + self.safeDictionaryAccess(header, 'FWHM_MAJ') + " + " + self.safeDictionaryAccess(header, 'FWHM_MIN') + " ) / 2.0 \ 324 ," + self.safeDictionaryAccess(header, 'FWHM_MAJ') + " \ 325 ," + self.safeDictionaryAccess(header, 'FWHM_MIN') + " \ 326 ," + self.safeDictionaryAccess(header, 'ANGLE') + " \ 327 ,-999 \ 328 ,-999 \ 329 ,-999 \ 330 ,-999 \ 331 ,-999 \ 332 ,-999 \ 333 ,-999 \ 334 ,-999 \ 335 ,-999 \ 336 ," + self.safeDictionaryAccess(header, 'APMIFIT') + " \ 337 ," + self.safeDictionaryAccess(header, 'DAPMIFIT') + " \ 338 ,'" + self.safeDictionaryAccess(header, 'DETECTOR') + "' \ 339 ," + str(self.scratchDb.getDvoImageFlags(header['IMAGEID'])) + " \ 340 ,'" + self.safeDictionaryAccess(header, 'DETREND.MASK') + "' \ 341 ,'" + self.safeDictionaryAccess(header, 'DETREND.DARK') + "' \ 342 ,'" + self.safeDictionaryAccess(header, 'DETREND.FLAT') + "' \ 343 ,' ' \ 344 ,' ' \ 345 ,' ' \ 346 ,' ' \ 347 ,' ' \ 348 ," + self.safeDictionaryAccess(self.header, 'ZPT_OBS') + " \ 349 ,'" + self.safeDictionaryAccess(header, 'CTYPE1') + "' \ 350 ,'" + self.safeDictionaryAccess(header, 'CTYPE2') + "' \ 351 ," + self.safeDictionaryAccess(header, 'CRVAL1') + " \ 352 ," + self.safeDictionaryAccess(header, 'CRVAL2') + " \ 353 ," + self.safeDictionaryAccess(header, 'CRPIX1') + " \ 354 ," + self.safeDictionaryAccess(header, 'CRPIX2') + " \ 355 ," + self.safeDictionaryAccess(header, 'CDELT1') + " \ 356 ," + self.safeDictionaryAccess(header, 'CDELT2') + " \ 357 ," + self.safeDictionaryAccess(header, 'PC001001') + " \ 358 ," + self.safeDictionaryAccess(header, 'PC001002') + " \ 359 ," + self.safeDictionaryAccess(header, 'PC002001') + " \ 360 ," + self.safeDictionaryAccess(header, 'PC002002') + " \ 361 ," + self.safeDictionaryAccess(header, 'NPLYTERM') + " \ 362 ," + self.safeDictionaryAccess(header, 'PCA1X3Y0') + " \ 363 ," + self.safeDictionaryAccess(header, 'PCA1X2Y1') + " \ 364 ," + self.safeDictionaryAccess(header, 'PCA1X1Y2') + " \ 365 ," + self.safeDictionaryAccess(header, 'PCA1X0Y3') + " \ 366 ," + self.safeDictionaryAccess(header, 'PCA1X2Y0') + " \ 367 ," + self.safeDictionaryAccess(header, 'PCA1X1Y1') + " \ 368 ," + self.safeDictionaryAccess(header, 'PCA1X0Y2') + " \ 369 ," + self.safeDictionaryAccess(header, 'PCA2X3Y0') + " \ 370 ," + self.safeDictionaryAccess(header, 'PCA2X2Y1') + " \ 371 ," + self.safeDictionaryAccess(header, 'PCA2X1Y2') + " \ 372 ," + self.safeDictionaryAccess(header, 'PCA2X0Y3') + " \ 373 ," + self.safeDictionaryAccess(header, 'PCA2X2Y0') + " \ 374 ," + self.safeDictionaryAccess(header, 'PCA2X1Y1') + " \ 375 ," + self.safeDictionaryAccess(header, 'PCA2X0Y2') + " \ 376 )" 205 sqlLine = sqlUtility("INSERT INTO " + tableName + "(") 206 207 sqlLine.group("frameID", str(self.expID)) 208 sqlLine.group("ccdID", str(ccdID)) 209 sqlLine.group("bias", str(self.bias)) 210 sqlLine.group("biasScat", str(self.biasScat)) 211 sqlLine.group("sky", self.getKeyFloat(header, "%.8f", 'MSKY_MN')) 212 sqlLine.group("skyScat", self.getKeyFloat(header, "%.8f", 'MSKY_SIG')) 213 sqlLine.group("magSat", self.getKeyFloat(header, "%.8f", 'FSATUR')) 214 sqlLine.group("completMag", self.getKeyFloat(header, "%.8f", 'FLIMIT')) 215 sqlLine.group("astroScat", self.getKeyFloat(header, "%.8f", 'CERROR')) 216 sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS')) 217 sqlLine.group("numAstroRef", self.getKeyValue(header, 'NASTRO')) 218 sqlLine.group("numPhotoRef", self.getKeyValue(header, 'NASTRO')) 219 sqlLine.group("nx", self.getKeyInt(header, 0, 'CNAXIS1')) 220 sqlLine.group("ny", self.getKeyInt(header, 0, 'CNAXIS2')) 221 sqlLine.group("psfFwhm", str(psfFwhm)) 222 sqlLine.group("psfWidMajor", psfFwhmMajor) 223 sqlLine.group("psfWidMinor", psfFwhmMinor) 224 sqlLine.group("psfTheta", self.getKeyFloat(header, "%.8f", 'ANGLE')) 225 sqlLine.group("momentMajor", self.getKeyFloat(header, "%.8f", 'IQ_FW1')) 226 sqlLine.group("momentMinor", self.getKeyFloat(header, "%.8f", 'IQ_FW2')) 227 sqlLine.group("momentM2C", self.getKeyFloat(header, "%.8f", 'IQ_M2C')) 228 sqlLine.group("momentM2S", self.getKeyFloat(header, "%.8f", 'IQ_M2S')) 229 sqlLine.group("momentM3", self.getKeyFloat(header, "%.8f", 'IQ_M3')) 230 sqlLine.group("momentM4", self.getKeyFloat(header, "%.8f", 'IQ_M4')) 231 sqlLine.group("apResid", self.getKeyFloat(header, "%.8f", 'APMIFIT')) 232 sqlLine.group("dapResid", self.getKeyFloat(header, "%.8f", 'DAPMIFIT')) 233 sqlLine.group("detectorID", self.getKeyValue(header, 'DETECTOR')) 234 sqlLine.group("qaFlags", str(self.scratchDb.getDvoImageFlags(header['IMAGEID']))) 235 sqlLine.group("detrend1", self.getKeyValue(header, 'DETREND.MASK')) 236 sqlLine.group("detrend2", self.getKeyValue(header, 'DETREND.DARK')) 237 sqlLine.group("detrend3", self.getKeyValue(header, 'DETREND.FLAT')) 238 # sqlLine.group("detrend4", " ") 239 # sqlLine.group("detrend5", " ") 240 # sqlLine.group("detrend6", " ") 241 # sqlLine.group("detrend7", " ") 242 # sqlLine.group("detrend8", " ") 243 sqlLine.group("photoZero", self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS')) 244 sqlLine.group("ctype1", self.getKeyValue(header, 'CTYPE1')) 245 sqlLine.group("ctype2", self.getKeyValue(header, 'CTYPE2')) 246 sqlLine.group("crval1", self.getKeyFloat(header, "%.8f", 'CRVAL1')) 247 sqlLine.group("crval2", self.getKeyFloat(header, "%.8f", 'CRVAL2')) 248 sqlLine.group("crpix1", self.getKeyFloat(header, "%.8f", 'CRPIX1')) 249 sqlLine.group("crpix2", self.getKeyFloat(header, "%.8f", 'CRPIX2')) 250 sqlLine.group("cdelt1", self.getKeyFloat(header, "%.8e", 'CDELT1')) 251 sqlLine.group("cdelt2", self.getKeyFloat(header, "%.8e", 'CDELT2')) 252 sqlLine.group("pc001001", self.getKeyFloat(header, "%.8e", 'PC001001')) 253 sqlLine.group("pc001002", self.getKeyFloat(header, "%.8e", 'PC001002')) 254 sqlLine.group("pc002001", self.getKeyFloat(header, "%.8e", 'PC002001')) 255 sqlLine.group("pc002002", self.getKeyFloat(header, "%.8e", 'PC002002')) 256 sqlLine.group("polyOrder", self.getKeyInt(header, 0, 'NPLYTERM')) 257 258 nPolyterm = int(self.getKeyValue(header, 'NPLYTERM')) 259 if nPolyterm > 1: 260 sqlLine.group("pca1x2y0", self.getKeyFloat(header, "%.8e", 'PCA1X2Y0')) 261 sqlLine.group("pca1x1y1", self.getKeyFloat(header, "%.8e", 'PCA1X1Y1')) 262 sqlLine.group("pca1x0y2", self.getKeyFloat(header, "%.8e", 'PCA1X0Y2')) 263 sqlLine.group("pca2x2y0", self.getKeyFloat(header, "%.8e", 'PCA2X2Y0')) 264 sqlLine.group("pca2x1y1", self.getKeyFloat(header, "%.8e", 'PCA2X1Y1')) 265 sqlLine.group("pca2x0y2", self.getKeyFloat(header, "%.8e", 'PCA2X0Y2')) 266 else: 267 sqlLine.group("pca1x2y0", "-999") 268 sqlLine.group("pca1x1y1", "-999") 269 sqlLine.group("pca1x0y2", "-999") 270 sqlLine.group("pca2x2y0", "-999") 271 sqlLine.group("pca2x1y1", "-999") 272 sqlLine.group("pca2x0y2", "-999") 273 274 if nPolyterm > 2: 275 sqlLine.group("pca1x3y0", self.getKeyFloat(header, "%.8e", 'PCA1X3Y0')) 276 sqlLine.group("pca1x2y1", self.getKeyFloat(header, "%.8e", 'PCA1X2Y1')) 277 sqlLine.group("pca1x1y2", self.getKeyFloat(header, "%.8e", 'PCA1X1Y2')) 278 sqlLine.group("pca1x0y3", self.getKeyFloat(header, "%.8e", 'PCA1X0Y3')) 279 sqlLine.group("pca2x3y0", self.getKeyFloat(header, "%.8e", 'PCA2X3Y0')) 280 sqlLine.group("pca2x2y1", self.getKeyFloat(header, "%.8e", 'PCA2X2Y1')) 281 sqlLine.group("pca2x1y2", self.getKeyFloat(header, "%.8e", 'PCA2X1Y2')) 282 sqlLine.group("pca2x0y3", self.getKeyFloat(header, "%.8e", 'PCA2X0Y3')) 283 else: 284 sqlLine.group("pca1x3y0", "-999") 285 sqlLine.group("pca1x2y1", "-999") 286 sqlLine.group("pca1x1y2", "-999") 287 sqlLine.group("pca1x0y3", "-999") 288 sqlLine.group("pca2x3y0", "-999") 289 sqlLine.group("pca2x2y1", "-999") 290 sqlLine.group("pca2x1y2", "-999") 291 sqlLine.group("pca2x0y3", "-999") 292 293 sql = sqlLine.make(") VALUES ( ", ")") 377 294 378 295 try: self.scratchDb.execute(sql) 379 except: print sql 296 except: 297 self.logger.errorPair('failed sql: ', sql) 298 raise 380 299 381 300 self.scratchDb.updateFilterID(tableName, self.filter) … … 407 326 results['SATDET'] = 0 408 327 409 # insert all detections into table 410 sql = "INSERT IGNORE INTO " + pspsTableName + " ( \ 411 ippDetectID \ 412 ,randomDetID \ 413 ,filterID \ 414 ,surveyID \ 415 ,obsTime \ 416 ,xPos \ 417 ,yPos \ 418 ,xPosErr \ 419 ,yPosErr \ 420 ,psfFlux \ 421 ,psfFluxErr \ 422 ,psfMajorFWHM \ 423 ,psfMinorFWHM \ 424 ,psfTheta \ 425 ,psfCore \ 426 ,psfLikelihood \ 427 ,psfQf \ 428 ,momentXX \ 429 ,momentXY \ 430 ,momentYY \ 431 ,momentR1 \ 432 ,momentRH \ 433 ,momentM3C \ 434 ,momentM3S \ 435 ,momentM4C \ 436 ,momentM4S \ 437 ,apFlux \ 438 ,kronFlux \ 439 ,kronFluxErr \ 440 ,psfQfPerfect \ 441 ,psfChiSq \ 442 ,infoFlag \ 443 ,sky \ 444 ,skyErr \ 445 ,pltScale \ 446 ,posAngle \ 447 ,dataRelease \ 448 ) \ 449 SELECT \ 450 IPP_IDET \ 451 , FLOOR(RAND()*9223372036854775807) \ 452 , " + str(self.filterID) + "\ 453 , " + str(self.surveyID) + " \ 454 ," + str(self.obsTime) + " \ 455 ,X_PSF \ 456 ,Y_PSF \ 457 ,X_PSF_SIG \ 458 ,Y_PSF_SIG \ 459 ,PSF_INST_FLUX / " + self.header['EXPTIME'] + " \ 460 ,PSF_INST_FLUX_SIG / " + self.header['EXPTIME'] + " \ 461 ,PSF_MAJOR \ 462 ,PSF_MINOR \ 463 ,PSF_THETA \ 464 ,-999 \ 465 ,psfLikelihood(EXT_NSIGMA) \ 466 ,PSF_QF \ 467 ,MOMENTS_XX \ 468 ,MOMENTS_XY \ 469 ,MOMENTS_YY \ 470 ,MOMENTS_R1 \ 471 ,MOMENTS_RH \ 472 ,MOMENTS_M3C \ 473 ,MOMENTS_M3S \ 474 ,MOMENTS_M4C \ 475 ,MOMENTS_M4S \ 476 ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + " \ 477 ,KRON_FLUX / " + self.header['EXPTIME'] + " \ 478 ,KRON_FLUX_ERR / " + self.header['EXPTIME'] + " \ 479 ,PSF_QF_PERFECT \ 480 ,PSF_CHISQ \ 481 ,FLAGS2 << 32 | FLAGS \ 482 ,SKY / " + self.header['EXPTIME'] + " \ 483 ,SKY_SIGMA / " + self.header['EXPTIME'] + " \ 484 ,PLTSCALE \ 485 ,POSANGLE \ 486 , " + str(self.skychunk.dataRelease) + "\ 487 FROM " + ippTableName 488 # self.logger.info(sql) 489 490 # these were used above (left over from before PSF_INST_FLUX, PSF_INST_FLUX_SIG were available? 491 # ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + " 492 # ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + ")) / 1.085736) 493 494 # I need a way to choose the sql above based on the cmf version: V3 (LAP.PV1) does not have AP_FLUX 495 # AP_FLUX is no ,AP_FLUX / " + self.header['EXPTIME'] + " 496 497 self.scratchDb.execute(sql) 498 499 # XXX EAM : I removed this old fix for invalid fluxes 500 # add a instFlux = 0.0 -> 0.000001 501 # XXX EAM : why is this done? 502 # sql="UPDATE " + pspsTableName + " SET psfFlux = 0.0000001 WHERE psfFlux = 0" 503 # self.scratchDb.execute(sql) 504 328 extTimeString = str(self.header['EXPTIME']) 329 330 # insert all detections into table : do NOT include the IGNORE unless we have to 331 # sqlLine = sqlUtility("INSERT IGNORE INTO " + pspsTableName + " (") 332 sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (") 333 334 # XXX WARNING: need to apply platescale to convert sizes to arcsec 335 # XXX apFluxF or apFluxNpix + apFluxRadius? 336 337 sqlLine.group("ippDetectID", "IPP_IDET") 338 sqlLine.group("randomDetID", "FLOOR(RAND()*9223372036854775807)") 339 sqlLine.group("filterID", str(self.filterID)) 340 sqlLine.group("surveyID", str(self.surveyID)) 341 sqlLine.group("obsTime", str(self.obsTime)) 342 sqlLine.group("xPos", "X_PSF") 343 sqlLine.group("yPos", "Y_PSF") 344 sqlLine.group("xPosErr", "X_PSF_SIG") 345 sqlLine.group("yPosErr", "Y_PSF_SIG") 346 sqlLine.group("pltScale", "PLTSCALE") 347 sqlLine.group("posAngle", "POSANGLE") 348 sqlLine.group("raErr", "X_PSF_SIG * abs(PLTSCALE)") 349 sqlLine.group("decErr", "Y_PSF_SIG * abs(PLTSCALE)") 350 sqlLine.group("psfFlux", "PSF_INST_FLUX / " + extTimeString) 351 sqlLine.group("psfFluxErr", "PSF_INST_FLUX_SIG / " + extTimeString) 352 sqlLine.group("psfMajorFWHM", "PSF_FWHM_MAJ") 353 sqlLine.group("psfMinorFWHM", "PSF_FWHM_MIN") 354 sqlLine.group("psfTheta", "PSF_THETA") 355 sqlLine.group("psfCore", "PSF_CORE") 356 sqlLine.group("psfQf", "PSF_QF") 357 sqlLine.group("psfQfPerfect", "PSF_QF_PERFECT") 358 sqlLine.group("psfChiSq", "PSF_CHISQ") 359 sqlLine.group("psfLikelihood", "psfLikelihood(EXT_NSIGMA)") 360 sqlLine.group("momentXX", "MOMENTS_XX") 361 sqlLine.group("momentXY", "MOMENTS_XY") 362 sqlLine.group("momentYY", "MOMENTS_YY") 363 sqlLine.group("momentR1", "MOMENTS_R1") 364 sqlLine.group("momentRH", "MOMENTS_RH") 365 sqlLine.group("momentM3C", "MOMENTS_M3C") 366 sqlLine.group("momentM3S", "MOMENTS_M3S") 367 sqlLine.group("momentM4C", "MOMENTS_M4C") 368 sqlLine.group("momentM4S", "MOMENTS_M4S") 369 sqlLine.group("apFlux", "AP_FLUX / " + extTimeString) 370 sqlLine.group("apFluxErr", "AP_FLUX_SIG / " + extTimeString) 371 sqlLine.group("apFillF", "AP_NPIX / (3.14159265359 * POW(AP_MAG_RADIUS - 0.5, 2))") 372 sqlLine.group("apRadius", "AP_MAG_RADIUS") 373 sqlLine.group("kronFlux", "KRON_FLUX / " + extTimeString) 374 sqlLine.group("kronFluxErr", "KRON_FLUX_ERR / " + extTimeString) 375 sqlLine.group("kronRad", "MOMENTS_R1 * 2.5") 376 sqlLine.group("sky", "SKY / " + extTimeString) 377 sqlLine.group("skyErr", "SKY_SIGMA / " + extTimeString) 378 sqlLine.group("infoFlag", "FLAGS") 379 sqlLine.group("infoFlag2", "FLAGS2") 380 sqlLine.group("dataRelease", str(self.skychunk.dataRelease)) 381 382 sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) 383 384 try: self.scratchDb.execute(sql) 385 except: 386 self.logger.errorPair("failed sql: ", sql) 387 raise 388 505 389 # NOTE : Flux limits : in the current PSPS schema, negative fluxes 506 390 # cause problems for sql queries which work in mags as SQL cannot do 507 391 # something like (f < 0.0) ? -999 : -2.5*log10(f) 508 392 # as a result, the negative fluxes here result in floating point errors 393 # XXX EAM 2014072 : Is this still a problem? 509 394 sql = "UPDATE " + pspsTableName + " SET psfFlux = 1e20 WHERE psfFlux <= 0.0" 510 395 self.scratchDb.execute(sql) … … 518 403 # we don't delete these anymore 519 404 results['NULLINSTFLUX'] = 0; 520 521 405 522 406 ''' … … 575 459 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) 576 460 577 578 579 '''580 Populates the DetectionCalib table for this OTA581 '''582 def populateDetectionCalibTableUpdateInsert(self, ota):583 584 tableName = "Detection_" + ota585 # drop then re-create table586 #self.scratchDb.dropTable(tableName)587 #sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib"588 #try: self.scratchDb.execute(sql)589 #except: pass590 591 # insert all detections into table592 sql = "INSERT INTO " + tableName + " ( \593 objID \594 ,detectID \595 ,ippObjID \596 ,ippDetectID \597 ,filterID \598 ,surveyID \599 ) \600 SELECT \601 objID \602 ,detectID \603 ,ippObjID \604 ,ippDetectID \605 ,filterID \606 ,surveyID \607 FROM Detection_" + ota608 # self.scratchDb.execute(sql)609 610 # insert calibration information from dvoDetections into the Table611 sql = "UPDATE " + tableName + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b \612 SET a.ra = b.ra, \613 a.dec = b.dec_, \614 a.raErr = b.raErr, \615 a.decErr = b.decErr, \616 a.zp = b.zp, \617 a.expTime = b.expTime, \618 a.airMass = b.airMass \619 WHERE a.objID = b.objID AND a.detectID = b.detectID"620 self.scratchDb.execute(sql)621 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease))622 623 '''624 Populates the DetectionCalib table for this OTA625 '''626 def populateDetectionCalibTable(self, ota):627 628 # target table name:629 tableName = "Detection_" + ota630 # drop then re-create table631 #self.scratchDb.dropTable(tableName)632 #sql = "CREATE TABLE " + tableName + " LIKE Detection"633 #try: self.scratchDb.execute(sql)634 #except: pass635 636 externID = self.imageIDs[ota]637 638 imageID = self.scratchDb.getImageIDFromExternID(externID)639 self.logger.infoPair("obtained imageID",imageID)640 self.logger.infoPair("obtained externID",externID)641 642 # insert all detections into table643 sql = "Update Detection_" + ota + " AS a, "+ self.scratchDb.dvoDetectionTable + " as b \644 set a.ra=b.ra, \645 a.dec = b.dec_, \646 a.raErr = b.raErr, \647 a.decErr = b.decErr, \648 a.zp = b.zp, \649 a.expTime = b.expTime, \650 a.airMass = b.airMass \651 WHERE (a.objID = b.objID AND a.detectID = b.detectID) \652 AND b.imageID = " + str(imageID) \653 654 655 try: self.scratchDb.execute(sql)656 except:657 self.logger.info("failed to update data for detectionCalib")658 self.logger.infoPair("sql: ", sql)659 raise660 661 '''662 Populates the Detection table for this OTA663 '''664 def populateObjectCalColorTable(self, ota):665 666 tableName = "ObjectCalColor_" + ota667 668 # drop then re-create table669 self.scratchDb.dropTable(tableName)670 sql = "CREATE TABLE " + tableName + " LIKE ObjectCalColor"671 try: self.scratchDb.execute(sql)672 except: pass673 674 # insert all detections into table675 sql = "INSERT INTO " + tableName + " ( \676 objID \677 ,ippObjID \678 ,filterID \679 ) \680 SELECT \681 objID \682 ,ippObjID \683 ,filterID \684 FROM Detection_" + ota685 self.scratchDb.execute(sql)686 687 self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))688 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease))689 690 461 ''' 691 462 Applies indexes and other constraints to the PSPS tables … … 710 481 711 482 # check we have valid sourceID/imageID pair from the header 712 if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": return False 713 if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": return False 714 715 # store sourceID/imageID combo in Db so DVO can look up later 716 if not self.useFullTables: 717 self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID']) 483 if self.getKeyValue(header, 'SOURCEID') == "NULL": return False 484 if self.getKeyValue(header, 'IMAGEID') == "NULL": return False 485 486 # store sourceID/imageID combo in Db so DVO can look up later (XXX deprecated?) 487 # self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID']) 718 488 719 489 # store these for later … … 746 516 self.logger.infoPair("Reading", "FITS headers") 747 517 self.logger.infoPair("Populating table", "ImageMeta") 518 519 if self.config.camera == "gpc1": 520 for x in range(self.startX, self.endX): 521 for y in range(self.startY, self.endY): 522 523 # dodge the corners 524 if x==0 and y==0: continue 525 if x==0 and y==7: continue 526 if x==7 and y==0: continue 527 if x==7 and y==7: continue 528 529 ota = "XY%d%d" % (x, y) 530 531 if self.config.test and not ((x == 0) and (y == 1)): 532 print "skipping ota: " + ota 533 continue 534 535 self.alterPspsTablesChip(ota, ota + ".hdr", x, y) 536 537 if self.config.camera == "simtest": 538 # try the test Chip 539 self.alterPspsTablesChip("Chip", "Chip.hdr", 0, 0) 540 541 # now run DVO code to get all IDs ( XXX deprecated, uses dvograbber) 542 # if not self.getIDsFromDVO(): return False 543 544 # dec is reserved in MySQL, so STILTS replaces if with dec_, which is not the name of 545 # the column in PSPS 546 self.scratchDb.execute("ALTER TABLE Detection CHANGE dec_ `dec` double") 547 548 return True 549 550 551 ''' 552 Applies indexes to the IPP tables 553 ''' 554 def indexIppTables(self): 555 556 self.logger.infoPair("Creating indexes on", "IPP tables") 557 748 558 for x in range(self.startX, self.endX): 749 559 for y in range(self.startY, self.endY): 750 560 751 561 # dodge the corners 752 562 if x==0 and y==0: continue … … 755 565 if x==7 and y==7: continue 756 566 757 ota = "XY%d%d" % (x, y)758 759 self.alterPspsTablesChip(ota, ota + ".hdr", x, y)760 761 # try the test Chip762 self.alterPspsTablesChip("Chip", "Chip.hdr", 0, 0)763 764 # now run DVO code to get all IDs765 if not self.useFullTables:766 if not self.getIDsFromDVO(): return False767 768 # dec is reserved in MySQL, so STILTS replaces if with dec_, which is not the name of769 # the column in PSPS770 self.scratchDb.execute("ALTER TABLE Detection CHANGE dec_ `dec` double")771 772 return True773 774 775 '''776 Applies indexes to the IPP tables777 '''778 def indexIppTables(self):779 780 self.logger.infoPair("Creating indexes on", "IPP tables")781 782 for x in range(self.startX, self.endX):783 for y in range(self.startY, self.endY):784 785 # dodge the corners786 if x==0 and y==0: continue787 if x==0 and y==7: continue788 if x==7 and y==0: continue789 if x==7 and y==7: continue790 791 567 extension = "XY%d%d_psf" % (x, y) 792 568 self.scratchDb.createIndex(extension, "IPP_IDET") … … 803 579 imageID = self.scratchDb.getImageIDFromExternID(externID) 804 580 self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID) 581 805 582 sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ 806 a.ippObjID = b.ippObjID, \ 807 a.detectID = b.detectID, \ 808 a.objID = b.objID, \ 809 a.infoFlag = b.flags << 45 | a.infoFlag \ 583 a.objID = b.objID, \ 584 a.detectID = b.detectID, \ 585 a.ippObjID = b.ippObjID, \ 586 a.dvoRegionID = b.catID, \ 587 a.ra = b.ra, \ 588 a.dec = b.dec_, \ 589 a.zp = b.zp, \ 590 a.telluricExt = b.telluricExt, \ 591 a.airmass = b.airmass, \ 592 a.expTime = b.expTime, \ 593 a.infoFlag2 = (b.flags << 13) | a.infoFlag2 \ 810 594 WHERE a.ippDetectID = b.ippDetectID \ 811 595 AND b.imageID = " + str(imageID) … … 817 601 ''' 818 602 def populatePspsTablesChip(self, chipname, x, y, results, tables): 819 # XXX EAM NOTE: drop tables (Detection_, SkinnyObject_, DetectionCalib_, Detection_)here so603 # XXX EAM NOTE: drop tables Detection_* here so 820 604 # they do not polute the db? 821 605 # XXX or put an explicit drop at the end of the loop? 822 606 823 # self.logger.infoTitle("Processing " + chipname)607 # self.logger.infoTitle("Processing " + chipname) 824 608 # this is a bit crude: if the chip is not present, this test will fail and the chip 825 609 # will be (correctly) skipped. would be better to carry that information explicitly ("chip is missing") … … 882 666 self.scratchDb.execute(sql) 883 667 884 #self.logger.info("updated imagedata")885 self.populateSkinnyObjectTable(chipname)886 self.logger.info("updated skinnyobject")887 888 self.populateDetectionCalibTable(chipname)889 #self.logger.info("updated detectioncalibtable")890 891 668 # add these to list of tables to export later 892 669 self.tablesToExport.append("ImageMeta_" + chipname) … … 896 673 self.logger.info("export Detection") 897 674 898 self.tablesToExport.append("SkinnyObject_" + chipname)899 self.logger.info("export Skinny")900 901 902 675 tables.append("Detection_" + chipname) 903 self.logger.info("updated detection calibtable")676 self.logger.info("updated detection") 904 677 905 678 self.validChips.append(chipname) … … 936 709 if ota not in self.imageIDs: continue 937 710 711 if self.config.test and not ((x == 0) and (y == 1)): 712 print "skipping ota " + ota 713 continue 714 938 715 if self.populatePspsTablesChip(ota, x, y, results, tables): otaCount = otaCount + 1 939 716 if self.skipBatch: … … 1045 822 ''' 1046 823 def importIppTables(self, filter=""): 1047 1048 ## if self.config.test: regex = "XY33.psf" 1049 ## else : regex = ".*.psf" 1050 regex = ".*.psf" 1051 1052 # XXX EAM NOTE : this is fragile : requires PS1_V4 1053 # columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG AP_FLUX KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" 1054 1055 # XXX EAM NOTE : this is fragile : requires PS1_V4 1056 columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ POSANGLE PLTSCALE" 1057 1058 return super(DetectionBatch, self).importIppTables(columns, regex) 1059 824 825 regex = ".*.psf" 826 if self.config.test and self.config.camera == "gpc1": 827 regex = "XY01.psf" 828 829 # XXX EAM NOTE : this is fragile : requires PS1_V4 830 # columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG AP_FLUX KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" 831 832 # XXX EAM NOTE : this is fragile : requires PS1_V5 833 columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG POSANGLE PLTSCALE PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_FWHM_MAJ PSF_FWHM_MIN PSF_THETA PSF_CORE PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_FLUX AP_FLUX_SIG AP_NPIX AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2" 834 835 return super(DetectionBatch, self).importIppTables(columns, regex) 1060 836 1061 837 ''' … … 1064 840 def exportPspsTablesToFits(self, regex="(.*)"): 1065 841 return super(DetectionBatch, self).exportPspsTablesToFits("([a-zA-Z]+)") 1066
Note:
See TracChangeset
for help on using the changeset viewer.
