- Timestamp:
- Sep 6, 2011, 11:00:22 AM (15 years ago)
- Location:
- branches/eam_branches/ipp-20110710/ippToPsps/jython
- Files:
-
- 2 edited
-
. (modified) (1 prop)
-
detectionbatch.py (modified) (17 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20110710/ippToPsps/jython
-
Property svn:ignore
set to
*.class
-
Property svn:ignore
set to
-
branches/eam_branches/ipp-20110710/ippToPsps/jython/detectionbatch.py
r31848 r32337 30 30 def __init__(self, 31 31 logger, 32 configPath, 32 33 configDoc, 33 34 gpc1Db, … … 37 38 super(DetectionBatch, self).__init__( 38 39 logger, 40 configPath, 39 41 configDoc, 40 42 gpc1Db, 41 43 ippToPspsDb, 42 44 camID, 43 "detection", 44 gpc1Db.getCameraStageSmf(camID), 45 #"MD04") # TODO 46 "3PI") 45 "P2", 46 gpc1Db.getCameraStageSmf(camID)) 47 47 48 48 if not self.everythingOK: return … … 50 50 # get camera meta data 51 51 meta = self.gpc1Db.getCameraStageMeta(self.id) 52 if not meta: 53 self.everythingOK = False 54 return 55 52 56 self.expID = meta[0]; 53 57 self.expName = meta[1]; … … 80 84 #self.endY = 8 81 85 86 # get a fre primary header values. if in test mode, then use defaults 87 if self.safeDictionaryAccessWithDefault(self.header, 'MJD-OBS', "1") == "NULL": 88 self.everythingOK = False 89 return 90 91 if self.safeDictionaryAccessWithDefault(self.header, 'EXPTIME', "1") == "NULL": 92 self.everythingOK = False 93 return 94 95 if self.safeDictionaryAccessWithDefault(self.header, 'FILTERID', "g.0000") == "NULL": 96 self.everythingOK = False 97 return 98 82 99 self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0) 83 100 84 101 # set up some defauts 85 102 self.calibModNum = 0 86 87 103 self.totalNumPhotoRef = 0 88 104 89 # get filterID using init table 105 90 106 self.filter = self.header['FILTERID'][0:1] 107 self.filterID = self.scratchDb.getFilterID(self.filter) 91 108 92 109 # insert what we know about this stack batch into the stack table 93 110 self.ippToPspsDb.insertDetectionMeta(self.batchID, self.expID, self.filter) 94 111 95 ''' 96 Prints metadata to the log 97 ''' 98 def printMe(self): 99 100 super(DetectionBatch, self).printMe() 101 102 self.logger.infoPair("Cam ID", "%d" % self.id) 103 self.logger.infoPair("Exp ID", "%d" % self.expID) 104 self.logger.infoPair("Exp name", "%s" % self.expName) 105 self.logger.infoPair("Distribution group", "%s" % self.distGroup) 112 # dump stuff to log 113 self.logger.infoPair("Cam ID", "%d" % self.id) 114 self.logger.infoPair("Exp ID", "%d" % self.expID) 115 self.logger.infoPair("Exp name", "%s" % self.expName) 116 self.logger.infoPair("Distribution group", "%s" % self.distGroup) 106 117 107 118 ''' … … 164 175 ,' ' \ 165 176 ,' ' \ 166 ," + self. header['ZPT_ERR']+ " \167 ," + self. header['MJD-OBS']+ " \168 ," + self. header['EXPREQ']+ " \169 ," + self. header['AIRMASS']+ " \170 ," + self. header['RA']+ " \171 ," + self. header['DEC']+ " \172 ,'" + self. header['CTYPE1']+ "' \173 ,'" + self. header['CTYPE2']+ "' \174 ," + self. header['CRVAL1']+ " \175 ," + self. header['CRVAL2']+ " \176 ," + self. header['CRPIX1']+ " \177 ," + self. header['CRPIX2']+ " \178 ," + self. header['CDELT1']+ " \179 ," + self. header['CDELT2']+ " \180 ," + self. header['PC001001']+ " \181 ," + self. header['PC001002']+ " \182 ," + self. header['PC002001']+ " \183 ," + self. header['PC002002']+ " \184 ," + self. header['NPLYTERM']+ " \185 ," + self. header['PCA1X3Y0']+ " \186 ," + self. header['PCA1X2Y1']+ " \187 ," + self. header['PCA1X1Y2']+ " \188 ," + self. header['PCA1X0Y3']+ " \189 ," + self. header['PCA1X2Y0']+ " \190 ," + self. header['PCA1X1Y1']+ " \191 ," + self. header['PCA1X0Y2']+ " \192 ," + self. header['PCA2X3Y0']+ " \193 ," + self. header['PCA2X2Y1']+ " \194 ," + self. header['PCA2X1Y2']+ " \195 ," + self. header['PCA2X0Y3']+ " \196 ," + self. header['PCA2X2Y0']+ " \197 ," + self. header['PCA2X1Y1']+ " \198 ," + self. header['PCA2X0Y2']+ " \177 ," + self.safeDictionaryAccess(self.header, 'ZPT_ERR') + " \ 178 ," + self.safeDictionaryAccess(self.header, 'MJD-OBS') + " \ 179 ," + self.safeDictionaryAccess(self.header, 'EXPREQ') + " \ 180 ," + self.safeDictionaryAccess(self.header, 'AIRMASS') + " \ 181 ," + self.safeDictionaryAccess(self.header, 'RA') + " \ 182 ," + self.safeDictionaryAccess(self.header, 'DEC') + " \ 183 ,'" + self.safeDictionaryAccess(self.header, 'CTYPE1') + "' \ 184 ,'" + self.safeDictionaryAccess(self.header, 'CTYPE2') + "' \ 185 ," + self.safeDictionaryAccess(self.header, 'CRVAL1') + " \ 186 ," + self.safeDictionaryAccess(self.header, 'CRVAL2') + " \ 187 ," + self.safeDictionaryAccess(self.header, 'CRPIX1') + " \ 188 ," + self.safeDictionaryAccess(self.header, 'CRPIX2') + " \ 189 ," + self.safeDictionaryAccess(self.header, 'CDELT1') + " \ 190 ," + self.safeDictionaryAccess(self.header, 'CDELT2') + " \ 191 ," + self.safeDictionaryAccess(self.header, 'PC001001') + " \ 192 ," + self.safeDictionaryAccess(self.header, 'PC001002') + " \ 193 ," + self.safeDictionaryAccess(self.header, 'PC002001') + " \ 194 ," + self.safeDictionaryAccess(self.header, 'PC002002') + " \ 195 ," + self.safeDictionaryAccess(self.header, 'NPLYTERM') + " \ 196 ," + self.safeDictionaryAccess(self.header, 'PCA1X3Y0') + " \ 197 ," + self.safeDictionaryAccess(self.header, 'PCA1X2Y1') + " \ 198 ," + self.safeDictionaryAccess(self.header, 'PCA1X1Y2') + " \ 199 ," + self.safeDictionaryAccess(self.header, 'PCA1X0Y3') + " \ 200 ," + self.safeDictionaryAccess(self.header, 'PCA1X2Y0') + " \ 201 ," + self.safeDictionaryAccess(self.header, 'PCA1X1Y1') + " \ 202 ," + self.safeDictionaryAccess(self.header, 'PCA1X0Y2') + " \ 203 ," + self.safeDictionaryAccess(self.header, 'PCA2X3Y0') + " \ 204 ," + self.safeDictionaryAccess(self.header, 'PCA2X2Y1') + " \ 205 ," + self.safeDictionaryAccess(self.header, 'PCA2X1Y2') + " \ 206 ," + self.safeDictionaryAccess(self.header, 'PCA2X0Y3') + " \ 207 ," + self.safeDictionaryAccess(self.header, 'PCA2X2Y0') + " \ 208 ," + self.safeDictionaryAccess(self.header, 'PCA2X1Y1') + " \ 209 ," + self.safeDictionaryAccess(self.header, 'PCA2X0Y2') + " \ 199 210 )" 200 211 self.scratchDb.execute(sql) … … 357 368 Populates the Detection table for this OTA 358 369 ''' 359 def populateDetectionTable(self, ota): 360 361 tableName = "Detection_" + ota 362 370 def populateDetectionTable(self, ota, results): 371 372 pspsTableName = "Detection_" + ota 373 ippTableName = ota + "_psf" 374 375 results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName) 376 363 377 # drop then re-create table 364 self.scratchDb.dropTable( tableName)365 sql = "CREATE TABLE " + tableName + " LIKE Detection"378 self.scratchDb.dropTable(pspsTableName) 379 sql = "CREATE TABLE " + pspsTableName + " LIKE Detection" 366 380 try: self.scratchDb.execute(sql) 367 381 except: pass 382 383 # delete all detections with PSF_INST_MAG < 17.5, as decreed by Gene 384 BEFORE = self.scratchDb.getRowCount(ippTableName) 385 sql = "DELETE FROM " + ippTableName + " WHERE PSF_INST_MAG < -17.5" 386 self.scratchDb.execute(sql) 387 #DELETED = BEFORE - self.scratchDb.getRowCount(ippTableName) 388 results['SATDET'] = BEFORE - self.scratchDb.getRowCount(ippTableName) 389 #self.logger.infoPair("Saturated detections", "%d deleted" % DELETED) 368 390 369 391 # insert all detections into table 370 sql = "INSERT IGNORE INTO " + tableName + " ( \392 sql = "INSERT IGNORE INTO " + pspsTableName + " ( \ 371 393 ippDetectID \ 394 ,filterID \ 395 ,surveyID \ 396 ,obsTime \ 372 397 ,xPos \ 373 398 ,yPos \ … … 380 405 ,psfWidMinor \ 381 406 ,psfTheta \ 407 ,psfLikelihood \ 382 408 ,psfCf \ 383 409 ,momentXX \ … … 391 417 ,skyErr \ 392 418 ,sgSep \ 419 ,activeFlag \ 420 ,assocDate \ 421 ,historyModNum \ 422 ,dataRelease \ 393 423 ) \ 394 424 SELECT \ 395 425 IPP_IDET \ 426 , " + str(self.filterID) + "\ 427 , " + str(self.surveyID) + " \ 428 ," + str(self.obsTime) + " \ 396 429 ,X_PSF \ 397 430 ,Y_PSF \ … … 404 437 ,PSF_MINOR \ 405 438 ,PSF_THETA \ 439 ,psfLikelihood(EXT_NSIGMA) \ 406 440 ,PSF_QF \ 407 441 ,MOMENTS_XX \ … … 415 449 ,SKY_SIGMA \ 416 450 ,EXT_NSIGMA \ 417 FROM " + ota + "_psf" 418 self.scratchDb.execute(sql) 419 420 # set obsTime 421 sql = "UPDATE " + tableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr) 422 self.scratchDb.execute(sql) 423 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) 424 self.scratchDb.updateAllRows(tableName, "historyModNum", "0") 425 426 self.scratchDb.updateAllRows(tableName, "surveyID", str(self.surveyID)) 427 self.scratchDb.updateFilterID(tableName, self.filter) 451 , 0 \ 452 , '" + self.dateStr + "' \ 453 , 0 \ 454 , " + str(self.dataRelease) + "\ 455 FROM " + ippTableName 456 self.scratchDb.execute(sql) 428 457 429 458 # now delete bad flux and bad chip positions 430 self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "instFlux") 431 self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "peakADU") 459 sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux 460 self.scratchDb.execute(sql) 461 462 # update cosmic ray and extended likelihoods 463 sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0" 464 self.scratchDb.execute(sql) 465 sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0" 466 self.scratchDb.execute(sql) 467 468 # remove detections will NULL inst flux or NULL peak ADU 469 results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux") 470 results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU") 432 471 433 472 ''' … … 499 538 self.scratchDb.makeColumnPrimaryKey("Detection", "ippDetectID") 500 539 501 '''502 Applies indexes to the IPP tables503 '''504 def indexIppTables(self):505 506 self.logger.infoPair("Creating indexes on", "IPP tables")507 508 for x in range(self.startX, self.endX):509 for y in range(self.startY, self.endY):510 511 # dodge the corners512 if x==0 and y==0: continue513 if x==0 and y==7: continue514 if x==7 and y==0: continue515 if x==7 and y==7: continue516 517 extension = "XY%d%d_psf" % (x, y)518 self.scratchDb.createIndex(extension, "IPP_IDET")519 520 '''521 Updates provided table with DVO IDs from DVO table522 '''523 def updateDvoIDs(self, table, sourceID, externID):524 525 imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID)526 self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)527 sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \528 a.ippObjID = b.ippObjID, \529 a.detectID = b.detectID, \530 a.objID = b.objID, \531 a.infoFlag = b.flags << 32 | a.infoFlag \532 WHERE a.ippDetectID = b.ippDetectID \533 AND b.sourceID = " + str(sourceID) + " \534 AND b.imageID = " + str(imageID)535 536 self.scratchDb.execute(sql)537 538 '''539 Generates psf, cosmic ray and extended source likelihoods540 '''541 def populateLikelihoods(self, tableName):542 543 sql = "SELECT ippDetectID, psfLikelihood, crLikelihood, extendedLikelihood, sgSep FROM " + tableName544 rs = self.scratchDb.executeUpdatableQuery(sql)545 546 sqrt2 = Math.sqrt(2)547 548 while rs.next():549 550 sgSep = rs.getDouble(5)551 psfLikelihood = Erf.erfc(Math.abs(sgSep)/sqrt2)552 553 if sgSep > 0:554 crLikelihood = 0555 extendedLikelihood = 1.0 - psfLikelihood556 else:557 crLikelihood = 1.0 - psfLikelihood558 extendedLikelihood = 0559 560 # update columns561 rs.updateDouble(2, psfLikelihood )562 rs.updateDouble(3, crLikelihood )563 rs.updateDouble(4, extendedLikelihood )564 565 # now 'commit' to database566 rs.updateRow();567 568 569 '''570 Does the processing, i.e. pulling stuff from IPP tables into PSPS tables571 '''572 def populatePspsTables(self):573 574 540 self.populateFrameMeta() 575 541 576 542 # dictionary objects to hold sourceIDs and imageIDs for later 577 s ourceIDs = {}578 imageIDs = {}543 self.sourceIDs = {} 544 self.imageIDs = {} 579 545 580 546 # loop through all OTAs and populate ImageMeta extensions … … 595 561 header = self.fits.findAndReadHeader(ota + ".hdr") 596 562 if not header: 597 self.logger.error ("No header found for OTA " +ota)563 self.logger.errorPair("No header found for OTA", ota) 598 564 continue 565 566 # check we have valid sourceID/imageID pair from the header 567 if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": continue 568 if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": continue 599 569 600 570 # store sourceID/imageID combo in Db so DVO can look up later 601 571 if not self.useFullTables: 602 self.scratchDb.insertNewDvoImage(header['SOURCEID'], header['IMAGEID']) 603 604 # check we have valid sourceID/imageID pair from the header 605 if 'SOURCEID' not in header or 'IMAGEID' not in header: 606 self.logger.error("Can't read SOURCEID/IMAGEID pair from " + ota + " header") 607 continue 572 self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID']) 608 573 609 574 # store these for later 610 s ourceIDs[ota] = header['SOURCEID']611 imageIDs[ota] = header['IMAGEID']575 self.sourceIDs[ota] = header['SOURCEID'] 576 self.imageIDs[ota] = header['IMAGEID'] 612 577 613 578 # populate ImageMeta … … 616 581 617 582 # now run DVO code to get all IDs 618 if not self.useFullTables: self.getIDsFromDVO() 583 if not self.useFullTables: 584 if not self.getIDsFromDVO(): return False 585 586 return True 587 588 589 ''' 590 Applies indexes to the IPP tables 591 ''' 592 def indexIppTables(self): 593 594 self.logger.infoPair("Creating indexes on", "IPP tables") 595 596 for x in range(self.startX, self.endX): 597 for y in range(self.startY, self.endY): 598 599 # dodge the corners 600 if x==0 and y==0: continue 601 if x==0 and y==7: continue 602 if x==7 and y==0: continue 603 if x==7 and y==7: continue 604 605 extension = "XY%d%d_psf" % (x, y) 606 self.scratchDb.createIndex(extension, "IPP_IDET") 607 608 ''' 609 Updates provided table with DVO IDs from DVO table 610 ''' 611 def updateDvoIDs(self, table, sourceID, externID): 612 613 imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID) 614 self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID) 615 sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ 616 a.ippObjID = b.ippObjID, \ 617 a.detectID = b.detectID, \ 618 a.objID = b.objID, \ 619 a.infoFlag = b.flags << 32 | a.infoFlag \ 620 WHERE a.ippDetectID = b.ippDetectID \ 621 AND b.sourceID = " + str(sourceID) + " \ 622 AND b.imageID = " + str(imageID) 623 624 self.scratchDb.execute(sql) 625 626 627 ''' 628 Does the processing, i.e. pulling stuff from IPP tables into PSPS tables 629 ''' 630 def populatePspsTables(self): 619 631 620 632 # loop through all OTAs again to update with DVO IDs … … 623 635 tables = [] 624 636 otaCount = 0 637 results = {} 638 totalOriginal = totalSatDet = totalNulIInstFlux = totalNullPeakFlux = totalNullObjID = totalDetections = 0 639 self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") 640 self.logger.info("| OTA | Initial total | Sat Det | NULL instFlux | NULL peak ADU | NULL obj ID | Remainder |") 641 self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") 625 642 for x in range(self.startX, self.endX): 626 643 for y in range(self.startY, self.endY): … … 633 650 634 651 ota = "XY%d%d" % (x, y) 635 if ota not in sourceIDs: continue 636 637 self.logger.infoMiniSeparator() 652 if ota not in self.sourceIDs: continue 653 654 #self.logger.infoTitle("Processing " + ota) 655 if not self.scratchDb.astrometricSolutionOK(ota): 656 self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % ota) 657 continue 638 658 639 659 # populate remainder of tables 640 self.populateDetectionTable(ota )660 self.populateDetectionTable(ota, results) 641 661 642 662 # now add DVO IDs 643 self.updateDvoIDs("Detection_" + ota, s ourceIDs[ota],imageIDs[ota])644 self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID")663 self.updateDvoIDs("Detection_" + ota, self.sourceIDs[ota], self.imageIDs[ota]) 664 results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID") 645 665 self.updateImageID("Detection_" + ota, x, y) 646 self.populateLikelihoods("Detection_" + ota) 647 648 # check we have something in this Detection table 649 if self.scratchDb.getRowCount("Detection_" + ota) < 1: 650 self.logger.infoPair("Skipping empty table for ota", ota) 666 rowCount = self.scratchDb.getRowCount("Detection_" + ota) 667 self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", 668 ota, 669 results['ORIGINALTOTAL'], 670 results['SATDET'], 671 results['NULLINSTFLUX'], 672 results['NULLPEAKADU'], 673 results['NULLOBJID'], 674 rowCount) 675 676 totalOriginal = totalOriginal + results['ORIGINALTOTAL'] 677 totalSatDet = totalSatDet + results['SATDET'] 678 totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX'] 679 totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU'] 680 totalNullObjID = totalNullObjID + results['NULLOBJID'] 681 totalDetections = totalDetections + rowCount 682 683 # check we have something in this Detection table TODO add this to table above 684 if rowCount < 1: 685 self.logger.debugPair("Skipping empty table for ota", ota) 651 686 continue; 652 687 653 688 # update ImageMeta with count of detections for this OTA and photoCodeID 654 689 sql = "UPDATE ImageMeta_" + ota + " \ 655 SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(s ourceIDs[ota],imageIDs[ota]))690 SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.sourceIDs[ota], self.imageIDs[ota])) 656 691 self.scratchDb.execute(sql) 657 692 … … 668 703 otaCount = otaCount + 1 669 704 705 # print totals 706 self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") 707 self.logger.info("| Total | %13d | %13d | %13d | %13d | %13d | %13d |", 708 totalOriginal, 709 totalSatDet, 710 totalNulIInstFlux, 711 totalNullPeakFlux, 712 totalNullObjID, 713 totalDetections) 714 self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") 715 670 716 # if we only have one table export, i.e. FrameMeta, then get out of here 671 717 if len(self.tablesToExport) == 1: … … 708 754 if self.testMode: regex = "XY33.psf" 709 755 else : regex = ".*.psf" 710 711 return super(DetectionBatch, self).importIppTables(regex) 756 757 columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS SKY SKY_SIGMA EXT_NSIGMA" 758 759 return super(DetectionBatch, self).importIppTables(columns, regex) 712 760 713 761
Note:
See TracChangeset
for help on using the changeset viewer.
