- Timestamp:
- Jan 20, 2013, 9:03:41 PM (13 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20121219/ippToPsps/jython/detectionbatch.py
r34926 r35003 214 214 ," + self.safeDictionaryAccess(self.header, 'PCA2X0Y2') + " \ 215 215 )" 216 217 print "frame meta sql: ", sql 218 216 219 self.scratchDb.execute(sql) 217 220 … … 593 596 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) 594 597 598 ''' 599 Applies indexes and other constraints to the PSPS tables 600 ''' 601 def alterPspsTablesChip(self, chipname, extname, x, y): 602 603 # load corresponding header into memory 604 header = self.fits.findAndReadHeader(extname) 605 if not header: 606 self.logger.errorPair("No header found for chip", chipname) 607 return False 608 609 # check we have valid sourceID/imageID pair from the header 610 if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": return False 611 if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": return False 612 613 # store sourceID/imageID combo in Db so DVO can look up later 614 if not self.useFullTables: 615 self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID']) 616 617 # store these for later 618 self.imageIDs[chipname] = header['IMAGEID'] 619 620 # populate ImageMeta 621 self.populateImageMetaTable(chipname, header) 622 self.updateImageID("ImageMeta_" + chipname, x, y) 623 624 return True 595 625 596 626 ''' … … 623 653 ota = "XY%d%d" % (x, y) 624 654 625 # load corresponding header into memory 626 header = self.fits.findAndReadHeader(ota + ".hdr") 627 if not header: 628 self.logger.errorPair("No header found for OTA", ota) 629 continue 630 631 # check we have valid sourceID/imageID pair from the header 632 if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": continue 633 if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": continue 634 635 # store sourceID/imageID combo in Db so DVO can look up later 636 if not self.useFullTables: 637 self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID']) 638 639 # store these for later 640 self.imageIDs[ota] = header['IMAGEID'] 641 642 # populate ImageMeta 643 self.populateImageMetaTable(ota, header) 644 self.updateImageID("ImageMeta_" + ota, x, y) 645 655 self.alterPspsTablesChip(ota, ota + ".hdr", x, y) 656 657 # try the test Chip 658 self.alterPspsTablesChip("Chip", "Chip.hdr", 0, 0) 659 646 660 # now run DVO code to get all IDs 647 661 if not self.useFullTables: … … 651 665 # the column in PSPS 652 666 self.scratchDb.execute("ALTER TABLE DetectionCalib CHANGE dec_ `dec` double") 653 654 655 667 656 668 return True … … 675 687 extension = "XY%d%d_psf" % (x, y) 676 688 self.scratchDb.createIndex(extension, "IPP_IDET") 689 # try the test Chip 690 self.scratchDb.createIndex(Chip_psf, "IPP_IDET") 691 677 692 self.logger.infoPair("created indexes on", "IPP tables") 693 678 694 ''' 679 695 Updates provided table with DVO IDs from DVO table … … 693 709 self.scratchDb.execute(sql) 694 710 711 ''' 712 Does the processing, i.e. pulling stuff from IPP tables into PSPS tables 713 ''' 714 def populatePspsTablesChip(self, chipname, x, y): 715 #self.logger.infoTitle("Processing " + chipname) 716 if not self.scratchDb.astrometricSolutionOK(chipname): 717 self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % chipname) 718 return False 719 self.logger.info("populate stuff "); 720 # populate remainder of tables 721 self.populateDetectionTable(chipname, results) 722 self.logger.info("successful populate "); 723 # now add DVO IDs 724 self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname]) 725 self.logger.info("updated dvoids") 726 results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID") 727 #self.logger.info("deleted nulls") 728 self.updateImageID("Detection_" + chipname, x, y) 729 self.logger.info("updateImageId") 730 rowCount = self.scratchDb.getRowCount("Detection_" + chipname) 731 self.logger.info("got row count") 732 self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", 733 chipname, 734 results['ORIGINALTOTAL'], 735 results['SATDET'], 736 results['NULLINSTFLUX'], 737 results['NULLPEAKADU'], 738 results['NULLOBJID'], 739 rowCount) 740 totalOriginal = totalOriginal + results['ORIGINALTOTAL'] 741 totalSatDet = totalSatDet + results['SATDET'] 742 totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX'] 743 totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU'] 744 totalNullObjID = totalNullObjID + results['NULLOBJID'] 745 totalDetections = totalDetections + rowCount 746 #self.logger.info("updated totals") 747 # check we have something in this Detection table TODO add this to table above 748 if rowCount < 1: 749 self.logger.debugPair("Skipping empty table for chipname", chipname) 750 return False; 751 752 # update ImageMeta with count of detections for this CHIPNAME and photoCodeID 753 sql = "UPDATE ImageMeta_" + chipname + " \ 754 SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + chipname), self.scratchDb.getPhotoCalID(self.imageIDs[chipname])) 755 self.scratchDb.execute(sql) 756 757 #self.logger.info("updated imagedata") 758 self.populateSkinnyObjectTable(chipname) 759 #self.logger.info("updated skinnyobject") 760 #self.populateObjectCalColorTable(chipname) 761 #self.logger.info("updated objectcalcolor") 762 self.populateDetectionCalibTable(chipname) 763 #self.logger.info("updated detectioncalibtable") 764 765 # add these to list of tables to export later 766 self.tablesToExport.append("ImageMeta_" + chipname) 767 self.tablesToExport.append("Detection_" + chipname) 768 self.tablesToExport.append("SkinnyObject_" + chipname) 769 #self.tablesToExport.append("ObjectCalColor_" + chipname) 770 self.tablesToExport.append("DetectionCalib_" + chipname) 771 tables.append("Detection_" + chipname) 772 return True 695 773 696 774 ''' … … 721 799 if ota not in self.imageIDs: continue 722 800 723 #self.logger.infoTitle("Processing " + ota) 724 if not self.scratchDb.astrometricSolutionOK(ota): 725 self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % ota) 726 continue 727 # self.logger.info("populate stuff "); 728 # populate remainder of tables 729 self.populateDetectionTable(ota, results) 730 # self.logger.info("successful populate "); 731 # now add DVO IDs 732 self.updateDvoIDs("Detection_" + ota, self.imageIDs[ota]) 733 # self.logger.info("updated dvoids") 734 results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID") 735 #self.logger.info("deleted nulls") 736 self.updateImageID("Detection_" + ota, x, y) 737 # self.logger.info("updateImageId") 738 rowCount = self.scratchDb.getRowCount("Detection_" + ota) 739 # self.logger.info("got row count") 740 self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", 741 ota, 742 results['ORIGINALTOTAL'], 743 results['SATDET'], 744 results['NULLINSTFLUX'], 745 results['NULLPEAKADU'], 746 results['NULLOBJID'], 747 rowCount) 748 totalOriginal = totalOriginal + results['ORIGINALTOTAL'] 749 totalSatDet = totalSatDet + results['SATDET'] 750 totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX'] 751 totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU'] 752 totalNullObjID = totalNullObjID + results['NULLOBJID'] 753 totalDetections = totalDetections + rowCount 754 #self.logger.info("updated totals") 755 # check we have something in this Detection table TODO add this to table above 756 if rowCount < 1: 757 self.logger.debugPair("Skipping empty table for ota", ota) 758 continue; 759 760 # update ImageMeta with count of detections for this OTA and photoCodeID 761 sql = "UPDATE ImageMeta_" + ota + " \ 762 SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.imageIDs[ota])) 763 self.scratchDb.execute(sql) 764 765 #self.logger.info("updated imagedata") 766 self.populateSkinnyObjectTable(ota) 767 #self.logger.info("updated skinnyobject") 768 #self.populateObjectCalColorTable(ota) 769 #self.logger.info("updated objectcalcolor") 770 self.populateDetectionCalibTable(ota) 771 #self.logger.info("updated detectioncalibtable") 772 773 # add these to list of tables to export later 774 self.tablesToExport.append("ImageMeta_" + ota) 775 self.tablesToExport.append("Detection_" + ota) 776 self.tablesToExport.append("SkinnyObject_" + ota) 777 #self.tablesToExport.append("ObjectCalColor_" + ota) 778 self.tablesToExport.append("DetectionCalib_" + ota) 779 tables.append("Detection_" + ota) 780 781 otaCount = otaCount + 1 782 #self.logger.info("end of ota") 801 if populatePspsTables(ota, x, y): otaCount = otaCount + 1 802 803 if populatePspsTables("Chip", 0, 0): otaCount = otaCount + 1 804 783 805 # print totals 784 806 self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") … … 833 855 else : regex = ".*.psf" 834 856 857 # XXX EAM NOTE : this is fragile : requires PS1_V_ 835 858 columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" 836 859
Note:
See TracChangeset
for help on using the changeset viewer.
