Changeset 35097 for trunk/ippToPsps/jython/detectionbatch.py
- Timestamp:
- Feb 6, 2013, 3:16:35 PM (13 years ago)
- Location:
- trunk/ippToPsps
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippToPsps
- Property svn:mergeinfo changed
-
trunk/ippToPsps/jython
- Property svn:ignore
-
old new 1 1 *.class 2 Makefile 3 Makefile.in
-
- Property svn:ignore
-
trunk/ippToPsps/jython/detectionbatch.py
r34879 r35097 3 3 import os.path 4 4 import sys 5 import glob 5 6 import time 6 7 import stilts 8 7 9 from java.lang import * 8 10 from java.sql import * … … 34 36 logger, 35 37 config, 38 skychunk, 36 39 gpc1Db, 37 40 ippToPspsDb, … … 44 47 logger, 45 48 config, 49 skychunk, 46 50 gpc1Db, 47 51 ippToPspsDb, … … 72 76 self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) 73 77 74 # if test mode 75 if self.config.test: 76 self.startX = 3 77 self.endX = 4 78 self.startY = 3 79 self.endY = 4 80 else: 81 self.startX = 0 82 self.endX = 8 83 self.startY = 0 84 self.endY = 8 78 self.startX = 0 79 self.endX = 8 80 self.startY = 0 81 self.endY = 8 85 82 86 83 #self.startX = 1 … … 89 86 #self.endY = 8 90 87 91 # get a f reprimary header values. if in test mode, then use defaults88 # get a few primary header values. if in test mode, then use defaults 92 89 if self.safeDictionaryAccessWithDefault(self.header, 'MJD-OBS', "1") == "NULL": 93 90 self.logger.errorPair("Could not get", "MJD-OBS") … … 214 211 ," + self.safeDictionaryAccess(self.header, 'PCA2X0Y2') + " \ 215 212 )" 213 214 print "frame meta sql: ", sql 215 216 216 self.scratchDb.execute(sql) 217 217 … … 219 219 self.scratchDb.updateFilterID("FrameMeta", self.filter) 220 220 self.scratchDb.updateAllRows("FrameMeta", "calibModNum", str(self.calibModNum)) 221 self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self. config.dataRelease))221 self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.skychunk.dataRelease)) 222 222 223 223 ''' … … 228 228 tableName = "ImageMeta_" + ota 229 229 230 # XXX we drop the table above so it is not left behind on failure 230 231 # drop then re-create table 231 self.scratchDb.dropTable(tableName)232 # self.scratchDb.dropTable(tableName) 232 233 sql = "CREATE TABLE " + tableName + " LIKE ImageMeta" 233 234 try: self.scratchDb.execute(sql) 234 235 except: pass 235 236 # insert all detections into table 236 if (ota[0:2] == "XY"): ccdID = ota[2:4] 237 else: ccdID = 0 238 239 # insert image metadata into table 237 240 sql = "INSERT INTO " + tableName + " ( \ 238 241 frameID \ … … 299 302 ) VALUES ( \ 300 303 " + str(self.expID) + " \ 301 ," + ota[2:4]+ " \304 ," + str(ccdID) + " \ 302 305 ," + str(self.bias) + " \ 303 306 ," + str(self.biasScat) + " \ … … 366 369 self.scratchDb.updateFilterID(tableName, self.filter) 367 370 self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) 368 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self. config.dataRelease))371 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) 369 372 if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO']) 370 373 self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0") … … 386 389 except: pass 387 390 388 # delete all detections with PSF_INST_MAG < 17.5, as decreed by Gene391 # delete all detections with PSF_INST_MAG < -17.5, as decreed by Gene 389 392 BEFORE = self.scratchDb.getRowCount(ippTableName) 390 393 #don't do this anymore … … 464 467 , '" + self.dateStr + "' \ 465 468 , 0 \ 466 , " + str(self. config.dataRelease) + "\469 , " + str(self.skychunk.dataRelease) + "\ 467 470 FROM " + ippTableName 468 471 # self.logger.info(sql) 472 469 473 self.scratchDb.execute(sql) 470 474 … … 517 521 self.scratchDb.execute(sql) 518 522 519 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self. config.dataRelease))523 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) 520 524 521 525 ''' 522 526 Populates the DetectionCalib table for this OTA 523 527 ''' 524 def populateDetectionCalibTable (self, ota):528 def populateDetectionCalibTableUpdateInsert(self, ota): 525 529 526 530 tableName = "DetectionCalib_" + ota … … 562 566 WHERE a.objID = b.objID AND a.detectID = b.detectID" 563 567 self.scratchDb.execute(sql) 564 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) 568 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) 569 570 ''' 571 Populates the DetectionCalib table for this OTA 572 ''' 573 def populateDetectionCalibTable(self, ota): 574 575 # target table name: 576 tableName = "DetectionCalib_" + ota 577 # drop then re-create table 578 self.scratchDb.dropTable(tableName) 579 sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib" 580 try: self.scratchDb.execute(sql) 581 except: pass 582 583 externID = self.imageIDs[ota] 584 585 imageID = self.scratchDb.getImageIDFromExternID(externID) 586 self.logger.infoPair("obtained","imageID") 587 588 # check for & create output directory first 589 datadumpDir = "/tmp/datadump" 590 try: 591 statinfo = os.stat(datadumpDir) 592 # check on the stat results? 593 except: 594 print "making the data dump directory ", datadumpDir 595 os.mkdir(datadumpDir) 596 os.chmod(datadumpDir, 0777) 597 statinfo = os.stat(datadumpDir) 598 599 dumpFile = datadumpDir + "/genetest.xx.dat" 600 files = glob.glob(dumpFile) 601 if len(files) > 0: 602 os.unlink(dumpFile) 603 604 # insert all detections into table 605 sql = "SELECT \ 606 a.objID, \ 607 a.detectID, \ 608 a.ippObjID, \ 609 a.ippDetectID, \ 610 a.filterID, \ 611 a.surveyID, \ 612 b.ra, \ 613 b.dec_, \ 614 b.raErr, \ 615 b.decErr, \ 616 b.zp, \ 617 b.zpErr, \ 618 b.expTime, \ 619 b.airMass, \ 620 " + str(self.skychunk.dataRelease) + " \ 621 FROM \ 622 Detection_" + ota + " as a \ 623 JOIN " + self.scratchDb.dvoDetectionTable + " as b \ 624 ON (a.objID = b.objID AND a.detectID = b.detectID) \ 625 WHERE b.imageID = " + str(imageID) + \ 626 " INTO OUTFILE '" + dumpFile + "'" 627 print "sql: ", sql 628 self.scratchDb.execute(sql) 629 630 sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName 631 print "sql: ", sql 632 self.scratchDb.execute(sql) 565 633 566 634 ''' … … 591 659 592 660 self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) 593 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) 594 661 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) 662 663 ''' 664 Applies indexes and other constraints to the PSPS tables 665 ''' 666 def alterPspsTablesChip(self, chipname, extname, x, y): 667 668 # drop the ImageMeta_ table first, or we can leave an invalid table behind 669 tableName = "ImageMeta_" + chipname 670 671 # drop then re-create table 672 self.scratchDb.dropTable(tableName) 673 674 # load corresponding header into memory 675 header = self.fits.findAndReadHeader(extname) 676 if not header: 677 self.logger.errorPair("No header found for chip", chipname) 678 return False 679 680 # check we have valid sourceID/imageID pair from the header 681 if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": return False 682 if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": return False 683 684 # store sourceID/imageID combo in Db so DVO can look up later 685 if not self.useFullTables: 686 self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID']) 687 688 # store these for later 689 self.imageIDs[chipname] = header['IMAGEID'] 690 691 # populate ImageMeta 692 self.populateImageMetaTable(chipname, header) 693 self.updateImageID("ImageMeta_" + chipname, x, y) 694 695 return True 595 696 596 697 ''' … … 623 724 ota = "XY%d%d" % (x, y) 624 725 625 # load corresponding header into memory 626 header = self.fits.findAndReadHeader(ota + ".hdr") 627 if not header: 628 self.logger.errorPair("No header found for OTA", ota) 629 continue 630 631 # check we have valid sourceID/imageID pair from the header 632 if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": continue 633 if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": continue 634 635 # store sourceID/imageID combo in Db so DVO can look up later 636 if not self.useFullTables: 637 self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID']) 638 639 # store these for later 640 self.imageIDs[ota] = header['IMAGEID'] 641 642 # populate ImageMeta 643 self.populateImageMetaTable(ota, header) 644 self.updateImageID("ImageMeta_" + ota, x, y) 645 726 self.alterPspsTablesChip(ota, ota + ".hdr", x, y) 727 728 # try the test Chip 729 self.alterPspsTablesChip("Chip", "Chip.hdr", 0, 0) 730 646 731 # now run DVO code to get all IDs 647 732 if not self.useFullTables: … … 651 736 # the column in PSPS 652 737 self.scratchDb.execute("ALTER TABLE DetectionCalib CHANGE dec_ `dec` double") 653 654 655 738 656 739 return True … … 675 758 extension = "XY%d%d_psf" % (x, y) 676 759 self.scratchDb.createIndex(extension, "IPP_IDET") 760 # try the test Chip 761 self.scratchDb.createIndex("Chip_psf", "IPP_IDET") 762 677 763 self.logger.infoPair("created indexes on", "IPP tables") 764 678 765 ''' 679 766 Updates provided table with DVO IDs from DVO table … … 693 780 self.scratchDb.execute(sql) 694 781 782 ''' 783 Does the processing, i.e. pulling stuff from IPP tables into PSPS tables 784 ''' 785 def populatePspsTablesChip(self, chipname, x, y, results, tables): 786 # XXX EAM NOTE: drop tables (Detection_, SkinnyObject_, DetectionCalib_, Detection_) here so 787 # they do not polute the db? 788 # XXX or put an explicit drop at the end of the loop? 789 790 #self.logger.infoTitle("Processing " + chipname) 791 # this is a bit crude: if the chip is not present, this test will fail and the chip 792 # will be (correctly) skipped. would be better to carry that information explicitly ("chip is missing") 793 if not self.scratchDb.astrometricSolutionOK(chipname): 794 self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % chipname) 795 return False 796 self.logger.info("populate stuff "); 797 # populate remainder of tables 798 self.populateDetectionTable(chipname, results) 799 self.logger.info("successful populate "); 800 # now add DVO IDs 801 self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname]) 802 self.logger.info("updated dvoids") 803 results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID") 804 #self.logger.info("deleted nulls") 805 self.updateImageID("Detection_" + chipname, x, y) 806 self.logger.info("updateImageId") 807 rowCount = self.scratchDb.getRowCount("Detection_" + chipname) 808 self.logger.info("got row count") 809 self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", 810 chipname, 811 results['ORIGINALTOTAL'], 812 results['SATDET'], 813 results['NULLINSTFLUX'], 814 results['NULLPEAKADU'], 815 results['NULLOBJID'], 816 rowCount) 817 self.totalOriginal = self.totalOriginal + results['ORIGINALTOTAL'] 818 self.totalSatDet = self.totalSatDet + results['SATDET'] 819 self.totalNulIInstFlux = self.totalNulIInstFlux + results['NULLINSTFLUX'] 820 self.totalNullPeakFlux = self.totalNullPeakFlux + results['NULLPEAKADU'] 821 self.totalNullObjID = self.totalNullObjID + results['NULLOBJID'] 822 self.totalDetections = self.totalDetections + rowCount 823 #self.logger.info("updated totals") 824 # check we have something in this Detection table TODO add this to table above 825 if rowCount < 1: 826 self.logger.infoPair("Skipping empty table for chipname", chipname) 827 return False; 828 829 # update ImageMeta with count of detections for this CHIPNAME and photoCodeID 830 sql = "UPDATE ImageMeta_" + chipname + " \ 831 SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + chipname), self.scratchDb.getPhotoCalID(self.imageIDs[chipname])) 832 self.scratchDb.execute(sql) 833 834 self.logger.info("updated imagedata") 835 self.populateSkinnyObjectTable(chipname) 836 self.logger.info("updated skinnyobject") 837 #self.populateObjectCalColorTable(chipname) 838 #self.logger.info("updated objectcalcolor") 839 self.populateDetectionCalibTable(chipname) 840 self.logger.info("updated detectioncalibtable") 841 842 # add these to list of tables to export later 843 self.tablesToExport.append("ImageMeta_" + chipname) 844 self.logger.info("export ImageMeta") 845 846 self.tablesToExport.append("Detection_" + chipname) 847 self.logger.info("export Detection") 848 849 self.tablesToExport.append("SkinnyObject_" + chipname) 850 self.logger.info("export Skinny") 851 852 #self.tablesToExport.append("ObjectCalColor_" + chipname) 853 self.tablesToExport.append("DetectionCalib_" + chipname) 854 self.logger.info("export DetectionCalib") 855 856 tables.append("Detection_" + chipname) 857 self.logger.info("updated detectioncalibtable") 858 859 return True 695 860 696 861 ''' … … 705 870 otaCount = 0 706 871 results = {} 707 totalOriginal = totalSatDet = totalNulIInstFlux = totalNullPeakFlux = totalNullObjID =totalDetections = 0872 self.totalOriginal = self.totalSatDet = self.totalNulIInstFlux = self.totalNullPeakFlux = self.totalNullObjID = self.totalDetections = 0 708 873 self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") 709 874 self.logger.info("| OTA | Initial total | Sat Det | NULL instFlux | NULL peak ADU | NULL obj ID | Remainder |") … … 721 886 if ota not in self.imageIDs: continue 722 887 723 #self.logger.infoTitle("Processing " + ota) 724 if not self.scratchDb.astrometricSolutionOK(ota): 725 self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % ota) 726 continue 727 # self.logger.info("populate stuff "); 728 # populate remainder of tables 729 self.populateDetectionTable(ota, results) 730 # self.logger.info("successful populate "); 731 # now add DVO IDs 732 self.updateDvoIDs("Detection_" + ota, self.imageIDs[ota]) 733 # self.logger.info("updated dvoids") 734 results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID") 735 #self.logger.info("deleted nulls") 736 self.updateImageID("Detection_" + ota, x, y) 737 # self.logger.info("updateImageId") 738 rowCount = self.scratchDb.getRowCount("Detection_" + ota) 739 # self.logger.info("got row count") 740 self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", 741 ota, 742 results['ORIGINALTOTAL'], 743 results['SATDET'], 744 results['NULLINSTFLUX'], 745 results['NULLPEAKADU'], 746 results['NULLOBJID'], 747 rowCount) 748 totalOriginal = totalOriginal + results['ORIGINALTOTAL'] 749 totalSatDet = totalSatDet + results['SATDET'] 750 totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX'] 751 totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU'] 752 totalNullObjID = totalNullObjID + results['NULLOBJID'] 753 totalDetections = totalDetections + rowCount 754 #self.logger.info("updated totals") 755 # check we have something in this Detection table TODO add this to table above 756 if rowCount < 1: 757 self.logger.debugPair("Skipping empty table for ota", ota) 758 continue; 759 760 # update ImageMeta with count of detections for this OTA and photoCodeID 761 sql = "UPDATE ImageMeta_" + ota + " \ 762 SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.imageIDs[ota])) 763 self.scratchDb.execute(sql) 764 765 #self.logger.info("updated imagedata") 766 self.populateSkinnyObjectTable(ota) 767 #self.logger.info("updated skinnyobject") 768 #self.populateObjectCalColorTable(ota) 769 #self.logger.info("updated objectcalcolor") 770 self.populateDetectionCalibTable(ota) 771 #self.logger.info("updated detectioncalibtable") 772 773 # add these to list of tables to export later 774 self.tablesToExport.append("ImageMeta_" + ota) 775 self.tablesToExport.append("Detection_" + ota) 776 self.tablesToExport.append("SkinnyObject_" + ota) 777 #self.tablesToExport.append("ObjectCalColor_" + ota) 778 self.tablesToExport.append("DetectionCalib_" + ota) 779 tables.append("Detection_" + ota) 780 781 otaCount = otaCount + 1 782 #self.logger.info("end of ota") 888 if self.populatePspsTablesChip(ota, x, y, results, tables): otaCount = otaCount + 1 889 890 if self.populatePspsTablesChip("Chip", 0, 0, results, tables): otaCount = otaCount + 1 891 783 892 # print totals 784 893 self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") 785 894 self.logger.info("| Total | %13d | %13d | %13d | %13d | %13d | %13d |", 786 totalOriginal,787 totalSatDet,788 totalNulIInstFlux,789 totalNullPeakFlux,790 totalNullObjID,791 totalDetections)895 self.totalOriginal, 896 self.totalSatDet, 897 self.totalNulIInstFlux, 898 self.totalNullPeakFlux, 899 self.totalNullObjID, 900 self.totalDetections) 792 901 self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") 793 902 … … 830 939 def importIppTables(self, filter=""): 831 940 832 if self.config.test: regex = "XY33.psf" 833 else : regex = ".*.psf" 941 ## if self.config.test: regex = "XY33.psf" 942 ## else : regex = ".*.psf" 943 regex = ".*.psf" 834 944 945 # XXX EAM NOTE : this is fragile : requires PS1_V4 835 946 columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" 836 947
Note:
See TracChangeset
for help on using the changeset viewer.
