- Timestamp:
- Dec 6, 2012, 6:32:19 PM (14 years ago)
- Location:
- branches/czw_branch/20120906/ippToPsps
- Files:
-
- 2 edited
-
. (modified) (1 prop)
-
jython/stackbatch.py (modified) (18 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/czw_branch/20120906/ippToPsps
- Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120905/ippToPsps (added) merged: 34428,34466,34567,34575,34735,34747
- Property svn:mergeinfo changed
-
branches/czw_branch/20120906/ippToPsps/jython/stackbatch.py
r34165 r34772 76 76 self.projectioncell = self.skycell 77 77 78 # proposed new values. Need to coordinate with the SkyCell table 79 # this fits in 32 bits for 0 < tessID < 214 80 # self.skycell = str( (tessID * (10000 * 1000)) + (int(self.projectioncell) * 1000) + cellID ) 81 78 82 self.analysisVer = meta[2]; 79 83 … … 89 93 self.scratchDb.dropTable("SkinnyObject") 90 94 self.scratchDb.dropTable("ObjectCalColor") 95 self.scratchDb.dropTable("StackDetectionCalib") 91 96 92 97 # delete IPP tables … … 140 145 141 146 sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET \ 142 "+prefix+"flxR1=b.APER_FLUX_1 , \143 "+prefix+"flxR1Err=b.APER_FLUX_ERR_1 , \144 "+prefix+"flxR1Std=b.APER_FLUX_STDEV_1 , \147 "+prefix+"flxR1=b.APER_FLUX_1 / " + str(self.expTime) + ", \ 148 "+prefix+"flxR1Err=b.APER_FLUX_ERR_1 / " + str(self.expTime) + ", \ 149 "+prefix+"flxR1Std=b.APER_FLUX_STDEV_1 / " + str(self.expTime) + ", \ 145 150 "+prefix+"flxR1Fill=b.APER_FILL_1, \ 146 "+prefix+"flxR2=b.APER_FLUX_2 , \147 "+prefix+"flxR2Err=b.APER_FLUX_ERR_2 , \148 "+prefix+"flxR2Std=b.APER_FLUX_STDEV_2 , \151 "+prefix+"flxR2=b.APER_FLUX_2 / " + str(self.expTime) + ", \ 152 "+prefix+"flxR2Err=b.APER_FLUX_ERR_2 / " + str(self.expTime) + ", \ 153 "+prefix+"flxR2Std=b.APER_FLUX_STDEV_2 / " + str(self.expTime) + ", \ 149 154 "+prefix+"flxR2Fill=b.APER_FILL_2, \ 150 "+prefix+"flxR3=b.APER_FLUX_3 , \151 "+prefix+"flxR3Err=b.APER_FLUX_ERR_3 , \152 "+prefix+"flxR3Std=b.APER_FLUX_STDEV_3 , \155 "+prefix+"flxR3=b.APER_FLUX_3 / " + str(self.expTime) + ", \ 156 "+prefix+"flxR3Err=b.APER_FLUX_ERR_3 / " + str(self.expTime) + ", \ 157 "+prefix+"flxR3Std=b.APER_FLUX_STDEV_3 / " + str(self.expTime) + ", \ 153 158 "+prefix+"flxR3Fill=b.APER_FILL_3, \ 154 "+prefix+"flxR4=b.APER_FLUX_4 , \155 "+prefix+"flxR4Err=b.APER_FLUX_ERR_4 , \156 "+prefix+"flxR4Std=b.APER_FLUX_STDEV_4 , \159 "+prefix+"flxR4=b.APER_FLUX_4 / " + str(self.expTime) + ", \ 160 "+prefix+"flxR4Err=b.APER_FLUX_ERR_4 / " + str(self.expTime) + ", \ 161 "+prefix+"flxR4Std=b.APER_FLUX_STDEV_4 / " + str(self.expTime) + ", \ 157 162 "+prefix+"flxR4Fill=b.APER_FILL_4, \ 158 "+prefix+"flxR5=b.APER_FLUX_5 , \159 "+prefix+"flxR5Err=b.APER_FLUX_ERR_5 , \160 "+prefix+"flxR5Std=b.APER_FLUX_STDEV_5 , \163 "+prefix+"flxR5=b.APER_FLUX_5 / " + str(self.expTime) + ", \ 164 "+prefix+"flxR5Err=b.APER_FLUX_ERR_5 / " + str(self.expTime) + ", \ 165 "+prefix+"flxR5Std=b.APER_FLUX_STDEV_5 / " + str(self.expTime) + ", \ 161 166 "+prefix+"flxR5Fill=b.APER_FILL_5, \ 162 "+prefix+"flxR6=b.APER_FLUX_6 , \163 "+prefix+"flxR6Err=b.APER_FLUX_ERR_6 , \164 "+prefix+"flxR6Std=b.APER_FLUX_STDEV_6 , \167 "+prefix+"flxR6=b.APER_FLUX_6 / " + str(self.expTime) + ", \ 168 "+prefix+"flxR6Err=b.APER_FLUX_ERR_6 / " + str(self.expTime) + ", \ 169 "+prefix+"flxR6Std=b.APER_FLUX_STDEV_6 / " + str(self.expTime) + ", \ 165 170 "+prefix+"flxR6Fill=b.APER_FILL_6, \ 166 "+prefix+"flxR7=b.APER_FLUX_7 , \167 "+prefix+"flxR7Err=b.APER_FLUX_ERR_7 , \168 "+prefix+"flxR7Std=b.APER_FLUX_STDEV_7 , \171 "+prefix+"flxR7=b.APER_FLUX_7 / " + str(self.expTime) + ", \ 172 "+prefix+"flxR7Err=b.APER_FLUX_ERR_7 / " + str(self.expTime) + ", \ 173 "+prefix+"flxR7Std=b.APER_FLUX_STDEV_7 / " + str(self.expTime) + ", \ 169 174 "+prefix+"flxR7Fill=b.APER_FILL_7, \ 170 "+prefix+"flxR8=b.APER_FLUX_8 , \171 "+prefix+"flxR8Err=b.APER_FLUX_ERR_8 , \172 "+prefix+"flxR8Std=b.APER_FLUX_STDEV_8 , \175 "+prefix+"flxR8=b.APER_FLUX_8 / " + str(self.expTime) + ", \ 176 "+prefix+"flxR8Err=b.APER_FLUX_ERR_8 / " + str(self.expTime) + ", \ 177 "+prefix+"flxR8Std=b.APER_FLUX_STDEV_8 / " + str(self.expTime) + ", \ 173 178 "+prefix+"flxR8Fill=b.APER_FILL_8, \ 174 "+prefix+"flxR9=b.APER_FLUX_9 , \175 "+prefix+"flxR9Err=b.APER_FLUX_ERR_9 , \176 "+prefix+"flxR9Std=b.APER_FLUX_STDEV_9 , \179 "+prefix+"flxR9=b.APER_FLUX_9 / " + str(self.expTime) + ", \ 180 "+prefix+"flxR9Err=b.APER_FLUX_ERR_9 / " + str(self.expTime) + ", \ 181 "+prefix+"flxR9Std=b.APER_FLUX_STDEV_9 / " + str(self.expTime) + ", \ 177 182 "+prefix+"flxR9Fill=b.APER_FILL_9, \ 178 "+prefix+"flxR10=b.APER_FLUX_10 , \179 "+prefix+"flxR10Err=b.APER_FLUX_ERR_10 , \180 "+prefix+"flxR10Std=b.APER_FLUX_STDEV_10 , \183 "+prefix+"flxR10=b.APER_FLUX_10 / " + str(self.expTime) + ", \ 184 "+prefix+"flxR10Err=b.APER_FLUX_ERR_10 / " + str(self.expTime) + ", \ 185 "+prefix+"flxR10Std=b.APER_FLUX_STDEV_10 / " + str(self.expTime) + ", \ 181 186 "+prefix+"flxR10Fill=b.APER_FILL_10 \ 182 187 WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition … … 196 201 sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ 197 202 "+model+"Radius=b.EXT_WIDTH_MAJ, \ 198 "+model+" Mag=b.EXT_INST_MAG, \199 "+model+" MagErr=b.EXT_INST_MAG_SIG,\203 "+model+"Flux=POW(10, -0.4 * b.EXT_INST_MAG) / " + str(self.expTime) + " , \ 204 "+model+"FluxErr=ABS(b.EXT_INST_MAG_SIG) * POW(10, -0.4 * b.EXT_INST_MAG) / " + str(self.expTime) + " / 1.085736, \ 200 205 "+model+"Ab=b.EXT_WIDTH_MAJ/b.EXT_WIDTH_MIN, \ 201 206 "+model+"Phi=b.EXT_THETA, \ … … 319 324 ,skyCellID \ 320 325 ,projectionCellID \ 321 ,obsTime \322 326 ,xPos \ 323 327 ,yPos \ 324 328 ,xPosErr \ 325 329 ,yPosErr \ 326 , instFlux \327 , instFluxErr \330 ,psfFlux \ 331 ,psfFluxErr \ 328 332 ,peakFlux \ 329 333 ,sky \ … … 335 339 ,psfLikelihood \ 336 340 ,infoFlag \ 337 ,psfCf \ 341 ,psfQf \ 342 ,psfQfPerfect \ 338 343 ,momentXX \ 339 344 ,momentXY \ … … 345 350 ,momentR1 \ 346 351 ,momentRH \ 347 ,ap Mag\348 ,ap MagErr \352 ,apFlux \ 353 ,apFluxErr \ 349 354 ,kronFlux \ 350 355 ,kronFluxErr \ 351 356 ,kronRad \ 352 , kronRadErr\357 ,psfChiSq \ 353 358 ,nFrames \ 354 359 ,assocDate \ … … 359 364 ," + self.skycell + " \ 360 365 ," + self.projectioncell + " \ 361 ," + self.header['MJD-OBS'] + " \362 366 ,X_PSF \ 363 367 ,Y_PSF \ … … 367 371 ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + ")) / 1.085736) \ 368 372 ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + str(self.expTime) + " \ 369 ,SKY \370 ,SKY_SIGMA \373 ,SKY / " + str(self.expTime) + " \ 374 ,SKY_SIGMA / " + str(self.expTime) + " \ 371 375 ,EXT_NSIGMA \ 372 376 ,PSF_MAJOR \ … … 374 378 ,PSF_THETA \ 375 379 ,psfLikelihood(EXT_NSIGMA) \ 376 , FLAGS << 32 | FLAGS2\380 , FLAGS2 << 32 | FLAGS \ 377 381 ,PSF_QF \ 382 ,PSF_QF_PERFECT \ 378 383 ,MOMENTS_XX \ 379 384 ,MOMENTS_XY \ … … 385 390 ,MOMENTS_R1 \ 386 391 ,MOMENTS_RH \ 387 , AP_MAG\388 , 392 ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + " \ 393 ,NULL \ 389 394 ,KRON_FLUX / " + str(self.expTime) + " \ 390 395 ,KRON_FLUX_ERR / " + str(self.expTime) + " \ 391 , NULL\392 , NULL\396 ,MOMENTS_R1 * 2.5 \ 397 ,PSF_CHISQ \ 393 398 ,N_FRAMES \ 394 399 , '" + self.dateStr + "' \ 395 400 ," + self.historyModNum + " \ 396 401 FROM SkyChip_psf" 397 398 self.scratchDb.execute(sql)402 self.scratchDb.execute(sql) 403 399 404 #it is possible to drop some detections from dvo (that are present in the cmf). when that happens we get a 0 for objid 400 405 #we drop those... … … 402 407 self.scratchDb.execute(sql) 403 408 self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0") 404 409 405 410 self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID)) 411 406 412 self.scratchDb.updateFilterID("StackDetection", self.filter) 413 407 414 self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.config.dataRelease)) 415 408 416 self.scratchDb.updateAllRows("StackDetection", "primaryF", "0") 417 409 418 self.scratchDb.updateAllRows("StackDetection", "activeFlag", "0") 419 410 420 self.updateStackMetaID("StackDetection") 421 411 422 self.updateStackTypeID("StackDetection") 412 self.updateDvoIDs("StackDetection") 423 424 self.updateDvoIDsAndFlags("StackDetection") 425 413 426 sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)" 427 414 428 self.scratchDb.execute(sql) 415 429 … … 418 432 #if deep stack and instFlux = null and err not null 419 433 sql = "UPDATE StackDetection AS a, SkyChip_psf AS b \ 420 SET instFlux = 2*b.PSF_INST_FLUX_SIG / " + str(self.expTime) + " \421 WHERE instFlux IS NULL \434 SET psfFlux = 2*b.PSF_INST_FLUX_SIG / " + str(self.expTime) + " \ 435 WHERE psfFlux IS NULL \ 422 436 AND a.ippDetectID = b.IPP_IDET \ 423 437 AND b.PSF_INST_FLUX_SIG IS NOT NULL" 438 424 439 self.scratchDb.execute(sql) 425 440 426 self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux") 441 #leave null instflux in 442 #self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux") 443 427 444 self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID") 445 428 446 sql="DELETE FROM StackDetection where objID = 0" 429 self.scratchDb.execute(sql) 430 self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0") 431 447 448 self.scratchDb.execute(sql) 449 self.logger.infoPair("Delleting", "entries with StackDetection.objID = 0") 450 432 451 433 452 ''' … … 460 479 petRadius=b.PETRO_RADIUS \ 461 480 ,petRadiusErr=b.PETRO_RADIUS_ERR \ 462 ,pet Mag=b.PETRO_MAG\463 ,pet MagErr=b.PETRO_MAG_ERR\481 ,petFlux=POW(10.0, -0.4 * b.PETRO_MAG) /" + str(self.expTime) + " \ 482 ,petFluxErr=ABS(b.PETRO_MAG_ERR) * POW(10.0, (-0.4*b.PETRO_MAG)) / " + str(self.expTime) + " / 1.085736 \ 464 483 ,petR50=b.PETRO_RADIUS_50 \ 465 484 ,petR50Err=b.PETRO_RADIUS_50_ERR \ … … 573 592 self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.config.dataRelease)) 574 593 594 ''' 595 Populates the StackDetectionCalib table 596 ''' 597 def populateStackDetectionCalib(self): 598 self.logger.infoPair("Processing table", "StackDetectionCalib") 599 tableName = "StackDetectionCalib" 600 # insert all detections into table 601 sql = "INSERT INTO " + tableName + " ( \ 602 objID \ 603 ,stackDetectID \ 604 ,ippObjID \ 605 ,ippDetectID \ 606 ,filterID \ 607 ,surveyID \ 608 ) \ 609 SELECT \ 610 objID \ 611 ,stackDetectID \ 612 ,ippObjID \ 613 ,ippDetectID \ 614 ,filterID \ 615 ,surveyID \ 616 FROM StackDetection" 617 self.scratchDb.execute(sql) 618 # insert calibration information from dvoDetections into the Table 619 sql = "UPDATE " + tableName + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b \ 620 SET a.ra = b.ra, \ 621 a.dec = b.dec_, \ 622 a.raErr = b.raErr, \ 623 a.decErr = b.decErr, \ 624 a.zp = b.zp, \ 625 a.zpErr = b.zpErr, \ 626 a.expTime = b.expTime, \ 627 a.airMass = b.airMass \ 628 WHERE a.stackDetectID = b.detectID" 629 self.scratchDb.execute(sql) 630 self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) 631 632 633 634 575 635 ''' 576 636 Populates the ObjectCalColor table … … 600 660 601 661 self.logger.debug("Altering PSPS tables") 662 # heather uncommented the unique line -- well that no worky 602 663 #self.scratchDb.makeColumnUnique("StackDetection", "objID") 603 664 self.scratchDb.createIndex("StackDetection", "ippDetectID") 604 665 self.scratchDb.createIndex("StackApFlx", "ippDetectID") 605 666 self.scratchDb.createIndex("StackModelFit", "ippDetectID") 667 668 # dec is reserved in MySQL, so STILTS replaces if with dec_, which is not the name of 669 # the column in PSPS 670 self.scratchDb.execute("ALTER TABLE StackDetectionCalib CHANGE dec_ `dec` double") 606 671 607 672 return True … … 624 689 ''' 625 690 def updateDvoIDs(self, table): 626 691 self.logger.infoPair("getting","imageID") 627 692 imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) 693 self.logger.infoPair("obtained","imageID") 628 694 self.logger.debug("Updating table '" + table + "' with DVO IDs...") 629 695 sql = "UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ 630 696 a.ippObjID = b.ippObjID, \ 631 a.stackDetectID = 10 * b.detectID + " + str(self.filterID) + ", \697 a.stackDetectID = b.detectID, \ 632 698 a.objID = b.objID \ 633 699 WHERE a.ippDetectID = b.ippDetectID \ 634 700 AND b.imageID = " + str(imageID) 635 701 self.scratchDb.execute(sql) 636 702 703 ''' 704 Updates Flags 705 ''' 706 def updateDvoIDsAndFlags(self, table): 707 self.logger.infoPair("getting","imageID") 708 imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) 709 self.logger.infoPair("obtained","imageID") 710 self.logger.debug("Updating table '" + table + "' with DVO IDs...") 711 sql = "UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ 712 a.ippObjID = b.ippObjID, \ 713 a.stackDetectID = b.detectID, \ 714 a.objID = b.objID, \ 715 a.infoFlag = b.flags << 45 | a.infoFlag \ 716 WHERE a.ippDetectID = b.ippDetectID \ 717 AND b.imageID = " + str(imageID) 718 self.scratchDb.execute(sql) 719 720 637 721 ''' 638 722 Does the processing, i.e. pulling stuff from IPP tables into PSPS tables … … 643 727 if not self.getIDsFromDVO(): 644 728 return False 645 729 #self.logger.infoPair("populating","StackMeta") 646 730 self.populateStackMeta() 731 #self.logger.infoPair("populating","StackDetection") 647 732 self.populateStackDetection() 648 733 649 734 if self.stackType != "NIGHTLY_STACK": 735 #self.logger.infoPair("populating","StackModelFit") 650 736 self.populateStackModelFit() 737 #self.logger.infoPair("populating","StackApFlx") 651 738 self.populateStackApFlx() 652 739 #self.logger.infoPair("populating","StackToImage") 653 740 self.populateStackToImage() 741 #self.logger.infoPair("populating","SkinnyObject") 654 742 self.populateSkinnyObject() 743 #self.logger.infoPair("populating","ObjectCalColor") 655 744 self.populateObjectCalColor() 745 #self.logger.infoPair("populating","StackDetectionCalib") 746 self.populateStackDetectionCalib() 656 747 657 748 self.setMinMaxObjID(["StackDetection"])
Note:
See TracChangeset
for help on using the changeset viewer.
