IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 35003


Ignore:
Timestamp:
Jan 20, 2013, 9:03:41 PM (14 years ago)
Author:
eugene
Message:

split psps table functions to allow non-XYnn chips; test Chip.hdr; parse header keyword values of form 1.00E+10; smfs need to be in PS1_V3 or higher; make pseudo mef cmf for test; turn on force mode (turn off before releasing)

Location:
branches/eam_branches/ipp-20121219/ippToPsps
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20121219/ippToPsps/doc/concerns.txt

    r34960 r35003  
    5151  (basically, the mods to the sql were wrong)
    5252
     53*** SMF header layout is hard-wired in detectionbatch.py (ie, XYnn)
     54    * need to have a way of defining / loading a camera layout, either
     55    * from the ippconfig info or elsewhere
     56
     57
    5358
    5459-----

  • branches/eam_branches/ipp-20121219/ippToPsps/jython/detectionbatch.py

    r34926 r35003  
    214214        ," + self.safeDictionaryAccess(self.header, 'PCA2X0Y2') + " \
    215215        )"
     216
     217        print "frame meta sql: ", sql
     218
    216219        self.scratchDb.execute(sql)
    217220
    … …  
    593596        self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease))
    594597
     598    '''
     599    Applies indexes and other constraints to the PSPS tables
     600    '''
     601    def alterPspsTablesChip(self, chipname, extname, x, y):
     602
     603        # load corresponding header into memory
     604        header = self.fits.findAndReadHeader(extname)
     605        if not header:
     606            self.logger.errorPair("No header found for chip", chipname)
     607            return False
     608       
     609        # check we have valid sourceID/imageID pair from the header
     610        if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": return False
     611        if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": return False
     612       
     613        # store sourceID/imageID combo in Db so DVO can look up later
     614        if not self.useFullTables:
     615            self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID'])
     616           
     617        # store these for later
     618        self.imageIDs[chipname] = header['IMAGEID']
     619           
     620        # populate ImageMeta
     621        self.populateImageMetaTable(chipname, header)
     622        self.updateImageID("ImageMeta_" + chipname, x, y)
     623           
     624        return True
    595625
    596626    '''
    … …  
    623653                ota = "XY%d%d" % (x, y)
    624654               
    625                 # load corresponding header into memory
    626                 header = self.fits.findAndReadHeader(ota + ".hdr")
    627                 if not header:
    628                     self.logger.errorPair("No header found for OTA", ota)
    629                     continue
    630 
    631                 # check we have valid sourceID/imageID pair from the header
    632                 if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": continue
    633                 if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": continue
    634 
    635                 # store sourceID/imageID combo in Db so DVO can look up later
    636                 if not self.useFullTables:
    637                     self.scratchDb.insertNewDvoExternID(header['SOURCEID'], header['IMAGEID'])
    638 
    639                 # store these for later
    640                 self.imageIDs[ota] = header['IMAGEID']
    641 
    642                 # populate ImageMeta
    643                 self.populateImageMetaTable(ota, header)
    644                 self.updateImageID("ImageMeta_" + ota, x, y)
    645              
     655                self.alterPspsTablesChip(ota, ota + ".hdr", x, y)
     656
     657        # try the test Chip
     658        self.alterPspsTablesChip("Chip", "Chip.hdr", 0, 0)
     659
    646660        # now run DVO code to get all IDs
    647661        if not self.useFullTables:
    … …  
    651665        # the column in PSPS
    652666        self.scratchDb.execute("ALTER TABLE DetectionCalib CHANGE dec_ `dec` double")
    653 
    654 
    655667
    656668        return True
    … …  
    675687                extension = "XY%d%d_psf" % (x, y)
    676688                self.scratchDb.createIndex(extension, "IPP_IDET")
     689        # try the test Chip
     690        self.scratchDb.createIndex(Chip_psf, "IPP_IDET")
     691
    677692        self.logger.infoPair("created indexes on", "IPP tables")     
     693
    678694    '''
    679695    Updates provided table with DVO IDs from DVO table
    … …  
    693709        self.scratchDb.execute(sql)
    694710
     711    '''
     712    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
     713    '''
     714    def populatePspsTablesChip(self, chipname, x, y):
     715        #self.logger.infoTitle("Processing " + chipname)
     716        if not self.scratchDb.astrometricSolutionOK(chipname):
     717            self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % chipname)
     718            return False
     719        self.logger.info("populate stuff ");
     720        # populate remainder of tables
     721        self.populateDetectionTable(chipname, results)
     722        self.logger.info("successful populate ");
     723        # now add DVO IDs
     724        self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname])
     725        self.logger.info("updated dvoids")
     726        results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID")
     727        #self.logger.info("deleted nulls")
     728        self.updateImageID("Detection_" + chipname, x, y)
     729        self.logger.info("updateImageId")
     730        rowCount = self.scratchDb.getRowCount("Detection_" + chipname)
     731        self.logger.info("got row count")
     732        self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |",
     733                chipname,
     734                results['ORIGINALTOTAL'],
     735                results['SATDET'],
     736                results['NULLINSTFLUX'],
     737                results['NULLPEAKADU'],
     738                results['NULLOBJID'],
     739                rowCount)
     740        totalOriginal = totalOriginal + results['ORIGINALTOTAL']
     741        totalSatDet = totalSatDet + results['SATDET']
     742        totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX']
     743        totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU']
     744        totalNullObjID = totalNullObjID + results['NULLOBJID']
     745        totalDetections = totalDetections + rowCount
     746        #self.logger.info("updated totals")
     747        # check we have something in this Detection table TODO add this to table above
     748        if rowCount < 1:
     749            self.logger.debugPair("Skipping empty table for chipname", chipname)
     750            return False;
     751
     752        # update ImageMeta with count of detections for this CHIPNAME and photoCodeID
     753        sql = "UPDATE ImageMeta_" + chipname + " \
     754               SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + chipname), self.scratchDb.getPhotoCalID(self.imageIDs[chipname]))
     755        self.scratchDb.execute(sql)
     756       
     757        #self.logger.info("updated imagedata")
     758        self.populateSkinnyObjectTable(chipname)
     759        #self.logger.info("updated skinnyobject")
     760        #self.populateObjectCalColorTable(chipname)
     761        #self.logger.info("updated objectcalcolor")
     762        self.populateDetectionCalibTable(chipname)
     763        #self.logger.info("updated detectioncalibtable")
     764       
     765        # add these to list of tables to export later
     766        self.tablesToExport.append("ImageMeta_" + chipname)
     767        self.tablesToExport.append("Detection_" + chipname)
     768        self.tablesToExport.append("SkinnyObject_" + chipname)
     769        #self.tablesToExport.append("ObjectCalColor_" + chipname)
     770        self.tablesToExport.append("DetectionCalib_" + chipname)
     771        tables.append("Detection_" + chipname)
     772        return True
    695773
    696774    '''
    … …  
    721799                if ota not in self.imageIDs: continue
    722800
    723                 #self.logger.infoTitle("Processing " + ota)
    724                 if not self.scratchDb.astrometricSolutionOK(ota):
    725                     self.logger.info("| %5s |            ------------- Bad astrometric solution : rejecting ------------                    |" % ota)
    726                     continue
    727                # self.logger.info("populate stuff ");
    728                 # populate remainder of tables
    729                 self.populateDetectionTable(ota, results)
    730                # self.logger.info("successful populate ");
    731                 # now add DVO IDs
    732                 self.updateDvoIDs("Detection_" + ota, self.imageIDs[ota])
    733                # self.logger.info("updated dvoids")
    734                 results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID")
    735                 #self.logger.info("deleted nulls")
    736                 self.updateImageID("Detection_" + ota, x, y)
    737                # self.logger.info("updateImageId")
    738                 rowCount = self.scratchDb.getRowCount("Detection_" + ota)
    739               #  self.logger.info("got row count")
    740                 self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |",
    741                         ota,
    742                         results['ORIGINALTOTAL'],
    743                         results['SATDET'],
    744                         results['NULLINSTFLUX'],
    745                         results['NULLPEAKADU'],
    746                         results['NULLOBJID'],
    747                         rowCount)
    748                 totalOriginal = totalOriginal + results['ORIGINALTOTAL']
    749                 totalSatDet = totalSatDet + results['SATDET']
    750                 totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX']
    751                 totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU']
    752                 totalNullObjID = totalNullObjID + results['NULLOBJID']
    753                 totalDetections = totalDetections + rowCount
    754                 #self.logger.info("updated totals")
    755                 # check we have something in this Detection table TODO add this to table above
    756                 if rowCount < 1:
    757                     self.logger.debugPair("Skipping empty table for ota", ota)
    758                     continue;
    759 
    760                 # update ImageMeta with count of detections for this OTA and photoCodeID
    761                 sql = "UPDATE ImageMeta_" + ota + " \
    762                        SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.imageIDs[ota]))
    763                 self.scratchDb.execute(sql)
    764                
    765                 #self.logger.info("updated imagedata")
    766                 self.populateSkinnyObjectTable(ota)
    767                 #self.logger.info("updated skinnyobject")
    768                 #self.populateObjectCalColorTable(ota)
    769                 #self.logger.info("updated objectcalcolor")
    770                 self.populateDetectionCalibTable(ota)
    771                 #self.logger.info("updated detectioncalibtable")
    772                
    773                 # add these to list of tables to export later
    774                 self.tablesToExport.append("ImageMeta_" + ota)
    775                 self.tablesToExport.append("Detection_" + ota)
    776                 self.tablesToExport.append("SkinnyObject_" + ota)
    777                 #self.tablesToExport.append("ObjectCalColor_" + ota)
    778                 self.tablesToExport.append("DetectionCalib_" + ota)
    779                 tables.append("Detection_" + ota)
    780            
    781                 otaCount = otaCount + 1
    782                 #self.logger.info("end of ota")
     801                if populatePspsTables(ota, x, y): otaCount = otaCount + 1
     802
     803        if populatePspsTables("Chip", 0, 0): otaCount = otaCount + 1
     804
    783805        # print totals
    784806        self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+")
    … …  
    833855       else : regex = ".*.psf"
    834856 
     857       # XXX EAM NOTE : this is fragile : requires PS1_V_
    835858       columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ"
    836859

  • branches/eam_branches/ipp-20121219/ippToPsps/jython/fits.py

    r33259 r35003  
    137137
    138138            # this regex will get param/value pairs for all header cards, ignoring comments and parsing out 'HIERACH' prefixes
    139             match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-_\.:\s@#]+)\'*\\/*', record)
     139            match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-+_\.:\s@#]+)\'*\\/*', record)
    140140            if match:
    141141

  • branches/eam_branches/ipp-20121219/ippToPsps/jython/ippjython

    r34989 r35003  
    4444shift
    4545
     46# XXX if you want to run a test on a local file, use this line:
     47# $JAVA -Xbootclasspath/a:$CLASSPATH -jar $JYTHON $classfile $*
     48
    4649$JAVA -Xbootclasspath/a:$CLASSPATH -jar $JYTHON $JYDIR/$classfile $*

  • branches/eam_branches/ipp-20121219/ippToPsps/jython/ipptopspsdb.py

    r34972 r35003  
    999999
    10001000            self.config.epoch = rs.getString(16)
     1001
    10011002            self.config.survey = rs.getString(17)
    10021003            self.config.pspsSurvey = rs.getString(18)
    … …  
    10051006            if rs.getInt(20) == 1: self.config.batchTypes.append("ST")
    10061007            if rs.getInt(21) == 1: self.config.batchTypes.append("OB")
    1007             self.config.force = False # TODO
     1008            self.config.force = True # TODO
    10081009            self.config.test = False # TODO
    10091010            self.config.isLoaded = True
    … …  
    11011102               GROUP BY ra_center LIMIT 1"
    11021103
     1104        # XXX print "sql: ", sql               
     1105
    11031106        try:
    11041107            rs = self.executeQuery(sql)

  • branches/eam_branches/ipp-20121219/ippToPsps/jython/queue.py

    r34954 r35003  
    3030        self.gpc1Db = Gpc1Db(self.logger, self.config)
    3131        self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb)
     32
    3233        try:
    3334            self.dvoObjects = DvoObjects(self.logger, self.config)
    … …  
    3940
    4041        if len(argv) > 2: self.parsePollTimeArg(sys.argv[2])
    41    
    4242
    4343    '''

  • branches/eam_branches/ipp-20121219/ippToPsps/test/mkgpc1data.dvo

    r35000 r35003  
    44
    55if (not($?VERBOSE)) set VERBOSE = 0
    6 
    7 macro test.all
    8   test.relphot PS1_V2 PS1_V2
    9 end 
    106
    117## this script must be run in this directory, output data goes to 'testdata'
    … …  
    4036macro mkfull
    4137  init.db
    42   mkcatdir.cam PS1_V2 PS1_V2
    43   mkcatdir.stk PS1_V2 PS1_V2
     38  mkcatdir.cam PS1_V3 PS1_V2
     39  mkcatdir.stk PS1_V3 PS1_V2
    4440  insert.stack.set
    4541end
    … …  
    9591    $offset = $word:3
    9692
    97     ctimes -ref 2008/01/01,$i\:00:00 -var mjd
     93    ctimes -abs 2008/01/01,$i\:00:00 -var mjd
     94
     95    # XXX for a simple test of ippToPsps, i need to generate smf files with some correspondence to gpc1 exposure smfs
     96    # this means:
     97    # 1) a PHU with some basic header data
     98    # 2) add extensions with EXTNAME of XYnn
     99
     100    # create an empty header and populate with the desired keywords
     101    mcreate dummy 0 0
     102    keyword dummy MJD-OBS  -wf $mjd
     103    keyword dummy FILTERID -w  r.00000
     104    keyword dummy EXPTIME  -wf 1.0
     105
     106    keyword dummy ZPT_ERR  -wf 0.01
     107    keyword dummy EXPREQ   -wf 1.0
     108    keyword dummy AIRMASS  -wf 1.0
     109    keyword dummy RA       -wf $RA
     110    keyword dummy DEC      -wf $DEC
     111    keyword dummy CTYPE1   -w "RA---DIS"
     112    keyword dummy CTYPE2   -w "DEC--DIS"
     113    keyword dummy CRVAL1   -wf $RA
     114    keyword dummy CRVAL2   -wf $DEC
     115    keyword dummy CRPIX1   -wf 0.0
     116    keyword dummy CRPIX2   -wf 0.0
     117    keyword dummy CDELT1   -wf {1.0/3600}
     118    keyword dummy CDELT2   -wf {1.0/3600}
     119    keyword dummy PC001001 -wf 1.0
     120    keyword dummy PC001002 -wf 0.0
     121    keyword dummy PC002001 -wf 0.0
     122    keyword dummy PC002002 -wf 1.0
     123    keyword dummy NPLYTERM -wd 0
     124    keyword dummy PCA1X3Y0 -wf 0.0
     125    keyword dummy PCA1X2Y1 -wf 0.0
     126    keyword dummy PCA1X1Y2 -wf 0.0
     127    keyword dummy PCA1X0Y3 -wf 0.0
     128    keyword dummy PCA1X2Y0 -wf 0.0
     129    keyword dummy PCA1X1Y1 -wf 0.0
     130    keyword dummy PCA1X0Y2 -wf 0.0
     131    keyword dummy PCA2X3Y0 -wf 0.0
     132    keyword dummy PCA2X2Y1 -wf 0.0
     133    keyword dummy PCA2X1Y2 -wf 0.0
     134    keyword dummy PCA2X0Y3 -wf 0.0
     135    keyword dummy PCA2X2Y0 -wf 0.0
     136    keyword dummy PCA2X1Y1 -wf 0.0
     137    keyword dummy PCA2X0Y2 -wf 0.0
     138    wd dummy $cmffile
     139
     140    exec ftable -list $cmffile
    98141
    99142    mkinput $offset $rawfile
    100143    exec mkcmf $rawfile $cmffile -date 2008/1/1 -time $i\:00:00 -radec $RA $DEC -type $1 -imageID $ID -sourceID 0
    101     exec rm -f header.dat
    102     output header.dat
    103     echo MJD-OBS = $mjd
    104     echo FILTERID= 'r.00000'
    105     output stdout
    106     exec fits_insert $cmffile header.dat
     144
     145    exec ftable -list $cmffile
     146
     147    # keywords which are read from image extension
     148    # MSKY_MN
     149    # MSKY_SIG
     150    # FSATUR
     151    # FLIMIT
     152    # CERROR
     153    # ZPT_OBS
     154    # NASTRO
     155    # NASTRO
     156    # CNAXIS1
     157    # CNAXIS2
     158    # FWHM_MAJ
     159    # FWHM_MIN
     160    # ANGLE
     161    # IQ_FW1
     162    # IQ_FW2
     163    # IQ_FW1
     164    # IQ_FW2
     165    # APMIFIT
     166    # DAPMIFIT
     167    # DETECTOR
     168    # IMAGEID
     169    # DETREND.MASK
     170    # DETREND.DARK
     171    # DETREND.FLAT
     172    # ZPT_OBS
     173    # CTYPE1
     174    # CTYPE2
     175    # CRVAL1
     176    # CRVAL2
     177    # CRPIX1
     178    # CRPIX2
     179    # CDELT1
     180    # CDELT2
     181    # PC001001
     182    # PC001002
     183    # PC002001
     184    # PC002002
     185    # NPLYTERM
     186    # PCA1X3Y0
     187    # PCA1X2Y1
     188    # PCA1X1Y2
     189    # PCA1X0Y3
     190    # PCA1X2Y0
     191    # PCA1X1Y1
     192    # PCA1X0Y2
     193    # PCA2X3Y0
     194    # PCA2X2Y1
     195    # PCA2X1Y2
     196    # PCA2X0Y3
     197    # PCA2X2Y0
     198    # PCA2X1Y1
     199    # PCA2X0Y2
     200
     201    echo addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
    107202    exec addstar -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
    108203  end
Note: See TracChangeset for help on using the changeset viewer.