Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/ippToPspsDbSchema.sql =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/ippToPspsDbSchema.sql (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/ippToPspsDbSchema.sql (revision 37801) @@ -1,5 +1,5 @@ -- MySQL dump 10.11 -- --- Host: ippdb01 Database: ipptopsps +-- Host: ipp06 Database: ipptopsps -- ------------------------------------------------------ -- Server version 5.0.77-log @@ -189,4 +189,19 @@ SET character_set_client = @saved_cs_client; -- +-- Table structure for table `forcedobject` +-- + +DROP TABLE IF EXISTS `forcedobject`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `forcedobject` ( + `batch_id` bigint(20) unsigned NOT NULL, + `region` varchar(50) default NULL, + KEY `batch_id` (`batch_id`), + CONSTRAINT `diffobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- -- Table structure for table `diffobject` -- Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/parse_vot.pl =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/parse_vot.pl (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/parse_vot.pl (revision 37801) @@ -3,6 +3,6 @@ # this thing will just parse the votables and print out in 'ken schema style' -$wiki=1; -#$csv = 1; +#$wiki=1; +$csv = 1; $infield = 0; $intable = 0; Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/settings.xml =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/settings.xml (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/settings.xml (revision 37801) @@ -70,5 +70,5 @@ gpc1 - ippdb01 + ippdb05 ipp ipp @@ -136,5 +136,5 @@ czardb - ippdb01 + ippdb05 ipp ipp Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.DF.vot =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.DF.vot (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.DF.vot (revision 37801) @@ -36,5 +36,5 @@ saturation magnitude level - + analysis version index @@ -94,6 +94,6 @@ ODM diff object identifier - - galactic coordinates ODM object identifier + + unique psps diff detection id, based on batch + row number @@ -244,7 +244,4 @@ Kron radius - - Kron radius error - NPix > 3sigma @@ -281,4 +278,7 @@ + flag indicating provenance information + + flag indicating provenance information Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.DO.vot =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.DO.vot (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.DO.vot (revision 37801) @@ -12,6 +12,6 @@ ODM object identifier index - - galactic coordinates ODM object identifier + + unique psps diff object id based on batch id and row number @@ -81,6 +81,18 @@ galactic latitude - - qf perfect + + qf perfect for g filter + + + qf perfect for r filter + + + qf perfect for i filter + + + qf perfect for z filter + + + qf perfect for y filter Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.FO.vot =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.FO.vot (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.FO.vot (revision 37801) @@ -9,6 +9,6 @@ ODM object identifier index - - galactic coordinates ODM object identifier + + unique psps id @@ -27,12 +27,12 @@ number of detections in g filter - + number of detections included in robust psf mean in g filter - + number of detections included in robust mean of Kron mag in g filter - - number of detections included in robust mean of Ap Mag in g filter + + number of detections included in robust mean of Ap Flux in g filter @@ -42,29 +42,29 @@ number of detections included in robust mean of ApFlx 6 in g filter - + robust mean g psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile - + estimated error of g psf AB magnitude mean(individual errors added in quadrature) - - standard deviation of measurements included in the robust mean - - + + standard deviation of measurements included in the robust mean + + mean g kron AB magnitude - + estimated error of g kron AB magnitude mean - - standard deviation of measurements included in the robust mean - - + + standard deviation of measurements included in the robust mean + + mean g aperture AB magnitude - + estimated error of g aperture AB magnitude mean - + standard deviation of measurements included in the robust mean @@ -171,12 +171,12 @@ number of detections in r filter - + number of detections included in robust psf mean in r filter - + number of detections included in robust mean of Kron mag in r filter - - number of detections included in robust mean of Ap Mag in r filter + + number of detections included in robust mean of Ap Flux in r filter @@ -186,29 +186,29 @@ number of detections included in robust mean of ApFlx 6 in r filter - + robust mean r psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile - + estimated error of g psf AB magnitude mean(individual errors added in quadrature) - - standard deviation of measurements included in the robust mean - - + + standard deviation of measurements included in the robust mean + + mean r kron AB magnitude - + estimated error of r kron AB magnitude mean - - standard deviation of measurements included in the robust mean - - + + standard deviation of measurements included in the robust mean + + mean r aperture AB magnitude - + estimated error of r aperture AB magnitude mean - + standard deviation of measurements included in the robust mean @@ -315,12 +315,12 @@ number of detections in i filter - + number of detections included in robust psf mean in i filter - + number of detections included in robust mean of Kron mag in i filter - - number of detections included in robust mean of Ap Mag in i filter + + number of detections included in robust mean of Ap Flux in i filter @@ -330,29 +330,29 @@ number of detections included in robust mean of ApFlx 6 in i filter - + robust mean i psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile - + estimated error of i psf AB magnitude mean(individual errors added in quadrature) - - standard deviation of measurements included in the robust mean - - + + standard deviation of measurements included in the robust mean + + mean i kron AB magnitude - + estimated error of i kron AB magnitude mean - - standard deviation of measurements included in the robust mean - - + + standard deviation of measurements included in the robust mean + + mean i aperture AB magnitude - + estimated error of i aperture AB magnitude mean - + standard deviation of measurements included in the robust mean @@ -459,12 +459,12 @@ number of detections in z filter - + number of detections included in robust psf mean in z filter - + number of detections included in robust mean of Kron mag in z filter - - number of detections included in robust mean of Ap Mag in z filter + + number of detections included in robust mean of Ap Flux in z filter @@ -474,29 +474,29 @@ number of detections included in robust mean of ApFlx 6 in z filter - + robust mean z psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile - + estimated error of z psf AB magnitude mean(individual errors added in quadrature) - - standard deviation of measurements included in the robust mean - - + + standard deviation of measurements included in the robust mean + + mean z kron AB magnitude - + estimated error of z kron AB magnitude mean - - standard deviation of measurements included in the robust mean - - + + standard deviation of measurements included in the robust mean + + mean z aperture AB magnitude - + estimated error of z aperture AB magnitude mean - + standard deviation of measurements included in the robust mean @@ -603,12 +603,12 @@ number of detections in y filter - + number of detections included in robust psf mean in y filter - + number of detections included in robust mean of Kron mag in y filter - - number of detections included in robust mean of Ap Mag in y filter + + number of detections included in robust mean of Ap Flux in y filter @@ -618,29 +618,29 @@ number of detections included in robust mean of ApFlx 6 in y filter - + robust mean y psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile - + estimated error of y psf AB magnitude mean(individual errors added in quadrature) - - standard deviation of measurements included in the robust mean - - + + standard deviation of measurements included in the robust mean + + mean y kron AB magnitude - + estimated error of y kron AB magnitude mean - - standard deviation of measurements included in the robust mean - - + + standard deviation of measurements included in the robust mean + + mean y aperture AB magnitude - + estimated error of y aperture AB magnitude mean - + standard deviation of measurements included in the robust mean Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.FW.vot =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.FW.vot (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.FW.vot (revision 37801) @@ -115,6 +115,6 @@ ODM object identifier - - galactic coordinates ODM object identifier + + unique psps id @@ -218,4 +218,83 @@ 4th moment sin(theta) + + Aperture Flux + + + Aperture Flux error + + + Aperture fill factor + + + Kron flux + + + Kron flux error + + + Kron radius + + + Kron radius error + + + PSF sky level at source + + + estimated error in sky + + + flag indicating provenance information + + + combination of cmf and dvo flags + + + Data release when this detection was originally taken. Recalibrations do not affect this value. + + + + + + + VOTable description of PSPS table ForcedWarpMeasurement + + + ODM object identifier + + + unique psps id + + + ODM detection identifier + + + object id generated by IPP + + + detection id generated by IPP + + + filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... + + + survey flag identifier + + + forced warp measurement meta identifer + + + random warp id + + + single float givijng PrimaryCell.SkyCell identifier + + + identifier to dvo spacial region + + + Time of mid observation of positive detection in Mean Julian Days + per-object smear polarizability X11 term @@ -278,25 +357,4 @@ psf model shear polarizability E2 term - - Aperture Flux - - - Aperture Flux error - - - Aperture fill factor - - - Kron flux - - - Kron flux error - - - Kron radius - - - Kron radius error - Flux inside r = 5 @@ -322,16 +380,4 @@ estimated filling factor at R6 - - - PSF sky level at source - - - estimated error in sky - - - flag indicating provenance information - - - Data release when this detection was originally taken. Recalibrations do not affect this value. Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.IN.full.vot =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.IN.full.vot (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.IN.full.vot (revision 37801) @@ -207,4 +207,6 @@ + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.IN.vot =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.IN.vot (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.IN.vot (revision 37801) @@ -207,4 +207,6 @@ + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.OB.vot =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.OB.vot (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.OB.vot (revision 37801) @@ -11,4 +11,7 @@ ODM object identifier index + + + unique psps id @@ -136,4 +139,7 @@ ODM object identifier index + + unique psps id + maximum psf QfPerfect from best available g band detections Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.P2.vot =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.P2.vot (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.P2.vot (revision 37801) @@ -383,4 +383,7 @@ ODM object identifier + + unique psps id + ODM detection identifier @@ -440,4 +443,8 @@ declination error + + sigma of how extended something is + + zeropoint Index: /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.ST.vot =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.ST.vot (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/config/tables.ST.vot (revision 37801) @@ -131,4 +131,7 @@ ODM object identifier + + unique psps id + IPP object identifier @@ -147,5 +150,5 @@ - random stack id + random for stack detections (not the same as random objid) @@ -164,5 +167,4 @@ Data release when this table was loaded. - @@ -205,7 +207,4 @@ estimated error of g kron AB magnitude from stack - - Excess extinction - PSF sky level at source (adu) @@ -264,7 +263,4 @@ estimated error of r kron AB magnitude from stack - - Excess extinction - PSF sky level at source (adu) @@ -323,7 +319,4 @@ estimated error of i kron AB magnitude from stack - - Excess extinction - PSF sky level at source (adu) @@ -382,7 +375,4 @@ estimated error of z kron AB magnitude from stack - - Excess extinction - PSF sky level at source (adu) @@ -440,7 +430,4 @@ estimated error of y kron AB magnitude from stack - - - Excess extinction @@ -464,6 +451,6 @@
25CAL13PS1 Cal 3C454.1 (Northern field) 225033+712919
28SSSPS1 Solar System Sweet Spot Survey
30SASPS1 SAS
254UNKPS1 undefined
25CAL13PS1 Cal 3C454.1 (Northern field) 225033+712919
28SSSPS1 Solar System Sweet Spot Survey
30SASPS1 SAS
254UNKPS1 undefined
- - + +
VOTable description of PSPS table StackObject @@ -471,4 +458,7 @@ ODM object identifier + + unique psps id + IPP object identifier @@ -508,13 +498,4 @@ estimated error in y - - - zero point offset - - - plate scale - - - exposure time (informational) @@ -572,7 +553,4 @@ Aperture fill factor - redundant with psfQF, keep? - - Aperture size - Kron flux @@ -583,4 +561,16 @@ Kron radius + + + exposure time (informational) + + + extnsigma is a measure of how extended something is + + + zero point offset + + + plate scale @@ -606,13 +596,4 @@ estimated error in y - - - zero point offset - - - plate scale - - - exposure time (informational) @@ -670,7 +651,4 @@ Aperture fill factor - redundant with psfQF, keep? - - Aperture size - Kron flux @@ -682,5 +660,16 @@ Kron radius - + + exposure time (informational) + + + extnsigma is a measure of how extended something is + + + zero point offset + + + plate scale + @@ -704,13 +693,4 @@ estimated error in y - - - zero point offset - - - plate scale - - - exposure time (informational) @@ -768,7 +748,4 @@ Aperture fill factor - redundant with psfQF, keep? - - Aperture size - Kron flux @@ -779,4 +756,16 @@ Kron radius + + + exposure time (informational) + + + extnsigma is a measure of how extended something is + + + zero point offset + + + plate scale @@ -802,13 +791,4 @@ estimated error in y - - - zero point offset - - - plate scale - - - exposure time (informational) @@ -866,7 +846,4 @@ Aperture fill factor - redundant with psfQF, keep? - - Aperture size - Kron flux @@ -878,6 +855,17 @@ Kron radius - - + + exposure time (informational) + + + extnsigma is a measure of how extended something is + + + zero point offset + + + plate scale + + ipp identifier @@ -900,13 +888,4 @@ estimated error in y - - - zero point offset - - - plate scale - - - exposure time (informational) @@ -964,7 +943,4 @@ Aperture fill factor - redundant with psfQF, keep? - - Aperture size - Kron flux @@ -976,5 +952,16 @@ Kron radius - + + exposure time (informational) + + + extnsigma is a measure of how extended something is + + + zero point offset + + + plate scale + @@ -989,4 +976,7 @@ ODM object identifier + + unique psps id + IPP object identifier @@ -1051,74 +1041,74 @@ estimated filling factor at R5 - + Flux inside r = 3 - - estimated error is c1flxR3 - - - estimated standard deviation in c1flxR3 - - + + estimated error is c6flxR3 + + + estimated standard deviation in c6flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c1flxR4 - - - estimated standard deviation in c1flxR4 - - + + estimated error is c6flxR4 + + + estimated standard deviation in c6flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c1flxR5 - - - estimated standard deviation in c1flxR5 - - + + estimated error is c6flxR5 + + + estimated standard deviation in c6flxR5 + + estimated filling factor at R5 - + Flux inside r = 3 - - estimated error in c2flxR3 - - - estimated standard deviation in c2flxR3 - - + + estimated error in c8flxR3 + + + estimated standard deviation in c8flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error in c2flxR4 - - - estimated standard deviation in c2flxR4 - - + + estimated error in c8flxR4 + + + estimated standard deviation in c8flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error in c2flxR5 - - - estimated standard deviation in c2flxR5 - - + + estimated error in c8flxR5 + + + estimated standard deviation in c8flxR5 + + estimated filling factor at R5 @@ -1170,74 +1160,74 @@ estimated filling factor at R5 - + Flux inside r = 3 - - estimated error is c1flxR3 - - - estimated standard deviation in c1flxR3 - - + + estimated error is c6flxR3 + + + estimated standard deviation in c6flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c1flxR4 - - - estimated standard deviation in c1flxR4 - - + + estimated error is c6flxR4 + + + estimated standard deviation in c6flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c1flxR5 - - - estimated standard deviation in c1flxR5 - - + + estimated error is c6flxR5 + + + estimated standard deviation in c6flxR5 + + estimated filling factor at R5 - + Flux inside r = 3 - - estimated error is c2flxR3 - - - estimated standard deviation in c2flxR3 - - + + estimated error is c8flxR3 + + + estimated standard deviation in c8flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c2flxR4 - - - estimated standard deviation in c2flxR4 - - + + estimated error is c8flxR4 + + + estimated standard deviation in c8flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c2flxR5 - - - estimated standard deviation in c2flxR5 - - + + estimated error is c8flxR5 + + + estimated standard deviation in c8flxR5 + + estimated filling factor at R5 @@ -1289,74 +1279,74 @@ estimated filling factor at R5 - + Flux inside r = 3 - - estimated error is c1flxR3 - - - estimated standard deviation in c1flxR3 - - + + estimated error is c6flxR3 + + + estimated standard deviation in c6flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c1flxR4 - - - estimated standard deviation in c1flxR4 - - + + estimated error is c6flxR4 + + + estimated standard deviation in c6flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c1flxR5 - - - estimated standard deviation in c1flxR5 - - + + estimated error is c6flxR5 + + + estimated standard deviation in c6flxR5 + + estimated filling factor at R5 - + Flux inside r = 3 - - estimated error is c2flxR3 - - - estimated standard deviation in c2flxR3 - - + + estimated error is c8flxR3 + + + estimated standard deviation in c8flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c2flxR4 - - - estimated standard deviation in c2flxR4 - - + + estimated error is c8flxR4 + + + estimated standard deviation in c8flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c2flxR5 - - - estimated standard deviation in c2flxR5 - - + + estimated error is c8flxR5 + + + estimated standard deviation in c8flxR5 + + estimated filling factor at R5 @@ -1408,74 +1398,74 @@ estimated filling factor at R5 - + Flux inside r = 3 - - estimated error is c1flxR3 - - - estimated standard deviation in c1flxR3 - - + + estimated error is c6flxR3 + + + estimated standard deviation in c6flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c1flxR4 - - - estimated standard deviation in c1flxR4 - - + + estimated error is c6flxR4 + + + estimated standard deviation in c6flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c1flxR5 - - - estimated standard deviation in c1flxR5 - - + + estimated error is c6flxR5 + + + estimated standard deviation in c6flxR5 + + estimated filling factor at R5 - + Flux inside r = 3 - - estimated error is c2flxR3 - - - estimated standard deviation in c2flxR3 - - + + estimated error is c8flxR3 + + + estimated standard deviation in c8flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c2flxR4 - - - estimated standard deviation in c2flxR4 - - + + estimated error is c8flxR4 + + + estimated standard deviation in c8flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c2flxR5 - - - estimated standard deviation in c2flxR5 - - + + estimated error is c8flxR5 + + + estimated standard deviation in c8flxR5 + + estimated filling factor at R5 @@ -1527,74 +1517,74 @@ estimated filling factor at R5 - + Flux inside r = 3 - - estimated error is c1flxR3 - - - estimated standard deviation in c1flxR3 - - + + estimated error is c6flxR3 + + + estimated standard deviation in c6flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c1flxR4 - - - estimated standard deviation in c1flxR4 - - + + estimated error is c6flxR4 + + + estimated standard deviation in c6flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c1flxR5 - - - estimated standard deviation in c1flxR5 - - + + estimated error is c6flxR5 + + + estimated standard deviation in c6flxR5 + + estimated filling factor at R5 - + Flux inside r = 3 - - estimated error is c2flxR3 - - - estimated standard deviation in c2flxR3 - - + + estimated error is c8flxR3 + + + estimated standard deviation in c8flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c2flxR4 - - - estimated standard deviation in c2flxR4 - - + + estimated error is c8flxR4 + + + estimated standard deviation in c8flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c2flxR5 - - - estimated standard deviation in c2flxR5 - - + + estimated error is c8flxR5 + + + estimated standard deviation in c8flxR5 + + estimated filling factor at R5 @@ -1611,4 +1601,7 @@ ODM object identifier + + unique psps id + IPP object identifier @@ -1799,4 +1792,7 @@ ODM object identifier + + unique psps id + IPP object identifier @@ -2092,4 +2088,7 @@ ODM object identifier + + unique psps id + IPP object identifier @@ -2385,4 +2384,7 @@ ODM object identifier + + unique psps id + IPP object identifier @@ -2702,5 +2704,5 @@ -
+
VOTable description of PSPS table StackApFlxFull @@ -2708,4 +2710,7 @@ ODM object identifier + + unique psps id + IPP object identifier @@ -3384,11 +3389,14 @@
- - - VOTable description of PSPS table StackApFlxFullC1 + +
+ VOTable description of PSPS table StackApFlxFullC6 ODM object identifier + + unique psps id + IPP object identifier @@ -3417,122 +3425,122 @@ stack identifier - + Flux inside r = 1 - - estimated error is c1flxR1 - - - estimated standard deviation in c1flxR1 - - + + estimated error is c6flxR1 + + + estimated standard deviation in c6flxR1 + + estimated filling factor at R1 - + Flux inside r = 2 - - estimated error is c1flxR2 - - - estimated standard deviation in c1flxR2 - - + + estimated error is c6flxR2 + + + estimated standard deviation in c6flxR2 + + estimated filling factor at R1 - + Flux inside r = 3 - - estimated error is c1flxR3 - - - estimated standard deviation in c1flxR3 - - + + estimated error is c6flxR3 + + + estimated standard deviation in c6flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c1flxR4 - - - estimated standard deviation in c1flxR4 - - + + estimated error is c6flxR4 + + + estimated standard deviation in c6flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c1flxR5 - - - estimated standard deviation in c1flxR5 - - + + estimated error is c6flxR5 + + + estimated standard deviation in c6flxR5 + + estimated filling factor at R5 - + Flux inside r = 6 - - estimated error is c1flxR6 - - - estimated standard deviation in c1flxR6 - - + + estimated error is c6flxR6 + + + estimated standard deviation in c6flxR6 + + estimated filling factor at R6 - + Flux inside r = 7 - - estimated error is c1flxR7 - - - estimated standard deviation in c1flxR7 - - + + estimated error is c6flxR7 + + + estimated standard deviation in c6flxR7 + + estimated filling factor at R7 - + Flux inside r = 8 - - estimated error is c1flxR8 - - - estimated standard deviation in c1flxR8 - - + + estimated error is c6flxR8 + + + estimated standard deviation in c6flxR8 + + estimated filling factor at R8 - + Flux inside r = 9 - - estimated error is c1flxR9 - - - estimated standard deviation in c1flxR9 - - + + estimated error is c6flxR9 + + + estimated standard deviation in c6flxR9 + + estimated filling factor at R9 - + Flux inside r = 10 - - estimated error is c1flxR10 - - - estimated standard deviation in c1flxR10 - - + + estimated error is c6flxR10 + + + estimated standard deviation in c6flxR10 + + estimated filling factor at R10 @@ -3548,122 +3556,122 @@ stack identifier - + Flux inside r = 1 - - estimated error is c1flxR1 - - - estimated standard deviation in c1flxR1 - - + + estimated error is c6flxR1 + + + estimated standard deviation in c6flxR1 + + estimated filling factor at R1 - + Flux inside r = 2 - - estimated error is c1flxR2 - - - estimated standard deviation in c1flxR2 - - + + estimated error is c6flxR2 + + + estimated standard deviation in c6flxR2 + + estimated filling factor at R1 - + Flux inside r = 3 - - estimated error is c1flxR3 - - - estimated standard deviation in c1flxR3 - - + + estimated error is c6flxR3 + + + estimated standard deviation in c6flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c1flxR4 - - - estimated standard deviation in c1flxR4 - - + + estimated error is c6flxR4 + + + estimated standard deviation in c6flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c1flxR5 - - - estimated standard deviation in c1flxR5 - - + + estimated error is c6flxR5 + + + estimated standard deviation in c6flxR5 + + estimated filling factor at R5 - + Flux inside r = 6 - - estimated error is c1flxR6 - - - estimated standard deviation in c1flxR6 - - + + estimated error is c6flxR6 + + + estimated standard deviation in c6flxR6 + + estimated filling factor at R6 - + Flux inside r = 7 - - estimated error is c1flxR7 - - - estimated standard deviation in c1flxR7 - - + + estimated error is c6flxR7 + + + estimated standard deviation in c6flxR7 + + estimated filling factor at R7 - + Flux inside r = 8 - - estimated error is c1flxR8 - - - estimated standard deviation in c1flxR8 - - + + estimated error is c6flxR8 + + + estimated standard deviation in c6flxR8 + + estimated filling factor at R8 - + Flux inside r = 9 - - estimated error is c1flxR9 - - - estimated standard deviation in c1flxR9 - - + + estimated error is c6flxR9 + + + estimated standard deviation in c6flxR9 + + estimated filling factor at R9 - + Flux inside r = 10 - - estimated error is c1flxR10 - - - estimated standard deviation in c1flxR10 - - + + estimated error is c6flxR10 + + + estimated standard deviation in c6flxR10 + + estimated filling factor at R10 @@ -3679,122 +3687,122 @@ stack identifier - + Flux inside r = 1 - - estimated error is c1flxR1 - - - estimated standard deviation in c1flxR1 - - + + estimated error is c6flxR1 + + + estimated standard deviation in c6flxR1 + + estimated filling factor at R1 - + Flux inside r = 2 - - estimated error is c1flxR2 - - - estimated standard deviation in c1flxR2 - - + + estimated error is c6flxR2 + + + estimated standard deviation in c6flxR2 + + estimated filling factor at R1 - + Flux inside r = 3 - - estimated error is c1flxR3 - - - estimated standard deviation in c1flxR3 - - + + estimated error is c6flxR3 + + + estimated standard deviation in c6flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c1flxR4 - - - estimated standard deviation in c1flxR4 - - + + estimated error is c6flxR4 + + + estimated standard deviation in c6flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c1flxR5 - - - estimated standard deviation in c1flxR5 - - + + estimated error is c6flxR5 + + + estimated standard deviation in c6flxR5 + + estimated filling factor at R5 - + Flux inside r = 6 - - estimated error is c1flxR6 - - - estimated standard deviation in c1flxR6 - - + + estimated error is c6flxR6 + + + estimated standard deviation in c6flxR6 + + estimated filling factor at R6 - + Flux inside r = 7 - - estimated error is c1flxR7 - - - estimated standard deviation in c1flxR7 - - + + estimated error is c6flxR7 + + + estimated standard deviation in c6flxR7 + + estimated filling factor at R7 - + Flux inside r = 8 - - estimated error is c1flxR8 - - - estimated standard deviation in c1flxR8 - - + + estimated error is c6flxR8 + + + estimated standard deviation in c6flxR8 + + estimated filling factor at R8 - + Flux inside r = 9 - - estimated error is c1flxR9 - - - estimated standard deviation in c1flxR9 - - + + estimated error is c6flxR9 + + + estimated standard deviation in c6flxR9 + + estimated filling factor at R9 - + Flux inside r = 10 - - estimated error is c1flxR10 - - - estimated standard deviation in c1flxR10 - - + + estimated error is c6flxR10 + + + estimated standard deviation in c6flxR10 + + estimated filling factor at R10 @@ -3810,122 +3818,122 @@ stack identifier - + Flux inside r = 1 - - estimated error is c1flxR1 - - - estimated standard deviation in c1flxR1 - - + + estimated error is c6flxR1 + + + estimated standard deviation in c6flxR1 + + estimated filling factor at R1 - + Flux inside r = 2 - - estimated error is c1flxR2 - - - estimated standard deviation in c1flxR2 - - + + estimated error is c6flxR2 + + + estimated standard deviation in c6flxR2 + + estimated filling factor at R1 - + Flux inside r = 3 - - estimated error is c1flxR3 - - - estimated standard deviation in c1flxR3 - - + + estimated error is c6flxR3 + + + estimated standard deviation in c6flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c1flxR4 - - - estimated standard deviation in c1flxR4 - - + + estimated error is c6flxR4 + + + estimated standard deviation in c6flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c1flxR5 - - - estimated standard deviation in c1flxR5 - - + + estimated error is c6flxR5 + + + estimated standard deviation in c6flxR5 + + estimated filling factor at R5 - + Flux inside r = 6 - - estimated error is c1flxR6 - - - estimated standard deviation in c1flxR6 - - + + estimated error is c6flxR6 + + + estimated standard deviation in c6flxR6 + + estimated filling factor at R6 - + Flux inside r = 7 - - estimated error is c1flxR7 - - - estimated standard deviation in c1flxR7 - - + + estimated error is c6flxR7 + + + estimated standard deviation in c6flxR7 + + estimated filling factor at R7 - + Flux inside r = 8 - - estimated error is c1flxR8 - - - estimated standard deviation in c1flxR8 - - + + estimated error is c6flxR8 + + + estimated standard deviation in c6flxR8 + + estimated filling factor at R8 - + Flux inside r = 9 - - estimated error is c1flxR9 - - - estimated standard deviation in c1flxR9 - - + + estimated error is c6flxR9 + + + estimated standard deviation in c6flxR9 + + estimated filling factor at R9 - + Flux inside r = 10 - - estimated error is c1flxR10 - - - estimated standard deviation in c1flxR10 - - + + estimated error is c6flxR10 + + + estimated standard deviation in c6flxR10 + + estimated filling factor at R10 @@ -3941,122 +3949,122 @@ stack identifier - + Flux inside r = 1 - - estimated error is c1flxR1 - - - estimated standard deviation in c1flxR1 - - + + estimated error is c6flxR1 + + + estimated standard deviation in c6flxR1 + + estimated filling factor at R1 - + Flux inside r = 2 - - estimated error is c1flxR2 - - - estimated standard deviation in c1flxR2 - - + + estimated error is c6flxR2 + + + estimated standard deviation in c6flxR2 + + estimated filling factor at R1 - + Flux inside r = 3 - - estimated error is c1flxR3 - - - estimated standard deviation in c1flxR3 - - + + estimated error is c6flxR3 + + + estimated standard deviation in c6flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c1flxR4 - - - estimated standard deviation in c1flxR4 - - + + estimated error is c6flxR4 + + + estimated standard deviation in c6flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c1flxR5 - - - estimated standard deviation in c1flxR5 - - + + estimated error is c6flxR5 + + + estimated standard deviation in c6flxR5 + + estimated filling factor at R5 - + Flux inside r = 6 - - estimated error is c1flxR6 - - - estimated standard deviation in c1flxR6 - - + + estimated error is c6flxR6 + + + estimated standard deviation in c6flxR6 + + estimated filling factor at R6 - + Flux inside r = 7 - - estimated error is c1flxR7 - - - estimated standard deviation in c1flxR7 - - + + estimated error is c6flxR7 + + + estimated standard deviation in c6flxR7 + + estimated filling factor at R7 - + Flux inside r = 8 - - estimated error is c1flxR8 - - - estimated standard deviation in c1flxR8 - - + + estimated error is c6flxR8 + + + estimated standard deviation in c6flxR8 + + estimated filling factor at R8 - + Flux inside r = 9 - - estimated error is c1flxR9 - - - estimated standard deviation in c1flxR9 - - + + estimated error is c6flxR9 + + + estimated standard deviation in c6flxR9 + + estimated filling factor at R9 - + Flux inside r = 10 - - estimated error is c1flxR10 - - - estimated standard deviation in c1flxR10 - - + + estimated error is c6flxR10 + + + estimated standard deviation in c6flxR10 + + estimated filling factor at R10 @@ -4067,11 +4075,14 @@
- - - VOTable description of PSPS table StackApFlxFullC2 + +
+ VOTable description of PSPS table StackApFlxFullC8 ODM object identifier + + unique psps id + IPP object identifier @@ -4100,122 +4111,122 @@ stack identifier - + Flux inside r = 1 - - estimated error is c2flxR1 - - - estimated standard deviation in c2flxR1 - - + + estimated error is c8flxR1 + + + estimated standard deviation in c8flxR1 + + estimated filling factor at R1 - + Flux inside r = 2 - - estimated error is c2flxR2 - - - estimated standard deviation in c2flxR2 - - + + estimated error is c8flxR2 + + + estimated standard deviation in c8flxR2 + + estimated filling factor at R1 - + Flux inside r = 3 - - estimated error is c2flxR3 - - - estimated standard deviation in c2flxR3 - - + + estimated error is c8flxR3 + + + estimated standard deviation in c8flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c2flxR4 - - - estimated standard deviation in c2flxR4 - - + + estimated error is c8flxR4 + + + estimated standard deviation in c8flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c2flxR5 - - - estimated standard deviation in c2flxR5 - - + + estimated error is c8flxR5 + + + estimated standard deviation in c8flxR5 + + estimated filling factor at R5 - + Flux inside r = 6 - - estimated error is c2flxR6 - - - estimated standard deviation in c2flxR6 - - + + estimated error is c8flxR6 + + + estimated standard deviation in c8flxR6 + + estimated filling factor at R6 - + Flux inside r = 7 - - estimated error is c2flxR7 - - - estimated standard deviation in c2flxR7 - - + + estimated error is c8flxR7 + + + estimated standard deviation in c8flxR7 + + estimated filling factor at R7 - + Flux inside r = 8 - - estimated error is c2flxR8 - - - estimated standard deviation in c2flxR8 - - + + estimated error is c8flxR8 + + + estimated standard deviation in c8flxR8 + + estimated filling factor at R8 - + Flux inside r = 9 - - estimated error is c2flxR9 - - - estimated standard deviation in c2flxR9 - - + + estimated error is c8flxR9 + + + estimated standard deviation in c8flxR9 + + estimated filling factor at R9 - + Flux inside r = 10 - - estimated error is c2flxR10 - - - estimated standard deviation in c2flxR10 - - + + estimated error is c8flxR10 + + + estimated standard deviation in c8flxR10 + + estimated filling factor at R10 @@ -4231,122 +4242,122 @@ stack identifier - + Flux inside r = 1 - - estimated error is c2flxR1 - - - estimated standard deviation in c2flxR1 - - + + estimated error is c8flxR1 + + + estimated standard deviation in c8flxR1 + + estimated filling factor at R1 - + Flux inside r = 2 - - estimated error is c2flxR2 - - - estimated standard deviation in c2flxR2 - - + + estimated error is c8flxR2 + + + estimated standard deviation in c8flxR2 + + estimated filling factor at R1 - + Flux inside r = 3 - - estimated error is c2flxR3 - - - estimated standard deviation in c2flxR3 - - + + estimated error is c8flxR3 + + + estimated standard deviation in c8flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c2flxR4 - - - estimated standard deviation in c2flxR4 - - + + estimated error is c8flxR4 + + + estimated standard deviation in c8flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c2flxR5 - - - estimated standard deviation in c2flxR5 - - + + estimated error is c8flxR5 + + + estimated standard deviation in c8flxR5 + + estimated filling factor at R5 - + Flux inside r = 6 - - estimated error is c2flxR6 - - - estimated standard deviation in c2flxR6 - - + + estimated error is c8flxR6 + + + estimated standard deviation in c8flxR6 + + estimated filling factor at R6 - + Flux inside r = 7 - - estimated error is c2flxR7 - - - estimated standard deviation in c2flxR7 - - + + estimated error is c8flxR7 + + + estimated standard deviation in c8flxR7 + + estimated filling factor at R7 - + Flux inside r = 8 - - estimated error is c2flxR8 - - - estimated standard deviation in c2flxR8 - - + + estimated error is c8flxR8 + + + estimated standard deviation in c8flxR8 + + estimated filling factor at R8 - + Flux inside r = 9 - - estimated error is c2flxR9 - - - estimated standard deviation in c2flxR9 - - + + estimated error is c8flxR9 + + + estimated standard deviation in c8flxR9 + + estimated filling factor at R9 - + Flux inside r = 10 - - estimated error is c2flxR10 - - - estimated standard deviation in c2flxR10 - - + + estimated error is c8flxR10 + + + estimated standard deviation in c8flxR10 + + estimated filling factor at R10 @@ -4362,122 +4373,122 @@ stack identifier - + Flux inside r = 1 - - estimated error is c2flxR1 - - - estimated standard deviation in c2flxR1 - - + + estimated error is c8flxR1 + + + estimated standard deviation in c8flxR1 + + estimated filling factor at R1 - + Flux inside r = 2 - - estimated error is c2flxR2 - - - estimated standard deviation in c2flxR2 - - + + estimated error is c8flxR2 + + + estimated standard deviation in c8flxR2 + + estimated filling factor at R1 - + Flux inside r = 3 - - estimated error is c2flxR3 - - - estimated standard deviation in c2flxR3 - - + + estimated error is c8flxR3 + + + estimated standard deviation in c8flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c2flxR4 - - - estimated standard deviation in c2flxR4 - - + + estimated error is c8flxR4 + + + estimated standard deviation in c8flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c2flxR5 - - - estimated standard deviation in c2flxR5 - - + + estimated error is c8flxR5 + + + estimated standard deviation in c8flxR5 + + estimated filling factor at R5 - + Flux inside r = 6 - - estimated error is c2flxR6 - - - estimated standard deviation in c2flxR6 - - + + estimated error is c8flxR6 + + + estimated standard deviation in c8flxR6 + + estimated filling factor at R6 - + Flux inside r = 7 - - estimated error is c2flxR7 - - - estimated standard deviation in c2flxR7 - - + + estimated error is c8flxR7 + + + estimated standard deviation in c8flxR7 + + estimated filling factor at R7 - + Flux inside r = 8 - - estimated error is c2flxR8 - - - estimated standard deviation in c2flxR8 - - + + estimated error is c8flxR8 + + + estimated standard deviation in c8flxR8 + + estimated filling factor at R8 - + Flux inside r = 9 - - estimated error is c2flxR9 - - - estimated standard deviation in c2flxR9 - - + + estimated error is c8flxR9 + + + estimated standard deviation in c8flxR9 + + estimated filling factor at R9 - + Flux inside r = 10 - - estimated error is c2flxR10 - - - estimated standard deviation in c2flxR10 - - + + estimated error is c8flxR10 + + + estimated standard deviation in c8flxR10 + + estimated filling factor at R10 @@ -4493,122 +4504,122 @@ stack identifier - + Flux inside r = 1 - - estimated error is c2flxR1 - - - estimated standard deviation in c2flxR1 - - + + estimated error is c8flxR1 + + + estimated standard deviation in c8flxR1 + + estimated filling factor at R1 - + Flux inside r = 2 - - estimated error is c2flxR2 - - - estimated standard deviation in c2flxR2 - - + + estimated error is c8flxR2 + + + estimated standard deviation in c8flxR2 + + estimated filling factor at R1 - + Flux inside r = 3 - - estimated error is c2flxR3 - - - estimated standard deviation in c2flxR3 - - + + estimated error is c8flxR3 + + + estimated standard deviation in c8flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c2flxR4 - - - estimated standard deviation in c2flxR4 - - + + estimated error is c8flxR4 + + + estimated standard deviation in c8flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c2flxR5 - - - estimated standard deviation in c2flxR5 - - + + estimated error is c8flxR5 + + + estimated standard deviation in c8flxR5 + + estimated filling factor at R5 - + Flux inside r = 6 - - estimated error is c2flxR6 - - - estimated standard deviation in c2flxR6 - - + + estimated error is c8flxR6 + + + estimated standard deviation in c8flxR6 + + estimated filling factor at R6 - + Flux inside r = 7 - - estimated error is c2flxR7 - - - estimated standard deviation in c2flxR7 - - + + estimated error is c8flxR7 + + + estimated standard deviation in c8flxR7 + + estimated filling factor at R7 - + Flux inside r = 8 - - estimated error is c2flxR8 - - - estimated standard deviation in c2flxR8 - - + + estimated error is c8flxR8 + + + estimated standard deviation in c8flxR8 + + estimated filling factor at R8 - + Flux inside r = 9 - - estimated error is c2flxR9 - - - estimated standard deviation in c2flxR9 - - + + estimated error is c8flxR9 + + + estimated standard deviation in c8flxR9 + + estimated filling factor at R9 - + Flux inside r = 10 - - estimated error is c2flxR10 - - - estimated standard deviation in c2flxR10 - - + + estimated error is c8flxR10 + + + estimated standard deviation in c8flxR10 + + estimated filling factor at R10 @@ -4624,122 +4635,122 @@ stack identifier - + Flux inside r = 1 - - estimated error is c2flxR1 - - - estimated standard deviation in c2flxR1 - - + + estimated error is c8flxR1 + + + estimated standard deviation in c8flxR1 + + estimated filling factor at R1 - + Flux inside r = 2 - - estimated error is c2flxR2 - - - estimated standard deviation in c2flxR2 - - + + estimated error is c8flxR2 + + + estimated standard deviation in c8flxR2 + + estimated filling factor at R1 - + Flux inside r = 3 - - estimated error is c2flxR3 - - - estimated standard deviation in c2flxR3 - - + + estimated error is c8flxR3 + + + estimated standard deviation in c8flxR3 + + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is c2flxR4 - - - estimated standard deviation in c2flxR4 - - + + estimated error is c8flxR4 + + + estimated standard deviation in c8flxR4 + + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is c2flxR5 - - - estimated standard deviation in c2flxR5 - - + + estimated error is c8flxR5 + + + estimated standard deviation in c8flxR5 + + estimated filling factor at R5 - + Flux inside r = 6 - - estimated error is c2flxR6 - - - estimated standard deviation in c2flxR6 - - + + estimated error is c8flxR6 + + + estimated standard deviation in c8flxR6 + + estimated filling factor at R6 - + Flux inside r = 7 - - estimated error is c2flxR7 - - - estimated standard deviation in c2flxR7 - - + + estimated error is c8flxR7 + + + estimated standard deviation in c8flxR7 + + estimated filling factor at R7 - + Flux inside r = 8 - - estimated error is c2flxR8 - - - estimated standard deviation in c2flxR8 - - + + estimated error is c8flxR8 + + + estimated standard deviation in c8flxR8 + + estimated filling factor at R8 - + Flux inside r = 9 - - estimated error is c2flxR9 - - - estimated standard deviation in c2flxR9 - - + + estimated error is c8flxR9 + + + estimated standard deviation in c8flxR9 + + estimated filling factor at R9 - + Flux inside r = 10 - - estimated error is c2flxR10 - - - estimated standard deviation in c2flxR10 - - + + estimated error is c8flxR10 + + + estimated standard deviation in c8flxR10 + + estimated filling factor at R10 @@ -4757,4 +4768,7 @@ ODM object identifier + + unique psps id + IPP object identifier Index: /branches/eam_branches/ipp-20140904/ippToPsps/doc/notes.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/doc/notes.txt (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/doc/notes.txt (revision 37801) @@ -43,7 +43,7 @@ mysql -h ipp006 -u dvo -p ipptopsps -e "select batch_id, stage_id from batch where batch_type = 'P2'" > P2.batches.dat -on ippdb01: cp P2.batches.dat /tmp/datadump/ +on ippdb05: cp P2.batches.dat /tmp/datadump/ -mysql -h ippdb01 -u dvo -p gpc1 +mysql -h ippdb05 -u dvo -p gpc1 mysql> create temporary table eam_p2_batches (batch_id int, cam_id int); mysql> create temporary table eam_st_batches (batch_id int, stack_id int); Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/Makefile.am (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/Makefile.am (revision 37801) @@ -14,4 +14,6 @@ diffobjectbatch.py \ dvodetections.py \ + dvoforceddetections.py \ + dvoforcedobjects.py \ dvoobjects.py \ dvo.py \ Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/batch.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/batch.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/batch.py (revision 37801) @@ -382,4 +382,31 @@ self.logger.infoPair("Min objID", "%ld" % self.minObjID) self.logger.infoPair("Max objID", "%ld" % self.maxObjID) + + + ''' + Sets min and max obj ID using the provided table, or list of tables + ''' + def setMinMaxDiffObjID(self, tables): + first = True + self.totalDetections = 0 + for table in tables: + sql = "SELECT MIN(diffObjID), MAX(diffObjID), COUNT(diffObjID) FROM " + table + rs = self.scratchDb.executeQuery(sql) + rs.first() + self.totalDetections = self.totalDetections + rs.getLong(3) + if first: + self.minObjID = rs.getLong(1) + self.maxObjID = rs.getLong(2) + else: + if rs.getLong(1) < self.minObjID: self.minObjID = rs.getLong(1) + if rs.getLong(2) > self.maxObjID: self.maxObjID = rs.getLong(2) + first = False + rs.close() + self.ippToPspsDb.updateDetectionStats(self.batchID, self.minObjID, self.maxObjID, self.totalDetections) + self.logger.infoPair("Total detections", "%ld" % self.totalDetections) + self.logger.infoPair("Min objID", "%ld" % self.minObjID) + self.logger.infoPair("Max objID", "%ld" % self.maxObjID) + + ''' @@ -579,5 +606,5 @@ # for OB, this imports object data from DVO if not self.importIppTables(): - if self.batchType == "OB": + if (self.batchType == "OB" or self.batchType =="DO" or self.batchType =="FO"): self.logger.errorPair("skipping this batch", "no dvo tables for this region") self.ippToPspsDb.updateProcessed(self.batchID, 1) Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/detectionbatch.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/detectionbatch.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/detectionbatch.py (revision 37801) @@ -214,5 +214,5 @@ sqlLine.group("completMag", self.getKeyFloat(header, "%.8f", 'FLIMIT')) sqlLine.group("astroScat", self.getKeyFloat(header, "%.8f", 'CERROR')) - sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS')) + sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR')) sqlLine.group("numAstroRef", self.getKeyValue(header, 'NASTRO')) sqlLine.group("numPhotoRef", self.getKeyValue(header, 'NASTRO')) @@ -336,5 +336,5 @@ sqlLine.group("ippDetectID", "IPP_IDET") - sqlLine.group("randomDetID", "FLOOR(RAND()*9223372036854775807)") + sqlLine.group("randomDetID", "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)") sqlLine.group("filterID", str(self.filterID)) sqlLine.group("surveyID", str(self.surveyID)) @@ -607,4 +607,18 @@ ''' + Updates table and generates pspsuniqueids + ''' + + def updatePspsUniqueIDs(self,table, ota): + sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, objID from \ + (select @r:=0) r, "+table+" t) as foo using (objID) set \ + uniquePspsP2id = ((" +str(self.batchID)+ "*1000000000 ) \ + + (" +str(ota) + "*10000000 )+rownum)" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + raise + + ''' Does the processing, i.e. pulling stuff from IPP tables into PSPS tables ''' @@ -637,4 +651,7 @@ results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID") #self.logger.info("deleted nulls") + self.logger.info("add psps unique p2 ids") + ota = x*10+y + self.updatePspsUniqueIDs("Detection_" + chipname, ota) self.updateImageID("Detection_" + chipname, x, y) #self.logger.info("updateImageId") Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/diffbatch.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/diffbatch.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/diffbatch.py (revision 37801) @@ -0,0 +1,539 @@ +#!/usr/bin/env jython + +import os.path +import sys +import glob +import time +import stilts +import re +from java.lang import * +from java.sql import * + +from java.lang import Math +from org.apache.commons.math.special import Erf + +from xml.etree.ElementTree import ElementTree, Element, tostring + +from batch import Batch +from gpc1db import Gpc1Db +from ipptopspsdb import IppToPspsDb +from scratchdb import ScratchDb +from sqlUtility import sqlUtility + +import logging.config + +''' +DiffBatch class + +This class, a sub-class of Batch, processes single exposures in the form of IPP smf files + +''' +class DiffBatch(Batch): + + ''' + Constructor + ''' + def __init__(self, + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + diffSkyFileID, + batchID): + + super(DiffBatch, self).__init__( + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + diffSkyFileID, + batchID, + "DF", + None) + # gpc1Db.getDiffStageCmf(skychunk.dvoLabel,diffSkyFileID)) + + # get diff meta data + # if error about catID you are usings an old ipptopsps_scratch db + self.logger.infoPair("getting diff" ," metadata") + meta = self.gpc1Db.getDiffStageMeta(diffSkyFileID) + self.logger.infoPair("ok, I guess that worked","now doing this") + self.diffSkyFileID = diffSkyFileID + if not meta: + self.logger.errorPair("Could not get", "diff metadata") + raise + self.logger.infoPair("so meta", "much amaze") + self.filterName = meta[0]; + self.difftypeID = meta[1]; + self.posImageID = meta[2]; + self.negImageID = meta[3]; + self.skycellName = meta[4]; + self.analVer = meta[5]; + self.expTime = meta[6]; + self.obsTime = meta[7]; + self.fits = gpc1Db.getDiffStageCmf(skychunk.dvoLabel, diffSkyFileID) + self.header = self.fits.getPrimaryHeader() + tessName = self.getKeyValue(self.header, 'TESS_ID') + self.tessID = self.scratchDb.getTessID(tessName) + + + # XXX EAM 20140812 : I am hardwiring the parsing logic for RINGS vs LOCAL + if tessName == 'RINGS.V3': + # skycell is, eg "skycell.1133.081" + # 0123456789012345 + self.projectionID = self.skycellName[8:12] + self.skycellID = self.skycellName[13:] + self.skycellID = self.skycellName[8:] + else: + # skycell is, eg "skycell.081" + # 01234567890 + self.projectionID = 0 + self.skycellID = self.skycellName[8:11] + self.skycellID=self.skycellName[8:] + if self.skycellID == "": + self.skycellID = -1 + + + # drop the existing tables + + self.dropTableVerbose("DiffMeta") + self.dropTableVerbose("DiffToImage") + self.dropTableVerbose("DiffDetection") + + + + + self.logger.infoPair("I can haz",diffSkyFileID) + + + + if not self.analVer: self.analVer = -999 + + # create an output filename, which is {expID}.FITS + self.outputFitsFile = "%08d.FITS" % self.diffSkyFileID + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + print "got here" + self.photoCalID = str(self.scratchDb.getPhotoCalID(self.diffSkyFileID)) + print "got photocalid" + self.calibModNum = 0; + self.dataRelease = skychunk.dataRelease; + + self.filterID = self.scratchDb.getFilterID(self.filterName[0]) + + print self.filterName + print self.filterName[0] + print self.filterID + # insert what we know about this stack batch into the stack table + self.ippToPspsDb.insertDiffMeta(self.batchID, self.filterName[0], self.difftypeID) + + # dump stuff to log + self.logger.infoPair("diff_skyfile_ID", "%d" % self.diffSkyFileID) + + + ''' + Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header + ''' + def populateDiffMeta(self): + self.logger.infoPair("Proccesing table", "DiffMeta") + + sqlLine = sqlUtility("INSERT INTO DiffMeta (") + + sqlLine.group("diffMetaID", str(self.diffSkyFileID)); + sqlLine.group("difftypeID", self.difftypeID); + sqlLine.group("posImageID", self.posImageID); + sqlLine.group("negImageID", self.negImageID); + sqlLine.group("skycellID", self.skycellID); + sqlLine.group("photoCalID", self.photoCalID); + sqlLine.group("magSat", self.getKeyFloat(self.header, "%.8f", 'FSATUR')); + sqlLine.group("analVer", str(self.analVer)); + sqlLine.group("expTime", self.expTime); + sqlLine.group("ctype1", self.getKeyValue(self.header, 'CTYPE1')); + sqlLine.group("ctype2", self.getKeyValue(self.header, 'CTYPE2')); + sqlLine.group("crval1", self.getKeyFloat(self.header, "%.8f", 'CRVAL1')); + sqlLine.group("crval2", self.getKeyFloat(self.header, "%.8f", 'CRVAL2')); + sqlLine.group("crpix1", self.getKeyFloat(self.header, "%.8f", 'CRPIX1')); + sqlLine.group("crpix2", self.getKeyFloat(self.header, "%.8f", 'CRPIX2')); + sqlLine.group("cdelt1", self.getKeyFloat(self.header, "%.8e", 'CDELT1')); + sqlLine.group("cdelt2", self.getKeyFloat(self.header, "%.8e", 'CDELT2')); + sqlLine.group("pc001001", self.getKeyFloat(self.header, "%.8e", 'PC001001')); + sqlLine.group("pc001002", self.getKeyFloat(self.header, "%.8e", 'PC001002')); + sqlLine.group("pc002001", self.getKeyFloat(self.header, "%.8e", 'PC002001')); + sqlLine.group("pc002002", self.getKeyFloat(self.header, "%.8e", 'PC002002')); + + sql = sqlLine.make(") VALUES ( ", ")") + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql: ', sql) + raise + self.logger.infoPair("DiffMeta","final stuff") + self.scratchDb.updateAllRows("DiffMeta", "batchID", str(self.batchID)) + self.scratchDb.updateAllRows("DiffMeta", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("DiffMeta", self.filterName[0]) + self.scratchDb.updateAllRows("DiffMeta", "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows("DiffMeta", "dataRelease", str(self.skychunk.dataRelease)) + self.logger.infoPair("DiffMeta","done") + + ''' + Populates the Diff table for this OTA + ''' + def populateDiffDetectionTable(self): + + pspsTableName = "DiffDetection" + ippTableName = "SkyChip_psf" + + #results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName) + + # drop then re-create table + #self.scratchDb.dropTable(pspsTableName) + #sql = "CREATE TABLE " + pspsTableName + " LIKE DiffDetection" + #try: self.scratchDb.execute(sql) + #except: pass + + # XXX we no longer delete detections with funny mags (eg PSF_INST_MAG < -17.5) + # for examples of code which do this, go to r35097 or earlier + + BEFORE = self.scratchDb.getRowCount(ippTableName) + #results['SATDET'] = 0 + + extTimeString = str(self.expTime) + + # insert all detections into table : do NOT include the IGNORE unless we have to + # sqlLine = sqlUtility("INSERT IGNORE INTO " + pspsTableName + " (") + sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (") + + # XXX WARNING: need to apply platescale to convert sizes to arcsec + # XXX apFluxF or apFluxNpix + apFluxRadius? + + sqlLine.group("ippDetectID", "IPP_IDET") + sqlLine.group("diffDetMetaID", str(self.diffSkyFileID)); + sqlLine.group("randomDiffID", "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)") + sqlLine.group("filterID", str(self.filterID)) + sqlLine.group("surveyID", str(self.surveyID)) + sqlLine.group("skycellID", str(self.skycellID)) +## sqlLine.group("obsTime", str(self.obsTime)) + sqlLine.group("expTime", str(self.expTime)) + sqlLine.group("obsTime", str(self.obsTime)) + sqlLine.group("xPos", "X_PSF") + sqlLine.group("yPos", "Y_PSF") + sqlLine.group("xPosErr", "X_PSF_SIG") + sqlLine.group("yPosErr", "Y_PSF_SIG") + if (self.id >= 982483): + sqlLine.group("pltScale", "PLTSCALE") + sqlLine.group("posAngle", "POSANGLE") +# sqlLine.group("raErr", "X_PSF_SIG * abs(PLTSCALE)") +# sqlLine.group("decErr", "Y_PSF_SIG * abs(PLTSCALE)") + else: + sqlLine.group("pltScale", "0.257") + sqlLine.group("posAngle", "-999") +# sqlLine.group("raErr", "X_PSF_SIG * 0.257") +# sqlLine.group("decErr", "Y_PSF_SIG * 0.257") + sqlLine.group("psfFlux", "PSF_INST_FLUX / " + extTimeString) + sqlLine.group("psfFluxErr", "PSF_INST_FLUX_SIG / " + extTimeString) +# if (self.id >= 982483): #we don't have PSF_FWHM_MAJ or MIN or PSF_CORE +# sqlLine.group("psfMajorFWHM", "PSF_FWHM_MAJ") +# sqlLine.group("psfMinorFWHM", "PSF_FWHM_MIN") +# sqlLine.group("psfCore", "PSF_CORE") + sqlLine.group("psfTheta", "PSF_THETA") + sqlLine.group("psfQf", "PSF_QF") + sqlLine.group("psfQfPerfect", "PSF_QF_PERFECT") + sqlLine.group("psfChiSq", "PSF_CHISQ") + sqlLine.group("psfLikelihood", "psfLikelihood(EXT_NSIGMA)") + sqlLine.group("momentXX", "MOMENTS_XX") + sqlLine.group("momentXY", "MOMENTS_XY") + sqlLine.group("momentYY", "MOMENTS_YY") + sqlLine.group("momentR1", "MOMENTS_R1") + sqlLine.group("momentRH", "MOMENTS_RH") +# sqlLine.group("momentM3C", "MOMENTS_M3C") +# sqlLine.group("momentM3S", "MOMENTS_M3S") +# sqlLine.group("momentM4C", "MOMENTS_M4C") +# sqlLine.group("momentM4S", "MOMENTS_M4S") + if (self.id >= 982483): + sqlLine.group("apFlux", "AP_FLUX / " + extTimeString) + sqlLine.group("apFluxErr", "AP_FLUX_SIG / " + extTimeString) + # sqlLine.group("apFillF", "AP_NPIX / (3.14159265359 * POW(AP_MAG_RADIUS - 0.5, 2))") + # sqlLine.group("apRadius", "AP_MAG_RADIUS") + sqlLine.group("kronFlux", "KRON_FLUX / " + extTimeString) + sqlLine.group("kronFluxErr", "KRON_FLUX_ERR / " + extTimeString) + sqlLine.group("kronRad", "MOMENTS_R1 * 2.5") + sqlLine.group("diffNPos", "DIFF_NPOS") + sqlLine.group("diffFRatio", "DIFF_FRATIO") + sqlLine.group("diffNBad", "DIFF_NRATIO_BAD") + sqlLine.group("diffNMask", "DIFF_NRATIO_MASK") + sqlLine.group("diffNAll", "DIFF_NRATIO_ALL") + sqlLine.group("diffPdist", "DIFF_R_P") + sqlLine.group("diffNdist", "DIFF_R_M") + sqlLine.group("diffPSN", "DIFF_SN_P") + sqlLine.group("diffNSN", "DIFF_SN_M") + sqlLine.group("sky", "SKY / " + extTimeString) + sqlLine.group("skyErr", "SKY_SIGMA / " + extTimeString) + sqlLine.group("infoFlag", "FLAGS") + sqlLine.group("infoFlag2", "FLAGS2") + sqlLine.group("dataRelease", str(self.skychunk.dataRelease)) + + sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName) + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed sql: ", sql) + raise + + # NOTE : Flux limits : in the current PSPS schema, negative fluxes + # cause problems for sql queries which work in mags as SQL cannot do + # something like (f < 0.0) ? -999 : -2.5*log10(f) + # as a result, the negative fluxes here result in floating point errors + # XXX EAM 2014072 : Is this still a problem? + # sql = "UPDATE " + pspsTableName + " SET psfFlux = 1e20 WHERE psfFlux <= 0.0" + # self.scratchDb.execute(sql) + + # sql = "UPDATE " + pspsTableName + " SET apFlux = 1e20 WHERE apFlux <= 0.0" + # self.scratchDb.execute(sql) + + # sql = "UPDATE " + pspsTableName + " SET kronFlux = 1e20 WHERE kronFlux <= 0.0" + # self.scratchDb.execute(sql) + + # we don't delete these anymore + #results['NULLINSTFLUX'] = 0; + + +# insert stuff from dvo + self.logger.infoPair("inserting dvo info:find imageID"," from externID") + imageID = self.scratchDb.getImageIDFromExternID(self.diffSkyFileID) + self.logger.infoPair("updating","diffdetection") + sqlLine = sqlUtility("UPDATE DiffDetection as a, " + self.scratchDb.dvoDetectionTable + " as b SET ") + sqlLine.group("a.diffObjID", "b.objId") + sqlLine.group("a.diffDetID", "b.detectID") + sqlLine.group("a.ippObjID","b.ippObjID") + sqlLine.group("a.dvoRegionID", "b.catID") + sqlLine.group("a.ra", "b.ra") + sqlLine.group("a.dec", "b.dec_") + sqlLine.group("a.raErr", "b.raErr") + sqlLine.group("a.decErr", "b.decErr") + + sqlLine.group("a.zp","b.zp") + sqlLine.group("a.telluricExt", "b.telluricExt") + sqlLine.group("a.airmass","b.airmass") + + + sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " +str( imageID)) + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed sql:",sql) + raise + self.logger.infoPair("need to cull"," nulls objid") + results= self.scratchDb.reportAndDeleteRowsWithNULLS("DiffDetection", "diffObjID") + self.logger.infoPair("adding","uniquePSPSDFid") + self.updatePspsUniqueIDs("DiffDetection") + + ''' + Populates the DiffToImage table + ''' + def populateDiffToImage(self): + + self.logger.infoPair("Procesing table", "DiffToImage") + + imageIDs = self.gpc1Db.getImageIDsForThisDiffID(self.diffSkyFileID) + + for imageID in imageIDs: + + sql = "INSERT INTO DiffToImage (diffMetaID, imageID) \ + VALUES (\ + " + str(self.diffSkyFileID) + ", " + imageID + ")" + self.scratchDb.execute(sql) + + # now update StackMeta with correct number of inputs + #sql = "UPDATE "+filter+"StackMeta SET nP2Images = (SELECT COUNT(distinct(floor(imageID/100))) FROM StackToImage where stackMetaID = "+str(stackmetaID)+")" + #self.scratchDb.execute(sql) + + #self.tablesToExport.append("DiffToImage") + + + + + + + + ''' + Applies indexes and other constraints to the PSPS tables + ''' + def alterPspsTables(self): + + self.logger.debug("Altering PSPS tables") + self.logger.debug("Creating indexes on PSPS tables") + self.scratchDb.makeColumnUnique("DiffDetection", "objID") + self.scratchDb.makeColumnPrimaryKey("DiffDetection", "ippDetectID") + + #self.populateDiffMeta() + + # dictionary objects to hold imageIDs for later + self.imageIDs = {} + self.sourceIDs = {} + + # loop through all OTAs and populate ImageMeta extensions + # the column in PSPS + self.scratchDb.execute("ALTER TABLE DiffDetection CHANGE dec_ `dec` double") + + return True + + + ''' + Applies indexes to the IPP tables + ''' + def indexIppTables(self): + + self.logger.infoPair("Creating indexes on", "IPP tables") + + #self.scratchDb.createIndex(extension, "IPP_IDET") + # try the test Chip + self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET") + + self.logger.infoPair("created indexes on", "IPP tables") + + ''' + Updates provided table with DVO IDs from DVO table + ''' + def updateDvoIDs(self, table, externID): + + imageID = self.scratchDb.getImageIDFromExternID(externID) + self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID) + + sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ + a.objID = b.objID, \ + a.detectID = b.detectID, \ + a.ippObjID = b.ippObjID, \ + a.dvoRegionID = b.catID, \ + a.ra = b.ra, \ + a.dec = b.dec_, \ + a.zp = b.zp, \ + a.telluricExt = b.telluricExt, \ + a.airmass = b.airmass, \ + a.expTime = b.expTime, \ + a.infoFlag2 = (b.flags << 13) | a.infoFlag2 \ + WHERE a.ippDetectID = b.ippDetectID \ + AND b.imageID = " + str(imageID) + + self.scratchDb.execute(sql) + + ''' + Updates table and generates pspsuniqueids + ''' + + def updatePspsUniqueIDs(self,table): + sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \ + (select @r:=0) r, "+table+" t) as foo using (diffobjID) set \ + uniquePspsDFid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + raise + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): + self.logger.infoPair("starting","populatePspsTables for diffBatch") + # loop through all OTAs again to update with DVO IDs + self.skipBatch = False + self.tablesToExport = [] + self.tablesToExport.append("DiffMeta") + self.populateDiffMeta() + self.logger.infoPair("populated","DiffMeta") + self.tablesToExport.append("DiffToImage") + self.populateDiffToImage() + self.logger.infoPair("populated","DiffToImage") + self.tablesToExport.append("DiffDetection") + self.populateDiffDetectionTable() + self.logger.infoPair("populated","DiffDetection") + + + + + self.setMinMaxDiffObjID(["DiffDetection"]) + + + #test = self.scratchDb.getRowCount("Object") + #return False + + return True + + + ''' + Imports IPP tables from FITS file + Accepts a regular expression to match the tables so not all tables need to be imported + (This regex feature is not currently used...) + ''' + def importIppTables(self, columns="*", tableRE=""): + + if self.config.retry: return True + + self.logger.infoPair("Importing DF tables with table match expression: ", tableRE) + + count = 0 + fileName = self.fits.getPath() + + self.logger.infoPair("using filename:",fileName) + + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False + + # count = 0 + for table in tables: + + match = re.match(tableRE, table.name) + if not match: continue + + self.logger.infoPair("Reading IPP table", table.name) + table = stilts.tpipe(table, cmd='addcol table_index $0') + table = stilts.tpipe(table, cmd='explodeall') + + # drop any previous tables before import + self.scratchDb.dropTable(table.name) + + + # IPP FITS files are littered with infinities, so remove these + self.logger.info("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + + try: + table.write(self.scratchDb.url + '#' + table.name) + count = count + 1 + except: + self.logger.exception("Problem writing table '" + table.name + "' to the database") + + self.logger.infoPair("Done. Imported", "%d tables" % count) + self.indexIppTables() + + return True + + + + + + + ''' + Overriding this method. Use regex to trim off, eg _XY33 extension + ''' + def exportPspsTablesToFits(self, regex="(.*)"): + return super(DiffBatch, self).exportPspsTablesToFits("([a-zA-Z]+)") + + + ''' + Drop a table and report hte drop + ''' + def dropTableVerbose(self, table): + self.logger.infoPair("dropping table:",table) + self.scratchDb.dropTable(table) + Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/diffobjectbatch.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/diffobjectbatch.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/diffobjectbatch.py (revision 37801) @@ -0,0 +1,418 @@ +#!/usr/bin/env jython + +import os.path +import sys + +import stilts +from java.lang import * +from java.sql import * + +from xml.etree.ElementTree import ElementTree, Element, tostring + +from batch import Batch +from gpc1db import Gpc1Db +from ipptopspsdb import IppToPspsDb +from scratchdb import ScratchDb +from dvoobjects import DvoObjects +from sqlUtility import sqlUtility + +import logging.config + +''' +DiffObjectBatch class + +This class, a sub-class of Batch, processes individual DVO region file pairs (cpt and cps files). The cpt file contains a list of all objects in that region, the cps file contains the magnitudes and so has a row per filter per object (so, generally, has rows = 5 x number of objects). The Photcode.dat table us used to figure out the number and order of filters as they appear in the cps file. + +''' +class DiffObjectBatch(Batch): + + ''' + Constructor + ''' + def __init__(self, + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + dvoID, + batchID): + + super(DiffObjectBatch, self).__init__( + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + dvoID, + batchID, + "DO", + None) + + try: + self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb) + except: + self.logger.errorPair("Unable to create instance of", "DvoObjects") + raise + + # create an output filename, which is {dvoINDEX}.FITS + self.outputFitsFile = "%08d.FITS" % self.id + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + + # dump stuff to log + self.logger.infoPair("DVO INDEX", "%d" % self.id) + + ''' + Overriden from batch base-class. We import from DVO directly for objects + ''' + def importIppTables(self, columns="*", filter=""): + + self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id) + self.ippToPspsDb.insertObjectMeta(self.batchID, self.region) + self.dvoObjects.nativeIngestRegion(self.region) + + cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") + if not self.scratchDb.tableExists(cptTableName): + return False + + return True + + ''' + Applies indexes to the PSPS tables + ''' + def alterPspsTables(self): + + return True + + ''' + Applies indexes to the IPP tables + ''' + def indexIppTables(self): + + # since dvopsps is now used, no action is needed here + # self.logger.infoPair("Creating indexes on", "IPP tables") + + return True + + ''' + Inserts stuff for all mags + ''' + def updateMeanObjectFromCps(self, cpsTable): + + # list of all filters PSPS is interested in + # XXX EAM : 2014.07.24 : this list should probably be in a config file somewhere + interestedFilters = ['g', 'r', 'i', 'z', 'y'] + + filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters) + + # get a count of the available filters + filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable() + + self.logger.infoPair("Available filters in Photcodes", filters) + + # the 'code' now defines the order in the cps file that the mags are listed for a given filter + self.logger.infoPair("Adding magnitudes from", "cps table") + for filter in filters: + + filterID = self.scratchDb.getFilterID(filter[1]) + + # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000) + # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags + # XXX EAM : 20140724 this manipulation is no longer needed : [grizy]Flags is now 4 byte (was 8 byte!) + + # set the MeanObject fields based largely on dvopsps cps fields: + + # XXX EAM 20140724 : filterCount is meant to match + # Nsecfilt, but is potentially not determined correctly. + # use a call to a dvo-native command which knows how to + # find Nsecfilt (or save in the db with dvopsps) + + # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting? + # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter + # " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + + sql = "UPDATE DiffDetObject JOIN \ + " + cpsTable + " AS cps ON (cps.row = (DiffDetObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + SET \ + DiffDetObject." + filter[1] + "QfPerfect = PSF_QF_PERF_MAX " + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed update DiffDetObject", sql) + raise + + ''' + Inserts stuff for all mags + ''' + def updateDiffObjectFromCps(self, cpsTable): + + # list of all filters PSPS is interested in + interestedFilters = ['g', 'r', 'i', 'z', 'y'] + + filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters) + + # get a count of the available filters + filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable() + # filterCount = len(filters) + + self.logger.infoPair("Available filters in Photcodes", filters) + + # the 'code' now defines the order in the cps file that the mags are listed for a given filter + self.logger.infoPair("Adding magnitudes from", "cps table") + for filter in filters: + + filterID = self.scratchDb.getFilterID(filter[1]) + + # XXX EAM 20140724 : this is quite awkward, add a objRow and ncode value to mysql db table? + # sqlLine = sqlUtility() + + sql = "UPDATE DiffDetObject JOIN " + sql += cpsTable + " AS cps " + sql += "ON (cps.row = (DiffDetObject.row * " + sql += str(filterCount) + ")-(" + sql += str(filterCount) + " - " + sql += str(filter[0]) + sql += ")) " + + sql += "SET DiffDetObject.n" + filter[1] + " = NCODE, " + sql += "DiffDetObject.nDetections = DiffDetObject.nDetections + cps.NSTACK_DET" + self.logger.info(sql) + self.scratchDb.execute(sql) + + # XXX this does not seem like a good thing to leave in SQL + self.logger.infoPair("Calculating nDetections from", "n[filters]") + for filter in filters: + # now do a sum of n[filters], but do not include the ones with -999 + sql = "UPDATE DiffDetObject " + sql += "SET nDetections = nDetections + n" + filter[1] + sql += " WHERE n" + filter[1] + " != -999" + self.scratchDb.execute(sql) + + ''' + give objectName to objectThin + XXX EAM 20140714 : This seems quite inefficient in SQL, move to dvopsps? + ''' + def updateDiffObjName(self): + ##this is a 2 part update: + + self.logger.infoPair("updating diffObjName", "using means") + ## use mean ra and dec (dec >= 0) + sql = "update DiffDetObject set diffObjName = concat('PS1.2 J', \ + lpad(floor(ra/15.),2,'0'), \ + lpad((ra/15.-(floor(ra/15.)))*60. ,2,'0'), \ + lpad(format(((ra/15.-(floor(ra/15.)))*60.-floor((ra/15.-(floor(ra/15.)))*60.))*60.,2),4,'0'), \ + '+', \ + lpad(floor(dec_),2,'0'), \ + lpad((dec_-(floor(dec_)))*60. ,2,'0'), \ + lpad(format(((dec_-(floor(dec_)))*60.-floor((dec_-(floor(dec_)))*60.))*60.,2),4,'0') \ + ) where ra > -999 and dec_ > -999 and dec_ >= 0" + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed sql", sql) + return False + + ## use mean ra and dec (dec < 0) + + sql = "update DiffDetObject set diffObjName = concat('PS1.2 J', \ + lpad(floor(ra/15.),2,'0'), \ + lpad((ra/15.-(floor(ra/15.)))*60. ,2,'0'), \ + lpad(format(((ra/15.-(floor(ra/15.)))*60.-floor((ra/15.-(floor(ra/15.)))*60.))*60.,2),4,'0'), \ + '-', \ + lpad(floor(dec_),2,'0'), \ + lpad((dec_-(floor(dec_)))*60. ,2,'0'), \ + lpad(format(((dec_-(floor(dec_)))*60.-floor((dec_-(floor(dec_)))*60.))*60.,2),4,'0') \ + ) where ra > -999 and dec_ > -999 and dec_ < 0" + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed sql", sql) + return False + + ''' + Populates the DiffObject table + ''' + def populateDiffObjectTable(self): + + cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") + cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps") + + if False: + # XXX EAM 20140724 : this is probably wrong : flux measurements can be 0.0 or negative: please review + self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON_ERR") + sql = "UPDATE " + cpsTableName + " set FLUX_KRON_ERR = NULL where FLUX_KRON_ERR < 1e-37 AND FLUX_KRON_ERR > -1e-37 " + self.exitProgram("review this code" + sql) + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't squash out of range stuff", sql) + return False + + self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON") + sql = "UPDATE " + cpsTableName + " set FLUX_KRON = NULL where FLUX_KRON < 1e-37 AND FLUX_KRON > -1e-37 " + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't squash out of range stuff", sql) + return False + + + self.logger.info("Populating DiffDetObject") + self.logger.info("Inserting objects from cpt file") + + # note "dec" is a reserved word in MySQL + # XXX EAM 20140724 : do not use INGORE unless we discover unavoidable problems... + # INSERT IGNORE INTO ObjectThin + + sqlLine = sqlUtility("INSERT INTO DiffDetObject (") + + sqlLine.group("diffObjID", "EXT_ID") + sqlLine.group("ippObjID", "OBJ_ID + (CAT_ID << 32)") # NOTE: shift by 32 bits exactly + sqlLine.group("surveyID", "'" + str(self.surveyID) + "'") + sqlLine.group("randomID", "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)") # XXX where does this number come from?? + sqlLine.group("batchID", "'" + str(self.batchID) + "'") + sqlLine.group("dvoRegionID", "CAT_ID") + sqlLine.group("skycellID", "SKYCELL_ID") + sqlLine.group("dataRelease", "'" + str(self.skychunk.dataRelease) + "'") + sqlLine.group("objInfoFlag", "FLAGS") + sqlLine.group("qualityFlag", "FLAGS >> 24 & 0xFF") + sqlLine.group("consistencyFlag", "'0'") + sqlLine.group("ra", "RA_MEAN") + sqlLine.group("dec_", "DEC_MEAN") + sqlLine.group("raErr", "RA_ERR") + sqlLine.group("decErr", "DEC_ERR") + sqlLine.group("nDetections", "'0'") + sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName) + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't populate DiffDetObject table", sql) + return False + + # add row count columns so we can perform joins to get colors + self.logger.infoPair("Adding 'row' columns to", "DiffDetObject and cps tables") + self.scratchDb.addRowCountColumn("DiffDetObject", "row") + self.scratchDb.addRowCountColumn(cpsTableName, "row") + +## self.insertMeanMags(cpsTableName) +## this is the new version + self.logger.infoPair("update diffObjName for ","DiffDetObject") + self.updateDiffObjName() + + self.logger.infoPair("update DiffDetObject from ","cps table") + + self.updateDiffObjectFromCps(cpsTableName) + + # XXX EAM 20140724 : is this necessary?? + #objects can have out of range ra dec in dvo - need to find and kill them at the end + + self.logger.infoPair("Determining", "ra/dec range") + + raMin = self.scratchDb.getFromdvoSkyTable("R_MIN",self.region) + raMax = self.scratchDb.getFromdvoSkyTable("R_MAX",self.region) + decMin = self.scratchDb.getFromdvoSkyTable("D_MIN",self.region) + decMax = self.scratchDb.getFromdvoSkyTable("D_MAX",self.region) + + self.logger.infoPair("R_MIN", raMin) + self.logger.infoPair("R_MAX", raMax) + self.logger.infoPair("D_MIN", decMin) + self.logger.infoPair("D_MAX", decMax) + #count out of range + + sql = "SELECT count(*) FROM DiffDetObject where \ + DiffDetObject.dec_ > (" + str(decMax) + " + .0033) \ + or DiffDetObject.dec_ < (" + str(decMin) + " - .0033) \ + or DiffDetObject.ra > (" + str(raMax) + " + .0033) \ + or DiffDetObject.ra < (" + str(raMin) + " - .0033)" + + rs = self.scratchDb.executeQuery(sql) + rs.first() + nToDelete = rs.getInt(1) + + #delete out of range + + + sql = "DELETE FROM DiffDetObject where \ + DiffDetObject.dec_ > (" + str(decMax) + " + .0033) or \ + DiffDetObject.dec_ < (" + str(decMin) + " - .0033) or \ + DiffDetObject.ra > (" + str(raMax) + " + .0033) or \ + DiffDetObject.ra < (" + str(raMin) + " - .0033)" + self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range") + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't cull outsiders from DiffDetObject table", sql) + return False + + self.logger.infoPair("Dropping row column from", "DiffDetObject table") + self.scratchDb.dropColumn("DiffDetObject", "row") + ##self.logger.infoPair("Purging from scratch Db", self.region + " region") + + ##Don't do this till after MeanObject + ##self.dvoObjects.purgeRegion(self.region) + + self.setMinMaxDiffObjID(["DiffDetObject"]) + + return True + + ''' + Updates table and generates pspsuniqueids + ''' + + def updatePspsUniqueIDs(self,table): + sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \ + (select @r:=0) r, "+table+" t) as foo using (diffobjID) set \ + uniquePspsDOid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + return + + + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): + + if not self.populateDiffObjectTable(): return False + + # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number + self.logger.infoPair("Forcing uniqueness on", "objID in DIffObject table") + rowCountBefore = self.scratchDb.getRowCount("DiffDetObject") + + # XXX EAM : note that in mysql versions later than 5.1, this fails + # unless the following is called first: + # set session old_alter_table=1 + # follow the command with + # set session old_alter_table=0 + # OF COURSE, this fails for mysql version < 5.5... + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=1") + + self.scratchDb.execute("ALTER IGNORE TABLE DiffDetObject ADD UNIQUE INDEX(diffObjID)") + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=0") + + rowCountAfter = self.scratchDb.getRowCount("DiffDetObject") + self.logger.infoPair("Number of duplicated diffobjIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + + self.scratchDb.execute("ALTER TABLE DiffDetObject CHANGE dec_ `dec` double") + + self.updatePspsUniqueIDs("DiffDetObject") + + self.dvoObjects.purgeRegion(self.region) + + #this is abuse of something but this is how I get the object batches to crash to further investigate them + + +# rowCountAfter = self.scratchDb.getRowCount("Object") + return True +# return False Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvodiffdetections.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvodiffdetections.py (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvodiffdetections.py (revision 37801) @@ -0,0 +1,167 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob + +from dvo import Dvo + + +''' +A class for ingesting DVO detections into MySQL +''' +class DvoDetections(Dvo): + + ''' + Constructor + ''' + def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb): + + super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) + + # declare DVO file types of interest + self.ingestFileTypes = ['cpm','cpt'] + + ''' + Ingests detections for this region + ''' + def ingestRegion(self, region): + + cpmPath = self.skychunk.dvoLocation + "/" + region + ".cpm" + cptPath = self.skychunk.dvoLocation + "/" + region + ".cpt" + + cpmTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpm") + cptTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpt") + + # if we have not already ingested the cpm table, then do it now + if not self.scratchDb.alreadyImportedThisDvoTable(cpmPath): + + # import cpm table and index + self.importFits( + cpmPath, + "IMAGE_ID DET_ID OBJ_ID CAT_ID EXT_ID DB_FLAGS PHOTCODE AIRMASS M_TIME M_CAL MAG_CAL_ERR D_RA D_DEC PLTSCALE X_CCD_ERR Y_CCD_ERR", + cpmTableName) + + self.scratchDb.createIndex(cpmTableName, "IMAGE_ID") + self.scratchDb.createIndex(cpmTableName, "CAT_ID") + self.scratchDb.createIndex(cpmTableName, "OBJ_ID") + + # XXX: Should we create index on PHOTCODE? + + # we can now report that we have imported this table + self.scratchDb.setImportedThisDvoTable(cpmPath, region) + + # get the id for this cpm file so we can insert it in the dvoDetection table + fileId = self.scratchDb.getDvoFileId(cpmPath) + + # import cpt table and index + self.importFits( + cptPath, + "OBJ_ID CAT_ID EXT_ID RA DEC", + cptTableName) + + self.scratchDb.createIndex(cptTableName, "IMAGE_ID") + self.scratchDb.createIndex(cptTableName, "CAT_ID") + self.scratchDb.createIndex(cptTableName, "OBJ_ID") + + # SA10 + # Columns needed for DetectionCalib and StackDetectionCalib + # zp = photcode.K * (AIRMASS - 1) + photcode.C_LAM * 0.001 - cpm.M_CAL + # zpErr = cpm.MAG_CAL_ERR + # airMass = cpm.AIRMASS + # expTime = POW(10.0, 0.4 * cpm.M_TIME) + # ra = cpt.RA - (cpm.D_RA / 3600.0) + # dec = cpt.DEC - (cpm.D_DEC / 3600.0) + # raErr = cpm.X_CCD_ERR * cpm.PLTSCALE + # decErr = cpm.Y_CCD_ERR * cpm.PLTSCALE + + self.logger.infoPair("Adding","PSPS objIDs") + # first try to add the new columns. catch failure and continue + try: + sql = "ALTER TABLE " + cpmTableName + " ADD COLUMN (PSPS_OBJ_ID BIGINT), ADD COLUMN (ZP REAL), ADD COLUMN (RA FLOAT), ADD COLUMN (DEC_ FLOAT)" + self.scratchDb.execute(sql) + except: + # failed, but may be due to mysql error + self.logger.infoPair("mysql error or already added PSPS_OBJ_ID column to", cpmTableName) + # self.logger.infoPair("sql command", sql) + + # shove PSPS objIDs from cpt table and the calibrated zero point and coordinates which are formed + # by combining values from multiple tables into cpt table + sql = "UPDATE " \ + + cpmTableName + " AS meas, " \ + + cptTableName + " AS ave, " \ + + self.scratchDb.dvoPhotcodesTable + " AS phot \ + SET \ + meas.PSPS_OBJ_ID = ave.EXT_ID, \ + meas.ZP = phot.C_LAM * 0.001 + phot.K * (meas.AIRMASS - 1) - meas.M_CAL, \ + meas.RA = ave.RA - (meas.D_RA / 3600.0), \ + meas.DEC_ = ave.DEC_ - (meas.D_DEC / 3600.0) \ + WHERE meas.CAT_ID = ave.CAT_ID \ + AND meas.OBJ_ID = ave.OBJ_ID \ + AND meas.PHOTCODE = phot.CODE" + + # EAM : add some debug I/O + # self.logger.infoPair("sql command", sql) + self.scratchDb.execute(sql) + + # NB we use an INSERT IGNORE here. This is because of a known issue where multiple DVO cpm + # files can include the same detection, with the same IMAGE_ID/DET_ID pairing, but different + # PSPS object IDs assigned in the corresponding cpt file. This is believed to be a chip-boundary + # issue within DVO. So, for now, we take the first IMAGE_ID/DET_ID detection we find, and ignore the rest + # XXX: Bills 2012-10031 Does this problem still exist? + self.logger.infoPair("Merging into", self.scratchDb.dvoDetectionTable) + sql = "INSERT IGNORE INTO " + self.scratchDb.dvoDetectionTable + " (\ + imageID \ + ,ippDetectID \ + ,detectID \ + ,ippObjID \ + ,objID \ + ,flags \ + ,zp \ + ,zpErr \ + ,airMass \ + ,expTime \ + ,ra \ + ,dec_ \ + ,raErr \ + ,decErr \ + ,fileID \ + ) SELECT \ + IMAGE_ID \ + ,DET_ID \ + ,EXT_ID \ + ,CAT_ID * 1000000000 + OBJ_ID \ + ,PSPS_OBJ_ID \ + ,DB_FLAGS \ + ,ZP \ + ,MAG_CAL_ERR \ + ,AIRMASS \ + ,POW(10.0, 0.4 * M_TIME) \ + ,RA \ + ,DEC_ \ + ,X_CCD_ERR*PLTSCALE \ + ,Y_CCD_ERR*PLTSCALE \ + , " + str(fileId) + " \ + FROM " + cpmTableName + + # print "sql :", sql + # response = raw_input("prepare to insert dvoDetectionTable ") + + try: + self.scratchDb.execute(sql) + except: + self.logger.error("FAILED: " + sql) + return False + finally: + # response = raw_input("insert dvoDetectionTable ") + self.logger.infoPair("Dropping tables", "%s and %s" % (cpmTableName, cptTableName)) + self.scratchDb.dropTable(cpmTableName) + self.scratchDb.dropTable(cptTableName) + + self.scratchDb.setImportedThisDvoTable(cptPath, region) + + return True Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvodiffobjects.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvodiffobjects.py (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvodiffobjects.py (revision 37801) @@ -0,0 +1,66 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob +from subprocess import call, PIPE, Popen + +from dvo import Dvo + +''' +A class for ingesting DVO detections into MySQL +''' +class DvoObjects(Dvo): + + ''' + Constructor + ''' + def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb): + + super(DvoObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) + + # declare DVO file types of interest + self.ingestFileTypes = ['cpt', 'cps'] + self.loadPhotcodes() + + ''' + ingest object data into MySQL database using the native DVO program dvopsps + Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this + includes purging detections outside sky area + ''' + def nativeIngestRegion(self, region): + + # drop cpt/cps/object table? + # create detections table? + + # TODO path to DVO prog hardcoded temporarily + cmd = "dvopsps objects" + cmd += " -dbhost " + self.scratchDb.dbHost + cmd += " -dbname " + self.scratchDb.dbName + cmd += " -dbuser " + self.scratchDb.dbUser + cmd += " -dbpass " + self.scratchDb.dbPass + cmd += " -D CATDIR " + self.skychunk.dvoLocation + cmd += " -cpt " + region + + if self.skychunk.parallel: + if self.scratchDb.dbHost == "localhost": + self.logger.info("dvopsps objects -parallel is incompatible with localhost for scratchdb") + raise + + cmd += " -parallel" + + self.logger.infoPair("Running dvopsps", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) + p.wait() + if p.returncode != 0: + self.logger.errorPair("dvopsps failed on ", region) + raise + + # update lists after attempted sync + # self.printSummary() + + return True Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvoforceddetections.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvoforceddetections.py (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvoforceddetections.py (revision 37801) @@ -0,0 +1,178 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob + +from dvo import Dvo + + +''' +A class for ingesting DVO detections into MySQL +''' +class DvoForcedDetections(Dvo): + + ''' + Constructor + ''' + def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb): + + super(DvoForcedDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) + + # declare DVO file types of interest + self.ingestFileTypes = ['cpm','cpt','cpx'] + + ''' + Ingests detections for this region + ''' + def ingestRegion(self, region): + + cpmPath = self.skychunk.dvoLocation + "/" + region + ".cpm" + cptPath = self.skychunk.dvoLocation + "/" + region + ".cpt" + cpxPath = self.skychunk.dvoLocation + "/" + region + ".cpx" + + cpmTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpm") + cptTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpt") + cpxTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpx") + + # if we have not already ingested the cpm table, then do it now + if not self.scratchDb.alreadyImportedThisDvoTable(cpxPath): + + # import cpm table and index + self.importFits( + cpmPath, + "IMAGE_ID DET_ID OBJ_ID CAT_ID EXT_ID AIRMASS M_CAL D_RA D_DEC PHOTCODE", + cpmTableName) + + self.scratchDb.createIndex(cpmTableName, "IMAGE_ID") + self.scratchDb.createIndex(cpmTableName, "CAT_ID") + self.scratchDb.createIndex(cpmTableName, "OBJ_ID") + + # XXX: Should we create index on PHOTCODE? + + # we can now report that we have imported this table + self.scratchDb.setImportedThisDvoTable(cpmPath, region) + + # get the id for this cpm file so we can insert it in the dvoDetection table + fileId = self.scratchDb.getDvoFileId(cpmPath) + + # import cpt table and index + self.importFits( + cptPath, + "OBJ_ID CAT_ID EXT_ID RA DEC" + cptTableName) + + self.scratchDb.createIndex(cptTableName, "IMAGE_ID") + self.scratchDb.createIndex(cptTableName, "CAT_ID") + self.scratchDb.createIndex(cptTableName, "OBJ_ID") + + # heather thinks this may be a problem (but I'm not entirely sure) - why create index on image_id? it's not an imported column. + + self.importFits( + cpxPath, + "X11_SM_OBJ X12_SM_OBJ X22_SM_OBJ E1_SM_OBJ E2_SM_OBJ X11_SH_OBJ X12_SH_OBJ X22_SH_OBJ E1_SH_OBJ E2_SH_OBJ \ + X11_SM_PSF X12_SM_PSF X22_SM_PSF E1_SM_PSF E2_SM_PSF X11_SH_PSF X12_SH_PSF X22_SH_PSF E1_SH_PSF E2_SH_PSF \ + FLUX_AP_R5 FLUX_ERR_AP_R5 FLUX_STD_AP_R5 FLUX_FIL_AP_R5 FLUX_AP_R6 FLUX_ERR_AP_R6 FLUX_STD_AP_R6 FLUX_FIL_AP_R6 \ + DET_ID OBJ_ID CAT_ID IMAGE_ID", + cpxTableName) + self.scratchDb.createIndex(cpxTableName, "IMAGE_ID") + self.scratchDb.createIndex(cpxTableName, "CAT_ID") + self.scratchDb.createIndex(cpxTableName, "OBJ_ID") + + + + + self.logger.infoPair("Adding","PSPS objIDs") + # first try to add the new columns. catch failure and continue + try: + sql = "ALTER TABLE " + cpmTableName + " ADD COLUMN (PSPS_OBJ_ID BIGINT), ADD COLUMN (ZP REAL), ADD COLUMN (RA FLOAT), ADD COLUMN (DEC_ FLOAT)" + self.scratchDb.execute(sql) + except: + # failed, but may be due to mysql error + self.logger.infoPair("mysql error or already added PSPS_OBJ_ID column to", cpmTableName) + # self.logger.infoPair("sql command", sql) + + + # shove PSPS objIDs from cpt table and the calibrated zero point and coordinates which are formed + # by combining values from multiple tables into cpt table + sql = "UPDATE " \ + + cpmTableName + " AS meas, " \ + + cptTableName + " AS ave, " \ + + self.scratchDb.dvoPhotcodesTable + " AS phot \ + SET \ + meas.PSPS_OBJ_ID = ave.EXT_ID, \ + meas.ZP = phot.C_LAM * 0.001 + phot.K * (meas.AIRMASS - 1) - meas.M_CAL, \ + meas.RA = ave.RA - (meas.D_RA / 3600.0), \ + meas.DEC_ = ave.DEC_ - (meas.D_DEC / 3600.0) \ + WHERE meas.CAT_ID = ave.CAT_ID \ + AND meas.OBJ_ID = ave.OBJ_ID \ + AND meas.PHOTCODE = phot.CODE" + + # EAM : add some debug I/O + # self.logger.infoPair("sql command", sql) + self.scratchDb.execute(sql) + +# HAF -- need to find out ZP Nominal and to do the columns here, otherwise this is done (up above is done) + + + # NB we use an INSERT IGNORE here. This is because of a known issue where multiple DVO cpm + # files can include the same detection, with the same IMAGE_ID/DET_ID pairing, but different + # PSPS object IDs assigned in the corresponding cpt file. This is believed to be a chip-boundary + # issue within DVO. So, for now, we take the first IMAGE_ID/DET_ID detection we find, and ignore the rest + # XXX: Bills 2012-10031 Does this problem still exist? + self.logger.infoPair("Merging into", self.scratchDb.dvoDetectionTable) + sql = "INSERT IGNORE INTO " + self.scratchDb.dvoDetectionTable + " (\ + imageID \ + ,ippDetectID \ + ,detectID \ + ,ippObjID \ + ,objID \ + ,flags \ + ,zp \ + ,zpErr \ + ,airMass \ + ,expTime \ + ,ra \ + ,dec_ \ + ,raErr \ + ,decErr \ + ,fileID \ + ) SELECT \ + IMAGE_ID \ + ,DET_ID \ + ,EXT_ID \ + ,CAT_ID * 1000000000 + OBJ_ID \ + ,PSPS_OBJ_ID \ + ,DB_FLAGS \ + ,ZP \ + ,MAG_CAL_ERR \ + ,AIRMASS \ + ,POW(10.0, 0.4 * M_TIME) \ + ,RA \ + ,DEC_ \ + ,X_CCD_ERR*PLTSCALE \ + ,Y_CCD_ERR*PLTSCALE \ + , " + str(fileId) + " \ + FROM " + cpmTableName + + # print "sql :", sql + # response = raw_input("prepare to insert dvoDetectionTable ") + + try: + self.scratchDb.execute(sql) + except: + self.logger.error("FAILED: " + sql) + return False + finally: + # response = raw_input("insert dvoDetectionTable ") + self.logger.infoPair("Dropping tables", "%s, %s and %s " % (cpmTableName, cptTableName, cpxTableName)) + self.scratchDb.dropTable(cpmTableName) + self.scratchDb.dropTable(cptTableName) + + self.scratchDb.setImportedThisDvoTable(cpxPath, region) + + return True Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvoforcedobjects.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvoforcedobjects.py (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/dvoforcedobjects.py (revision 37801) @@ -0,0 +1,67 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob +from subprocess import call, PIPE, Popen + +from dvo import Dvo + +''' +A class for ingesting DVO detections into MySQL +''' +class DvoForcedObjects(Dvo): + + ''' + Constructor + ''' + def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb): + + super(DvoForcedObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) + + # declare DVO file types of interest + self.ingestFileTypes = ['cpt', 'cps', 'cpy'] + self.loadPhotcodes() + + ''' + ingest forced object data into MySQL database using the native DVO program dvopsps + Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this + includes purging detections outside sky area + ''' + def nativeIngestRegion(self, region): + + # drop cpt/cps/cpy/object table? + # create detections table? + + # TODO path to DVO prog hardcoded temporarily + # TODO this needs to be modified for forced objects + cmd = "dvopsps objects" + cmd += " -dbhost " + self.scratchDb.dbHost + cmd += " -dbname " + self.scratchDb.dbName + cmd += " -dbuser " + self.scratchDb.dbUser + cmd += " -dbpass " + self.scratchDb.dbPass + cmd += " -D CATDIR " + self.skychunk.dvoLocation + cmd += " -cpt " + region + + if self.skychunk.parallel: + if self.scratchDb.dbHost == "localhost": + self.logger.info("dvopsps objects -parallel is incompatible with localhost for scratchdb") + raise + + cmd += " -parallel" + + self.logger.infoPair("Running dvopsps", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) + p.wait() + if p.returncode != 0: + self.logger.errorPair("dvopsps failed on ", region) + raise + + # update lists after attempted sync + # self.printSummary() + + return True Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/forcedobjectbatch.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/forcedobjectbatch.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/forcedobjectbatch.py (revision 37801) @@ -0,0 +1,360 @@ +#!/usr/bin/env jython + +import os.path +import sys + +import stilts +from java.lang import * +from java.sql import * + +from xml.etree.ElementTree import ElementTree, Element, tostring + +from batch import Batch +from gpc1db import Gpc1Db +from ipptopspsdb import IppToPspsDb +from scratchdb import ScratchDb +from dvoforcedobjects import DvoForcedObjects # need one specific for cpy ? +from sqlUtility import sqlUtility + +import logging.config + +''' +ForcedObjectBatch class + +This class, a sub-class of Batch, processes individual DVO region file pairs (cpt, cpy, and cps files). The cpt file contains a list of all objects in that region, the cps file contains the magnitudes and so has a row per filter per object (so, generally, has rows = 5 x number of objects). The Photcode.dat table us used to figure out the number and order of filters as they appear in the cps file. + +''' +class ForcedMeanObjectBatch(Batch): + + ''' + Constructor + ''' + def __init__(self, + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + dvoID, + batchID): + + super(ForcedMeanObjectBatch, self).__init__( + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + dvoID, + batchID, + "FO", + None) + + try: + self.dvoForcedObjects = DvoForcedObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb) + except: + self.logger.errorPair("Unable to create instance of", "DvoForcedObjects") + raise + + # create an output filename, which is {dvoINDEX}.FITS + self.outputFitsFile = "%08d.FITS" % self.id + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + + # dump stuff to log + self.logger.infoPair("DVO INDEX", "%d" % self.id) + + ''' + Overriden from batch base-class. We import from DVO directly for forced mean objects + ''' + def importIppTables(self, columns="*", filter=""): + + self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id) + self.ippToPspsDb.insertForcedMeanObjectMeta(self.batchID, self.region) + self.dvoForcedObjects.nativeIngestRegion(self.region) + + cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") + + if not self.scratchDb.tableExists(cptTableName): + self.logger.infoPair("can't find in scratch db:","cpt table") + return False + + cpyTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpy") + + if not self.scratchDb.tableExists(cpyTableName): + self.logger.infoPair("can't find in scratch db:","cpy table") + return False + + return True + + ''' + Applies indexes to the PSPS tables + ''' + def alterPspsTables(self): + + return True + + ''' + Applies indexes to the IPP tables + ''' + def indexIppTables(self): + + # since dvopsps is now used, no action is needed here + # self.logger.infoPair("Creating indexes on", "IPP tables") + + return True + + ''' + Inserts stuff for all mags + ''' + def updateForcedMeanObjectFromCps(self, cpsTable): + + # list of all filters PSPS is interested in + # XXX EAM : 2014.07.24 : this list should probably be in a config file somewhere + interestedFilters = ['g', 'r', 'i', 'z', 'y'] + + filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters) + + # get a count of the available filters + filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable() + + self.logger.infoPair("Available filters in Photcodes", filters) + + # the 'code' now defines the order in the cps file that the mags are listed for a given filter + self.logger.infoPair("Adding magnitudes from", "cps table") + for filter in filters: + + filterID = self.scratchDb.getFilterID(filter[1]) + + # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000) + # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags + # XXX EAM : 20140724 this manipulation is no longer needed : [grizy]Flags is now 4 byte (was 8 byte!) + + # set the MeanObject fields based largely on dvopsps cps fields: + + # XXX EAM 20140724 : filterCount is meant to match + # Nsecfilt, but is potentially not determined correctly. + # use a call to a dvo-native command which knows how to + # find Nsecfilt (or save in the db with dvopsps) + + # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting? + # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter + # " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + + sql = "UPDATE ForcedMeanObject JOIN \ + " + cpsTable + " AS cps ON (cps.row = (ForcedMeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + SET \ + ForcedMeanObject." + filter[1] + "nTotal = NWARP \ + ,ForcedMeanObject." + filter[1] + "nIncPSFMag = NUSED_WRP \ + ,ForcedMeanObject." + filter[1] + "nIncKronMag = NUSED_KRON_WRP \ + ,ForcedMeanObject." + filter[1] + "nIncApMag = NUSED_AP_WRP \ + ,ForcedMeanObject." + filter[1] + "PSFMag = MAG_PSF_WRP \ + ,ForcedMeanObject." + filter[1] + "PSFMagErr = MAG_PSF_WRP_ERR \ + ,ForcedMeanObject." + filter[1] + "PSFMagStd = MAG_PSF_WRP_STD \ + ,ForcedMeanObject." + filter[1] + "KronMag = MAG_KRON_WRP \ + ,ForcedMeanObject." + filter[1] + "KronMagErr = MAG_KRON_WRP_ERR \ + ,ForcedMeanObject." + filter[1] + "KronMagStd = MAG_KRON_WRP_STD \ + ,ForcedMeanObject." + filter[1] + "ApMag = MAG_AP_WRP \ + ,ForcedMeanObject." + filter[1] + "ApMagErr = MAG_AP_WRP_ERR \ + ,ForcedMeanObject." + filter[1] + "ApMagStd = MAG_AP_WRP_STD \ + ,ForcedMeanObject." + filter[1] + "Flags = FLAGS " + + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed update MeanObject", sql) + raise + + # now set to null all MeanMagErr values > 0.5 (cut set by Gene, 2012-04-12) + # XXX EAM 20140724 : keep this cut? + cut = 0.5 + self.logger.infoPair("Setting to NULL all MeanMagErr value >", "%f" % cut) + for filter in filters: + + sql = "UPDATE ForcedMeanObject \ + SET " + filter[1] + "PSFMagErr = null \ + WHERE " + filter[1] + "PSFMagErr > " + str(cut) + self.scratchDb.execute(sql) + + self.logger.infoPair("Calculating nDetections from", "n[filters]") + for filter in filters: + # now do a sum of n[filters], but do not include the ones with -999 + sql = "UPDATE ObjectThin " + sql += "SET nDetections = nDetections + n" + filter[1] + sql += " WHERE n" + filter[1] + " != -999" + self.scratchDb.execute(sql) + + ''' + Inserts stuff for all mags from the cpy files + ''' + def updateForcedMeanObjectFromCpy(self, cpyTable): + + # list of all filters PSPS is interested in + # XXX EAM : 2014.07.24 : this list should probably be in a config file somewhere + interestedFilters = ['g', 'r', 'i', 'z', 'y'] + + filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters) + + # get a count of the available filters + filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable() + + self.logger.infoPair("Available filters in Photcodes", filters) + + # the 'code' now defines the order in the cps file that the mags are listed for a given filter + self.logger.infoPair("Adding magnitudes from", "cpy table") + for filter in filters: + + filterID = self.scratchDb.getFilterID(filter[1]) + + # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000) + # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags + # XXX EAM : 20140724 this manipulation is no longer needed : [grizy]Flags is now 4 byte (was 8 byte!) + + # set the MeanObject fields based largely on dvopsps cps fields: + + # XXX EAM 20140724 : filterCount is meant to match + # Nsecfilt, but is potentially not determined correctly. + # use a call to a dvo-native command which knows how to + # find Nsecfilt (or save in the db with dvopsps) + + # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting? + # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter + # " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + + sql = "UPDATE ForcedMeanObject JOIN \ + " + cpyTable + " AS cpy ON (cpy.row = (ForcedMeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + SET \ + ForcedMeanObject." + filter[1] + "nIncR5 = NMEAS \ + ,ForcedMeanObject." + filter[1] + "nIncR6 = NMEAS \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR5 = FLUX_AP_R5 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR5Err = FLUX_ERR_AP_R5 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR5Std = FLUX_STD_AP_R5 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR5Fill = FLUX_FIL_AP_R5 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR6 = FLUX_AP_R6 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR6Err = FLUX_ERR_AP_R6 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR6Std = FLUX_STD_AP_R6 \ + ,ForcedMeanObject." + filter[1] + "FmeanflxR6Fill = FLUX_FIL_AP_R6 \ + ,ForcedMeanObject." + filter[1] + "LensObjSmearX11 = X11_SM_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjSmearX12 = X12_SM_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjSmearX22 = X22_SM_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjSmearE1 = E1_SM_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjSmearE2 = E2_SM_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjShearX11 = X11_SH_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjShearX12 = X12_SH_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjShearX22 = X22_SH_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjShearE1 = E1_SH_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensObjShearE2 = E2_SH_OBJ \ + ,ForcedMeanObject." + filter[1] + "LensPSFSmearX11 = X11_SM_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFSmearX12 = X12_SM_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFSmearX22 = X22_SM_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFSmearE1 = E1_SM_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFSmearE2 = E2_SM_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFShearX11 = X11_SH_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFShearX12 = X12_SH_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFShearX22 = X22_SH_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFShearE1 = E1_SH_PSF \ + ,ForcedMeanObject." + filter[1] + "LensPSFShearE2 = E2_SH_PSF \ + ,ForcedMeanObject." + filter[1] + "Gamma = GAMMA \ + ,ForcedMeanObject." + filter[1] + "E1 = E1 \ + ,ForcedMeanObject." + filter[1] + "E2 = E2 " + + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed update on ForcedMeanObject using cpy table", sql) + raise + + + ''' + Populates the ForcedMeanObject table + ''' + def populateForcedMeanObjectTable(self): + + cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") + cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps") + cpyTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpy") + + + self.logger.info("Populating ForcedMeanObject") + self.logger.info("Inserting objects from cpt file") + + # note "dec" is a reserved word in MySQL + # XXX EAM 20140724 : do not use INGORE unless we discover unavoidable problems... + # INSERT IGNORE INTO ObjectThin + + sqlLine = sqlUtility("INSERT INTO ForcedMeanObject (") + + sqlLine.group("objID", "EXT_ID") + sqlLine.group("ippObjID", "OBJ_ID + (CAT_ID << 32)") # NOTE: shift by 32 bits exactly + sqlLine.group("batchID", "'" + str(self.batchID) + "'") + sqlLine.group("nDetections", "'0'") + sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName) + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't populate Object table from cpt", sql) + return False + + # add row count columns so we can perform joins to get colors + self.logger.infoPair("Adding 'row' columns to", "Object and cps tables") + self.scratchDb.addRowCountColumn("ForcedMeanObject", "row") + self.scratchDb.addRowCountColumn(cpsTableName, "row") + + + self.logger.infoPair("update ForcedMeanObject from ","cps table") + + self.updateForcedMeanObjectFromCps(cpsTableName) + + self.logger.infoPair("update ForcedMeanObject from ","cpy table") + + self.updateForcedMeanObjectFromCpy(cpyTableName) + + + self.logger.infoPair("Dropping row column from", "ForcedMeanObject table") + self.scratchDb.dropColumn("ForcedMeanObject", "row") + + ##self.logger.infoPair("Purging from scratch Db", self.region + " region") + + ##Don't do this till after MeanObject + ##self.dvoForcedObjects.purgeRegion(self.region) + + self.setMinMaxObjID(["ForcedMeanObject"]) + + return True + + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): + + if not self.populateForcedMeanObjectTable(): return False + + # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number + self.logger.infoPair("Forcing uniqueness on", "objID in ForcedMeanObject table") + rowCountBefore = self.scratchDb.getRowCount("ForcedMeanObject") + + # XXX EAM : note that in mysql versions later than 5.1, this fails + # unless the following is called first: + # set session old_alter_table=1 + # follow the command with + # set session old_alter_table=0 + # OF COURSE, this fails for mysql version < 5.5... + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=1") + + self.scratchDb.execute("ALTER IGNORE TABLE ForcedMeanObject ADD UNIQUE INDEX(objID)") + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=0") + + rowCountAfter = self.scratchDb.getRowCount("ForcedMeanObject") + self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + + self.dvoForcedObjects.purgeRegion(self.region) + + #this is abuse of something but this is how I get the object batches to crash to further investigate them + +# rowCountAfter = self.scratchDb.getRowCount("Object") + return True +# return False Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/forcedwarpbatch.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/forcedwarpbatch.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/forcedwarpbatch.py (revision 37801) @@ -0,0 +1,359 @@ +#!/usr/bin/env jython + +import os.path +import sys +import glob +import time +import stilts + +from java.lang import * +from java.sql import * + +from java.lang import Math +from org.apache.commons.math.special import Erf + +from xml.etree.ElementTree import ElementTree, Element, tostring + +from batch import Batch +from gpc1db import Gpc1Db +from ipptopspsdb import IppToPspsDb +from scratchdb import ScratchDb +from sqlUtility import sqlUtility + +import logging.config + + +''' +ForcedWarpBatch class + +This class, a sub-class of Batch, processes single exposures in the form of IPP smf files + +''' +class ForcedWarpBatch(Batch): + + ''' + Constructor + ''' + def __init__(self, + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + forcedWarpID, + batchID): + + super(ForcedWarpBatch, self).__init__( + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + forcedWarpID, + batchID, + "P2", + gpc1Db.getForcedWarpStageSmf(forcedWarpID) + + # get camera meta data + meta = self.gpc1Db.getForcedWarpStageMeta(self.id) + if not meta: + self.logger.errorPair("Could not get", "forced warp metadata") + raise + + self.expID = meta[0]; + self.expName = meta[1]; + self.distGroup = meta[2]; + self.analysisVer = meta[3]; + self.bias = meta[4]; + self.biasScat = meta[5]; + + if not self.analysisVer: self.analysisVer = -999 + + # create an output filename, which is {expID}.FITS + self.outputFitsFile = "%08d.FITS" % self.expID + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + self.dropTableVerbose("ForcedWarpMeasurement") + self.dropTableVerbose("ForcedWarpExtended") + self.dropTableVerbose("ForcedWarpMeta") + self.dropTableVerbose("ForcedWarpToImage") + if not self.config.retry: + self.scratchDb.dropTable("ForcedWarpMeasurement") + self.scratchDb.dropTable("ForcedWarpExtended") + self.scratchDb.dropTable("ForcedWarpMeta") + self.scratchDb.dropTable("ForcedWarpToImage") + self.historyModNum = "0" + + # get a few primary header values. if in test mode, then use defaults + if self.safeDictionaryAccessWithDefault(self.header, 'MJD-OBS', "1") == "NULL": + self.logger.errorPair("Could not get", "MJD-OBS") + raise + + + if self.safeDictionaryAccessWithDefault(self.header, 'EXPTIME', "1") == "NULL": + self.logger.errorPair("Could not get", "EXPTIME") + raise + + if self.safeDictionaryAccessWithDefault(self.header, 'FILTERID', "g.0000") == "NULL": + self.logger.errorPair("Could not get", "FILTERID") + raise + + # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2 + self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0) + + # set up some defauts + self.calibModNum = 0 + self.totalNumPhotoRef = 0 + + + self.filter = self.header['FILTERID'][0:1] + self.filterID = self.scratchDb.getFilterID(self.filter) + + # insert what we know about this stack batch into the stack table + self.ippToPspsDb.insertDetectionMeta(self.batchID, self.expID, self.filter) + + # dump stuff to log + self.logger.infoPair("Cam ID", "%d" % self.id) + self.logger.infoPair("Exp ID", "%d" % self.expID) + self.logger.infoPair("Exp name", "%s" % self.expName) + self.logger.infoPair("Distribution group", "%s" % self.distGroup) + + + + + ''' + Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header + ''' + def populateForcedWarpMeta(self): + self.logger.infoPair("Processing table", "ForcedWarpMeta") + + photoCalID = str(self.scratchDb.getPhotoCalID(forcedWarpID)) + + + sqlLine = sqlUtility("INSERT INTO ForcedWarpMeta (") + #batchID (below) + #surveyID (below) + #filterdID (below) + sqlLine.group("skyCellID", str(self.skycellID)) + sqlLine.group("photoCalID", photoCalID) + sqlLine.group("magSat", self.getKeyFloat(header, "%.8f", 'FSATUR')) + sqlLine.group("analVer", self.analysisVer); + sqlLine.group("expTime", self.getKeyFloat(self.header, "%.8f", 'EXPREQ')); + sqlLine.group("completMag", self.getKeyFloat(header, "%.8f", 'FLIMIT')) + sqlLine.group("astroScat", self.getKeyFloat(header, "%.8f", 'CERROR')) + sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR')); + sqlLine.group("nAstroRef", self.getKeyValue(header, 'NASTRO')) + sqlLine.group("nPhotoRef", self.getKeyValue(header, 'NASTRO')) + sqlLine.group("psfModelID", self.getKeyValue(header, 'PSFMODEL')) + sqlLine.group("psfFwhm_mean", self.getKeyFloat(header, "%.8f", 'FWHM_MAJ')) + sqlLine.group("psfFwhm_max", self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ')) + sqlLine.group("photoZero", self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS')) + #photoColor -- how do I set this? it's also not set in stack meta + sqlLine.group("ctype1", self.getKeyValue(self.header, 'CTYPE1')); + sqlLine.group("ctype2", self.getKeyValue(self.header, 'CTYPE2')); + sqlLine.group("crval1", self.getKeyFloat(self.header, "%.8f", 'CRVAL1')); + sqlLine.group("crval2", self.getKeyFloat(self.header, "%.8f", 'CRVAL2')); + sqlLine.group("crpix1", self.getKeyFloat(self.header, "%.8f", 'CRPIX1')); + sqlLine.group("crpix2", self.getKeyFloat(self.header, "%.8f", 'CRPIX2')); + sqlLine.group("cdelt1", self.getKeyFloat(self.header, "%.8e", 'CDELT1')); + sqlLine.group("cdelt2", self.getKeyFloat(self.header, "%.8e", 'CDELT2')); + sqlLine.group("pc001001", self.getKeyFloat(self.header, "%.8e", 'PC001001')); + sqlLine.group("pc001002", self.getKeyFloat(self.header, "%.8e", 'PC001002')); + sqlLine.group("pc002001", self.getKeyFloat(self.header, "%.8e", 'PC002001')); + sqlLine.group("pc002002", self.getKeyFloat(self.header, "%.8e", 'PC002002')); + + + sql = sqlLine.make(") VALUES ( ", ")") + + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql: ', sql) + raise + + self.scratchDb.updateAllRows("ForcedWarpMeta", "batchID", str(self.batchID)) + self.scratchDb.updateAllRows("ForcedWarpMeta", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("ForcedWarpMeta", self.filter) + self.scratchDb.updateAllRows("ForcedWarpMeta", "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows("ForcedWarpMeta", "dataRelease", str(self.skychunk.dataRelease)) + self.tablesToExport.append("ForcedWarpMeta") + + ''' + Populates the ForcedWarpToImage table + ''' + def populateForcedWarpToImage(self): + + self.logger.infoPair("Procesing table", "ForcedWarpToImage") + + for filter in self.filters: + + forcedwarpID = self.forcedwarpID + if forcedwarpID > 0: + + imageIDs = self.gpc1Db.getImageIDsForThisForcedWarpID(forcedwarpID) + + for imageID in imageIDs: + + sql = "INSERT INTO ForcedWarpToImage (forcedwarpID, imageID) \ + VALUES (\ + " + str(forcedwarpID) + ", " + imageID + ")" + self.scratchDb.execute(sql) + + # now update StackMeta with correct number of inputs + + self.tablesToExport.append("ForcedWarpToImage") + + + + + + ''' + Populates the ForcedWarpMeasurement table + ''' + def populateForcedWarpMeasurementTable(self, results): + + pspsTableName = "ForcedWarpMeasurement" + ippTableName = "ForcedWarpMeasurement_psf" + + + self.logger.infoPair("Procesing table", "ForcedWarpMeasurement") + + # insert the per-object data (IDs, random stack ID, dataRelease, etc) + self.selectDvoObjIDs() + + self.generateRandomIDs() + + self.generateUniquePspsIDs() + + # add indexes ForcedWarpMeasurement + self.scratchDb.createIndex("ForcedWarpMeasurement", "objID") + + + + + + + + + + self.tablesToExport.append("ForcedWarpMeasurement") + + + ''' + Populates the ForcedWarpExtended table + ''' + def populateForcedWarpExtendedTable(self, results): + + pspsTableName = "ForcedWarpExtended" + ippTableName = "ForcedWarpExtended_psf" + + + self.tablesToExport.append("ForcedWarpExtended") + + # get dvo ids, + # add in the psps unique ids from stuff + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): + self.logger.infoPair("starting","populatePspsTables"); + self.skipBatch = False + # each of the "populate*" methods below add their tables to the list: + self.tablesToExport=[] + + #this should be done + self.populateForcedWarpMeta() + + self.populateForcedWarpMeasurement() + self.populateForcedWarpExtended() + self.populateForcedWarpToImage() + + self.setMinMaxObjID(["ForcedWarpMeasurement"]) + + self.logger.infoPair("finishing","populatePspsTables"); + return True + + + + ''' + This function reads the cmf/smf file and loads it into the database as its own table + ''' + def importIppTables(self, columns="*", tableRE=""): + + if self.config.retry: return True + self.logger.infoPair("Importing FW tables with table match expression: ", tableRE) + fileName = self.fits.getPath() + + self.logger.infoPair("using filename:",fileName) + + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False + for table in tables: + + match = re.match(tableRE, table.name) + if not match: continue + self.logger.infoPair("Reading IPP table", table.name) + table = stilts.tpipe(table, cmd='addcol table_index $0') + table = stilts.tpipe(table, cmd='explodeall') + + # drop any previous tables before import + self.scratchDb.dropTable(table.name) + + # IPP FITS files are littered with infinities, so remove these + self.logger.info("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + + try: + table.write(self.scratchDb.url + '#' + table.name) + count = count + 1 + except: + self.logger.exception("Problem writing table '" + table.name + "' to the database") + + self.logger.infoPair("Done. Imported", "%d tables" % count) + self.indexIppTables() + + return True + + + + + + + + + + def generateRandomIDs(self): + sql = "UPDATE ForcedWarpMeasurement set randomWarpID = FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + return + + + ''' + Updates table and generates pspsuniqueids + ''' + + def generatePspsUniqueIDs(self): + sql = "UPDATE ForcedWarpMeasurement join (select @r:=@r+1 rownum, objID from \ + (select @r:=0) r, ForcedWarpMeasurement t) as foo using (objID) set \ + uniquePspsFWid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + return + + + + + + + + def exportPspsTablesToFits(self, regex="(.*)"): + return super(DetectionBatch, self).exportPspsTablesToFits("(ForcedWarp.*)") Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/gpc1db.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/gpc1db.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/gpc1db.py (revision 37801) @@ -108,13 +108,64 @@ AND mjd_obs <= (to_days('" + tend + "') - 678941) " # self.logger.infoPair("sql",sql) - elif batchType == "DF": + elif batchType == "DF": # this is specifically only for WARPSTACK types of diffs - + # it only makes sense for the date to come from the warp1 + # may need to be more clever if we do other types of diffs stage = "diff" - sql = "SELECT stuff from stufftable" + sql = "SELECT DISTINCT stage_extra1, radeg,decdeg \ + FROM addRun \ + JOIN addProcessedExp using (add_id) \ + JOIN minidvodbRun using (minidvodb_name) \ + JOIN minidvodbProcessed using (minidvodb_id) \ + JOIN mergedvodbRun using (minidvodb_id) \ + JOIN mergedvodbProcessed using (merge_id) \ + JOIN diffInputSkyfile \ + ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 \ + AND stage = 'diff') \ + JOIN skycell using (skycell_id, tess_id) \ + JOIN warpRun on (warp1 = warp_id \ + AND diffInputSkyfile.skycell_id = skycell_id) \ + JOIN fakeRun using (fake_id) \ + JOIN camRun using (cam_id) \ + JOIN chipRun using (chip_id) \ + JOIN rawExp using (exp_id) \ + WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \ + AND minidvodbRun.state = 'merged' \ + AND minidvodbProcessed.fault = 0 \ + AND mergedvodbRun.state = 'full' \ + AND mergedvodbProcessed.fault = 0 \ + AND addRun.stage = '" + stage + "' \ + AND addRun.state = 'full' \ + AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ + AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ + AND dateobs >= '" + tstart + "'\ + AND dateobs <= '" + tend + "'" + elif batchType == "FW": stage = "forcedwarp" - sql = "SELECT differentStuff from Stufftable" + sql = "SELECT DISTINCT sky_id, radeg, decdeg FROM fullForceInput \ + JOIN addRun ON(fullForceInput.ff_id = addRun.stage_id, fullForceInput.warp_id = addRun.stage_extra1) \ + JOIN minidvodbRun USING(minidvodb_name) \ + JOIN minidvodbProcessed USING(minidvodb_id) \ + JOIN mergedvodbRun USING(minidvodb_id) \ + JOIN mergedvodbProcessed USING (merge_id) \ + JOIN fullForceRun USING(ff_id) \ + JOIN skycalRun using (skycal_id) \ + JOIN stackRun USING(stack_id) \ + JOIN stackSumSkyfile using (stack_id) \ + JOIN skycell USING(skycell_id) \ + WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \ + AND minidvodbRun.state = 'merged' \ + AND minidvodbProcessed.fault = 0 \ + AND mergedvodbRun.state = 'full' \ + AND mergedvodbProcessed.fault = 0 \ + AND addRun.stage = '" + stage + "' \ + AND addRun.state = 'full' \ + AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ + AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ + AND mjd_obs >= (to_days('" + tstart + "')-678941) \ + AND mjd_obs <= (to_days('" + tend + "') - 678941) " try: @@ -166,4 +217,9 @@ return imageIDs + + + + + ''' Gets some stack-stage meta data for this sky_id # TODO this SQL could surely be improved @@ -196,4 +252,131 @@ return meta + + ''' + Gets some stack-stage meta data for this sky_id # TODO this SQL could surely be improved + ''' + def getDiffStageMeta(self, diffSkyfileID): + + self.logger.debug("Querying GPC1 for stack meta data") + + meta = [] + sql = "SELECT \ + filter,\ + diff_mode, \ + warp1, \ + stack2, \ + skycell_id, \ + diffRun.software_ver, \ + exp_time, \ + (to_days(dateobs)-678941 + time_to_sec(dateobs)/86400.) \ + FROM diffRun join diffInputSkyfile using (diff_id) \ + JOIN warpRun on (warp1=warp_id) \ + JOIN fakeRun using (fake_id) \ + JOIN camRun using (cam_id) \ + JOIN chipRun using (chip_id) \ + JOIN rawExp using (exp_id) \ + WHERE diff_skyfile_id = %d" % diffSkyfileID + #this is a crappy way to get the mjd from the diff but we don't save it anywhere + try: + rs = self.executeQuery(sql) + rs.first() + except: + self.logger.infoPair ("this sql is broken:",sql) + self.logger.errorPair("Can't query for", "stack meta data") + + try: + meta.append(rs.getString(1)) + meta.append(rs.getString(2)) + meta.append(rs.getString(3)) + meta.append(rs.getString(4)) + meta.append(rs.getString(5)) + meta.append(rs.getString(6)) + meta.append(rs.getString(7)) + meta.append(rs.getString(8)) + except: + self.logger.errorPair("getDiffStageMeta()", "empty meta data") + + return meta + + ''' + get diff stage cmf name + ''' + def getDiffStageCmf(self, dvoDb, diffSkyfileID): + print "got here, diffstagecmf" + sql = "SELECT DISTINCT diffSkyfile.path_base \ + FROM mergedvodbRun join minidvodbRun using (minidvodb_id) \ + JOIN addRun using (minidvodb_name) \ + JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 \ + AND diff_id = stage_id AND stage = 'diff') \ + JOIN diffSkyfile using (diff_id, skycell_id) \ + WHERE diff_skyfile_id = " + str(diffSkyfileID) + " \ + AND minidvodbRun.state = 'merged' \ + AND mergedvodbRun.state = 'full' \ + AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \ + AND stage = 'diff'" + + try: + rs = self.executeQuery(sql) + rs.first() + except: + self.logger.infoPair("failed sql", sql) + self.logger.errorPair("can't query for diff cmf", "dfif_skyfile_id = %d" %diff_skyfile_id) + return None + print "query worked" + try: + pathBase = rs.getString(1) + except: + self.logger.errorPair("no diff cms found for diff_skyfile_id = %d" %diff_skyfile_id) + return None + print pathBase + files = [] + if pathBase.startswith("neb"): + print "open" + f=os.popen("neb-ls -p " + pathBase + ".cmf") + for i in f.readlines(): + print "i",i + files.append(i.rstrip()) + print "append" + print i + if len(files) < 1: return None + fits = Fits(self.logger, self.config, files[0]) + return fits + + ''' + Gets a list of PSPS image IDs for this diff ID + ''' + def getImageIDsForThisDiffID(self, diffID): + + self.logger.debug("Querying GPC1 for image IDs for diff ID: " + str(diffID)) + + # JOIN stackInputSkyfile shoule use stack_id as well as warp_id, right? (maybe not -- the join to warp is first) + sql = "SELECT DISTINCT CONCAT(exp_id, SUBSTR(class_id, 3, 4)) FROM ( \ + SELECT DISTINCT exp_id,class_id \ + FROM warpSkyCellMap \ + JOIN warpRun USING(warp_id) \ + JOIN diffInputSkyfile on warp1 = warp_id \ + JOIN fakeRun using (fake_id) \ + JOIN camRun using (cam_id) \ + JOIN chipRun using (chip_id) \ + WHERE diffInputSkyfile.diff_skyfile_id = " + str(diffID) + ") AS a" + + try: + rs = self.executeQuery(sql) + except: + self.logger.infoPair("failed sql:",sql) + self.logger.exception("Can't query for imageIDs") + + + imageIDs = [] + while (rs.next()): + imageIDs.append(rs.getString(1)) + rs.close() + + return imageIDs + + + + + ''' Gets some camera-stage meta data for this cam_id @@ -368,19 +551,4 @@ def getStackStageCmf(self, dvoDb, stackID): - ''' - sql = "SELECT DISTINCT staticskyResult.path_base \ - FROM staticskyRun \ - JOIN staticskyInput USING(sky_id) \ - JOIN staticskyResult USING(sky_id) \ - JOIN stackRun USING(stack_id) \ - JOIN stackSumSkyfile USING(stack_id) \ - JOIN addRun ON sky_id = stage_id \ - JOIN minidvodbRun USING (minidvodb_name) \ - WHERE stack_id = " + str(stackID) + " \ - AND minidvodbRun.minidvodb_group = '" + dvoDb + "' \ - AND minidvodbRun.state = 'merged' \ - AND stage = 'staticsky'" - ''' - sql = "SELECT DISTINCT skycalResult.path_base \ FROM skycalRun \ @@ -432,41 +600,4 @@ fits = Fits(self.logger, self.config, files[0]) return fits - - # XXX validate the file? - # if fits.getPrimaryHeaderValue("STK_ID") == str(stackID): - - # now find and loop through all cmf files at this path_base - - ## XXX the code below was needed when we ingested from staticsky (N files per run) - ## XXX ingesting from skycal solves this problem (1 file per run) - ## files=os.popen("neb-ls " + pathBase + "%cmf") - ## for i in files.readlines(): - ## nebPath = i.rstrip() - ## - ## # now attempt to run 'neb-ls -p' to actually read this cmf file - ## try: - ## self.logger.debug("Trying to neb-ls -p on '" + nebPath + "'") - ## paths=os.popen("neb-ls -p " + nebPath) - ## path = paths.readline().rstrip() - ## if len(path) < 1: - ## self.logger.debug("zero length path - skipping") - ## raise - ## - ## print "starint stack stage cmf 3" - ## - ## # if we get here, then the cmf is readable, now check the stack_id - ## fits = Fits(self.logger, self.config, path) - ## if fits.getPrimaryHeaderValue("STK_ID") == str(stackID): - ## # we have the right file! - ## self.logger.debug("SUCCESS!: %s == %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID))) - ## return fits - ## else: - ## # this is not the correct file - ## self.logger.debug("STACK ID: %s != %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID))) - ## - ## # an exception here means that nebulous could not read this file, so we just skip it - ## except: - ## self.logger.debug("NEB FAILED SKIPPING") - ## pass ''' Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/initbatch.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/initbatch.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/initbatch.py (revision 37801) @@ -16,4 +16,7 @@ from stackbatch import StackBatch from detectionbatch import DetectionBatch +#from diffbatch import DiffBatch +#from forcedwarpbatch import ForcedWarpBatch +#why no object stuffs ''' Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/ippjython =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/ippjython (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/ippjython (revision 37801) @@ -27,4 +27,5 @@ if (! -r $JYTHON) then echo "jython jar file not found, check jython installation" + echo $JYTHON exit 2 endif Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/ipptopspsdb.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/ipptopspsdb.py (revision 37801) @@ -756,4 +756,81 @@ self.execute(sql) + + ''' + Inserts some forced detection metadata for this batch ID + ''' + def insertForcedWarpMeta(self, batchID, expID, filter): + + sql = "INSERT INTO forcedwarp ( \ + batch_id \ + ,exp_id \ + ,filter \ + ) VALUES ( \ + " + str(batchID) + " \ + ," + str(expID) + " \ + ,'" + filter + "' \ + )" + + self.execute(sql) + + + + ''' + Inserts some forcedobject metadata for this batch ID + ''' + def insertForcedObjectMeta(self, batchID, region): + + sql = "INSERT INTO forcedobject ( \ + batch_id \ + ,region \ + ) VALUES ( \ + " + str(batchID) + " \ + ,'" + region + "' \ + )" + + self.execute(sql) + + + + + ''' + Inserts some diff metadata for this batch ID + ''' + def insertDiffMeta(self, batchID, filter, diffType): + + sql = "INSERT INTO diff ( \ + batch_id \ + ,filter \ + ,diff_type \ + ) VALUES ( \ + " + str(batchID) + " \ + ,'" + filter + "' \ + ,'" + diffType + "')" + + self.execute(sql) + + + + + ''' + Inserts some diffobject metadata for this batch ID + ''' + def insertDiffObjectMeta(self, batchID, region): + + sql = "INSERT INTO diffobject ( \ + batch_id \ + ,region \ + ) VALUES ( \ + " + str(batchID) + " \ + ,'" + region + "' \ + )" + + self.execute(sql) + + + + + + ''' Resets a batch ready for re-loading to the datastore Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/loader.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/loader.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/loader.py (revision 37801) @@ -24,4 +24,11 @@ from objectbatch import ObjectBatch +from diffbatch import DiffBatch +from diffobjectbatch import DiffObjectBatch + +#from forcedwarpbatch import ForcedWarpBatch +#from forcedobjectbatch import ForcedObjectBatch +# +# ''' Loader class @@ -191,8 +198,8 @@ # loop round IDs of all items to be processed self.ippToPspsDb.lockBatchTable() - #self.logger.infoPair("heather:","lockbatchtable") + self.logger.infoPair("heather:","lockbatchtable") unattemptedCount = 0 for id in ids: - #self.logger.infoPair("heather:","in ids") + self.logger.infoPair("heather:","in ids") #self.logger.infoPair("heather:id",str(id)) batchID = self.ippToPspsDb.createNewBatch(batchType, id) @@ -202,5 +209,5 @@ unattemptedCount += 1 continue - #self.logger.infoPair("heather:","passed logic") + self.logger.infoPair("heather:","passed logic") self.ippToPspsDb.unlockTables() @@ -244,4 +251,5 @@ elif batchType == "DF": + self.logger.infoPair("wegothere!","we are awesome") batch = DiffBatch(self.logger, self.config, Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/objectbatch.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/objectbatch.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/objectbatch.py (revision 37801) @@ -87,4 +87,19 @@ return True + ''' + Updates table and generates pspsuniqueids + ''' + + def updatePspsUniqueIDs(self): + sql = "UPDATE ObjectThin join (select @r:=@r+1 rownum, objID from \ + (select @r:=0) r, ObjectThin t) as foo using (objID) set \ + uniquePspsOBid = ((" +str(self.batchID) +"*1000000000 ) + rownum)"; + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + raise + + + ''' Applies indexes to the IPP tables @@ -336,5 +351,5 @@ sqlLine.group("ippObjID", "OBJ_ID + (CAT_ID << 32)") # NOTE: shift by 32 bits exactly sqlLine.group("surveyID", "'" + str(self.surveyID) + "'") - sqlLine.group("randomID", "FLOOR(RAND()*9223372036854775807)") # XXX where does this number come from?? + sqlLine.group("randomID", "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)") # XXX where does this number come from?? sqlLine.group("batchID", "'" + str(self.batchID) + "'") sqlLine.group("dvoRegionID", "CAT_ID") @@ -410,8 +425,8 @@ sql = "DELETE FROM ObjectThin where \ - ObjectThin.decMean > " + str(decMax) + " or \ - ObjectThin.decMean < " + str(decMin) + " or \ - ObjectThin.raMean > " + str(raMax) + " or \ - ObjectThin.raMean < " + str(raMin) + ObjectThin.decMean > (" + str(decMax) + " + .0033) or \ + ObjectThin.decMean < (" + str(decMin) + " - .0033) or \ + ObjectThin.raMean > (" + str(raMax) + " + .0033) or \ + ObjectThin.raMean < (" + str(raMin) + " - .0033)" self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range") @@ -429,4 +444,6 @@ ##self.dvoObjects.purgeRegion(self.region) + self.updatePspsUniqueIDs() + self.setMinMaxObjID(["ObjectThin"]) @@ -439,7 +456,7 @@ cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps") - self.logger.infoPair("Populating MeanObject from ", "ObjectThin") - - sql = "INSERT INTO MeanObject ( objID ) SELECT objID FROM ObjectThin" + self.logger.infoPair("Populating MeanObject.objid and uniquePspsOBid from ", "ObjectThin") + + sql = "INSERT INTO MeanObject ( objID, uniquePspsOBid ) SELECT objID, uniquePspsOBid FROM ObjectThin" try: self.scratchDb.execute(sql) @@ -450,5 +467,8 @@ self.logger.infoPair("Adding 'row' columns to", "MeanObject table") self.scratchDb.addRowCountColumn("MeanObject", "row") - + + + + ##self.scratchDb.addRowCountColumn(cpsTableName, "row") Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/scratchdb.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/scratchdb.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/scratchdb.py (revision 37801) @@ -111,5 +111,5 @@ except: self.logger.errorPair("No survey ID found for survey", name) - surveyID = -1 + surveyID = 254 return surveyID Index: /branches/eam_branches/ipp-20140904/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/jython/stackbatch.py (revision 37800) +++ /branches/eam_branches/ipp-20140904/ippToPsps/jython/stackbatch.py (revision 37801) @@ -73,5 +73,5 @@ # we just need one valid filter to get the generic metadata validFilter = 'none' - + validFilterCount = 0 # get a list of stackIDs, identify the CMFs, read the headers: for filter in self.filters: @@ -81,5 +81,6 @@ stackID = gpc1Db.getStackIDFromSkyIDAndFilter(skychunk.dvoLabel, skyID, filterName) self.stackIDs[filter] = stackID - + if (stackID > 0): + validFilterCount=validFilterCount + 1 if (stackID <= 0): continue @@ -115,4 +116,10 @@ self.logger.errorPair("can't find any valid stacks...?? ","??") raise + # did we find any valid stacks? + if validFilterCount == 0: + self.logger.errorPair("can't find any valid stacks...?? ","??") + raise + else: + self.logger.infoPair("number of filters found:",validFilterCount) validStackID = self.stackIDs[validFilter] @@ -154,5 +161,5 @@ self.dropTableVerbose("StackObjectThin") - self.dropTableVerbose("StackObjectRaw") + self.dropTableVerbose("StackObjectAttributes") self.dropTableVerbose("StackModelFitExtra") @@ -162,7 +169,7 @@ self.dropTableVerbose("StackApFlx") - self.dropTableVerbose("StackApFlxFull") - self.dropTableVerbose("StackApFlxFullC1") - self.dropTableVerbose("StackApFlxFullC2") + self.dropTableVerbose("StackApFlxEGUnc") + self.dropTableVerbose("StackApFlxEGCon6") + self.dropTableVerbose("StackApFlxEGCon8") self.dropTableVerbose("StackPetrosian") @@ -284,4 +291,8 @@ self.selectDvoObjIDs() + self.generateRandomIDs() + + self.generateStackDetectRowIDsAndPspsUniqueIDs() + # add indexes StackObjectThin self.scratchDb.createIndex("StackObjectThin", "objID") @@ -325,5 +336,5 @@ sqlLine.group("a."+filter+"ApMag", "b.Map") sqlLine.group("a."+filter+"ApMagErr", "b.dMap") - sqlLine.group("a."+filter+"telluricExt", "b.telluricExt") + # sqlLine.group("a."+filter+"telluricExt", "b.telluricExt") sqlLine.group("a."+filter+"infoFlag2", "(b.flags << 13)") @@ -365,13 +376,13 @@ ''' - Populates the StackObjectRaw table - ''' - def populateStackObjectRaw(self): - - tablename = "StackObjectRaw" + Populates the StackObjectAttributes table + ''' + def populateStackObjectAttributes(self): + + tablename = "StackObjectAttributes" self.logger.infoPair("Procesing table", tablename) self.insertDvoIDs(tablename, "StackObjectThin") - + # if we are going to add a key, do it here so it is useful self.scratchDb.createIndex(tablename, "objID") @@ -420,5 +431,5 @@ sqlLine.group("a."+filter+"KronFluxErr", "b.KRON_FLUX_ERR / " + str(exptime)) sqlLine.group("a."+filter+"ApFillFac", "b.AP_NPIX / (3.14159265359 * POW(b.AP_MAG_RADIUS - 0.5, 2))") - sqlLine.group("a."+filter+"ApRadius", "b.AP_MAG_RADIUS") +# sqlLine.group("a."+filter+"ApRadius", "b.AP_MAG_RADIUS") sqlLine.group("a."+filter+"KronRad", "b.MOMENTS_R1 * 2.5") sqlLine.group("a."+filter+"PlateScale", "b.PLTSCALE") @@ -430,5 +441,5 @@ except: self.logger.errorPair('failed sql',sql) - return + raise # insert detection information for each filter based on the DVO contents @@ -446,5 +457,5 @@ except: self.logger.errorPair('failed sql',sql) - return + raise @@ -549,15 +560,13 @@ # insert detection information for each filter based on the # DVO contents. Since I already have extracted - # StackObjectRaw, this is probably faster using that table: + # StackObjectAttributes, this is probably faster using that table: sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + self.scratchDb.dvoDetectionTable + " as b SET") sqlLine.group("a." + filter + model + "Mag", "a." + filter + model + "Mag + b.zp") - sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID") - + sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID" + " AND b.imageID = " + str(self.imageIDs[filter])) try: self.scratchDb.execute(sql) except: self.logger.errorPair('failed sql',sql) - return - + raise # we have (modulo signs): # gRaExp = gra + (X_EXT - gxPos)*gPlateScale / cos(gdec) @@ -583,5 +592,5 @@ # insert detection information for each filter based on the DVO contents - sqlLine = sqlUtility("UPDATE " + tablename + " AS a, StackObjectThin as b, StackObjectRaw as c SET ") + sqlLine = sqlUtility("UPDATE " + tablename + " AS a, StackObjectThin as b, StackObjectAttributes as c SET ") sqlLine.group(extR, rawR + " - (" + extR + " - " + rawX + ")*" + rawS + " / 3600.0 / cos(radians(" + rawD + "))") sqlLine.group(extD, rawD + " + (" + extD + " - " + rawY + ")*" + rawS + " / 3600.0") @@ -596,5 +605,5 @@ except: self.logger.errorPair('failed sql',sql) - return + raise ''' @@ -655,5 +664,5 @@ sqlLine.group("a." + filter + "petCf", "b.PETRO_FILL") - sql = sqlLine.makeEquals("WHERE a." + filter + "ippDetectID = b.IPP_IDET") + sql = sqlLine.makeEquals("WHERE a." + filter + "ippDetectID = b.IPP_IDET ") try: self.scratchDb.execute(sql) @@ -670,5 +679,5 @@ except: self.logger.errorPair('failed sql',sql) - return + raise ''' @@ -690,7 +699,7 @@ for filter in self.filters: - self.populateStackApFlxFilter(tablename, "", filter) - self.populateStackApFlxFilter(tablename, "C1", filter) - self.populateStackApFlxFilter(tablename, "C2", filter) + self.populateStackApFlxFilter(tablename, "Unc", filter) + self.populateStackApFlxFilter(tablename, "Con6", filter) + self.populateStackApFlxFilter(tablename, "Con8", filter) self.deleteRowsWithNoStackData(tablename) @@ -698,9 +707,9 @@ ''' - Populates the StackApFlxFull table - ''' - def populateStackApFlxFull(self, version): - - tablename = "StackApFlxFull" + version + Populates the StackApFlxEG table + ''' + def populateStackApFlxEG(self, version): + + tablename = "StackApFlxEG" + version if self.config.camera == "simtest": @@ -721,5 +730,5 @@ ''' - Updates aperture fluxes for StackApFkx table + Updates aperture fluxes for StackApFlx table ''' def populateStackApFlxFilter(self, tablename, version, filter): @@ -728,13 +737,13 @@ # order of the 3 convolutions is fixed (raw, C1, C2) = (1, 2, 3) - if version == "": + if version == "Unc": psfCondition = "(b.table_index % 3 = 1)" prefix = "" - if version == "C1": + if version == "Con6": psfCondition = "(b.table_index % 3 = 2)" - prefix = "c1" - if version == "C2": + prefix = "c6" + if version == "Con8": psfCondition = "(b.table_index % 3 = 0)" - prefix = "c2" + prefix = "c8" if prefix == "none": @@ -769,5 +778,5 @@ except: self.logger.errorPair('failed sql',sql) - return + raise # set the flux values from the cmf file @@ -833,5 +842,5 @@ sqlLine.group(field + "Err", field + "err" + " * POW(10, 3.56 - 0.4* b.zp)") sqlLine.group(field + "Std", field + "Std" + " * POW(10, 3.56 - 0.4* b.zp)") - sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID") + sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID AND b.imageID = " + str(self.imageIDs[filter]) ) self.scratchDb.execute(sql) @@ -931,4 +940,23 @@ return True + + def generateRandomIDs(self): + sql = "UPDATE StackObjectThin set randomStackObjID = FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + raise + + + def generateStackDetectRowIDsAndPspsUniqueIDs(self): + sql = "UPDATE StackObjectThin join (select @r:=@r+1 rownum, objID from \ + (select @r:=0) r, StackObjectThin t) as foo using (objID) set \ + stackDetectRowID = ((" + str(self.batchID) + " << 32 ) + rownum), \ + uniquePspsSTid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"; + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + raise + ''' @@ -951,5 +979,5 @@ sqlLine.group("skyCellID", "'" + str(self.skycellID) + "'") sqlLine.group("dataRelease", "'" + str(self.skychunk.dataRelease) + "'") - sqlLine.group("randomStackObjID", "FLOOR(RAND()*9223372036854775807)") + # sqlLine.group("randomStackObjID", "FLOOR(RAND()*9223372036854775807)") whereClause = " WHERE (" @@ -971,5 +999,5 @@ except: self.logger.errorPair('failed sql',sql) - return + raise ''' @@ -982,5 +1010,5 @@ self.scratchDb.addColumn(table, filter + "haveData", "tinyint") - fields = "objID, ippObjID, randomStackObjID, primaryDetection, bestDetection, " + fields = "objID, ippObjID, randomStackObjID, primaryDetection, bestDetection, uniquePspsSTID, " for i in range(len(self.filters)): @@ -988,8 +1016,10 @@ fields += filter + "ippDetectID, " fields += filter + "stackDetectID, " - fields += filter + "stackMetaID" # careful here: do not add comma to last entry + fields += filter + "stackMetaID " # careful here: do not add comma to last entry if (i < len(self.filters) - 1): fields += ", " - + + if (table == "StackObjectAttributes"): + fields += ", stackDetectRowID " sql = "INSERT INTO " + table + " (" + fields + ") " sql += "SELECT " + fields + " FROM " + mainTable @@ -998,6 +1028,8 @@ except: self.logger.errorPair('failed sql',sql) - return - + raise + + + ''' Does the processing, i.e. pulling stuff from IPP tables into PSPS tables @@ -1036,5 +1068,5 @@ self.populateStackObjectThin() - self.populateStackObjectRaw() + self.populateStackObjectAttributes() if self.stackType != "NIGHTLY_STACK": @@ -1053,7 +1085,7 @@ self.populateStackApFlx() - self.populateStackApFlxFull("") - self.populateStackApFlxFull("C1") - self.populateStackApFlxFull("C2") + self.populateStackApFlxEG("Unc") + self.populateStackApFlxEG("Con6") + self.populateStackApFlxEG("Con8") # delete rows with no content. for the other tables, this is done in @@ -1130,4 +1162,4 @@ self.logger.infoPair("Done. Imported", "%d tables" % count) self.indexIppTables() - +# return False return True Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema10.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema10.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema10.txt (revision 37801) @@ -0,0 +1,132 @@ +Notes on Schema Version 9 + + +Proposed Image Products and PSPS Tables and Views + +IMAGE PRODUCTS + +Survey | Description | Image Products Saved + +3pi | 3pi | 3pi warps, 3pi stacks +CNP | Celestial North Pole (3pi)| CNP warps, stacks +SS | ecliptic w band data | w warps, w stack +MDD | Medium Deep Deep | MD warps, deep stacks +MDT | Medium Deep Template | template stacks +MDN | Medium Deep Nightly Stack | nightly stacks +MDN | Medium Deep Off Year | off year template stacks +M31 | M31 | M31 warps, M31 stacks, template +STS | STS | STS warps, STS template - best iq stack + + +Survey | Description | Image Products NOT Saved, but analyzed for difference detections and discarded + +3pi | difference |3pi warps - stack +CNP | difference |CNP warps - stack +SS | difference image | w warps - stack +MDD | difference image | MD warps - off year template stacks +MDO | difference image | MDN stacks - off year template stacks +M31 | difference image | M31 warps - stack +STS | difference image | STS warps - template + + + +PSPS TABLES + + +Fundamental IPP Data Products Origin + +Detection detections found in detrended images +ObjectThin positonal and flag data, no fluxes +MeanObject extracted Mean properties as determined from individual Detections. +StackObject Stack Detections in all filters, with forced upperlimits, separate rows for primary, secondary, best stack detections (80% time only primary) +StackApFlx all filters in one table, unconvolved and convolved apertures (up to 6 , 8, 10(?) pixels in all bands) +StackModelFit all filters in one table. Models of high signal to noise objects in low density regions. +DiffDet difference detections found in difference image = image - template +DiffObject associations of Difference Detections +ForcedWarpMeasure forced measurements on each warp in all filters at the position of an object found in a stack. Includes StackApFlx-like table +ForcedMeanObject like MeanObject, but using the Forced Mean Measurements to form the Mean. + +w band TBD + + +Derived Data Product Tables + +pcsPhotoZ populated from PCS +pcsPhotoZProbability populated from PCS +pcsStarGalQSOSep populated from PCS +pcsStellarParams populated from PCS +mopsMovingObject populated from MOPS database +ppmObj proper motion and parallax populated from IPP +tcsClassification populated from transient server database +galExtinction populated from Dust team +stargal populated from various algorithms, combos of ipp parameters, fits to population as function of magnitude, or Support Vector Machine +GoodObject selected and populated post-facto +StarGalaxySep selected and populated post-facto + + +Observational Metadata and System MetaData Tables + +CameraConfig +DetectionFlags +DiffMeta +Filter +FitModel +ForcedWarpMeta +FrameMeta +ImageFlags +ImageMeta add psf 3nd and 4th moments +Mask +MovingObjectToDet +MovingObjectToDiffDet +MovingObjToObject +ObjectFilterFlags flag info not in current schema but in PSI interface... +ObjectInfoFlags +ObjectQualityFlags +PhotoCal +ProjectionCell +ProcessingVersion +SkyCell +StackDetectionFlags +StackMeta +StackToImage +StackType +Survey +TelescopeConfig +fitModel +pcsGalaxyModel +pcsPhotoZRecipe +pcsSGSepRecipe +pcsStarModel +pcsStellarParamsRecipe + + + +PSPS VIEWS - examples + +ObjectMean = ObjectThin join MeanObject +ObjectStack = ObjectThin join StackObject +ObjectApFlx = ObjectThin join StackApFlx +ObjectModel = ObjectThin join StackModelFit +ObjectStackAll = ObjectThin join StackObject and StackAp and StackModelFit +ObjectForcedMean = ObjectThin join ForcedMeanObject +ObjectPrimary = ObjectThin join StackObject join StackApFlx where primaryDection is true +ObjectBest = ObjectThin join StackObject join StackApFlx where bestDetection is true + + +PSPS Indicies + +DetectionID +ObjectID +BatchID +StackTypeID +MovingObjID +DiffObjID +RandomID between 1 and 2^16 = 4,294,967,296 so you pick how many random objects you want , + i.e 10,000 pick randomid < 10,000 and if indexed, this will be very fast. +RandomStackID between 1 and 2^16 = 4,294,967,296 so you pick how many random objects you want , + i.e 10,000 pick randomid < 10,000 and if indexed, this will be very fast. + + + + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema11.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema11.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema11.txt (revision 37801) @@ -0,0 +1,132 @@ +Notes on Schema Version 9 + + +Proposed Image Products and PSPS Tables and Views + +IMAGE PRODUCTS + +Survey | Description | Image Products Saved + +3pi | 3pi | 3pi warps, 3pi stacks +CNP | Celestial North Pole (3pi)| CNP warps, stacks +SS | ecliptic w band data | w warps, w stack +MDD | Medium Deep Deep | MD warps, deep stacks +MDT | Medium Deep Template | template stacks +MDN | Medium Deep Nightly Stack | nightly stacks +MDN | Medium Deep Off Year | off year template stacks +M31 | M31 | M31 warps, M31 stacks, template +STS | STS | STS warps, STS template - best iq stack + + +Survey | Description | Image Products NOT Saved, but analyzed for difference detections and discarded + +3pi | difference |3pi warps - stack +CNP | difference |CNP warps - stack +SS | difference image | w warps - stack +MDD | difference image | MD warps - off year template stacks +MDO | difference image | MDN stacks - off year template stacks +M31 | difference image | M31 warps - stack +STS | difference image | STS warps - template + + + +PSPS TABLES + + +Fundamental IPP Data Products Origin + +Detection detections found in detrended images +ObjectThin positonal and flag data, no fluxes +MeanObject extracted Mean properties as determined from individual Detections. +StackObject Stack Detections in all filters, with forced upperlimits, separate rows for primary, secondary, best stack detections (80% time only primary) +StackApFlx all filters in one table, unconvolved and convolved apertures (up to 6 , 8, 10(?) pixels in all bands) +StackModelFit all filters in one table. Models of high signal to noise objects in low density regions. +DiffDet difference detections found in difference image = image - template +DiffObject associations of Difference Detections +ForcedWarpMeasure forced measurements on each warp in all filters at the position of an object found in a stack. Includes StackApFlx-like table +ForcedMeanObject like MeanObject, but using the Forced Mean Measurements to form the Mean. + +w band TBD + + +Derived Data Product Tables + +pcsPhotoZ populated from PCS +pcsPhotoZProbability populated from PCS +pcsStarGalQSOSep populated from PCS +pcsStellarParams populated from PCS +mopsMovingObject populated from MOPS database +ppmObj proper motion and parallax populated from IPP +tcsClassification populated from transient server database +galExtinction populated from Dust team +stargal populated from various algorithms, combos of ipp parameters, fits to population as function of magnitude, or Support Vector Machine +GoodObject selected and populated post-facto +StarGalaxySep selected and populated post-facto + + +Observational Metadata and System MetaData Tables + +CameraConfig +DetectionFlags +DiffMeta +Filter +FitModel +ForcedWarpMeta +FrameMeta +ImageFlags +ImageMeta add psf 3nd and 4th moments +Mask +MovingObjectToDet +MovingObjectToDiffDet +MovingObjToObject +ObjectFilterFlags flag info not in current schema but in PSI interface... +ObjectInfoFlags +ObjectQualityFlags +PhotoCal +ProjectionCell +ProcessingVersion +SkyCell +StackDetectionFlags +StackMeta +StackToImage +StackType +Survey +TelescopeConfig +fitModel +pcsGalaxyModel +pcsPhotoZRecipe +pcsSGSepRecipe +pcsStarModel +pcsStellarParamsRecipe + + + +PSPS VIEWS - examples + +ObjectMean = ObjectThin join MeanObject +ObjectStack = ObjectThin join StackObject +ObjectApFlx = ObjectThin join StackApFlx +ObjectModel = ObjectThin join StackModelFit +ObjectStackAll = ObjectThin join StackObject and StackAp and StackModelFit +ObjectForcedMean = ObjectThin join ForcedMeanObject +ObjectPrimary = ObjectThin join StackObject join StackApFlx where primaryDection is true +ObjectBest = ObjectThin join StackObject join StackApFlx where bestDetection is true + + +PSPS Indicies + +DetectionID +ObjectID +BatchID +StackTypeID +MovingObjID +DiffObjID +RandomID between 1 and 2^16 = 4,294,967,296 so you pick how many random objects you want , + i.e 10,000 pick randomid < 10,000 and if indexed, this will be very fast. +RandomStackID between 1 and 2^16 = 4,294,967,296 so you pick how many random objects you want , + i.e 10,000 pick randomid < 10,000 and if indexed, this will be very fast. + + + + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema12.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema12.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema12.txt (revision 37801) @@ -0,0 +1,2 @@ +Heather cleaned up the column names and types to make them consistent. + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema13.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema13.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/Notes_on_Schema13.txt (revision 37801) @@ -0,0 +1,8 @@ +changes by EAM: +* added lensing terms to ForcedWarpMeasurement & ForcedMeanObject +* removed fluxPeak and momentRH generally +* changed psfWidMajor,Minor to psfMajorFWHM,psfMinorFWHM +* added psfCore (k term in psf model : 1 / (1 + kz + k^1.667) +* fixed stack flux & mag names +* set units for moments to be in arcsec everywhere. + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/GoodObject =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/GoodObject (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/GoodObject (revision 37801) @@ -0,0 +1,4 @@ +objName string VAR 8 NA IAU Name constructed from objID postion PS1.x JHHMMSS.SS+/-DDMMSS.S where x is leading number of procesing version +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +goodObjectID dimensionless BIGINT 8 NA index for post-facto "good objects" subset from ObjID Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/StarGalaxySep =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/StarGalaxySep (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/StarGalaxySep (revision 37801) @@ -0,0 +1,9 @@ +objName string VAR 8 NA IAU Name constructed from objID postion PS1.x JHHMMSS.SS+/-DDMMSS.S where x is leading number of procesing version +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +sgbinarySep1 dimensionless TINYINT 2 255 binary flag for star galaxy separator - technique 1 +sgbinarySep2 dimensionless TINYINT 2 255 binary flag for star galaxy separator - technique 2 +sgbinarySep3 dimensionless TINYINT 2 255 binary flag for star galaxy separator - technique 3 +sgSep1 dimensionless REAL 4 -999 probility of star/galaxy technique 1 +sgSep2 dimensionless REAL 4 -999 probility of star/galaxy technique 2 +sgSep3 dimensionless REAL 4 -999 probility of star/galaxy technique 3 Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/galExtinction =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/galExtinction (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/galExtinction (revision 37801) @@ -0,0 +1,28 @@ +galExtinction +Name Unit Data Type Size Default Value Description + +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number + +From Doug and Eddie ? + +E(B-V) + +gGalext +rGalext +iGalext +zGalext +yGalext +wGalext +probability vector + +stellar parameters +Teff +g_surf +[Fe/H] + + + + + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/mopsMovingObject =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/mopsMovingObject (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/mopsMovingObject (revision 37801) @@ -0,0 +1,28 @@ +Name Unit Data Type Size Default Value Description + +MovingObjName dimensionless STRING x -999 MPC Name +MovingObjType dimensionless STRING x -999 Type of moving object (NEO, PHA, COMET, ...) +MovingObjID dimensionless BIGINT 8 -999 Moving Object identifier +OrbEpoch date DATE -999 The epoch of osculation of the orbital elements. +Moid au REAL 4 -999 Minimum orbit intersection distance +OrbM dimensionles REAL 4 -999 Mean anomaly at the epoch. +OrbT date DATE -999 Date of perihelion passage. +ORBn deg/day REAL 4 -999 Mean daily motion (in degrees/day). +ORBa AU REAL 4 -999 Semimajor axis (in AU). +ORBz 1/AU REAL 4 -999 Reciprocal semimajor axis (in 1/AU). +ORBq AU REAL 4 -999 Perihelion distance (in AU). +ORBe dimensionless REAL 4 -999 Orbital eccentricity. +OrbP years REAL 4 -999 Orbital period (in years). +OrbPeri degrees REAL 4 -999 The J2000.0 argument of perihelion (in degrees). +OrbNode degrees REAL 4 -999 The J2000.0 longitude of the ascending node (in degrees). +OrbIncl degrees REAL 4 -999 The J2000.0 inclination (in degrees). +P1 dimensionless REAL 4 -999 P1 first component of P vector +P2 dimensionless REAL 4 -999 P2 second component of P vector +P3 dimensionless REAL 4 -999 P3 third component of P vector +Q1 dimensionless REAL 4 -999 Q1 first component of Q vector +Q2 dimensionless REAL 4 -999 Q2 second component of Q vector +Q3 dimensionless REAL 4 -999 Q3 third component of Q vector +U dimensionless REAL 4 -999 Uncertainty parameter. +H dimensionless REAL 4 -999 Absolute visual magnitude. A table converting H to a diameter is available. +G dimensionless REAL 4 -999 Slope parameter. + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZ =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZ (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZ (revision 37801) @@ -0,0 +1,21 @@ +pcsPhotoZ +Name Unit Data Type Size Default Value Description + +objID BIGINT 8 NA ODM object identifier index +ippObjID BIGINT 8 NA IPP object number +pcsPhotoZRecipeID INT 4 -999 Recipe of the photoZ calculation +galExt REAL 4 -999 Galactic extinction E(B-V) used for photoz determination (derived from Schlegel Maps) +photoz_1 REAL 4 -999 PhotoZ for best fitting template +photozErr_1 REAL 4 -999 Estimated error for this tempalte +chi2_1 REAL 4 -999 Chi^2 for this template +modelID_1 INT 4 -999 Identifier for the template for this recipe +photoz_2 REAL 4 -999 PhotoZ for second template +photozErr_2 REAL 4 -999 Estimated error for this tempalte +chi2_2 REAL 4 -999 Chi^2 for this template +modelID_2 INT 4 -999 Identifier for the template for this recipe +flags_gtfit INT 4 -999 warning flags (e.g. missing band(s), unreliable galExt value etc.), TBD +av REAL 4 -999 determined extinction +chi2 REAL 4 -999 chi2 for the best fitting stellar template +modelID INT 4 -999 modelID of the best fitting stellar template +flags_stfit INT 4 -999 warning flags from stellar template fitting +starGalQSOSep INT 4 -999 0: Object is a star (chi2 Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZProbabiity =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZProbabiity (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZProbabiity (revision 37801) @@ -0,0 +1,37 @@ +PhotoZProbability +Name Unit Data Type Size Default Value Description + +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +pcsPhotoZRecipeID dimensionless INT 4 -999 recipe for the photoz determination +modelID dimensionless INT 4 -999 modelID of the template +sp1 dimensionless REAL 4 -999 spline params 1 +sp2 dimensionless REAL 4 -999 spline params 2 +sp3 dimensionless REAL 4 -999 spline params 3 +sp4 dimensionless REAL 4 -999 spline params 4 +sp5 dimensionless REAL 4 -999 spline params 5 +sp6 dimensionless REAL 4 -999 spline params 6 +sp7 dimensionless REAL 4 -999 spline params 7 +sp8 dimensionless REAL 4 -999 spline params 8 +sp9 dimensionless REAL 4 -999 spline params 9 +sp10 dimensionless REAL 4 -999 spline params 10 +sp11 dimensionless REAL 4 -999 spline params 11 +sp12 dimensionless REAL 4 -999 spline params 12 +sp13 dimensionless REAL 4 -999 spline params 13 +sp14 dimensionless REAL 4 -999 spline params 14 +sp15 dimensionless REAL 4 -999 spline params 15 +sp16 dimensionless REAL 4 -999 spline params 16 +sp17 dimensionless REAL 4 -999 spline params 17 +sp18 dimensionless REAL 4 -999 spline params 18 +sp19 dimensionless REAL 4 -999 spline params 19 +sp20 dimensionless REAL 4 -999 spline params 20 +sp21 dimensionless REAL 4 -999 spline params 21 +sp22 dimensionless REAL 4 -999 spline params 22 +sp23 dimensionless REAL 4 -999 spline params 23 +sp24 dimensionless REAL 4 -999 spline params 24 +sp25 dimensionless REAL 4 -999 spline params 25 +sp26 dimensionless REAL 4 -999 spline params 26 +sp27 dimensionless REAL 4 -999 spline params 27 +sp28 dimensionless REAL 4 -999 spline params 28 +sp29 dimensionless REAL 4 -999 spline params 29 +sp30 dimensionless REAL 4 -999 spline params 30 Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStarGalQSOSep =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStarGalQSOSep (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStarGalQSOSep (revision 37801) @@ -0,0 +1,10 @@ +pcsStarGalQSOSep +Name Unit Data Type Size Default Value Description + +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +pcsSGSepRecipeIDdimensionless INT 4 NA recipe for the star/galaxy/quasar separation +probStar percentage FLOAT 8 -999 probability object is a star +probGalaxy percentage FLOAT 8 -999 probability object is a galaxy +probQuasar percentage FLOAT 8 -999 probability object is a quasar +flags dimensionless INT 4 -999 flags Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStellarParams =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStellarParams (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStellarParams (revision 37801) @@ -0,0 +1,11 @@ +pcsStellarParams s +Name Unit Data Type Size Default Value Description + +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +pcsStellarParamsRecipeID dimensionless INT 4 -999 recipe for the star/galaxy/quasar separation +Teff dimensionless REAL 4 -999 N/A +A0 dimensionless REAL 4 -999 N/A +Fe_H dimensionless REAL 4 -999 N/A +flags dimensionless INT 4 -999 flags + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/ppmObj =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/ppmObj (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/ppmObj (revision 37801) @@ -0,0 +1,25 @@ +PPM +Description: Proper motions and parallax associated with defined objects. + +Name Unit Data Type Size Default Value Description +objID BIGINT 8 NA ODM object identifier +ippObjID BIGINT 8 NA IPP object identifier index +ppmID BIGINT 8 NA ppm object identifier ( ppmToObject metatable?) +meanEpoch FLOAT 8 -999 mean epoch of observations in Mean Julian Days +elapseTime REAL 4 -999 Elapsed time of astrometric observations +muRa REAL 4 -999 proper motion in RA +muRaErr REAL 4 -999 error in proper motion in RA +muDec REAL 4 -999 proper motion in DEC +muDecErr REAL 4 -999 error in proper motion in Dec +mu REAL 4 -999 total proper motion +muErr REAL 4 -999 error in total proper motion +theta REAL 4 -999 angle of parallax vector from N in degrees +plx REAL 4 -999 total parallax +plxErr REAL 4 -999 error in total parallax +nAstro SMALLINT 2 -999 number of measures in astrometric fit +rmsRA REAL 4 -999 rms of astrometric error in RA +rmsDec REAL 4 -999 rms of astrometric error in Dec +wiggle TINYINT 1 255 flag/pointer to astrometric orbit + + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/tcsClassification =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/tcsClassification (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Derived_Data_Product_Tables/tcsClassification (revision 37801) @@ -0,0 +1,16 @@ +galExtinction +Name Unit Data Type Size Default Value Description + +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +diffobjID dimensionless BIGINT 8 NA IPP object number + +From Stephen and Ken? + +TransientName eg. PS1-10jh +transientClass TDE +ATEL -999 Astronomical Telegram number + + + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/Detection =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/Detection (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/Detection (revision 37801) @@ -0,0 +1,65 @@ +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsP2id dimensionless BIGINT 8 NA unique identifier for psps based on batch_id, OTA, and counter (per detection) +detectID dimensionless BIGINT 8 NA ODM detection identifier +ippObjID dimensionless BIGINT 8 NA detection id generated by IPP +ippDetectID dimensionless BIGINT 8 NA detection id generated by IPP +filterID dimensionless TINYINT 1 NA filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... +surveyID dimensionless TINYINT 1 NA survey flag identifier +imageID dimensionless BIGINT 8 NA image id hash of frame identifier & OTA/CCD number +randomDetID dimensionless BIGINT 8 NA random detection Id +dvoRegionID dimensionless INT 4 0 identifier to dvo spatial region +obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation in Mean Julian Days +xPos raw pixels REAL 4 -999 measured x on CCD from PSF fit +yPos raw pixels REAL 4 -999 measured y on CCD from PSF fit +xPosErr raw pixels REAL 4 -999 estimated error in x +yPosErr raw pixels REAL 4 -999 estimated error in y +pltScale arcsec/pix REAL 4 -999 plate scale at this location (from DVO) +posAngle degrees REAL 4 -999 position angle (sky-to-chip) at this location (from DVO) +------------------------------------------------------ Note - next 8 lines came from previous DetCalib table +ra degrees FLOAT 8 NA Right Ascension +dec degrees FLOAT 8 NA Declination +raErr arcsec REAL 4 NA Right Ascension error +decErr arcsec REAL 4 NA Declination error +extNSigma dimensionless REAL 4 -999 measure of extendedness +zp magnitude REAL 4 0 zeropoint (only thing needed to convert adu/sec to AB mag), note no zpErr +telluricExt magnitude REAL 4 NA estimated extinction due to non-photometric conditions +expTime seconds REAL 4 -999 exposure time (informational) +airMass dimensionless REAL 4 0 airmass (informational) +------------------------------------------------------ +psfFlux adu/seconds REAL 4 -999 PSF instrumental flux (in analog-to-digital units/sec) or adu/sec +psfFluxErr adu/seconds REAL 4 -999 estimated error in flux +psfMajorFWHM arcsec REAL 4 -999 model PSF width in major axis +psfMinorFWHM arcsec REAL 4 -999 model PSF width in minor axis +psfTheta degrees REAL 4 -999 model PSF orientation angle degrees East of North +psfCore dimensionless REAL 4 -999 model PSF core parameter [k in (1 + kz + z^1.66)] +psfQf dimensionless REAL 4 -999 PSF coverage factor +psfQfPerfect dimensionless REAL 4 -999 PSF-weighted fraction of pixels totally unmasked +psfChiSq dimensionless REAL 4 -999 reduced-chisq of the psf model fit. (ndof?, npix?) +psfLikelihood dimensionless REAL 4 -999 PSF likelihood +momentXX arcsec^2 REAL 4 -999 moment XX +momentXY arcsec^2 REAL 4 -999 moment XY +momentYY arcsec^2 REAL 4 -999 momeny YY +momentR1 arcsec REAL 4 -999 first radial moment +momentRH arcsec REAL 4 -999 half light radius +momentM3C arcsec^n REAL 4 -999 3rd moment sin(theta) +momentM3S arcsec^n REAL 4 -999 3rd moment cos(theta) +momentM4C arcsec^n REAL 4 -999 4th moment sin(theta) +momentM4S arcsec^n REAL 4 -999 4th moment cos(theta) +apFlux adu/seconds REAL 4 -999 Aperture flux +apFluxErr adu/seconds REAL 4 -999 Aperture flux error +apFillF dimensionless REAL 4 -999 Aperture fill factor +apRadius arcsec REAL 4 -999 Aperture radius +kronFlux adu/seconds REAL 4 -999 Kron flux +kronFluxErr adu/seconds REAL 4 -999 Kron flux error +kronRad arcsec REAL 4 -999 Kron radius +sky adu/sec/pixel REAL 4 -999 PSF sky level at source +skyErr adu/sec/pixel REAL 4 -999 estimated error in sky +infoFlag dimensionless INT 4 -999 flag indicating provenance information +infoFlag2 dimensionless INT 4 -999 flag indicating provenance information +dataRelease dimensionless TINYINT 1 NA Data release when this detection was originally taken. Recalibrations do not affect this value. +-- +-- [231 bytes/det] + +### do we want apRadius? +### relevant code spots: +### config/tables.P2.vot, detectionbatch.py:315 Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/Detection.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/Detection.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/Detection.txt (revision 37801) @@ -0,0 +1,59 @@ + +objID + dvo = average.extID [calculated in addstar] +uniquePspsP2id + ipptopsps : combination of batch_id, ota, and counter +detectID + dvo = measure.extID [calculated in addstar] +ippObjID + dvo = average.objID + (average.catID << 32) [NOTE: this is a change in definition] +ippDetectID + smf:IPP_IDET +filterID + ipptopsps : internal, lookup from filter (grizy) +surveyID + ipptopsps : internal, from survey name +imageID - ipptopsps : expID x y (*** is this right???) (smf:IMAGEID / gpc1.chipRun.chip_image_id)_ +randomDetID + ipptopsps : RANDOM from sql +dvoRegionID + dvo:average.catID +obsTime + smf:MJD-OBS + 0.5 EXPTIME / 84600 +xPos + smf:X_PSF +yPos + smf:Y_PSF +xPosErr + smf:X_PSF_SIG +yPosErr + smf:Y_PSF_SIG +pltScale + smf:PLTSCALE (negative -> sky parity) +posAngle + smf:POSANGLE +--------------- +ra + dvo:measure.ra +dec + dvo:measure.dec +raErr + smf:X_PSF_SIG * abs(PLTSCALE) +decErr + smf:Y_PSF_SIG * abs(PLTSCALE) +extNSigma + smf:EXT_NSIGMA +zp + dvo / dvopsps : zp +telluricExt + dvo / dvopsps : zp - zpNominal (dvopsps) +expTime + dvo:measure.dt [converted to seconds] +airMass + dvo:measure.airmass +--------------- +psfFlux + smf:PSF_INST_FLUX / EXPTIME +psfFluxErr + smf:PSF_INST_FLUX_SIG / EXPTIME +psfMajorFWHM + smf:PSF_FWHM_MAJ +psfMinorFWHM + smf:PSF_FWHM_MIN +psfTheta + smf:PSF_THETA +psfCore + smf:PSF_CORE +psfQf + smf:PSF_QF +psfQfPerfect + smf:PSF_QF_PERFECT +psfChiSq + smf:PSF_CHISQ +psfLikelihood + smf:EXT_NSIGMA +momentXX + smf:MOMENTS_XX +momentXY + smf:MOMENTS_XY +momentYY + smf:MOMENTS_YY +momentR1 + smf:MOMENTS_R1 +momentRH + smf:MOMENTS_RH +momentM3C + smf:MOMENTS_M3C +momentM3S + smf:MOMENTS_M3S +momentM4C + smf:MOMENTS_M4C +momentM4S + smf:MOMENTS_M4S +apFlux + smf:AP_FLUX +apFluxErr + smf:AP_FLUX_SIG +apFillF + smf:AP_NPIX / (pi * (AP_MAG_RADIUS - 0.5)^2) +kronFlux + smf:KRON_FLUX +kronFluxErr + smf:KRON_FLUX_ERR +kronRad + smf:2.5*MOMENTS_R1 +sky + smf:SKY / exptime +skyErr + smf:SKY_SIGMA / exptime +infoFlag + smf:FLAGS +infoFlag2 + smf:FLAGS2 | (dvo.measure.flags << 13) +dataRelease + ippToPsps : user input (skychunk.dataRelease) Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject (revision 37801) @@ -0,0 +1,38 @@ + +diffObjName string VAR 8 NA IAU Name constructed from objID postion PS1.x JHHMMSS.SS+/-DDMMSS.S x procesing version +diffObjID dimensionless BIGINT 8 NA ODM object identifier index +uniquePspsDOid dimentionless BIGINT 8 NA unique id provided by ipptopsps, based on batch_Id + counter +ippObjID dimensionless BIGINT 8 NA IPP object number +surveyID dimensionless TINYINT 1 255 Survey ID +skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +htmID dimensionless BIGINT 8 NA HTM index +zoneID dimensionless INT 4 NA Zone index +randomID dimensionless BIGINT 8 0 Random integer between 0 and 2^32 = 4,394,967,296 remaining ramdom integer 0 +batchID dimensionless BIGINT 8 -999 +dvoRegionID dimensionless INT 4 -999 identifier to dvo spatial region +objInfoFlag dimensionless BIGINT 8 -999 flag indicating odd properties +qualityFlag dimensionless TINYINT 1 0 subset of objInfoFlag - flags related to goodness of object +consistencyFlag dimensionless SMALLINT2 0 Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation (DROP) +ra degrees FLOAT 8 NA Right Ascension +dec degrees FLOAT 8 NA Declination +raErr arcsec REAL 4 0.0 Right Ascension error +decErr arcsec REAL 4 0.0 Declination error +cx dimensionless FLOAT 8 NA Cartesian x +cy dimensionless FLOAT 8 NA Cartesian y +cz dimensionless FLOAT 8 NA Cartesian z +lambda degrees FLOAT 8 -999 ecliptic longitude +beta degrees FLOAT 8 -999 ecliptic latitude +l degrees FLOAT 8 -999 galactic longitude +b degrees FLOAT 8 -999 galactic latitude +gQfPerfect dimensionless REAL 4 -999 qfPerfect for g filter +rQfPerfect dimensionless REAL 4 -999 qfPerfect for r filter +iQfPerfect dimensionless REAL 4 -999 qfPerfect for i filter +zQfPerfect dimensionless REAL 4 -999 qfPerfect for z filter +yQfPerfect dimensionless REAL 4 -999 qfPerfect for y filter +dataRelease dimensionless TINYINT 1 NA Data release Version of this table +nDetections dimensionless SMALLINT2 -999 total number of detection measurements in all filters +ng dimensionless SMALLINT2 -999 number of detections in g filter +nr dimensionless SMALLINT2 -999 number of detections in g filter +ni dimensionless SMALLINT2 -999 number of detections in g filter +nz dimensionless SMALLINT2 -999 number of detections in g filter +ny dimensionless SMALLINT2 -999 number of detections in g filter Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject.txt (revision 37801) @@ -0,0 +1,38 @@ + +diffObjName = ipptopsps +diffObjID = dvo / addstar +uniquePspsDOid = ipptopsps +ippObjID = dvo / addstar +surveyID = ipptopsps +skyCellID = smf header +htmID = PSPS +zoneID = PSPS +randomID = ipptopsps +batchID = ipptopsps +dvoRegionID = dvo catID +objInfoFlag = average.flags +qualityFlag = average.flags >> 24 & 0xff (needed??) +consistencyFlag = ? +ra = dvo / relastro +dec = dvo / relastro +raErr = dvo / relastro +decErr = dvo / relastro +cx = PSPS +cy = PSPS +cz = PSPS +lambda = PSPS +beta = PSPS +l = PSPS +b = PSPS +gQfPerfect = dvo / relphot / secfilt. [XXX Not calculated] +rQfPerfect = dvo / relphot / secfilt. [XXX Not calculated] +iQfPerfect = dvo / relphot / secfilt. [XXX Not calculated] +zQfPerfect = dvo / relphot / secfilt. [XXX Not calculated] +yQfPerfect = dvo / relphot / secfilt. [XXX Not calculated] +dataRelease = ipptopsps +nDetections = dvo / relphot +ng = dvo / relphot +nr = dvo / relphot +ni = dvo / relphot +nz = dvo / relphot +ny = dvo / relphot Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetection =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetection (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetection (revision 37801) @@ -0,0 +1,70 @@ + +diffObjID dimensionless BIGINT 8 NA diff object identifier +uniquePspsDFid dimensionless BIGINT 8 NA unique psps id (combination of batch_id, and counter) provided by ipptopsps +diffDetID dimensionless BIGINT 8 NA diff detection identifier +diffDetMetaID dimensionless BIGINT 8 NA diff detection meta identifier +ippDetectID dimensionless BIGINT 8 NA ipp detection id generated by IPP +ippObjID dimensionless BIGINT 8 NA ipp object id generated by IPP +filterID dimensionless TINYINT 1 NA filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... +surveyID dimensionless TINYINT 1 NA survey flag identifierr +randomDiffID dimensionless BIGINT 8 NA random diff id +skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +dvoRegionID dimensionless INT 4 0 identifier to dvo spatial region +obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation of positive detection in Mean Julian Days of positive detection +xPos pixels REAL 4 -999 measured x on diff image sky cell PSF fit +yPos pixels REAL 4 -999 measured y on diff image sky cell from PSF fit +xPosErr pixels REAL 4 -999 estimated error in x +yPosErr pixels REAL 4 -999 estimated error in y +pltScale arcsec/pix REAL 4 -999 plate scale at this location (from DVO) +posAngle degrees REAL 4 -999 position angle (sky-to-chip) at this location (from DVO) +---------------------------------------------------- next 8 came from old DiffDetCalib, zp derived from cpt tables of BOTH images +ra degrees FLOAT 8 NA Right Ascension +dec degrees FLOAT 8 NA Declination +raErr arcsec REAL 4 NA Right Ascension error -> posErr because RA,Dec same? +decErr arcsec REAL 4 NA Declination error +zp magnitude REAL 4 0 zeropoint (all that is needed to convert adu/sec to AB mag), no zpErr +telluricExt magnitude REAL 4 NA estimated extinction due to non-photometric conditions +expTime seconds REAL 4 -999 exposure time (informational) +airMass dimensionless REAL 4 0 airmass (informational) +----------------------------------------------------- attributes of positive detection +psfFlux adu/seconds REAL 4 -999 PSF instrumental flux (in analog-to-digital units/sec) or adu/sec +psfFluxErr adu/seconds REAL 4 -999 estimated error in flux +psfMajorFWHM arcsec REAL 4 -999 model PSF width in major axis +psfMinorFWHM arcsec REAL 4 -999 model PSF width in minor axis +psfTheta degrees REAL 4 -999 model PSF orientation angle degrees East of North +psfCore dimensionless REAL 4 -999 model PSF core parameter [k in (1 + kz + z^1.66)] +psfQf dimensionless REAL 4 -999 PSF coverage factor +psfQfPerfect dimensionless REAL 4 -999 PSF-weighted fraction of pixels totally unmasked +psfChiSq dimensionless REAL 4 -999 reduced-chisq of the psf model fit. (ndof?, npix?) +psfLikelihood dimensionless REAL 4 -999 PSF likelihood +momentXX arcsec^2 REAL 4 -999 moment XX +momentXY arcsec^2 REAL 4 -999 moment XY +momentYY arcsec^2 REAL 4 -999 momeny YY +momentR1 arcsec REAL 4 -999 first radial moment +momentRH arcsec REAL 4 -999 half light radius +momentM3C arcsec^3 REAL 4 -999 3rd moment sin(theta) +momentM3S arcsec^3 REAL 4 -999 3rd moment cos(theta) +momentM4C arcsec^4 REAL 4 -999 4th moment sin(theta) +momentM4S arcsec^4 REAL 4 -999 4th moment cos(theta) +apFlux adu/seconds REAL 4 -999 Aperture flux +apFluxErr adu/seconds REAL 4 -999 Aperture flux error +apFillF dimensionless REAL 4 -999 Aperture fill factor +kronFlux adu/seconds REAL 4 -999 Kron flux +kronFluxErr adu/seconds REAL 4 -999 Kron flux error +kronRad raw pixels REAL 4 -999 Kron radius +------------------------------------------------------- from diff image cmf/smf files +diffNPos dimensionless REAL 4 -999 npix > 3sigma +diffFRatio dimensionless REAL 4 -999 fPos/(fPos +fNeg) +diffNBad dimensionless REAL 4 -999 nPos/(nPos+nNeg) +diffNMask dimensionless REAL 4 -999 nPos/(nPos + nMask) +diffNAll dimensionless REAL 4 -999 nPos/(nPos + nMask + nNeg) +diffPdist arcsec REAL 4 -999 distance to positive match source +diffNdist arcsec REAL 4 -999 distance to negative match source +diffPSN dimensionless REAL 4 -999 signal to noise of pos match source +diffNSN dimensionless REAL 4 -999 signal to noise of neg match source +------------------------------------------------------- +sky adu/sec/pixel REAL 4 -999 PSF sky level at source +skyErr adu/sec/pixel REAL 4 -999 estimated error in sky +infoFlag dimensionless INT 4 -999 flag indicating provenance information +infoFlag2 dimensionless INT 4 -999 flag indicating provenance information +dataRelease dimensionless TINYINT 1 NA Data release when this detection was originally taken. Recalibrations do not affect this value. Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetection.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetection.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetection.txt (revision 37801) @@ -0,0 +1,71 @@ + +diffObjID = dvo / addstar +uniquePspsDFid = ipptopsps +diffDetID = dvo / addstar +diffDetMetaID = dvo / addstar +ippDetectID = dvo / addstar +ippObjID = dvo / addstar +filterID = smf / ipptopsps +surveyID = ipptopsps +randomDiffID = ipptopsps +skyCellID = smf header +dvoRegionID = dvo / catID +obsTime = smf +xPos = smf +yPos = smf +xPosErr = smf +yPosErr = smf +pltScale = smf +posAngle = smf +---------------- = +ra = dvo / relastro? [allow / require relastro recalibration?] +dec = dvo / relastro? +raErr = dvo / relastro? +decErr = dvo / relastro? +zp = dvo / relphot +telluricExt = (zp - zpNominal) dvopsps +expTime = smf +airMass = smf +---------------- = +psfFlux = smf +psfFluxErr = smf +psfMajorFWHM = [PSF_MAJOR is Sxx parameter, NOT FWHM] [ADD PSF_MAJ_FWHM to smf, keep PSF_MAJOR] +psfMinorFWHM = [PSF_MAJOR is Sxx parameter, NOT FWHM] [ditto] +psfTheta = smf +psfCore = [XXX : not currently written out!!] +psfQf = smf +psfQfPerfect = smf +psfChiSq = smf +psfLikelihood = ipptopsps +momentXX = smf +momentXY = smf +momentYY = smf +momentR1 = smf +momentRH = smf [XXX : MOMENTS_RH is NOT half-light radius] +momentM3C = smf +momentM3S = smf +momentM4C = smf +momentM4S = smf +apFlux = smf [PS1_DV2 or later] +apFluxErr = smf [PS1_DV2 or later] +apFillF = [XXX : not currently calculated] +kronFlux = smf +kronFluxErr = smf +kronRad = (2.5*R1) +---------------- = +diffNPos = smf +diffFRatio = smf +diffNBad = smf +diffNMask = smf +diffNAll = smf +diffPdist = smf +diffNdist = smf +diffPSN = smf +diffNSN = smf +---------------- = +sky = smf +skyErr = smf +infoFlag = smf +infoFlag2 = smf +dataRelease = ipptopsps + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject (revision 37801) @@ -0,0 +1,63 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier index +uniquePspsFOid dimensionless BIGINT 8 NA unique id supplied by ipptopsps +ippObjID dimensionless BIGINT 8 NA IPP object identifier +nDetections dimensionless SMALLINT 2 -999 total number of detection measurements in all filters +batchID dimensionless BIGINT 8 NA batch id +----------------------------------------------------- replicate to bottom for each filter +gStackID dimensionless INT 4 stack image id (how to carry through IPP + DVO) +gnTotal dimensionless SMALLINT 2 -999 number of detections in g filter +gnIncPSFFlux dimensionless SMALLINT 2 -999 number of detections included in robust psf mean in g filter +gnIncKronFlux dimensionless SMALLINT 2 -999 number of detections included in robust mean of Kron flux in g filter +gnIncApFlux dimensionless SMALLINT 2 -999 number of detections included in robust mean of Ap flux in g filter +gnIncR5 dimensionless SMALLINT 2 -999 number of detections included in robust mean of ApFlx 5 in g filter +gnIncR6 dimensionless SMALLINT 2 -999 number of detections included in robust mean of ApFlx 6 in g filter +------------------------------------------------------- one of the above also corresponds to the moments +gPSFFlux adu/seconds REAL 4 -999 robust mean g psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number sumary? 1/12 is ~9th percentile +gPSFFluxErr adu/seconds REAL 4 -999 estimated error of g psf AB magnitude mean (individual errors added in quadrature) +gPSFFluxStd dimensionless REAL 4 -999 standard deviation of measurements included in the robust mean +gKronFlux adu/seconds REAL 4 -999 mean g kron AB magnitude +gKronFluxErr adu/seconds REAL 4 -999 estimated error of g kron AB magnitude mean +gKronFluxStd dimensionless REAL 4 -999 standard deviation of measurements included in the robust mean +gApFlux adu/seconds REAL 4 -999 mean g kron AB magnitude +gApFluxErr adu/seconds REAL 4 -999 estimated error of g kron AB magnitude mean +gApFluxStd dimensionless REAL 4 -999 standard deviation of measurements included in the robust mean +------------------------------------------------------ +gFmeanflxR5 adu/seconds REAL 4 -999 Flux inside r = 5 in g unconvolved stack +gFmeanflxR5Err adu/seconds REAL 4 -999 estimated error is flxR5 +gFmeanflxR5Std adu/seconds REAL 4 -999 estimated standard deviation in flxR5 +gFmeanflxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +gFmeanflxR6 adu/seconds REAL 4 -999 Flux inside r = 6 in g unconvolved stack +gFmeanflxR6Err adu/seconds REAL 4 -999 estimated error is flxR6 +gFmeanflxR6Std adu/seconds REAL 4 -999 estimated standard deviation in flxR6 +gFmeanflxR6Fill dimensionless REAL 4 -999 estimated filling factor at R6 +gFlags dimensionless BIGINT 8 NA flags for g forced +------------------------------------------------------ +gLensObjSmearX11 arcsec^-2 REAL 4 -999 per-object smear polarizability X11 term +gLensObjSmearX12 arcsec^-2 REAL 4 -999 per-object smear polarizability X12 term +gLensObjSmearX22 arcsec^-2 REAL 4 -999 per-object smear polarizability X22 term +gLensObjSmearE1 arcsec^-2 REAL 4 -999 per-object smear polarizability E1 term +gLensObjSmearE2 arcsec^-2 REAL 4 -999 per-object smear polarizability E2 term +gLensObjShearX11 dimensionless REAL 4 -999 per-object shear polarizability X11 term +gLensObjShearX12 dimensionless REAL 4 -999 per-object shear polarizability X12 term +gLensObjShearX22 dimensionless REAL 4 -999 per-object shear polarizability X22 term +gLensObjShearE1 dimensionless REAL 4 -999 per-object shear polarizability E1 term +gLensObjShearE2 dimensionless REAL 4 -999 per-object shear polarizability E2 term +gLensPSFSmearX11 arcsec^-2 REAL 4 -999 psf model smear polarizability X11 term +gLensPSFSmearX12 arcsec^-2 REAL 4 -999 psf model smear polarizability X12 term +gLensPSFSmearX22 arcsec^-2 REAL 4 -999 psf model smear polarizability X22 term +gLensPSFSmearE1 arcsec^-2 REAL 4 -999 psf model smear polarizability E1 term +gLensPSFSmearE2 arcsec^-2 REAL 4 -999 psf model smear polarizability E2 term +gLensPSFShearX11 dimensionless REAL 4 -999 psf model shear polarizability X11 term +gLensPSFShearX12 dimensionless REAL 4 -999 psf model shear polarizability X12 term +gLensPSFShearX22 dimensionless REAL 4 -999 psf model shear polarizability X22 term +gLensPSFShearE1 dimensionless REAL 4 -999 psf model shear polarizability E1 term +gLensPSFShearE2 dimensionless REAL 4 -999 psf model shear polarizability E2 term +------------------------------------------------------ +gGamma arcsec^2 REAL 4 -999 second moment (Mxx + Myy) +gE1 dimensionless REAL 4 -999 polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy) +gE2 dimensionless REAL 4 -999 polarization, KSB-corrected 2Mxy / (Mxx + Myy) + +-- duplicated for other filters +-- [184 byte/obj/filter + 22 byte/obj] + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject.txt (revision 37801) @@ -0,0 +1,68 @@ + +objID = cpt.extID +uniquePspsFOid = ipptopsps +ippObjID = cpt.objID | cpt.catID << 32 +nDetections = sql sum of cps.NCODE (xnTotal) +batchID = ipptopsps +---------------- = +gStackID = ipptopsps / gpc1 db +gnTotal = cps.NWARP +gnIncPSFFlux = cps.NUSED_WRP +gnIncKronFlux = cps.NUSED_KRON_WRP +gnIncApFlux = cps.NUSED_AP_WRP +gnIncR5 = cpy.NMEAS +gnIncR6 = cpy.NMEAS +---------------- = +gPSFFlux = cps.FLUX_PSF_WRP +gPSFFluxErr = cps.FLUX_PSF_WRP_ERR +gPSFFluxStd = cps.FLUX_PSF_WRP_STD +gKronFlux = cps.FLUX_KRON_WRP +gKronFluxErr = cps.FLUX_KRON_WRP_ERR +gKronFluxStd = cps.FLUX_KRON_WRP_STD +gApFlux = cps.FLUX_AP_WRP +gApFluxErr = cps.FLUX_AP_WRP_ERR +gApFluxStd = cps.FLUX_AP_WRP_STD +---------------- = +gFmeanflxR5 = cpy.FLUX_AP_R5 +gFmeanflxR5Err = cpy.FLUX_ERR_AP_R5 +gFmeanflxR5Std = cpy.FLUX_STD_AP_R5 +gFmeanflxR5Fill = cpy.FLUX_FIL_AP_R5 +gFmeanflxR6 = cpy.FLUX_AP_R6 +gFmeanflxR6Err = cpy.FLUX_ERR_AP_R6 +gFmeanflxR6Std = cpy.FLUX_STD_AP_R6 +gFmeanflxR6Fill = cpy.FLUX_FIL_AP_R6 +gFlags = cps.FLAGS +---------------- = +gLensObjSmearX11 = cpy.X11_SM_OBJ +gLensObjSmearX12 = cpy.X12_SM_OBJ +gLensObjSmearX22 = cpy.X22_SM_OBJ +gLensObjSmearE1 = cpy.E1_SM_OBJ +gLensObjSmearE2 = cpy.E2_SM_OBJ +gLensObjShearX11 = cpy.X11_SH_OBJ +gLensObjShearX12 = cpy.X12_SH_OBJ +gLensObjShearX22 = cpy.X22_SH_OBJ +gLensObjShearE1 = cpy.E1_SH_OBJ +gLensObjShearE2 = cpy.E2_SH_OBJ +gLensPSFSmearX11 = cpy.X11_SM_PSF +gLensPSFSmearX12 = cpy.X12_SM_PSF +gLensPSFSmearX22 = cpy.X22_SM_PSF +gLensPSFSmearE1 = cpy.E1_SM_PSF +gLensPSFSmearE2 = cpy.E2_SM_PSF +gLensPSFShearX11 = cpy.X11_SH_PSF +gLensPSFShearX12 = cpy.X12_SH_PSF +gLensPSFShearX22 = cpy.X22_SH_PSF +gLensPSFShearE1 = cpy.E1_SH_PSF +gLensPSFShearE2 = cpy.E2_SH_PSF +---------------- = +gGamma = cpy.GAMMA +gE1 = cpy.E1 +gE2 = cpy.E2 + +[NOTES: + + * PSF Flux, Kron Flux, Ap Flux stats are all + calculated from values in the existing Measure table + + * R5 & R6 ap flux values & lensing parameters need + a new table + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended (revision 37801) @@ -0,0 +1,47 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsFWid dimentionless BIGINT 8 NA unique id supplied by ipptopsps +detectID dimensionless BIGINT 8 NA ODM detection identifier +ippObjID dimensionless BIGINT 8 NA ippobject id generated by IPP +ippDetectID dimensionless BIGINT 8 NA detection id generated by IPP +filterID dimensionless TINYINT 1 NA filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... +surveyID dimensionless TINYINT 1 NA survey flag identifier +forcedWarpID dimensionless BIGINT 8 NA IPP warp_id +randomWarpID dimensionless BIGINT 8 NA Random warp_id +skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +dvoRegionID dimensionless INT 4 -999 identifier to dvo spatial region +obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation in Mean Julian Days +------------------------------------------------------ +lensObjSmearX11 arcsec^-2 REAL 4 -999 per-object smear polarizability X11 term +lensObjSmearX12 arcsec^-2 REAL 4 -999 per-object smear polarizability X12 term +lensObjSmearX22 arcsec^-2 REAL 4 -999 per-object smear polarizability X22 term +lensObjSmearE1 arcsec^-2 REAL 4 -999 per-object smear polarizability E1 term +lensObjSmearE2 arcsec^-2 REAL 4 -999 per-object smear polarizability E2 term +lensObjShearX11 dimensionless REAL 4 -999 per-object shear polarizability X11 term +lensObjShearX12 dimensionless REAL 4 -999 per-object shear polarizability X12 term +lensObjShearX22 dimensionless REAL 4 -999 per-object shear polarizability X22 term +lensObjShearE1 dimensionless REAL 4 -999 per-object shear polarizability E1 term +lensObjShearE2 dimensionless REAL 4 -999 per-object shear polarizability E2 term +lensPSFSmearX11 arcsec^-2 REAL 4 -999 psf model smear polarizability X11 term +lensPSFSmearX12 arcsec^-2 REAL 4 -999 psf model smear polarizability X12 term +lensPSFSmearX22 arcsec^-2 REAL 4 -999 psf model smear polarizability X22 term +lensPSFSmearE1 arcsec^-2 REAL 4 -999 psf model smear polarizability E1 term +lensPSFSmearE2 arcsec^-2 REAL 4 -999 psf model smear polarizability E2 term +lensPSFShearX11 dimensionless REAL 4 -999 psf model shear polarizability X11 term +lensPSFShearX12 dimensionless REAL 4 -999 psf model shear polarizability X12 term +lensPSFShearX22 dimensionless REAL 4 -999 psf model shear polarizability X22 term +lensPSFShearE1 dimensionless REAL 4 -999 psf model shear polarizability E1 term +lensPSFShearE2 dimensionless REAL 4 -999 psf model shear polarizability E2 term +------------------------------------------------------ +flxR5 adu/seconds REAL 4 -999 Flux inside r = 5 +flxR5Err adu/seconds REAL 4 -999 estimated error is flxR5 +flxR5Std adu/seconds REAL 4 -999 estimated standard deviation in flxR5 +flxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +flxR6 adu/seconds REAL 4 -999 Flux inside r = 6 +flxR6Err adu/seconds REAL 4 -999 estimated error is flxR6 +flxR6Std adu/seconds REAL 4 -999 estimated standard deviation in flxR6 +flxR6Fill dimensionless REAL 4 -999 estimated filling factor at R6 +------------------------------------------------------ + +-- [295 byte/det] +-- this table is only for non-Galatic-plane data Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended.txt (revision 37801) @@ -0,0 +1,44 @@ + +objID = cpt.extID +uniquePspsFWid = ipptopsps +detectID = cpm.extID (???) +ippObjID = cpt.obj_ID | cpx.cat_ID << 32 +ippDetectID = cmf.IPP_IDET +filterID = ipptopsps / gpc1 +surveyID = ipptopsps +forcedWarpID = ipptopsps / gpc1 +randomWarpID = ipptopsps +skyCellID = ipptopsps / gpc1 +dvoRegionID = ipptopsps / gpc1 +obsTime = cmf +---------------- = +lensObjSmearX11 = cmf.X11_SM_OBJ +lensObjSmearX12 = cmf.X12_SM_OBJ +lensObjSmearX22 = cmf.X22_SM_OBJ +lensObjSmearE1 = cmf.E1_SM_OBJ +lensObjSmearE2 = cmf.E2_SM_OBJ +lensObjShearX11 = cmf.X11_SH_OBJ +lensObjShearX12 = cmf.X12_SH_OBJ +lensObjShearX22 = cmf.X22_SH_OBJ +lensObjShearE1 = cmf.E1_SH_OBJ +lensObjShearE2 = cmf.E2_SH_OBJ +lensPSFSmearX11 = cmf.X11_SM_PSF +lensPSFSmearX12 = cmf.X12_SM_PSF +lensPSFSmearX22 = cmf.X22_SM_PSF +lensPSFSmearE1 = cmf.E1_SM_PSF +lensPSFSmearE2 = cmf.E2_SM_PSF +lensPSFShearX11 = cmf.X11_SH_PSF +lensPSFShearX12 = cmf.X12_SH_PSF +lensPSFShearX22 = cmf.X22_SH_PSF +lensPSFShearE1 = cmf.E1_SH_PSF +lensPSFShearE2 = cmf.E2_SH_PSF +---------------- = +flxR5 = cmf.FLX_AP_R5 +flxR5Err = cmf.FLX_ERR_AP_R5 +flxR5Std = cmf.FLX_STD_AP_R5 +flxR5Fill = cmf.FLX_FIL_AP_R5 +flxR6 = cmf.FLX_AP_R6 +flxR6Err = cmf.FLX_ERR_AP_R6 +flxR6Std = cmf.FLX_STD_AP_R6 +flxR6Fill = cmf.FLX_FIL_AP_R6 +---------------- = Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement (revision 37801) @@ -0,0 +1,54 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsFWid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +detectID dimensionless BIGINT 8 NA ODM detection identifier +ippObjID dimensionless BIGINT 8 NA ippobject id generated by IPP +ippDetectID dimensionless BIGINT 8 NA detection id generated by IPP +filterID dimensionless TINYINT 1 NA filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... +surveyID dimensionless TINYINT 1 NA survey flag identifier +forcedWarpID dimensionless BIGINT 8 NA IPP warp_id +randomWarpID dimensionless BIGINT 8 NA Random warp_id +skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +dvoRegionID dimensionless INT 4 -999 identifier to dvo spatial region +obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation in Mean Julian Days +------------------------------------------------------ +zp magnitude REAL 4 0 zeropoint (only thing needed to convert adu/sec to AB mag), note no zpErr +telluricExt magnitude REAL 4 NA estimated extinction due to non-photometric conditions +expTime seconds REAL 4 -999 exposure time (informational) +airMass dimensionless REAL 4 0 airmass (informational) +------------------------------------------------------ +psfFlux adu/seconds REAL 4 -999 PSF instrumental flux (in analog-to-digital units/sec) or adu/sec +psfFluxErr adu/seconds REAL 4 -999 estimated error in flux +psfMajorFWHM arcsec REAL 4 -999 model PSF width in major axis +psfMinorFWHM arcsec REAL 4 -999 model PSF width in minor axis +psfTheta degrees REAL 4 -999 model PSF orientation angle degrees East of North +psfCore dimensionless REAL 4 -999 model PSF core parameter [k in (1 + kz + z^1.66)] +psfQf dimensionless REAL 4 -999 PSF coverage factor +psfQfPerfect dimensionless REAL 4 -999 PSF-weighted fraction of pixels totally unmasked +psfChiSq dimensionless REAL 4 -999 reduced-chisq of the psf model fit. (ndof?, npix?) +psfLikelihood dimensionless REAL 4 -999 PSF likelihood +momentXX arcsec^2 REAL 4 -999 moment XX +momentXY arcsec^2 REAL 4 -999 moment XY +momentYY arcsec^2 REAL 4 -999 momeny YY +momentR1 arcsec^1 REAL 4 -999 first radial moment +momentRH arcsec REAL 4 -999 to be half light radius (DROP) +momentM3C arcsec^3 REAL 4 -999 3rd moment sin(theta) +momentM3S arcsec^3 REAL 4 -999 3rd moment cos(theta) +momentM4C arcsec^4 REAL 4 -999 4th moment sin(theta) +momentM4S arcsec^4 REAL 4 -999 4th moment cos(theta) + +apFlux adu/seconds REAL 4 -999 Aperture flux +apFluxErr adu/seconds REAL 4 -999 Aperture flux error +apFillF dimensionless REAL 4 -999 Aperture fill factor +kronFlux adu/seconds REAL 4 -999 Kron flux +kronFluxErr adu/seconds REAL 4 -999 Kron flux error +kronRad arcsec REAL 4 -999 Kron radius +kronRadErr arcsed REAL 4 -999 Kron radius error +------------------------------------------------------ +sky adu/sec/pixel REAL 4 -999 PSF sky level at source +skyErr adu/sec/pixel REAL 4 -999 estimated error in sky +infoFlag dimensionless BIGINT 8 -999 flag indicating provenance information +infoFlag2 dimensionless BIGINT 8 -999 FLAG2 | flags << 13 +dataRelease dimensionless TINYINT 1 NA Data release when this detection was originally taken. Recalibrations do not affect this value. + +-- [295 byte/det] Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement.txt (revision 37801) @@ -0,0 +1,51 @@ + +objID = cpt.extID (???) +uniquePspsFWid = ipptopsps +detectID = cpm.extID (???) +ippObjID = cpt.obj_id | cpt.cat_id << 32 +ippDetectID = smf:IPP_IDET +filterID = ipptopsps /gpc1 +surveyID = ipptopsps +forcedWarpID = ipptopsps / gpc1 +randomWarpID = ipptopsps +skyCellID = ipptopsps / gpc1 +dvoRegionID = ipptopsps / gpc1 +obsTime = smf +---------------- = +zp = dvopsps zp +telluricExt = dvopsps (zp - zpNominal) +expTime = smf +airMass = smf +---------------- = +psfFlux = smf:PSF_INST_FLUX / EXPTIME +psfFluxErr = smf:PSF_INST_FLUX_SIG / EXPTIME +psfMajorFWHM = smf:PSF_FWHM_MAJ +psfMinorFWHM = smf:PSF_FWHM_MIN +psfTheta = smf:PSF_THETA +psfCore = smf:PSF_CORE +psfQf = smf:PSF_QF +psfQfPerfect = smf:PSF_QF_PERFECT +psfChiSq = smf:PSF_CHISQ +psfLikelihood = smf:NEXT_NSIGMA +momentXX = smf:MOMENTS_XX +momentXY = smf:MOMENTS_XY +momentYY = smf:MOMENTS_YY +momentR1 = smf:MOMENTS_R1 +momentRH = smf:MOMENTS_RH +momentM3C = smf:MOMENTS_M3C +momentM3S = smf:MOMENTS_M3S +momentM4C = smf:MOMENTS_M4C +momentM4S = smf:MOMENTS_M4S + = +apFlux = smf:AP_FLUX +apFluxErr = smf:AP_FLUX_SIG +apFillF = smf:AP_NPIX / (pi * (AP_MAG_RADIUS - 0.5)^2) +kronFlux = smf:KRON_FLUX +kronFluxErr = smf:KRON_FLUX_ERR +kronRad = smf:2.5*MOMENTS_R1 +---------------- = +sky = smf:SKY / exptime +skyErr = smf:SKY_SIGMA / exptime +infoFlag = smf:FLAGS +infoFlag2 = smf:FLAGS2 | (dvo.measure.flags << 13) +dataRelease = ipptopsps Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/MeanObject =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/MeanObject (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/MeanObject (revision 37801) @@ -0,0 +1,28 @@ + +-- The "best available" measurements refer first to those objects with qfPerfect > 0.85 and good flags, +-- but descends down a "waterfall logic" in ipp so that there exists a mean mag for less than ideal objects. The logic is given in Magnier etal 2013. + +objID dimensionless BIGINT 8 NA ODM object identifier index +uniquePspsOBid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +gQfPerfect dimensionless REAL 4 -999 maximum psf qfPerfect from best available g band detections +gMeanPSFMag AB_magnitude REAL 4 -999 mean g psf AB magnitude from best available measurements (best available as current defined by ipp "waterfall logic" +gMeanPSFMagErr AB_magnitude REAL 4 -999 estimated error of best availalbe g-band psf AB magnitudes (individual errors added in quadrature) +gMeanPSFMagStd dimensioless REAL 4 -999 standard deviation of best available measurements - compared with MeanErr this is one measure of true variability +gMeanPSFMagMin AB_magnitude REAL 4 -999 minimum best available PSFMag - min and max are given for PSFMag as they may indicate variability. +gMeanPSFMagMax AB_magnitude REAL 4 -999 maximum best available PSFMag +gMeanPSFMagNpt dimensionless SMALLINT 2 -999 number of best available detections included in mean psfMag + +gMeanKronMag AB_magnitude REAL 4 -999 mean g kron AB magnitude from best available measurements +gMeanKronMagErr AB_magnitude REAL 4 -999 estimated error of mean g kron AB magnitude +gMeanKronMagStd dimensionless REAL 4 -999 standard deviation of best available measurements +gMeanKronMagNpt dimensionless SMALLINT 2 -999 number of best available detections included mean Kron mag + +gMeanApMag AB_magnitude REAL 4 -999 mean g kron AB magnitude from best available measurements +gMeanApMagErr AB_magnitude REAL 4 -999 estimated error of g kron AB magnitude mean +gMeanApMagStd dimensioless REAL 4 -999 standard deviation of best available measurements +gMeanApMagNpt dimensionless SMALLINT 2 -999 number of detections included in best available mean of Kron mag in g filter + +gFlags dimensionless INT 4 0 flags for g mean [XXX secfilt.flags is 4byte] +-- duplicate for other filters + +[62 byte/obj/filter + 8 byte/filter] Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/MeanObject.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/MeanObject.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/MeanObject.txt (revision 37801) @@ -0,0 +1,24 @@ + +*** I've checked that these are plausible, not exact + +objID + dvo / addstar +uniquePspsOBid = ipptopsps +gQfPerfect + dvo / relphot / secfilt.psfQfMax +gMeanPSFMag + dvo / relphot / secfilt.M +gMeanPSFMagErr + dvo / relphot / secfilt.dM +gMeanPSFMagStd + dvo / relphot / secfilt.Mstdev +gMeanPSFMagMin + dvo / relphot / secfilt.Min +gMeanPSFMagMax + dvo / relphot / secfilt.Max +gMeanPSFMagNpt + dvo / relphot / secfilt.Nused + +gMeanKronMag + dvo / relphot / secfilt.Mkron +gMeanKronMagErr + dvo / relphot / secfilt.dMkron +gMeanKronMagStd + dvo / relphot / secfilt.sMkron +gMeanKronMagNpt + dvo / relphot / secfilt.NusedKron + +gMeanApMag + dvo / relphot / secfilt.Map +gMeanApMagErr + dvo / relphot / secfilt.dMap +gMeanApMagStd + dvo / relphot / secfilt.sMap +gMeanApMagNpt + dvo / relphot / secfilt.NusedAp + +gFlags + dvo / relphot / secfilt.flags Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ObjectThin =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ObjectThin (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ObjectThin (revision 37801) @@ -0,0 +1,52 @@ + +objName string VAR 8 NA IAU Name constructed from objID postion PS1.x JHHMMSS.SS+/-DDMMSS.S where x is leading number of procesing version +objID dimensionless BIGINT 8 NA ODM object identifier index +uniquePspsOBid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +ippObjID dimensionless BIGINT 8 NA IPP object number +surveyID dimensionless TINYINT 1 255 Survey ID +htmID dimensionless BIGINT 8 NA HTM index +zoneID dimensionless INT 4 NA Zone index +tessID dimensionless TINYINT 1 -1 ID mapping to tessellation names (eg, RINGS.V3, MD01.V3, etc) +projectionID dimensionless SMALLINT 2 -999 valid values 0 - < 10000 +skyCellID dimensionless TINYINT 1 255 valid values 0 - 143 (12x12 grid) +randomID dimensionless BIGINT 8 0 Random integer between 0 and 2^32 = 4,394,967,296 remaining ramdom integer 0 +batchID dimensionless BIGINT 8 -999 batch id +dvoRegionID dimensionless INT 4 -999 identifier of dvo spatial region - dvo cat id +dataRelease dimensionless TINYINT 1 NA Data release Version of this table +objInfoFlag dimensionless BIGINT 8 -999 flag indicating odd properties +qualityFlag dimensionless TINYINT 1 0 subset of objInfoFlag - flags related to goodness of object +consistencyFlag dimensionless SMALLINT 2 0 Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation (DROP) +raStack degrees FLOAT 8 -999 Right Ascension - best weighted RA across filters from stack WHENEVER an object is detected in the stack +decStack degrees FLOAT 8 -999 Declination - best weighted Dec across filters from stack WHENEVER an object is detected in the stack +raStackErr arcsec REAL 4 -999 Error in stack Right Ascension, note will have a systematic floor, and is not necessary the best ra,dec for all objects +decStackErr arcsec REAL 4 -999 Error in stack Declination, note will have a systematic floor, and is not necessarily the best ra,dec for all objects +raMean degrees FLOAT 8 -999 Right Ascension from the Mean of RA from individual detections if it exists. This is for information only +decMean degrees FLOAT 8 -999 Declination from the Mean of Dec from individual detections if it exists This is for information only. +raMeanErr arcsec REAL 4 -999 Combination of the measurement Errors in the Right Ascension from the mean detections if they exist +decMeanErr arcsec REAL 4 -999 Combination of the measurement Errors in the Declination measurements if they exist +posMeanChisq dimensionless REAL 4 -999 chisq of mean position +-------------------------------------------------------------------------------------------------------- +-- Note: the following three sets of coordinate systems are based on stack if it exists, and mean if it does not. +-- This means cone or box searches work on the stack and consistent with Primary and Best stack detection. +-- For objects without a stack postion, (moving, transient, or false dectection) a cone search may be incomplete for +-- an object within less than 1 arc sec of the boundary edge. But science cases here will want to work from Detection +-- Table anyway, or can vary cone search by 1'' radius to check for incompleteness. +cx dimensionless FLOAT 8 NA Cartesian x from Stack RA,Dec WHENEVER a stack detection exists, and Mean RA, Dec (mean of one?) when it doesn't +cy dimensionless FLOAT 8 NA Cartesian y from Stack RA,Dec WHENEVER a stack detection exists, and mean RA, Dec (mean of one?) when it doesn't +cz dimensionless FLOAT 8 NA Cartesian z from Stack RA,Dec WHENEVER a stack detection exists, ditto +lambda degrees FLOAT 8 -999 ecliptic longitude from Stack RA,Dec WHENEVER A STACK MEASURMENT EXISTS and Mean RA,Dec (mean of one?) otherwise +beta degrees FLOAT 8 -999 ecliptic latitude ditto +l degrees FLOAT 8 -999 galactic longitude ditto +b degrees FLOAT 8 -999 galactic latitude ditto +--------------------------------------------------------------------------------- +nStackObjectRows dimensionless SMALLINT 2 -999 number of independent StackObjectRows this object has. +-- zero if not detected in the Stack, 1 if there is a primary, 2 or more if it is in an overlap region +nStackDetections dimensionless TINYINT 2 255 which filter bands it is detected in (not forced) or if it was masked in a band. +nDetections dimensionless SMALLINT 2 -999 total number of detection measurements in all filters +ng dimensionless SMALLINT 2 -999 number of detections in individual g filter exposures +nr dimensionless SMALLINT 2 -999 number of detections in individual r filter exposures +ni dimensionless SMALLINT 2 -999 number of detections in individual i filter exposures +nz dimensionless SMALLINT 2 -999 number of detections in individual z filter exposures +ny dimensionless SMALLINT 2 -999 number of detections in individual y filter exposures + +-- [197 byte/obj] Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ObjectThin.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ObjectThin.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ObjectThin.txt (revision 37801) @@ -0,0 +1,50 @@ + +objName - ipptopsps : move to dvopsps? +objID + dvo:average.extID +uniquePspsOBid = ipptopsps +ippObjID + dvo:(average.objID | average.catID << 32) +surveyID + ipptopsps : internal, from survey name +htmID - PSPS +zoneID - PSPS +tessID + average.tessID +projectionID + average.projectionID +skycellID + average.skycellID +randomID + ipptopsps : RANDOM from sql +batchID + ipptopsps : internal +dvoRegionID + dvo:average.catID +dataRelease + ipptopsps : user input (skychunk.dataRelease) +objInfoFlag + dvo:average.flags +qualityFlag - (dvo:average.flags >> 24) & 0xFF *** correct?? +consistencyFlag - *** hard-wired in ipptopsps to 0 +raStack + dvo:average.Rstk *** not calculated? +decStack + dvo:average.Dstk *** not calculated? +raStackErr + dvo:average.dRstk *** not calculated? +decStackErr + dvo:average.dDstk *** not calculated? +raMean + dvo:average.R +decMean + dvo:average.D +raMeanErr + dvo:average.dR +decMeanErr + dvo:average.dD +posMeanChisq + dvo:average.ChiSqAve +----------------- +-- +-- +-- +-- +-- +cx - PSPS +cy - PSPS +cz - PSPS +lambda - PSPS +beta - PSPS +l - PSPS +b - PSPS +----------------- +nStackObjectRows - do in dvopsps +-- +nStackDetections - sum of secfilt.NstackDet (sum in sql) +nDetections + ipptopsps:Sum of ng+nr in sql +ng + dvo:secfilt.ncode ** check that this is right +nr + dvo:secfilt.ncode ** check that this is right +ni + dvo:secfilt.ncode ** check that this is right +nz + dvo:secfilt.ncode ** check that this is right +ny + dvo:secfilt.ncode ** check that this is right Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlx =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlx (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlx (revision 37801) @@ -0,0 +1,56 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsSTid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +ippObjID dimensionless BIGINT 8 NA ipp object identifier +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +------------------------------------------------------- replicate to end for each filter g,r,i,z,y +gippDetectID dimensionless BIGINT 8 NA stack identifier +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +--------------------------------------------------------unconvolved +gflxR3 Jy REAL 4 -999 Flux inside r = 3 +gflxR3Err Jy REAL 4 -999 estimated error is flxR3 +gflxR3Std Jy REAL 4 -999 estimated standard deviation in flxR3 +gflxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gflxR4 Jy REAL 4 -999 Flux inside r = 4 +gflxR4Err Jy REAL 4 -999 estimated error is flxR4 +gflxR4Std Jy REAL 4 -999 estimated standard deviation in flxR4 +gflxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gflxR5 Jy REAL 4 -999 Flux inside r = 5 +gflxR5Err Jy REAL 4 -999 estimated error is flxR5 +gflxR5Std Jy REAL 4 -999 estimated standard deviation in flxR5 +gflxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +---------------------------------------------------- convolved across all bands to 6 or 8 pixels +gc6flxR3 Jy REAL 4 -999 Flux inside r = 3 +gc6flxR3Err Jy REAL 4 -999 estimated error is c1flxR3 +gc6flxR3Std Jy REAL 4 -999 estimated standard deviation in c1flxR3 +gc6flxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gc6flxR4 Jy REAL 4 -999 Flux inside r = 4 +gc6flxR4Err Jy REAL 4 -999 estimated error is c1flxR4 +gc6flxR4Std Jy REAL 4 -999 estimated standard deviation in c1flxR4 +gc6flxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gc6flxR5 Jy REAL 4 -999 Flux inside r = 5 +gc6flxR5Err Jy REAL 4 -999 estimated error is c1flxR5 +gc6flxR5Std Jy REAL 4 -999 estimated standard deviation in c1flxR5 +gc6flxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +---------------------------------------------------- convolved across all bands to 6 or 8 pixels +gc8flxR3 Jy REAL 4 -999 Flux inside r = 3 +gc8flxR3Err Jy REAL 4 -999 estimated error in c2flxR3 +gc8flxR3Std Jy REAL 4 -999 estimated standard deviation in c2flxR3 +gc8flxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gc8flxR4 Jy REAL 4 -999 Flux inside r = 4 +gc8flxR4Err Jy REAL 4 -999 estimated error in c2flxR4 +gc8flxR4Std Jy REAL 4 -999 estimated standard deviation in c2flxR4 +gc8flxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gc8flxR5 Jy REAL 4 -999 Flux inside r = 5 +gc8flxR5Err Jy REAL 4 -999 estimated error in c2flxR5 +gc8flxR5Std Jy REAL 4 -999 estimated standard deviation in c2flxR5 +gc8flxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 + +-- repeat for other filters with appropriate leading prefex. +-- [168 byte/filter/obj + 44 byte/obj] + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlx.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlx.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlx.txt (revision 37801) @@ -0,0 +1,52 @@ + +objID + from StackObjectThin +uniquePspsSTid = ipptopsps +ippObjID + from StackObjectThin +randomStackObjID + from StackObjectThin +stackDetectRowID - PSPS +primaryDetection - from StackObjectThin NOT YET TESTED +bestDetection - from StackObjectThin CANNOT BE SET until after PV3 +-- +---------------- +gippDetectID + from StackObjectThin +gstackDetectID + from StackObjectThin +gstackMetaID + from StackObjectThin +---------------- +gflxR3 + cmf : APER_FLUX_3 +gflxR3Err + cmf : APER_FLUX_ERR_3 +gflxR3Std - cmf : APER_FLUX_STDEV_3 : ** badly defined in cmf +gflxR3Fill + cmf : APER_FILL_3 +gflxR4 + cmf : APER_FLUX_4 +gflxR4Err + cmf : APER_FLUX_ERR_4 +gflxR4Std - cmf : APER_FLUX_STDEV_4 : ** badly defined in cmf +gflxR4Fill + cmf : APER_FILL_4 +gflxR5 + cmf : APER_FLUX_5 +gflxR5Err + cmf : APER_FLUX_ERR_5 +gflxR5Std - cmf : APER_FLUX_STDEV_5 : ** badly defined in cmf +gflxR5Fill + cmf : APER_FILL_5 +---------------- +gc6flxR3 + cmf : APER_FLUX_3 +gc6flxR3Err + cmf : APER_FLUX_ERR_3 +gc6flxR3Std - cmf : APER_FLUX_STDEV_3 : ** badly defined in cmf +gc6flxR3Fill + cmf : APER_FILL_3 +gc6flxR4 + cmf : APER_FLUX_4 +gc6flxR4Err + cmf : APER_FLUX_ERR_4 +gc6flxR4Std - cmf : APER_FLUX_STDEV_4 : ** badly defined in cmf +gc6flxR4Fill + cmf : APER_FILL_4 +gc6flxR5 + cmf : APER_FLUX_5 +gc6flxR5Err + cmf : APER_FLUX_ERR_5 +gc6flxR5Std - cmf : APER_FLUX_STDEV_5 : ** badly defined in cmf +gc6flxR5Fill + cmf : APER_FILL_5 +---------------- +gc8flxR3 + cmf : APER_FLUX_3 +gc8flxR3Err + cmf : APER_FLUX_ERR_3 +gc8flxR3Std - cmf : APER_FLUX_STDEV_3 : ** badly defined in cmf +gc8flxR3Fill + cmf : APER_FILL_3 +gc8flxR4 + cmf : APER_FLUX_4 +gc8flxR4Err + cmf : APER_FLUX_ERR_4 +gc8flxR4Std - cmf : APER_FLUX_STDEV_4 : ** badly defined in cmf +gc8flxR4Fill + cmf : APER_FILL_4 +gc8flxR5 + cmf : APER_FLUX_5 +gc8flxR5Err + cmf : APER_FLUX_ERR_5 +gc8flxR5Std - cmf : APER_FLUX_STDEV_5 : ** badly defined in cmf +gc8flxR5Fill + cmf : APER_FILL_5 Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6 =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6 (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6 (revision 37801) @@ -0,0 +1,56 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsSTid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +ippObjID dimensionless BIGINT 8 NA object ID generated by IPP +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +-------------------------------------------------------- replicate below for all filters (grizy) +gippDetectID dimensionless BIGINT 8 NA ipp detect identifier +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +---------------------------------------------------------------------------------------- +gc6flxR1 Jy REAL 4 -999 Flux inside r = 1 +gc6flxR1Err Jy REAL 4 -999 estimated error is c1flxR1 +gc6flxR1Std Jy REAL 4 -999 estimated standard deviation in c1flxR1 +gc6flxR1Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gc6flxR2 Jy REAL 4 -999 Flux inside r = 2 +gc6flxR2Err Jy REAL 4 -999 estimated error is c1flxR2 +gc6flxR2Std Jy REAL 4 -999 estimated standard deviation in c1flxR2 +gc6flxR2Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gc6flxR3 Jy REAL 4 -999 Flux inside r = 3 +gc6flxR3Err Jy REAL 4 -999 estimated error is c1flxR3 +gc6flxR3Std Jy REAL 4 -999 estimated standard deviation in c1flxR3 +gc6flxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gc6flxR4 Jy REAL 4 -999 Flux inside r = 4 +gc6flxR4Err Jy REAL 4 -999 estimated error is c1flxR4 +gc6flxR4Std Jy REAL 4 -999 estimated standard deviation in c1flxR4 +gc6flxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gc6flxR5 Jy REAL 4 -999 Flux inside r = 5 +gc6flxR5Err Jy REAL 4 -999 estimated error is c1flxR5 +gc6flxR5Std Jy REAL 4 -999 estimated standard deviation in c1flxR5 +gc6flxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +gc6flxR6 Jy REAL 4 -999 Flux inside r = 6 +gc6flxR6Err Jy REAL 4 -999 estimated error is c1flxR6 +gc6flxR6Std Jy REAL 4 -999 estimated standard deviation in c1flxR6 +gc6flxR6Fill dimensionless REAL 4 -999 estimated filling factor at R6 +gc6flxR7 Jy REAL 4 -999 Flux inside r = 7 +gc6flxR7Err Jy REAL 4 -999 estimated error is c1flxR7 +gc6flxR7Std Jy REAL 4 -999 estimated standard deviation in c1flxR7 +gc6flxR7Fill dimensionless REAL 4 -999 estimated filling factor at R7 +gc6flxR8 Jy REAL 4 -999 Flux inside r = 8 +gc6flxR8Err Jy REAL 4 -999 estimated error is c1flxR8 +gc6flxR8Std Jy REAL 4 -999 estimated standard deviation in c1flxR8 +gc6flxR8Fill dimensionless REAL 4 -999 estimated filling factor at R8 +gc6flxR9 Jy REAL 4 -999 Flux inside r = 9 +gc6flxR9Err Jy REAL 4 -999 estimated error is c1flxR9 +gc6flxR9Std Jy REAL 4 -999 estimated standard deviation in c1flxR9 +gc6flxR9Fill dimensionless REAL 4 -999 estimated filling factor at R9 +gc6flxR10 Jy REAL 4 -999 Flux inside r = 10 +gc6flxR10Err Jy REAL 4 -999 estimated error is c1flxR10 +gc6flxR10Std Jy REAL 4 -999 estimated standard deviation in c1flxR10 +gc6flxR10Fill dimensionless REAL 4 -999 estimated filling factor at R10 + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6.txt (revision 37801) @@ -0,0 +1,54 @@ + +objID + from StackObjectThin +uniquePspsSTid = ipptopsps +ippObjID + from StackObjectThin +randomStackObjID + from StackObjectThin +stackDetectRowID - PSPS +primaryDetection - from StackObjectThin NOT YET TESTED +bestDetection - from StackObjectThin CANNOT BE SET until after PV3 +-- +---------------- +gippDetectID + from StackObjectThin +gstackDetectID + from StackObjectThin +gstackMetaID + from StackObjectThin +---------------- +gc6flxR1 + cmf : APER_FLUX_1 +gc6flxR1Err + cmf : APER_FLUX_ERR_1 +gc6flxR1Std - cmf : APER_FLUX_STDEV_1 : ** badly defined in cmf +gc6flxR1Fill + cmf : APER_FILL_1 +gc6flxR2 + cmf : APER_FLUX_2 +gc6flxR2Err + cmf : APER_FLUX_ERR_2 +gc6flxR2Std - cmf : APER_FLUX_STDEV_2 : ** badly defined in cmf +gc6flxR2Fill + cmf : APER_FILL_2 +gc6flxR3 + cmf : APER_FLUX_3 +gc6flxR3Err + cmf : APER_FLUX_ERR_3 +gc6flxR3Std - cmf : APER_FLUX_STDEV_3 : ** badly defined in cmf +gc6flxR3Fill + cmf : APER_FILL_3 +gc6flxR4 + cmf : APER_FLUX_4 +gc6flxR4Err + cmf : APER_FLUX_ERR_4 +gc6flxR4Std - cmf : APER_FLUX_STDEV_4 : ** badly defined in cmf +gc6flxR4Fill + cmf : APER_FILL_4 +gc6flxR5 + cmf : APER_FLUX_5 +gc6flxR5Err + cmf : APER_FLUX_ERR_5 +gc6flxR5Std - cmf : APER_FLUX_STDEV_5 : ** badly defined in cmf +gc6flxR5Fill + cmf : APER_FILL_5 +gc6flxR6 + cmf : APER_FLUX_6 +gc6flxR6Err + cmf : APER_FLUX_ERR_6 +gc6flxR6Std - cmf : APER_FLUX_STDEV_6 : ** badly defined in cmf +gc6flxR6Fill + cmf : APER_FILL_6 +gc6flxR7 + cmf : APER_FLUX_7 +gc6flxR7Err + cmf : APER_FLUX_ERR_7 +gc6flxR7Std - cmf : APER_FLUX_STDEV_7 : ** badly defined in cmf +gc6flxR7Fill + cmf : APER_FILL_7 +gc6flxR8 + cmf : APER_FLUX_8 +gc6flxR8Err + cmf : APER_FLUX_ERR_8 +gc6flxR8Std - cmf : APER_FLUX_STDEV_8 : ** badly defined in cmf +gc6flxR8Fill + cmf : APER_FILL_8 +gc6flxR9 + cmf : APER_FLUX_9 +gc6flxR9Err + cmf : APER_FLUX_ERR_9 +gc6flxR9Std - cmf : APER_FLUX_STDEV_9 : ** badly defined in cmf +gc6flxR9Fill + cmf : APER_FILL_9 +gc6flxR10 - cmf : APER_FLUX_10 ** does not exist in cmf +gc6flxR10Err - cmf : APER_FLUX_ERR_10 ** does not exist in cmf +gc6flxR10Std - cmf : APER_FLUX_STDEV_10 ** does not exist in cmf +gc6flxR10Fill - cmf : APER_FILL_10 ** does not exist in cmf Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8 =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8 (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8 (revision 37801) @@ -0,0 +1,56 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsSTid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +ippObjID dimensionless BIGINT 8 NA object ID generated by IPP +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +-------------------------------------------------------- replicate below for all filters (grizy) +gippDetectID dimensionless BIGINT 8 NA ipp detect identifier +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +---------------------------------------------------------------------------------------- +gc8flxR1 Jy REAL 4 -999 Flux inside r = 1 +gc8flxR1Err Jy REAL 4 -999 estimated error is c2flxR1 +gc8flxR1Std Jy REAL 4 -999 estimated standard deviation in c2flxR1 +gc8flxR1Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gc8flxR2 Jy REAL 4 -999 Flux inside r = 2 +gc8flxR2Err Jy REAL 4 -999 estimated error is c2flxR2 +gc8flxR2Std Jy REAL 4 -999 estimated standard deviation in c2flxR2 +gc8flxR2Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gc8flxR3 Jy REAL 4 -999 Flux inside r = 3 +gc8flxR3Err Jy REAL 4 -999 estimated error is c2flxR3 +gc8flxR3Std Jy REAL 4 -999 estimated standard deviation in c2flxR3 +gc8flxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gc8flxR4 Jy REAL 4 -999 Flux inside r = 4 +gc8flxR4Err Jy REAL 4 -999 estimated error is c2flxR4 +gc8flxR4Std Jy REAL 4 -999 estimated standard deviation in c2flxR4 +gc8flxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gc8flxR5 Jy REAL 4 -999 Flux inside r = 5 +gc8flxR5Err Jy REAL 4 -999 estimated error is c2flxR5 +gc8flxR5Std Jy REAL 4 -999 estimated standard deviation in c2flxR5 +gc8flxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +gc8flxR6 Jy REAL 4 -999 Flux inside r = 6 +gc8flxR6Err Jy REAL 4 -999 estimated error is c2flxR6 +gc8flxR6Std Jy REAL 4 -999 estimated standard deviation in c2flxR6 +gc8flxR6Fill dimensionless REAL 4 -999 estimated filling factor at R6 +gc8flxR7 Jy REAL 4 -999 Flux inside r = 7 +gc8flxR7Err Jy REAL 4 -999 estimated error is c2flxR7 +gc8flxR7Std Jy REAL 4 -999 estimated standard deviation in c2flxR7 +gc8flxR7Fill dimensionless REAL 4 -999 estimated filling factor at R7 +gc8flxR8 Jy REAL 4 -999 Flux inside r = 8 +gc8flxR8Err Jy REAL 4 -999 estimated error is c2flxR8 +gc8flxR8Std Jy REAL 4 -999 estimated standard deviation in c2flxR8 +gc8flxR8Fill dimensionless REAL 4 -999 estimated filling factor at R8 +gc8flxR9 Jy REAL 4 -999 Flux inside r = 9 +gc8flxR9Err Jy REAL 4 -999 estimated error is c2flxR9 +gc8flxR9Std Jy REAL 4 -999 estimated standard deviation in c2flxR9 +gc8flxR9Fill dimensionless REAL 4 -999 estimated filling factor at R9 +gc8flxR10 Jy REAL 4 -999 Flux inside r = 10 +gc8flxR10Err Jy REAL 4 -999 estimated error is c2flxR10 +gc8flxR10Std Jy REAL 4 -999 estimated standard deviation in c2flxR10 +gc8flxR10Fill dimensionless REAL 4 -999 estimated filling factor at R10 + +-- repeat for other filters with appropriate leading prefex. Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8.txt (revision 37801) @@ -0,0 +1,54 @@ + +objID + from StackObjectThin +uniquePspsSTid = ipptopsps +ippObjID + from StackObjectThin +randomStackObjID + from StackObjectThin +stackDetectRowID - PSPS +primaryDetection - from StackObjectThin NOT YET TESTED +bestDetection - from StackObjectThin CANNOT BE SET until after PV3 +-- +---------------- +gippDetectID + from StackObjectThin +gstackDetectID + from StackObjectThin +gstackMetaID + from StackObjectThin +---------------- +gc8flxR1 + cmf : APER_FLUX_1 +gc8flxR1Err + cmf : APER_FLUX_ERR_1 +gc8flxR1Std - cmf : APER_FLUX_STDEV_1 : ** badly defined in cmf +gc8flxR1Fill + cmf : APER_FILL_1 +gc8flxR2 + cmf : APER_FLUX_2 +gc8flxR2Err + cmf : APER_FLUX_ERR_2 +gc8flxR2Std - cmf : APER_FLUX_STDEV_2 : ** badly defined in cmf +gc8flxR2Fill + cmf : APER_FILL_2 +gc8flxR3 + cmf : APER_FLUX_3 +gc8flxR3Err + cmf : APER_FLUX_ERR_3 +gc8flxR3Std - cmf : APER_FLUX_STDEV_3 : ** badly defined in cmf +gc8flxR3Fill + cmf : APER_FILL_3 +gc8flxR4 + cmf : APER_FLUX_4 +gc8flxR4Err + cmf : APER_FLUX_ERR_4 +gc8flxR4Std - cmf : APER_FLUX_STDEV_4 : ** badly defined in cmf +gc8flxR4Fill + cmf : APER_FILL_4 +gc8flxR5 + cmf : APER_FLUX_5 +gc8flxR5Err + cmf : APER_FLUX_ERR_5 +gc8flxR5Std - cmf : APER_FLUX_STDEV_5 : ** badly defined in cmf +gc8flxR5Fill + cmf : APER_FILL_5 +gc8flxR6 + cmf : APER_FLUX_6 +gc8flxR6Err + cmf : APER_FLUX_ERR_6 +gc8flxR6Std - cmf : APER_FLUX_STDEV_6 : ** badly defined in cmf +gc8flxR6Fill + cmf : APER_FILL_6 +gc8flxR7 + cmf : APER_FLUX_7 +gc8flxR7Err + cmf : APER_FLUX_ERR_7 +gc8flxR7Std - cmf : APER_FLUX_STDEV_7 : ** badly defined in cmf +gc8flxR7Fill + cmf : APER_FILL_7 +gc8flxR8 + cmf : APER_FLUX_8 +gc8flxR8Err + cmf : APER_FLUX_ERR_8 +gc8flxR8Std - cmf : APER_FLUX_STDEV_8 : ** badly defined in cmf +gc8flxR8Fill + cmf : APER_FILL_8 +gc8flxR9 + cmf : APER_FLUX_9 +gc8flxR9Err + cmf : APER_FLUX_ERR_9 +gc8flxR9Std - cmf : APER_FLUX_STDEV_9 : ** badly defined in cmf +gc8flxR9Fill + cmf : APER_FILL_9 +gc8flxR10 - cmf : APER_FLUX_10 ** does not exist in cmf +gc8flxR10Err - cmf : APER_FLUX_ERR_10 ** does not exist in cmf +gc8flxR10Std - cmf : APER_FLUX_STDEV_10 ** does not exist in cmf +gc8flxR10Fill - cmf : APER_FILL_10 ** does not exist in cmf Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc (revision 37801) @@ -0,0 +1,54 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsSTid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +ippObjID dimensionless BIGINT 8 NA object ID generated by IPP +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +-------------------------------------------------------- replicate below for all filters (grizy) +gippDetectID dimensionless BIGINT 8 NA ipp detect identifier +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +---------------------------------------------------------------------------------------- +gflxR1 Jy REAL 4 -999 Flux inside r = 1 +gflxR1Err Jy REAL 4 -999 estimated error is flxR1 +gflxR1Std Jy REAL 4 -999 estimated standard deviation in flxR1 +gflxR1Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gflxR2 Jy REAL 4 -999 Flux inside r = 2 +gflxR2Err Jy REAL 4 -999 estimated error is flxR2 +gflxR2Std Jy REAL 4 -999 estimated standard deviation in flxR2 +gflxR2Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gflxR3 Jy REAL 4 -999 Flux inside r = 3 +gflxR3Err Jy REAL 4 -999 estimated error is flxR3 +gflxR3Std Jy REAL 4 -999 estimated standard deviation in flxR3 +gflxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gflxR4 Jy REAL 4 -999 Flux inside r = 4 +gflxR4Err Jy REAL 4 -999 estimated error is flxR4 +gflxR4Std Jy REAL 4 -999 estimated standard deviation in flxR4 +gflxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gflxR5 Jy REAL 4 -999 Flux inside r = 5 +gflxR5Err Jy REAL 4 -999 estimated error is flxR5 +gflxR5Std Jy REAL 4 -999 estimated standard deviation in flxR5 +gflxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +gflxR6 Jy REAL 4 -999 Flux inside r = 6 +gflxR6Err Jy REAL 4 -999 estimated error is flxR6 +gflxR6Std Jy REAL 4 -999 estimated standard deviation in flxR6 +gflxR6Fill dimensionless REAL 4 -999 estimated filling factor at R6 +gflxR7 Jy REAL 4 -999 Flux inside r = 7 +gflxR7Err Jy REAL 4 -999 estimated error is flxR7 +gflxR7Std Jy REAL 4 -999 estimated standard deviation in flxR7 +gflxR7Fill dimensionless REAL 4 -999 estimated filling factor at R7 +gflxR8 Jy REAL 4 -999 Flux inside r = 8 +gflxR8Err Jy REAL 4 -999 estimated error is flxR8 +gflxR8Std Jy REAL 4 -999 estimated standard deviation in flxR8 +gflxR8Fill dimensionless REAL 4 -999 estimated filling factor at R8 +gflxR9 Jy REAL 4 -999 Flux inside r = 9 +gflxR9Err Jy REAL 4 -999 estimated error is flxR9 +gflxR9Std Jy REAL 4 -999 estimated standard deviation in flxR9 +gflxR9Fill dimensionless REAL 4 -999 estimated filling factor at R9 +gflxR10 Jy REAL 4 -999 Flux inside r = 10 +gflxR10Err Jy REAL 4 -999 estimated error is flxR10 +gflxR10Std Jy REAL 4 -999 estimated standard deviation in flxR10 +gflxR10Fill dimensionless REAL 4 -999 estimated filling factor at R10 Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc.txt (revision 37801) @@ -0,0 +1,54 @@ + +objID + from StackObjectThin +uniquePspsSTid = ipptopsps +ippObjID + from StackObjectThin +randomStackObjID + from StackObjectThin +stackDetectRowID - PSPS +primaryDetection - from StackObjectThin NOT YET TESTED +bestDetection - from StackObjectThin CANNOT BE SET until after PV3 +-- +---------------- +gippDetectID + from StackObjectThin +gstackDetectID + from StackObjectThin +gstackMetaID + from StackObjectThin +---------------- +gflxR1 + cmf : APER_FLUX_1 +gflxR1Err + cmf : APER_FLUX_ERR_1 +gflxR1Std - cmf : APER_FLUX_STDEV_1 : ** badly defined in cmf +gflxR1Fill + cmf : APER_FILL_1 +gflxR2 + cmf : APER_FLUX_2 +gflxR2Err + cmf : APER_FLUX_ERR_2 +gflxR2Std - cmf : APER_FLUX_STDEV_2 : ** badly defined in cmf +gflxR2Fill + cmf : APER_FILL_2 +gflxR3 + cmf : APER_FLUX_3 +gflxR3Err + cmf : APER_FLUX_ERR_3 +gflxR3Std - cmf : APER_FLUX_STDEV_3 : ** badly defined in cmf +gflxR3Fill + cmf : APER_FILL_3 +gflxR4 + cmf : APER_FLUX_4 +gflxR4Err + cmf : APER_FLUX_ERR_4 +gflxR4Std - cmf : APER_FLUX_STDEV_4 : ** badly defined in cmf +gflxR4Fill + cmf : APER_FILL_4 +gflxR5 + cmf : APER_FLUX_5 +gflxR5Err + cmf : APER_FLUX_ERR_5 +gflxR5Std - cmf : APER_FLUX_STDEV_5 : ** badly defined in cmf +gflxR5Fill + cmf : APER_FILL_5 +gflxR6 + cmf : APER_FLUX_6 +gflxR6Err + cmf : APER_FLUX_ERR_6 +gflxR6Std - cmf : APER_FLUX_STDEV_6 : ** badly defined in cmf +gflxR6Fill + cmf : APER_FILL_6 +gflxR7 + cmf : APER_FLUX_7 +gflxR7Err + cmf : APER_FLUX_ERR_7 +gflxR7Std - cmf : APER_FLUX_STDEV_7 : ** badly defined in cmf +gflxR7Fill + cmf : APER_FILL_7 +gflxR8 + cmf : APER_FLUX_8 +gflxR8Err + cmf : APER_FLUX_ERR_8 +gflxR8Std - cmf : APER_FLUX_STDEV_8 : ** badly defined in cmf +gflxR8Fill + cmf : APER_FILL_8 +gflxR9 + cmf : APER_FLUX_9 +gflxR9Err + cmf : APER_FLUX_ERR_9 +gflxR9Std - cmf : APER_FLUX_STDEV_9 : ** badly defined in cmf +gflxR9Fill + cmf : APER_FILL_9 +gflxR10 - cmf : APER_FLUX_10 ** does not exist in cmf +gflxR10Err - cmf : APER_FLUX_ERR_10 ** does not exist in cmf +gflxR10Std - cmf : APER_FLUX_STDEV_10 ** does not exist in cmf +gflxR10Fill - cmf : APER_FILL_10 ** does not exist in cmf Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitDeV =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitDeV (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitDeV (revision 37801) @@ -0,0 +1,28 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsSTid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +ippObjID dimensionless BIGINT 8 NA object ID generated by IPP +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +-------------------------------------------------------- replicate below for all filters (grizy) +gippDetectID dimensionless BIGINT 8 NA ipp detect identifier +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +---------------------------------------------------------------------------------------- +gDeVRadius arcsec REAL 4 -999 DeVaucouleurs radius +gDeVRadiusErr arcsec REAL 4 -999 estimated error in DeVaucouleurs radius +gDeVMag AB_mag REAL 4 -999 DeVaucouleurs flux +gDeVMagErr AB_mag REAL 4 -999 estimated error in DeVaucouleurs flux +gDeVAb dimensionless REAL 4 -999 DeVaucoulerus axis ratio +gDeVAbErr dimensionless REAL 4 -999 estimated error in DeVaucoulerus axis ratio +gDeVPhi degrees REAL 4 -999 estmated phi of DeVaucouleurs axis. Degrees East of North. +gDeVPhiErr degrees REAL 4 -999 estmated error of phi of DeVaucouleurs axis +graDeVOff arcsec REAL 4 -999 Offset in RA of DeVaucouleurs fit from PSF RA +gdecDeVOff arcsec REAL 4 -999 Offset in DEC of DeVaucouleurs fit from PSF DEC +graDeVOffErr arcsec REAL 4 -999 estimated error in ra offset +gdecDeVOffErr arcsec REAL 4 -999 estimated error in dec offset +gDeVCf dimensionless REAL 4 -999 DeVaucouleurs fit coverage factor + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitDeV.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitDeV.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitDeV.txt (revision 37801) @@ -0,0 +1,29 @@ + +objID + from StackObjectThin +uniquePspsSTid = ipptopsps +ippObjID + from StackObjectThin +randomStackObjID + from StackObjectThin +stackDetectRowID - PSPS +primaryDetection - from StackObjectThin NOT YET TESTED +bestDetection - from StackObjectThin CANNOT BE SET until after PV3 +-- +---------------- +gippDetectID + from StackObjectThin +gstackDetectID + from StackObjectThin +gstackMetaID + from StackObjectThin +---------------- +gDeVRadius + cmf : EXT_WIDTH_MAJ +gDeVRadiusErr - cmf : EXT_WIDTH_MAJ_SIG ** NULL in cmf +gDeVMag + cmf : EXT_INST_MAG + 2.5log(exptime) + dvo.zp +gDeVMagErr + cmf : EXT_INST_MAG_SIG ** seems kind of small in CMF +gDeVAb + cmf : EXT_WIDTH_MAJ/EXT_WIDTH_MIN ** should this be Minor / Major? +gDeVAbErr - cmf : EXT_WIDTH_MIN_ERR +gDeVPhi + cmf : EXT_THETA +gDeVPhiErr - cmf : EXT_THETA_ERR +gDeVRa + cmf : use X_EXT and X_PSF, PLTSCALE to offset StackObjectThin.gra +gDeVDec + cmf : use Y_EXT and Y_PSF, PLTSCALE to offset StackObjectThin.gdec +gDeVRaErr + cmf : X_EXT_SIG * PLTSCALE +gDeVDecErr + cmf : Y_EXT_SIG * PLTSCALE +gDeVCf - ** I do not have a source for this in the CMF +gDeVChisq - cmf : EXT_CHISQ / EXT_NDOF +-- gDeVLikelihood - ** replace with chisq? Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitExp =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitExp (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitExp (revision 37801) @@ -0,0 +1,29 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsSTid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +ippObjID dimensionless BIGINT 8 NA object ID generated by IPP +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +------------------------------------------------------- replicate below for all filters (grizy) +gippDetectID dimensionless BIGINT 8 NA ipp detect identifier +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +---------------------------------------------------------------------------------------- +gExpRadius arcsec REAL 4 -999 Exponential fit radius +gExpRadiusErr arcsec REAL 4 -999 estimated error in Exponential fit radius +gExpMag AB_mag REAL 4 -999 Exponential fit flux +gExpMagErr AB_mag REAL 4 -999 estimated error in expFlux +gExpAb dimensionless REAL 4 -999 Exponential fit axis ratio (minor / major) +gExpAbErr dimensionless REAL 4 -999 estimated error in Exponential fit axis ratio +gExpPhi degrees REAL 4 -999 estmated phi of Exponential axis +gExpPhiErr degrees REAL 4 -999 estmated error of phi of Exponential axis +gExpRa degrees DOUBLE 8 -999 RA of Exponential fit +gExpDec degrees DOUBLE 8 -999 DEC of Exponential fit +gExpRaErr arcsec REAL 4 -999 estimated error in RA +gExpDecErr arcsec REAL 4 -999 estimated error in DEC +gExpCf dimensionless REAL 4 -999 Exponential fit coverage factor +gExpChisq dimensionless REAL 4 -999 Exponential fit chi square +-- gExpLikelihood dimensionless REAL 4 -999 Exponential fit likelihood factor Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitExp.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitExp.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitExp.txt (revision 37801) @@ -0,0 +1,28 @@ + +objID + from StackObjectThin +uniquePspsSTid = ipptopsps +ippObjID + from StackObjectThin +randomStackObjID + from StackObjectThin +stackDetectRowID - PSPS +primaryDetection - from StackObjectThin NOT YET TESTED +bestDetection - from StackObjectThin CANNOT BE SET until after PV3 +-- +---------------- +gippDetectID + from StackObjectThin +gstackDetectID + from StackObjectThin +gstackMetaID + from StackObjectThin +---------------- +gExpRadius + cmf : EXT_WIDTH_MAJ +gExpRadiusErr - cmf : EXT_WIDTH_MAJ_SIG ** NULL in cmf +gExpMag + cmf : EXT_INST_MAG + 2.5log(exptime) + dvo.zp +gExpMagErr + cmf : EXT_INST_MAG_SIG ** seems kind of small in CMF +gExpAb + cmf : EXT_WIDTH_MIN/EXT_WIDTH_MAJ +gExpAbErr - cmf : EXT_WIDTH_MIN_ERR +gExpPhi + cmf : EXT_THETA ** in radians, not degrees (fixed, check) +gExpPhiErr - cmf : EXT_THETA_ERR +gExpRa + cmf : use X_EXT and X_PSF, PLTSCALE to offset StackObjectThin.gra +gExpDec + cmf : use Y_EXT and Y_PSF, PLTSCALE to offset StackObjectThin.gdec +gExpRaErr + cmf : X_EXT_SIG * PLTSCALE +gExpDecErr + cmf : Y_EXT_SIG * PLTSCALE +gExpCf - ** I do not have a source for this in the CMF +gExpChisq - cmf : EXT_CHISQ / EXT_NDOF Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitExtra =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitExtra (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitExtra (revision 37801) @@ -0,0 +1,22 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsSTid dimensionless BIGINT 8 NA unique ipptopsps id +ippObjID dimensionless BIGINT 8 NA object ID generated by IPP +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +-------------------------------------------------------- replicate below for all filters (grizy) +gippDetectID dimensionless BIGINT 8 NA ipp detect identifier +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +-------------------------------------------------------- +gS2 dimensionless REAL 4 -999 (un)smoothness (Cheng 2012, Simard 2002) +glogRT dimensionless REAL 4 -999 total residual (from ellipcally symmetric model) +glogRA dimensionless REAL 4 -999 asymmetric residual (from ellipcally symmetric model) +glogC dimensionless REAL 4 -999 concentration index (Petrosian R_50 / R_90) +glogA dimensionless REAL 4 -999 log of asymmetry index (Gyory & Bell 2010) +gbumpy dimensionless REAL 4 -999 Blakeslee bumpiness +ghalfLightRad arcsec REAL 4 -999 half-light radius +-------------------------------------------------------- replicate below for all filters (grizy) Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitSer =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitSer (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitSer (revision 37801) @@ -0,0 +1,31 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsSTid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +ippObjID dimensionless BIGINT 8 NA object ID generated by IPP +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +-------------------------------------------------------- replicate below for all filters (grizy) +gippDetectID dimensionless BIGINT 8 NA ipp detect identifier +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +---------------------------------------------------------------------------------------- +gSerRadius arcsec REAL 4 -999 Sersic radius +gSerRadiusErr arcsed REAL 4 -999 estimated error in Sersic radius +gSerMag AB_mag REAL 4 -999 Sersic Flux +gSerMagErr AB_mag REAL 4 -999 estimated error in serFlux +gSerAb dimensionless REAL 4 -999 Sersic axis ratio +gSerAbErr dimensionlessn REAL 4 -999 estimated error in Sersic axis ratio +gSerNu dimensionlessn REAL 4 -999 Sersic index +gSerNuErr diemensionless REAL 4 -999 estimated error in Sersic index +gSerPhi degrees REAL 4 -999 estmated phi of Sersic axis +gSerPhiErr degrees REAL 4 -999 estmated error of phi of Sersic axis +gSerRa dimensionless REAL 4 -999 Offset in RA of Sersic fit from PSF RA +gSerDec dimensionless REAL 4 -999 Offset in DEC of Sersic fit from PSF DEC +gSerRaErr dimensionless REAL 4 -999 estimated error in raSerOff +gSerDecErr dimensionless REAL 4 -999 estimated error in decSerOff +gserCf dimensionless REAL 4 -999 Sersic fit coverage factor + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitSer.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitSer.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitSer.txt (revision 37801) @@ -0,0 +1,31 @@ + +objID + from StackObjectThin +uniquePspsSTid = ipptopsps +ippObjID + from StackObjectThin +randomStackObjID + from StackObjectThin +stackDetectRowID - PSPS +primaryDetection - from StackObjectThin NOT YET TESTED +bestDetection - from StackObjectThin CANNOT BE SET until after PV3 +-- +---------------- +gippDetectID + from StackObjectThin +gstackDetectID + from StackObjectThin +gstackMetaID + from StackObjectThin +---------------- +gSerRadius + cmf : EXT_WIDTH_MAJ +gSerRadiusErr - cmf : EXT_WIDTH_MAJ_SIG ** NULL in cmf +gSerMag + cmf : EXT_INST_MAG + 2.5log(exptime) + dvo.zp +gSerMagErr + cmf : EXT_INST_MAG_SIG ** seems kind of small in CMF +gSerAb - cmf : EXT_WIDTH_MAJ/EXT_WIDTH_MIN ** should this be Minor / Major? +gSerAbErr - cmf : EXT_WIDTH_MIN_ERR +gSerNu + cmf : EXT_PAR_07 +gSerNuErr - cmf : 0.1 ** NOTE this is hard-wired +gSerPhi - cmf : EXT_THETA ** in radians, not degrees +gSerPhiErr - cmf : EXT_THETA_ERR +gSerRa + cmf : use X_EXT and X_PSF, PLTSCALE to offset StackObjectThin.gra +gSerDec + cmf : use Y_EXT and Y_PSF, PLTSCALE to offset StackObjectThin.gdec +gSerRaErr + cmf : X_EXT_SIG * PLTSCALE +gSerDecErr + cmf : Y_EXT_SIG * PLTSCALE +gSerCf - ** I do not have a source for this in the CMF +gSerChisq - cmf : EXT_CHISQ / EXT_NDOF +--gSerLikelihood - ** replace with chisq? Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectAttributes =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectAttributes (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectAttributes (revision 37801) @@ -0,0 +1,43 @@ + +objID dimensionless BIGINT 8 NA ODM object identifie +uniquePspsSTid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +ippObjID dimensionless BIGINT 8 NA object ID generated by IPP +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. + the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +------------------------------------------------------------------------------------------ +gippDetectID dimensionless BIGINT 8 NA detection ID generated by IPP +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +------------------------------------------------------------------------------------------ +gxPos pixels REAL 4 -999 measured x in sky Cell from PSF fit +gyPos pixels REAL 4 -999 measured y in sky Cell from PSF fit +gxPosErr pixels REAL 4 -999 estimated error in x +gyPosErr pixels REAL 4 -999 estimated error in y +gpsfMajorFWHM arcsec REAL 4 -999 PSF width in major axis +gpsfMinorFWHM arcsec REAL 4 -999 PSF width in minor axis +gpsfTheta degrees REAL 4 -999 PSF orientation angle. Degrees East of North. +gpsfCore dimensionless REAL 4 -999 PSF core parameter [k in (1 + kz + z^1.66)] +gpsfLikelihood dimensionless REAL 4 -999 PSF likelihood +gpsfQf dimensionless REAL 4 -999 PSF coverage factor +gpsfQfPerfect dimensionless REAL 4 -999 PSF-weighted fraction of arcsec totally unmasked +gpsfChiSq dimensionless REAL 4 -999 PSF-weighted fraction of arcsec to. +gmomentXX arcsec^2 REAL 4 -999 moment XX +gmomentXY arcsec^2 REAL 4 -999 moment XY +gmomentYY arcsec^2 REAL 4 -999 moment YY +gmomentR1 arcsec REAL 4 -999 moment R1 +gmomentRH arcsec REAL 4 -999 moment RH - or half light radius (REMOVE) +gPSFFlux adu/sec REAL 4 -999 g psf flux from stack +gPSFFluxErr adu/sec REAL 4 -999 estimated error of g psf flux from stack +gApFlux adu/seconds REAL 4 -999 Aperture magnitude +gApFluxErr adu/seconds REAL 4 -999 Aperture magnitude error +gApFillFac dimensionless REAL 4 -999 Aperture fill factor - ~ redundant with psfQF, keep? +gKronFlux adu/seconds REAL 4 -999 Kron flux +gKronFluxErr adu/seconds REAL 4 -999 Kron flux error +gKronRad arcsec REAL 4 -999 Kron radius +gexpTime seconds REAL 4 -999 total exposure time (informational) +gExtNSigma dimensionless REAL 4 -999 EXT_NSIGMA, how extended something is +gzp magnitude REAL 4 0 zeropoint , no error given +gPlateScale arcsec/pixel REAL 4 0 plate scale Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectAttributes.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectAttributes.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectAttributes.txt (revision 37801) @@ -0,0 +1,44 @@ + +objID + from StackObjectThin +uniquePspsSTid = ipptopsps +ippObjID + from StackObjectThin +randomStackObjID + from StackObjectThin +stackDetectRowID - PSPS +primaryDetection - from StackObjectThin NOT YET TESTED +bestDetection - from StackObjectThin CANNOT BE SET until after PV3 + +------------------ +gippDetectID + from StackObjectThin +gstackDetectID + from StackObjectThin +gstackMetaID + from StackObjectThin +------------------ +gxPos + cmf : X_PSF +gyPos + cmf : Y_PSF +gxPosErr + cmf : X_PSF_SIG +gyPosErr + cmf : Y_PSF_SIG +gpsfMajorFWHM + cmf : PSF_FWHM_MAJ +gpsfMinorFWHM + cmf : PSF_FWHM_MIN +gpsfTheta + cmf : PSF_THETA +gpsfCore + cmf : PSF_CORE +gpsfLikelihood - cmf : psfLikelihood(EXT_NSIGMA) ** problem with calculation? plot +gpsfQf + cmf : PSF_QF +gpsfQfPerfect + cmf : PSF_QF_PERFECT +gpsfChiSq + cmf : PSF_CHISQ / PSF_NDOF +gmomentXX + cmf : MOMENTS_XX +gmomentXY + cmf : MOMENTS_XY +gmomentYY + cmf : MOMENTS_YY +gmomentR1 + cmf : MOMENTS_R1 +gmomentRH + cmf : MOMENTS_RH +gPSFFlux + cmf : PSF_INST_FLUX / exptime +gPSFFluxErr + cmf : PSF_INST_FLUX_SIG / exptime +gApFlux + cmf : AP_FLUX / exptime +gApFluxErr + cmf : AP_FLUX_SIG / exptime +gApFillFac + cmf : AP_NPIX / (3.14159265359 * POW(b.AP_MAG_RADIUS - 0.5, 2)) ** adjusted formula, re-check +gApRadius + cmf : AP_MAG_RADIUS +gKronFlux + cmf : KRON_FLUX / exptime +gKronFluxErr + cmf : KRON_FLUX_ERR / exptime +gKronRad + cmf : MOMENTS_R1 * 2.5 +gexpTime + cmf : header EXPTIME +gExtNsigma + cmg : EXT_NSIGMA +gzp + dvo : zp ** clarify in dvopsps +gPlateScale + cmf : PLTSCALE Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectThin =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectThin (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectThin (revision 37801) @@ -0,0 +1,40 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsSTid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +ippObjID dimensionless BIGINT 8 NA object ID generated by IPP [this is a per-object, not per-filter value] +surveyID dimensionless TINYINT 1 NA survey flag identifier +tessID dimensionless TINYINT 1 -1 ID mapping to tessellation names (eg, RINGS.V3, MD01.V3, etc) +projectionID dimensionless SMALLINT 2 -999 valid values 0 - < 10000 +skyCellID dimensionless TINYINT 1 255 valid values 0 - 143 (12x12 grid) +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +dvoRegionID dimensionless INT 4 -999 identifier to dvo spatial region +dataRelease dimensionless TINYINT 1 NA Data release when this detection was originally taken. +------------------------------------------------------------------------------------------ +-- attributes for all stack detections in all filters (these are currently forced anyway) +-- Replicate the section below for each filter, changing the prefex ("g" in this example) +------------------------------------------------------------------------------------------ +gippDetectID dimensionless BIGINT 8 NA detection ID generated by IPP +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +------------------------------------------------------------------------------------------ +gra degrees FLOAT 8 NA Right Ascension +gdec degrees FLOAT 8 NA Declination +graErr arcsec REAL 4 NA Right Ascension error +gdecErr arcsec REAL 4 NA Declination error (same as RA error - need?) +gPSFMag AB_magnitude REAL 4 -999 g psf AB magnitude from stack +gPSFMagErr AB_magnitude REAL 4 -999 estimated error of g psf AB magnitude from stack +gApMag AB_magnitude REAL 4 -999 g Aperture AB magnitude from stack +gApMagErr AB_magnitude REAL 4 -999 estimated error of g Ap Mag AB magnitude from stack +gKronMag AB_magnitude REAL 4 -999 g kron AB magnitude from stack +gKronMagErr AB_magnitude REAL 4 -999 estimated error of g kron AB magnitude from stack +gsky adu/secs/arcsec REAL 4 -999 PSF sky level at source (adu) +gskyErr adu/secs/arcsec REAL 4 -999 estimated error in sky +ginfoFlag dimensionless INT 4 0 informational bits +ginfoFlag2 dimensionless INT 4 0 informational bits +gnFrames dimensionless INT 4 -999 Number of frames contributing to source + +-- [220 byte/filter/obj + 44 byte/obj] Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectThin.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectThin.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObjectThin.txt (revision 37801) @@ -0,0 +1,40 @@ + +-- fields marked with + below have been verified by EAM in the ippToPsps output mysql tables + +objID + dvo = average.extID [calculated in addstar] +uniquePspsSTid = ipptopsps +ippObjID + dvo = average.objID + (average.catID << 32) [NOTE: this is a change in definition] +surveyID + ipptopsps : internal, from survey name +tessID + smf : TESS_ID (table mapping) +projectionID + smf : parse SKYCELL (skycell.NNN or skycell.MMMM.NNN) +skycellID + smf : parse SKYCELL (skycell.NNN or skycell.MMMM.NNN) +randomStackObjID + ipptopsps (RANDOM on sql into mysql) +stackDetectRowID - set by PSPS +primaryDetection + (dvo:measure.dbFlags >> 16) +bestDetection - dvo * THIS CANNOT BE SET UNTIL LATER (after PV3 initial ingest to psps) + +dvoRegionID + dvo = measure.catID +dataRelease + ippToPsps : user input (skychunk.dataRelease) +-- +-- +-- +-- +gippDetectID + dvopsps.ippDetectID = dvo:measure.detID = smf / IPP_IDET +gstackDetectID + dvopsps.detectID = dvo:measure.extID [calculated in addstar: psps stack detect ID] +gstackMetaID + ipptopsps : gpc1 db, matches cmf IMAGEID +-- +gra + dvo = measure.ra +gdec + dvo = measure.dec +graErr + dvo = X_PSF_SIG * PLTSCALE +gdecErr + dvo = X_PSF_SIG * PLTSCALE +gPSFMag + dvo = measure.Mpsf +gPSFMagErr + dvo = measure.dMpsf +gApMag + dvo = measure.Map +gApMagErr + dvo = measure.dMap +gKronMag + dvo = measure.Mkron +gKronMagErr + dvo = measure.dMkron +gsky + smf : SKY / exptime +gskyErr + smf : SKY_SIGMA / exptime +ginfoFlag + smf : FLAGS +ginfoFlag2 - smf + dvo : FLAGS2 | measure.flags << 13 *** we only include measure.dbflags, not secfilt.flags +gnFrames + smf : N_FRAMES Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackPetrosian =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackPetrosian (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackPetrosian (revision 37801) @@ -0,0 +1,23 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +uniquePspsSTid dimensionless BIGINT 8 NA unique identifier provided by ipptopsps +ippbjID dimensionless BIGINT 8 NA object ID generated by IPP +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +-------------------------------------------------------- replicate below for all filters (grizy) +gippDetectID dimensionless BIGINT 8 NA ipp detect identifier +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +---------------------------------------------------------------------------------------- +gpetRadius arcsec REAL 4 -999 Petrosian radius +gpetRadiusErr arcsec REAL 4 -999 estimated error inPetrosian radius +gpetMag AB_mag REAL 4 -999 Petrosian Flux +gpetMagErr AB_mag REAL 4 -999 estimated error in petFlux +gpetR50 arcsec REAL 4 -999 Petrosian radius at 50% light +gpetR50Err arcsec REAL 4 -999 estimated error inPetrosian radius at 50% light +gpetR90 arcsec REAL 4 -999 Petrosian radius at 90% light +gpetR90Err arcsec REAL 4 -999 estimated error in Petrosian radius at 90% light +gpetCf dimensionless REAL 4 -999 Petrosian fit coverage factor Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackPetrosian.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackPetrosian.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackPetrosian.txt (revision 37801) @@ -0,0 +1,23 @@ + +objID + from StackObjectThin +uniquePspsSTid = ipptopsps +ippobjID + from StackObjectThin +randomStackObjID + from StackObjectThin +stackDetectRowID - PSPS +primaryDetection - from StackObjectThin NOT YET TESTED +bestDetection - from StackObjectThin CANNOT BE SET until after PV3 +-- +---------------- +gippDetectID + from StackObjectThin +gstackDetectID + from StackObjectThin +gstackMetaID + from StackObjectThin +---------------- +gpetRadius + cmf : PETRO_RADIUS +gpetRadiusErr + cmf : PETRO_RADIUS_ERR +gpetMag + cmf : PETRO_MAG - header:FPA.ZP + dvo.zp +gpetMagErr + cmf : PETRO_MAG_ERR +gpetR50 + cmf : PETRO_RADIUS_50 +gpetR50Err + cmf : PETRO_RADIUS_50_ERR +gpetR90 + cmf : PETRO_RADIUS_90 +gpetR90Err + cmf : PETRO_RADIUS_90_ERR +gpetCf - cmf : PETRO_FILL ** consistently > 1.0?? [problem in cmf] Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/changes.r36847 =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/changes.r36847 (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/changes.r36847 (revision 37801) @@ -0,0 +1,199 @@ +Index: Detection +=================================================================== +--- Detection (revision 36838) ++++ Detection (working copy) +@@ -7,7 +7,7 @@ + surveyID dimensionless TINYINT 1 NA survey flag identifier + imageID dimensionless BIGINT 8 NA image id hash of frame identifier & OTA/CCD number + randomDetID dimensionless BIGINT 8 NA random detection Id +-dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region ++dvoRegionID dimensionless INT 4 0 identifier to dvo spatial region + obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation in Mean Julian Days + xPos raw pixels REAL 4 -999 measured x on CCD from PSF fit + yPos raw pixels REAL 4 -999 measured y on CCD from PSF fit +@@ -53,7 +53,8 @@ + kronRadErr arcsec REAL 4 -999 Kron radius error + sky adu/sec/pixel REAL 4 -999 PSF sky level at source + skyErr adu/sec/pixel REAL 4 -999 estimated error in sky +-infoFlag dimensionless BIGINT 8 -999 flag indicating provenance information ++infoFlag dimensionless INT 4 -999 flag indicating provenance information ++infoFlag2 dimensionless INT 4 -999 flag indicating provenance information + dataRelease dimensionless TINYINT 1 NA Data release when this detection was originally taken. Recalibrations do not affect this value. + -- + -- [231 bytes/det] +Index: StackApFlx +=================================================================== +--- StackApFlx (revision 36838) ++++ StackApFlx (working copy) +@@ -1,6 +1,7 @@ + + objID dimensionless BIGINT 8 NA ODM object identifier + gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier ++ippObjID dimensionless BIGINT 8 NA ipp object identifier + surveyID dimensionless TINYINT 1 0 stack type identifier + skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier + randomStackObjID dimensionless BIGINT 8 NA random stack id +@@ -8,13 +9,12 @@ + primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. + bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. + -- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +-dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region ++dvoRegionID dimensionless INT 4 -999 identifier to dvo spatial region + dataRelease dimensionless TINYINT 1 NA Data release when this table was loaded. + ------------------------------------------------------- replicate to end for each filter g,r,i,z,y ++gippDetectID dimensionless BIGINT 8 NA stack identifier + gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier + gstackMetaID dimensionless BIGINT 8 NA stack identifier +-gippObjID dimensionless BIGINT 8 NA ipp object identifier +-gippDetectID dimensionless BIGINT 8 NA stack identifier + --------------------------------------------------------unconvolved + gflxR3 adu/seconds REAL 4 -999 Flux inside r = 3 + gflxR3Err adu/seconds REAL 4 -999 estimated error is flxR3 +Index: ForcedWarpMeasurement +=================================================================== +--- ForcedWarpMeasurement (revision 36838) ++++ ForcedWarpMeasurement (working copy) +@@ -8,7 +8,7 @@ + forcedWarpID dimensionless BIGINT 8 NA IPP warp_id + randomWarpID dimensionless BIGINT 8 NA Random warp_id + skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +-dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region ++dvoRegionID dimensionless INT 4 -999 identifier to dvo spatial region + obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation in Mean Julian Days + ------------------------------------------------------ + zp magnitude REAL 4 0 zeropoint (only thing needed to convert adu/sec to AB mag), note no zpErr +Index: DiffDetection +=================================================================== +--- DiffDetection (revision 36838) ++++ DiffDetection (working copy) +@@ -1,3 +1,4 @@ ++ + diffObjID dimensionless BIGINT 8 NA diff object identifier + diffgcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier + diffDetID dimensionless BIGINT 8 NA diff detection identifier +@@ -8,7 +9,7 @@ + surveyID dimensionless TINYINT 1 NA survey flag identifierr + randomDiffID dimensionless BIGINT 8 NA random diff id + skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +-dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region ++dvoRegionID dimensionless INT 4 0 identifier to dvo spatial region + obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation of positive detection in Mean Julian Days of positive detection + xPos pixels REAL 4 -999 measured x on diff image sky cell PSF fit + yPos pixels REAL 4 -999 measured y on diff image sky cell from PSF fit +@@ -65,5 +66,6 @@ + ------------------------------------------------------- + sky adu/sec/pixel REAL 4 -999 PSF sky level at source + skyErr adu/sec/pixel REAL 4 -999 estimated error in sky +-infoFlag dimensionless BIGINT 8 -999 flag indicating provenance information ++infoFlag dimensionless INT 4 -999 flag indicating provenance information ++infoFlag2 dimensionless INT 4 -999 flag indicating provenance information + dataRelease dimensionless TINYINT 1 NA Data release when this detection was originally taken. Recalibrations do not affect this value. +Index: ObjectThin +=================================================================== +--- ObjectThin (revision 36838) ++++ ObjectThin (working copy) +@@ -9,7 +9,7 @@ + zoneID dimensionless INT 4 NA Zone index + randomID dimensionless BIGINT 8 0 Random integer between 0 and 2^32 = 4,394,967,296 remaining ramdom integer 0 + batchID dimensionless BIGINT 8 -999 batch id +-dvoRegionID dimensionless REAL 4 -999 identifier of dvo spatial region - dvo cat id ++dvoRegionID dimensionless INT 4 -999 identifier of dvo spatial region - dvo cat id + dataRelease dimensionless TINYINT 1 NA Data release Version of this table + objInfoFlag dimensionless BIGINT 8 -999 flag indicating odd properties + qualityFlag dimensionless TINYINT 1 0 subset of objInfoFlag - flags related to goodness of object +Index: StackModelFit +=================================================================== +--- StackModelFit (revision 36838) ++++ StackModelFit (working copy) +@@ -1,6 +1,7 @@ + + objID dimensionless BIGINT 8 NA ODM object identifier + gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier ++ippObjID dimensionless BIGINT 8 NA ipp object identifier + surveyID dimensionless TINYINT 1 NA survey flag identifier + skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier + randomStackObjID dimensionless BIGINT 8 NA random stack id +@@ -8,17 +9,17 @@ + primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. + bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. + -- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +-dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region ++dvoRegionID dimensionless INT 4 -999 identifier to dvo spatial region + dataRelease dimensionless TINYINT 1 NA Data release when this table was loaded + -------------------------------------------------------- replicate below for all filters (grizy) + glogC dimensionless REAL 4 -999 concentration index + glogA dimensionless REAL 4 -999 asymmetry index + gclump dimensionless REAL 4 -999 clumpiness factor + -------------------------------------------------------- replicate below for all filters (grizy) ++gippDetectID dimensionless BIGINT 8 NA ipp detect identifier + gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier + gstackMetaID dimensionless BIGINT 8 NA stack identifier +-gippObjID dimensionless BIGINT 8 NA ipp object identifier +-gippDetectID dimensionless BIGINT 8 NA ipp detect identifier ++---------------------------------------------------------------------------------------- + gdeVRadius arcsec REAL 4 -999 deVaucouleurs radius + gdeVRadiusErr arcsec REAL 4 -999 estimated error in deVaucouleurs radius + gdeVFlux adu/seconds REAL 4 -999 deVaucouleurs flux +Index: ForcedMeanObject +=================================================================== +--- ForcedMeanObject (revision 36838) ++++ ForcedMeanObject (working copy) +@@ -1,3 +1,4 @@ ++ + objID dimensionless BIGINT 8 NA ODM object identifier index + gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier + ippObjID dimensionless BIGINT 8 NA IPP object identifier +Index: DiffDetObject +=================================================================== +--- DiffDetObject (revision 36838) ++++ DiffDetObject (working copy) +@@ -9,7 +9,7 @@ + zoneID dimensionless INT 4 NA Zone index + randomID dimensionless BIGINT 8 0 Random integer between 0 and 2^32 = 4,394,967,296 remaining ramdom integer 0 + batchID dimensionless BIGINT 8 -999 +-dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region ++dvoRegionID dimensionless INT 4 -999 identifier to dvo spatial region + objInfoFlag dimensionless BIGINT 8 -999 flag indicating odd properties + qualityFlag dimensionless TINYINT 1 0 subset of objInfoFlag - flags related to goodness of object + consistencyFlag dimensionless SMALLINT2 0 Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation +Index: StackObject +=================================================================== +--- StackObject (revision 36838) ++++ StackObject (working copy) +@@ -1,6 +1,7 @@ + + objID dimensionless BIGINT 8 NA ODM object identifier + gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier ++ippObjID dimensionless BIGINT 8 NA object ID generated by IPP + surveyID dimensionless TINYINT 1 NA survey flag identifier + skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier + randomStackObjID dimensionless BIGINT 8 NA random stack id +@@ -8,16 +9,16 @@ + primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. + bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. + the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +-dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region ++dvoRegionID dimensionless INT 4 -999 identifier to dvo spatial region + dataRelease dimensionless TINYINT 1 NA Data release when this detection was originally taken. + ------------------------------------------------------------------------------------------ + -- attributes for all stack detections in all filters (these are currently forced anyway) + -- Replicate the section below for each filter, changing the prefex ("g" in this example) + ------------------------------------------------------------------------------------------ +-gippObjID dimensionless BIGINT 8 NA object ID generated by IPP + gippDetectID dimensionless BIGINT 8 NA detection ID generated by IPP + gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier + gstackMetaID dimensionless BIGINT 8 NA stack identifier ++------------------------------------------------------------------------------------------ + gxPos arcsec REAL 4 -999 measured x in sky Cell from PSF fit + gyPos arcsec REAL 4 -999 measured y in sky Cell from PSF fit + gxPosErr arcsec REAL 4 -999 estimated error in x +@@ -57,7 +58,8 @@ + gKronMagErr AB magnitude REAL 4 -999 estimated error of g kron AB magnitude from stack + gsky adu/secs/arcsec REAL 4 -999 PSF sky level at source (adu) + gskyErr adu/secs/arcsec REAL 4 -999 estimated error in sky +-ginfoFlag dimensionless BIGINT 8 -999 indicator of strange propeties ++ginfoFlag dimensionless INT 4 -999 indicator of strange propeties ++ginfoFlag2 dimensionless INT 4 -999 indicator of strange propeties + gnFrames dimensionless INT 4 -999 Number of frames contributing to source + + -- [220 byte/filter/obj + 44 byte/obj] Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/notes.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/notes.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/notes.txt (revision 37801) @@ -0,0 +1,22 @@ + +tables processed by the different batch types: + +objectbatch.py: + ObjectThin + MeanObject + +detectionbatch.py: + Detection + +stackbatch.py: + StackObjectThin - populated for all stack detections * checked + StackObjectRaw - populated for all stack detections * checked + StackModelFitThin - populated for Galactic Cap regions + StackModelFitDeV - populated for Galactic Cap regions * checked + StackModelFitExp - populated for Galactic Cap regions * checked + StackModelFitSer - populated for Galactic Cap regions * checked + StackApFlx - populated for all stack detections + StackApFlxFull - populated for Galactic Cap regions * checked + StackApFlxFullC1 - populated for Galactic Cap regions * checked + StackApFlxFullC2 - populated for Galactic Cap regions * checked + StackPetrosian - populated for Galactic Cap regions * checked Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/schema18.notes.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/schema18.notes.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/schema18.notes.txt (revision 37801) @@ -0,0 +1,160 @@ + +This document describes the new PSPS schema (18.0) and identifies existing problems. + +ObjectThin + + Changes since Schema 16.0: + + * DROPPED gcObjID (concept rejected by DRAVG) + * replaced skycellID (REAL) with combination of tessID, projectionID, skycellID (1,2,1 byte INTs) + * replaced raMeanStd, decMeanStd (not calculated) with posMeanChisq + + Concerns: + + * objName is generated by SQL in ippToPsps and is probably fragile: move to dvopsps? + * fields calculated by PSPS and not tested (htmID, zoneID, cx,cy,cz, lambda,beta, l,b) + * nStackObjectRows is not calculated + * should nStackDetections be split per filter? + * quality flag is currently the upper byte of dvo:average.flags, is this reasonable? + * consistency flag is hard-wired as 0 + * raStack, etc are not yet calculated by relastro (set to match raMean, etc) + +MeanObject + + Changes since Schema 16.0: + + * DROPPED gcObjID (concept rejected by DRAVG) + * CHANGED gFlags to be 4 byte not 8 byte (secfilt.flags is 4 byte) + * renamed fields to have consistent naming structure: + * gMeanTYPEMagVALUE, where TYPE = (PSF, Kron, Ap) and VALUE = (Err, Std, Min Max, Npt) + + Concerns: + + * NONE + +Detections + + Changes since Schema 16.0: + + * ADDED apRadius + * DROPPED gcObjID (concept rejected by DRAVG) + * DROPPED kronRadErr (not measured by psphot, probably not needed?) + + Concerns: + + * imageID : this is currently generated from expID + ccd X,Y. + shouldn't we use gpc1.chipImfile.chip_imfile_id? (IMAGE_ID in + header) + * double-check reported value for zero point (needs to include airmass term, etc) + + Implementation Changes: + + * raErr,decErr are now taken from the SMF file not from dvo (x,y errors truncated in dvo) + * dvoRegion now uses CAT_ID from dvopsps + * psfMajor,MinorFWHM use PSF_FWHM_MJA,MIN + * added PSF_CORE to smf + * apFlux, kronFlux use FLUX values directly from smf + +Stacks + + Measurements related to the stacks are now split into 11 separate + tables depending on the measurements and, in some cases, the Galactic + latitude. The granularity of the split is partly mandated by the + inability of PSPS or its ingest system to handle excessively wide + tables. Note that in all of the tables below, a single row represents + the set of measurements from all 5 filters (grizy) for the aparition + of an astronomical object in a single skycell. The primaryDetection + flag is set to identify the row which comes from the primary skycell + for that location on the sky. + + Several of the tables are meant to be filled only for the Galactic cap + regions. As of 2014.08.09, ippToPsps is not setting this limit. + These tables are: + + StackModelFitDeV + StackModelFitExp + StackModelFitSer + StackModelFitExtra + StackApFlxFull + StackApFlxFullC1 + StackApFlxFullC2 + StackPetrosian + + Other general concerns: + + * various size-related fields are reported in pixels not arcseconds. + + * I need to double-check the reported zero point and telluric + extinction values. + + * are we defining too many radial apertures? + + The tables are divided as follows: + + StackObjectThin : basic properties of the stack object detection. + * stackDetectRowID : this field will be set by PSPS and cannot be + tested in the ippToPsps code + + * bestDetection : this field cannot be set until the entire PSPS is + populated. this will be deferred until the end of + the ingest. + + * primaryDetection : NOT YET TESTED + * xinfoFlag2 : currently only includes bits from + dvo:measure.dbflags but not dvo:secfilt.flags + + * NOTE : stackDetectRowID, bestDetection, primaryDetection are + inherited by the following tables from StackObjectThin, so the issue + above are present in all Stack tables + + StackObjectRaw : raw measurement values from the PSF fits + * xpsfLikelihood : some values seem to be negative, which should + not be allowed + + StackModelFitExtra : extra properties derived from extended source + fits. NOTE: this table is not being generated + + StackModelFitExp : Exponential model fit parameters + * xExpRadiusErr : EXT_WIDTH_MAJ_SIG is not being set in the CMF file + * xExpAbErr : EXT_WIDTH_MIN_SIG is not being set in the CMF file + * xExpPhiErr : not set in CMF + * xExpRa,Dec : this position is currently an approximation based on + X_EXT, X_PSF and the plate scale + * xExpRa,DecErr : circularly symmetric approximation based on S/N + * xExpCf : not calculated in psphot + * xExpChisq : the fit chisq is provided, not the likelihood + + StackModelFitDeV : DeVaucouleur model fit parameters + * NOTE same issues as for Exp + + StackModelFitSer : Sersic model fit parameters + * NOTE same issues as for Exp, also: + * xSerNuErr : value is hard-wired at 0.1, and not well-measured + in CMF + + StackApFlx : flux measured in radial apertures (only 3 radii; raw and + both convolutions) + + * standard deviation fields (eg, gflxR3Std) are measured in psphot as + the standard deviation across the full object. Should this perhaps + be a measurement of the standard deviation after the radial profile + is subtracted? + + StackApFlxFull : flux measured in radial apertures (raw, all radii) + + * see note above on Standard Deviation + * radius 10 is not measured or reported in the CMF + + StackApFlxFullC1 : flux measured in radial apertures (first convolution, all radii) + + * see note above on Standard Deviation + * radius 10 is not measured or reported in the CMF + + StackApFlxFullC2 : flux measured in radial apertures (second convolution, all radii) + + * see note above on Standard Deviation + * radius 10 is not measured or reported in the CMF + + StackPetrosian : petrosian analysis data + * xpetCf : fill factor is mis-calculated in psphot (consistently > 1.0) + * NOTE that PETRO_MAG_ERR is inverted in pmSourceIO_CMF.c.in r37244 Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/CameraConfig =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/CameraConfig (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/CameraConfig (revision 37801) @@ -0,0 +1,12 @@ +CameraConfig +Description: Camera configuration + +Name Unit DataType Size Default ValueDescription + +cameraID dimensionless SMALLINT 2 -999 camera identifier (i.e. GPC1, ISP) +cameraConfigID dimensionless SMALLINT 2 -999 camera configuration +endDate dimensionless REAL 4 -999 ending date of this configuration in MJD +configDetails string VARCHAR(MAX) 8000 0x00 NA text/table giving the details of the camera configuration + NEED TO CHANGE and pass camera x from raw data file + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DetectionFlags =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DetectionFlags (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DetectionFlags (revision 37801) @@ -0,0 +1,10 @@ +DetectionFlags + +Description: Flags used to interpret the information flag provided +with each detection (infoFlag in 'Detection' table) + +Name Unit Data Type Size Default Value Description + +name VARCHAR(32) 32 NA Name of flag +value BIGINT 8 NA Flag value +description VARCHAR(2000) 2000 NA Description of the flag Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DiffMeta =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DiffMeta (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DiffMeta (revision 37801) @@ -0,0 +1,26 @@ +diffMetaID dimensionless BIGINT 8 NA diff identifier +batchID dimensionless BIGINT 8 NA batch identifier +surveyID dimensionless TINYINT 1 NA survey flag identifier +filterID dimensionless TINYINT 1 NA filter id +diffTypeID dimensionless TINYINT 1 0 diff type identifier +posImageID dimensionless BIGINT 8 NA positive image identifier +negImageID dimensionless BIGINT 8 NA negative image identifier +skyCellID dimensionless REAL 4 NA skycell region identifier +photoCalID dimensionless INT 4 NA photometry code +magSat dimensionless REAL 4 -999 saturation magnitude level +analVer alphanumeric VARCHAR(100) 100 -999 analysis version index +expTime seconds REAL 4 -999 exposure time of positive image +ctype1 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in RA +ctype2 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in DEC +crval1 degrees FLOAT 8 -999 RA corresponding to reference pixel +crval2 degrees FLOAT 8 -999 DEC corresponding to reference pixel +crpix1 dimensionless FLOAT 8 -999 reference pixel value for RA +crpix2 dimensionless FLOAT 8 -999 reference pixel value for DEC +cdelt1 deg/pixel FLOAT 8 -999 scale factor for RA +cdelt2 deg/pixel FLOAT 8 -999 scale factor for DEC +pc001001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc001002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +calibModNum dimensionless SMALLINT 2 NA calibration modification number +dataRelease dimensionless TINYINT 1 NA Data release Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DiffToImage =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DiffToImage (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/DiffToImage (revision 37801) @@ -0,0 +1,2 @@ +diffMetaID BIGINT 8 NA diff identifier +imageID BIGINT 8 NA hashed exposure-ccdID identifier Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Filter =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Filter (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Filter (revision 37801) @@ -0,0 +1,3 @@ +FilterID dimensionless TINYNT 1 NA filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... +filterType VARCHAR(2) 2 NA filter name: g,r,i,z,y, etc. +filterSpec VARCHAR(MAX) 8000 0x00 No comment Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/FitModel =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/FitModel (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/FitModel (revision 37801) @@ -0,0 +1,42 @@ +fitModelID SMALLINT 2 NA indentifier of alternate model fit +name VARCHAR(30) 30 model name +description VARCHAR(MAX) 8000 string describing the model +param1 VARCHAR(MAX) 8000 description of model parameter 1 +param2 VARCHAR(MAX) 8000 description of model parameter 2 +param3 VARCHAR(MAX) 8000 description of model parameter 3 +param4 VARCHAR(MAX) 8000 description of model parameter 4 +param5 VARCHAR(MAX) 8000 description of model parameter 5 +param6 VARCHAR(MAX) 8000 description of model parameter 6 +param7 VARCHAR(MAX) 8000 description of model parameter 7 +param8 VARCHAR(MAX) 8000 description of model parameter 8 +param9 VARCHAR(MAX) 8000 description of model parameter 9 +param10 VARCHAR(MAX) 8000 description of model parameter 10 +param11 VARCHAR(MAX) 8000 description of model parameter 11 +param12 VARCHAR(MAX) 8000 description of model parameter 12 +param13 VARCHAR(MAX) 8000 description of model parameter 13 +param14 VARCHAR(MAX) 8000 description of model parameter 14 +param15 VARCHAR(MAX) 8000 description of model parameter 15 +param16 VARCHAR(MAX) 8000 description of model parameter 16 +param17 VARCHAR(MAX) 8000 description of model parameter 17 +param18 VARCHAR(MAX) 8000 description of model parameter 18 +param19 VARCHAR(MAX) 8000 description of model parameter 19 +param20 VARCHAR(MAX) 8000 description of model parameter 20 +param21 VARCHAR(MAX) 8000 description of model parameter 21 +param22 VARCHAR(MAX) 8000 description of model parameter 22 +param23 VARCHAR(MAX) 8000 description of model parameter 23 +param24 VARCHAR(MAX) 8000 description of model parameter 24 +param25 VARCHAR(MAX) 8000 description of model parameter 25 +param26 VARCHAR(MAX) 8000 description of model parameter 26 +param27 VARCHAR(MAX) 8000 description of model parameter 27 +param28 VARCHAR(MAX) 8000 description of model parameter 28 +param29 VARCHAR(MAX) 8000 description of model parameter 29 +param30 VARCHAR(MAX) 8000 description of model parameter 30 +param31 VARCHAR(MAX) 8000 description of model parameter 31 +param32 VARCHAR(MAX) 8000 description of model parameter 32 +param33 VARCHAR(MAX) 8000 description of model parameter 33 +param34 VARCHAR(MAX) 8000 description of model parameter 34 +param35 VARCHAR(MAX) 8000 description of model parameter 35 +param36 VARCHAR(MAX) 8000 description of model parameter 36 +param37 VARCHAR(MAX) 8000 description of model parameter 37 +param38 VARCHAR(MAX) 8000 description of model parameter 38 +param39 VARCHAR(MAX) 8000 description of model parameter 39 Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ForcedWarpMeta =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ForcedWarpMeta (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ForcedWarpMeta (revision 37801) @@ -0,0 +1,33 @@ +forcedWarpID dimensionless BIGINT 8 NA stack identifier +batchID dimensionless BIGINT 8 NA batch identifier +surveyID dimensionless TINYINT 1 NA survey flag identifier +filterID dimensionless TINYINT 1 NA fnilutemr eidriecnatl ifidier +skyCellID dimensionless REAL 4 NA skycell region identifier +photoCalID dimensionless INT 4 NA photometry code +magSat dimensionless REAL 4 -999 saturation magnitude level +analVer dimensionless SMALLINT 2 -999 analysis version index +expTime seconds REAL 4 -999 exposure time +completMag magnitudes REAL 4 -999 95% completion level in mag +astroScat dimensionless REAL 4 -999 astrometric scatter for chip +photoScat dimensionless REAL 4 -999 photometric scatter for chip +nAstroRef dimensionless INT 4 -999 number of astrometric reference sources +nPhoRef dimensionless INT 4 -999 number of photometric reference sources +psfModelID dimensionless VARCHAR(MAX) 8000 PSF model identifier +psfFwhm_mean arcsec FLOAT 8 -999 mean PSF FWHM in the stack +psfFwhm_max arcsec FLOAT 8 -999 maximum PSF FWHM in the stack +photoZero magnitudes REAL 4 -999 local derived photometric zero point +photoColor magnitudes REAL 4 -999 local derived photometric color term +ctype1 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in RA +ctype2 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in DEC +crval1 degrees FLOAT 8 -999 RA corresponding to reference pixel +crval2 degrees FLOAT 8 -999 DEC corresponding to reference pixel +crpix1 dimensionless FLOAT 8 -999 reference pixel value for RA +crpix2 dimensionless FLOAT 8 -999 reference pixel value for DEC +cdelt1 deg/pixel FLOAT 8 -999 scale factor for RA +cdelt2 deg/pixel FLOAT 8 -999 scale factor for DEC +pc001001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc001002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +calibModNum dimensionless SMALLINT 2 NA calibration modification number +dataRelease dimensionless TINYINT 1 NA Data release Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ForcedWarpToImage =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ForcedWarpToImage (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ForcedWarpToImage (revision 37801) @@ -0,0 +1,2 @@ +forcedWarpID BIGINT 8 NA forced warp identifier +imageID BIGINT 8 NA hashed exposure-ccdID identifier Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/FrameMeta =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/FrameMeta (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/FrameMeta (revision 37801) @@ -0,0 +1,49 @@ +frameID dimensionless INT 4 NA unique exposure/frame identifier. +frameName dimensionless VARCHAR(32) 32 NA frame name provided by camera software +surveyID dimensionless TINYINT 1 NA survey flag identifier +filterID dimensionless TINYINT 1 NA filter identifier +cameraID dimensionless SMALLINT 2 NA camera identifier +cameraConfigID dimensionless SMALLINT 2 NA camera configuration identifier +telescopeID dimensionless SMALLINT 2 NA telescope identifier +analysisVer alphanumeric VARCHAR(MAX) 8000 NA IPP software analysis release +p1Recip alphanumeric VARCHAR(MAX) 8000 NA IPP phase 1 MD5 Checksum +p2Recip alphanumeric VARCHAR(MAX) 8000 NA IPP phase 2 MD5 Checksum +p3Recip alphanumeric VARCHAR(MAX) 8000 NA IPP phase 3 MD5 Checksum +nOTA alphanumeric SMALLINT 2 -999 number of valid OTA/CCD images in this exposure +photoScat magnitude REAL 4 -999 global photometric scatter +numPhotoRef dimensionless INT 4 -999 number of photometric reference sources +expStart days FLOAT 8 -999 exposure start time in MJD +expTime seconds REAL 4 -999 exposure time +airmass dimensionless REAL 4 -999 airmass at mid-exposure +raBore degrees FLOAT 8 -999 RA of telescope boresight +decBore degrees FLOAT 8 -999 DEC of telescope boresight +ctype1 alphanumeric VARCHAR(MAX) 8000 NA name of astrometric projection in RA +ctype2 alphanumeric VARCHAR(MAX) 8000 NA name of astrometric projection in DEC +crval1 degrees FLOAT 8 -999 RA corresponding to reference pixel +crval2 degrees FLOAT 8 -999 DEC corresponding to reference pixel +crpix1 dimensionless FLOAT 8 -999 reference pixel value for RA +crpix2 dimensionless FLOAT 8 -999 reference pixel value for DEC +cdelt1 degrees/pixel FLOAT 8 -999 scale factor for RA +cdelt2 degrees/pixel FLOAT 8 -999 scale factor for DEC +pc001001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc001002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +polyOrder dimensionless TINYINT 1 255 polynomial order of astrometry fit +pca1x3y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x2y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x1y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x0y3 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x2y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x1y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x0y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x3y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x2y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x1y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x0y3 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x2y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x1y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x0y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +calibModNum dimensionless SMALLINT 2 NA calibration modification number +batchID dimensionless BIGINT 8 NA batch ID +dataRelease dimensionless TINYINT 1 NA Data release Version of this table Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/FrameMeta.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/FrameMeta.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/FrameMeta.txt (revision 37801) @@ -0,0 +1,50 @@ +frameID | ipptopsps : gpc1.rawExp.exp_id +frameName | ipptopsps : gpc1.rawExp.exp_name +surveyID | ipptopsps : internal, from survey name +filterID | ipptopsps : internal, lookup from filter (grizy) +cameraID | ipptopsps : hardwired to 1 *** +cameraConfigID | ipptopsps : hardwired to 1 *** +telescopeID | ipptopsps : hardwired to 1 *** +analysisVer | ipptopsps : gpc1.camRun.software_ver +p1Recip | * NOT SET +p2Recip | * NOT SET +p3Recip | * NOT SET +nOTA | ipptopsps : count of OTAs +photoScat | cmf: ZPT_ERR +numPhotoRef | sum(cmf:NASTRO) +expStart | cmf:MJD-OBS +expTime | cmf:EXPREQ +airmass | cmf:AIRMASS +raBore | cmf:RA +decBore | cmf:DEC +ctype1 | cmf:CTYPE1 +ctype2 | cmf:CTYPE2 +crval1 | cmf:CRVAL1 +crval2 | cmf:CRVAL2 +crpix1 | cmf:CRPIX1 +crpix2 | cmf:CRPIX2 +cdelt1 | cmf:CDELT1 +cdelt2 | cmf:CDELT2 +pc001001 | cmf:PC001001 +pc001002 | cmf:PC001002 +pc002001 | cmf:PC002001 +pc002002 | cmf:PC002002 +polyOrder | cmf:NPLYTERM +pca1x3y0 | cmf:PCA1X3Y0 +pca1x2y1 | cmf:PCA1X2Y1 +pca1x1y2 | cmf:PCA1X1Y2 +pca1x0y3 | cmf:PCA1X0Y3 +pca1x2y0 | cmf:PCA1X2Y0 +pca1x1y1 | cmf:PCA1X1Y1 +pca1x0y2 | cmf:PCA1X0Y2 +pca2x3y0 | cmf:PCA2X3Y0 +pca2x2y1 | cmf:PCA2X2Y1 +pca2x1y2 | cmf:PCA2X1Y2 +pca2x0y3 | cmf:PCA2X0Y3 +pca2x2y0 | cmf:PCA2X2Y0 +pca2x1y1 | cmf:PCA2X1Y1 +pca2x0y2 | cmf:PCA2X0Y2 +calibModNum | ipptopsps: hardwired to 0 *** +batchID | ipptopsps: internal +dataRelease | ipptopsps: user input (skychunk.dataRelease) + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ImageFlags =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ImageFlags (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ImageFlags (revision 37801) @@ -0,0 +1,3 @@ +name VARCHAR(32) 32 NA Name of flag +value BIGINT 8 NA Flag value +description VARCHAR(2000) 2000 Description of the flag Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ImageMeta =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ImageMeta (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ImageMeta (revision 37801) @@ -0,0 +1,71 @@ +imageID dimensionless BIGINT 8 NA unique ID for each image, hashed from frameID and ccdID +frameID dimensionless INT 4 NA frame identifier, also known as exposure number +ccdID dimensionless SMALLINT 2 NA ccd cell identifier in the focal plane +photoCalID dimensionless INT 4 NA photometry reduction code identifier +filterID dimensionless TINYINT 1 NA filter ID +bias ADU REAL 4 -999 detector bias level +biasScat ADU REAL 4 -999 scatter in bias level +sky ADU/pixel REAL 4 -999 mean sky level +skyScat ADU/pixel REAL 4 -999 scatter in mean sky level +nDetect dimensionless INT 4 -999 number of detections on CCD +magSat magnitude REAL 4 -999 saturation magnitude level +completMag magnitude REAL 4 -999 95% completion level in mag +astroScat arcsec REAL 4 -999 astrometric scatter for chip +photoScat magitude REAL 4 -999 photometric scatter for chip +numAstroRef dimensionless INT 4 -999 number of astrometric reference sources +numPhotoRef dimensionless INT 4 -999 number of photometric reference sources +nx pixels SMALLINT 2 -999 chip dimension in x +ny pixels SMALLINT 2 -999 chip dimension in y +psfModelID dimensionless INT 4 -999 PSF model identifier +psfFwhm arcsec REAL 4 -999 model psf full width at half maximum at chip center +psfWidMajor arcsec REAL 4 -999 model PSF parameters at chip center +psfWidMinor arcsec REAL 4 -999 model PSF parameters at chip center +psfTheta degrees REAL 4 -999 model PSF parameters at chip center +momentMajor pixels REAL 4 -999 momentMajor +momentMinor pixels REAL 4 -999 momentMinor +momentM2C pixels squared REAL 4 -999 momentM2C +momentM2S pixels squared REAL 4 -999 momentM2S +momentM3 pixels squared REAL 4 -999 momentM3 +momentM4 pixels squared REAL 4 -999 momentM4 +apResid magnitude REAL 4 -999 corrected aperture residual +dapResid magnitude REAL 4 -999 scatter of aperture corrections +detectorID alphanumeric VARCHAR(32) 32 identifier for actual CCD chip +qaFlags dimensionless BIGINT 8 -999 Q/A flags for this OTA +detrend1 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 1 +detrend2 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 2 +detrend3 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 3 +detrend4 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 4 +detrend5 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 5 +detrend6 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 6 +detrend7 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 7 +detrend8 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 8 +photoZero magnitude REAL 4 -999 local derived photometric zero point +ctype1 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in RA +ctype2 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in DEC +crval1 degrees FLOAT 8 -999 RA corresponding to reference pixel +crval2 degrees FLOAT 8 -999 DEC corresponding to reference pixel +crpix1 pixels FLOAT 8 -999 reference pixel value for RA +crpix2 pixels FLOAT 8 -999 reference pixel value for DEC +cdelt1 degrees/pixel FLOAT 8 -999 scale factor for RA +cdelt2 degrees/pixel FLOAT 8 -999 scale factor for DEC +pc001001 unknown FLOAT 8 -999 elements of rotation/Dcale matrix +pc001002 unknown FLOAT 8 -999 elements of rotation/Dcale matrix +pc002001 unknown FLOAT 8 -999 elements of rotation/Dcale matrix +pc002002 unknown FLOAT 8 -999 elements of rotation/Dcale matrix +polyOrder dimensionless TINYINT 1 255 polynomial order of astrometry fit +pca1x3y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x2y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x1y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x0y3 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x2y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x1y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x0y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x3y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x2y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x1y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x0y3 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x2y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x1y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x0y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +calibModNum dimensionless SMALLINT 2 NA calibration modification number +dataRelease dimensionless TINYINT 1 NA Data release Version of this table Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ImageMeta.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ImageMeta.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ImageMeta.txt (revision 37801) @@ -0,0 +1,71 @@ +imageID | ipptopsps : expID x y (*** is this right???) (smf:IMAGEID / gpc1.chipRun.chip_image_id)_ +frameID | ipptopsps : gpc1.rawExp.exp_id +ccdID | ipptopsps : from XYnn +photoCalID | dvo:photcode (from photcode table) +filterID | ipptopsps : internal, lookup from filter (grizy) +bias | gpc1.rawExp.bg *** WRONG! +biasScat | gpc1.rawExp.bg_stdev *** WRONG! +sky | MSKY_MN +skyScat | MSKY_SIG +nDetect | ipptopsps : rows(Detection table) +magSat | FSATUR +completMag | FLIMIT +astroScat | CERROR +photoScat | ZPT_OBS ** wrong +numAstroRef | NASTRO +numPhotoRef | NASTRO +nx | CNAXIS1 +ny | CNAXIS2 +psfModelID | ** NOT SET +psfFwhm | 0.5*(FWHM_MAJ + FWHM_MIN) +psfWidMajor | FWHM_MAJ +psfWidMinor | FWHM_MIN +psfTheta | ANGLE +momentMajor | IQ_FW1 +momentMinor | IQ_FW1 +momentM2C | IQ_M2C +momentM2S | IQ_M2S +momentM3 | IQ_M3 +momentM4 | IQ_M4 +apResid | APMIFIT +dapResid | DAPMIFIT +detectorID | DETECTOR +qaFlags | dvo:image.flags +detrend1 | DETREND.MASK +detrend2 | DETREND.DARK +detrend3 | DETREND.FLAT +detrend4 | ** NOT SET +detrend5 | ** NOT SET +detrend6 | ** NOT SET +detrend7 | ** NOT SET +detrend8 | ** NOT SET +photoZero | ZPT_OBS +ctype1 | CTYPE1 +ctype2 | CTYPE2 +crval1 | CRVAL1') +crval2 | CRVAL2') +crpix1 | CRPIX1') +crpix2 | CRPIX2') +cdelt1 | CDELT1') +cdelt2 | CDELT2') +pc001001 | PC001001 +pc001002 | PC001002 +pc002001 | PC002001 +pc002002 | PC002002 +polyOrder | NPLYTERM +pca1x2y0 | PCA1X2Y0 +pca1x1y1 | PCA1X1Y1 +pca1x0y2 | PCA1X0Y2 +pca2x2y0 | PCA2X2Y0 +pca2x1y1 | PCA2X1Y1 +pca2x0y2 | PCA2X0Y2 +pca1x3y0 | PCA1X3Y0 +pca1x2y1 | PCA1X2Y1 +pca1x1y2 | PCA1X1Y2 +pca1x0y3 | PCA1X0Y3 +pca2x3y0 | PCA2X3Y0 +pca2x2y1 | PCA2X2Y1 +pca2x1y2 | PCA2X1Y2 +pca2x0y3 | PCA2X0Y3 +calibModNum | ipptopsps: hardwired to 0 *** +dataRelease | ipptopsps: user input (skychunk.dataRelease) Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Mask =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Mask (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Mask (revision 37801) @@ -0,0 +1,9 @@ +maskID BIGINT 8 NA mask identifier +partitionKey BIGINT 8 NA fget(mask_center_ra, mask_center_dec, zoneH) +frameID INT 4 NA reference to the P2FrameMeta table where the mask is defined +imageID BIGINT 8 NA reference to the P2FrameMeta table where the mask is +ra FLOAT 8 NA rdae finin ethde center +dec FLOAT 8 NA dec in the center +radius FLOAT 8 NA No comment +maskString VARCHAR(MAX) 8000 string representation fo the mask region +maskBinary VARBINARY(MAX) 8000 0x Binary object managed by the spatial library Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/MovingObjToDet =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/MovingObjToDet (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/MovingObjToDet (revision 37801) @@ -0,0 +1,2 @@ +MovingObjID BIGINT 8 NA Moving Object identifier +DetectionID BIGINT 8 NA detection identifier Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/MovingObjToDiffDet =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/MovingObjToDiffDet (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/MovingObjToDiffDet (revision 37801) @@ -0,0 +1,2 @@ +MovingObjID BIGINT 8 NA Moving Object identifier +DiffDetID BIGINT 8 NA difference image detection identifier Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/MovingObjToObject =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/MovingObjToObject (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/MovingObjToObject (revision 37801) @@ -0,0 +1,2 @@ +MovingObjID BIGINT 8 NA Moving Object identifier +ObjID BIGINT 8 NA Object identifier Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ObjectFilterFlags =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ObjectFilterFlags (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ObjectFilterFlags (revision 37801) @@ -0,0 +1,3 @@ +name dimensionless VARCHAR(32) 32 Name of flag +value dimensionless BIGINT 8 NA Flag value +description dimensionless VARCHAR(2000) 2000 Description of the flag Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ObjectInfoFlags =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ObjectInfoFlags (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ObjectInfoFlags (revision 37801) @@ -0,0 +1,3 @@ +name dimensionless VARCHAR(32) 32 Name of flag +value dimensionless BIGINT 8 NA Flag value +description dimensionless VARCHAR(2000) 2000 Description of the flag Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ObjectQualityFlags =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ObjectQualityFlags (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ObjectQualityFlags (revision 37801) @@ -0,0 +1,3 @@ +name dimensionless VARCHAR(32) 32 Name of flag +value dimensionless BIGINT 8 NA Flag value +description dimensionless VARCHAR(2000) 2000 Description of the flag Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/PhotoCal =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/PhotoCal (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/PhotoCal (revision 37801) @@ -0,0 +1,11 @@ +photoCalID INT 4 NA numerical code for this reduction +filterID TINYINT 1 NA filter id +photoCodeDesc VARCHAR(MAX) 8000 Photometry reduction code name +AB REAL 4 1.0 AB magnitude +zeropoint REAL 4 -999 photometric zero point (excluding long term variation) +extinction REAL 4 -999 photometric extinction term (excluding long term variation) +colorterm REAL 4 -999 photometric color term (excluding long term variation) +colorExtn REAL 4 -999 photometric color dependent extinction (excluding long term variation) +orphanCalColor REAL 4 -999 color adopted for magnitude calculation +orphanCalColorErr REAL 4 -999 error in calibrating color +startDate day REAL 4 NA starting date of this configuration in MJD Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ProjectionCell =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ProjectionCell (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/ProjectionCell (revision 37801) @@ -0,0 +1,22 @@ +projectionCellID INT 4 NA projection cell unique identifier +partitionKey BIGINT 8 NA fGetPanObjID(crval1,crval2) +primaryCellRegionID BIGINT 8 NA reference to the primary cell region definition in the Region table +projectionCellRegionID BIGINT 8 NA reference to the projectio cell region definition in the +ctype1 VARCHAR(MAX) 8000 NA nRaemgieo no ft aabsltero)metric projection in RA +ctype2 VARCHAR(MAX) 8000 NA name of astrometric projection in DEC +crval1 deg FLOAT 8 NA RA corresponding to reference pixel +crval2 deg FLOAT 8 NA DEC corresponding to reference pixel +crpix1 FLOAT 8 NA reference pixel value for RA +crpix2 FLOAT 8 NA reference pixel value for DEC +pc001001 FLOAT 8 NA elements of rotation/Dcale matrix +pc001002 FLOAT 8 NA elements of rotation/Dcale matrix +pc002001 FLOAT 8 NA elements of rotation/Dcale matrix +pc002002 FLOAT 8 NA elements of rotation/Dcale matrix +raNEcornerPC degrees FLOAT 8 -999 ra of NE corner +decNEcornerPC degrees FLOAT 8 -999 dec of NE corner +raNWcornerPC degrees FLOAT 8 -999 ra of NE corner +decNWcornerPC degrees FLOAT 8 -999 dec of NE corner +raSEcornerPC degrees FLOAT 8 -999 ra of NE corner +decSEcornerPC degrees FLOAT 8 -999 dec of NE corner +raSWcornerPC degrees FLOAT 8 -999 ra of NE corner +decSWcornerPC degrees FLOAT 8 -999 dec of NE corner Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/PsConstant =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/PsConstant (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/PsConstant (revision 37801) @@ -0,0 +1,4 @@ +name VARCHAR(32) 32 NA constant's name +value FLOAT 8 NA constant's value +unit VARCHAR(32) 32 unit of constant +description VARCHAR(2000) 2000 description of the constant Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Region =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Region (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Region (revision 37801) @@ -0,0 +1,7 @@ +regionID dimensionless BIGINT 8 NA Unique Region identifier +id dimensionless BIGINT 8 NA Custom Identifier +type string VARCHAR(16) 16 NA Region type +comment string VARCHAR(MAX) 8000 Comment +area sqdeg? FLOAT 8 0 Area covered by the region +region dimensionless Binary VARBINARY(MAX) 8000 0x Binary object managed by the spatial library + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/RegionPatch =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/RegionPatch (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/RegionPatch (revision 37801) @@ -0,0 +1,15 @@ +Unit Data Type Size Default Value Description +regionid BIGINT 8 NA No comment +convexid BIGINT 8 NA No comment +patchid INT 4 NA No comment +type (16) NA No comment +radius FLOAT 8 0 No comment +ra degree FLOAT 8 0 Right Ascension +dec FLOAT 8 0 No comment +x FLOAT 8 0 No comment +y FLOAT 8 0 No comment +z FLOAT 8 0 No comment +c FLOAT 8 0 No comment +htmid BIGINT 8 0 No comment +area FLOAT 8 0 No comment +convexString (MAX) No comment Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/SkyCell =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/SkyCell (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/SkyCell (revision 37801) @@ -0,0 +1,12 @@ +skyCellID dimensionless I INT 4 NA indentifier of the sky cell +projectionCellID INT 4 -999 indentifier of the projection cell +partitionKey BIGINT 8 -999 fgetPANobjID(skyCell_center_ra, skyCell_center_dec, zHeight) +regionID BIGINT 8 -999 region identifier in the Region table +raNEcorner degrees FLOAT 8 -999 ra of NE corner +decNEcorner degrees FLOAT 8 -999 dec of NE corner +raNWcorner degrees FLOAT 8 -999 ra of NE corner +decNWcorner degrees FLOAT 8 -999 dec of NE corner +raSEcorner degrees FLOAT 8 -999 ra of NE corner +decSEcorner degrees FLOAT 8 -999 dec of NE corner +raSWcorner degrees FLOAT 8 -999 ra of NE corner +decSWcorner degrees FLOAT 8 -999 dec of NE corner Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackDetectionFlags =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackDetectionFlags (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackDetectionFlags (revision 37801) @@ -0,0 +1,3 @@ +Name dimensionless VARCHAR(32) 32 Name of flag +value dimensionless BIGINT 8 NA Flag value +description dimensionless VARCHAR(2000) 2000 Description of the flag Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackMeta =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackMeta (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackMeta (revision 37801) @@ -0,0 +1,37 @@ +stackMetaID dimensionless BIGINT 8 NA stack identifier +batchID dimensionless BIGINT 8 NA batch identifier +surveyID dimensionless TINYINT 1 NA survey flag identifier +filterID dimensionless TINYINT 1 NA filter number identifier +stackTypeID dimensionless TINYINT 1 0 stack type identifier +tessID dimensionless TINYINT 1 -1 ID mapping to tessellation names (eg, RINGS.V3, MD01.V3, etc) +projectionID dimensionless SMALLINT 2 -999 valid values 0 - < 10000 +skyCellID dimensionless TINYINT 1 255 valid values 0 - 143 (12x12 grid) +photoCalID dimensionless INT 4 NA photometry code +magSat dimensionless REAL 4 -999 saturation magnitude level +analysisVer dimensionless INT 4 -999 analysis version index x +expTime seconds REAL 4 -999 exposure time +nP2Images dimensionless SMALLINT 2 -999 number of P2 images contributing to this cell +completMag magnitudes REAL 4 -999 95% completion level in mag +astroScat dimensionless REAL 4 -999 astrometric scatter for chip +photoScat dimensionless REAL 4 -999 photometric scatter for chip +nAstroRef dimensionless INT 4 -999 number of astrometric reference sources +nPhotomRef dimensionless INT 4 -999 number of photometric reference sources +psfModelID dimensionless VARCHAR(MAX) 8000 PSF model identifier +psfFwhm_mean arcsec FLOAT 8 -999 mean PSF FWHM in the stack +psfFwhm_max arcsec FLOAT 8 -999 maximum PSF FWHM in the stack +photoZero magnitudes REAL 4 -999 local derived photometric zero point +photoColor magnitudes REAL 4 -999 local derived photometric color term +ctype1 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in RA +ctype2 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in DEC +crval1 degrees FLOAT 8 -999 RA corresponding to reference pixel +crval2 degrees FLOAT 8 -999 DEC corresponding to reference pixel +crpix1 dimensionless FLOAT 8 -999 reference pixel value for RA +crpix2 dimensionless FLOAT 8 -999 reference pixel value for DEC +cdelt1 deg/pixel FLOAT 8 -999 scale factor for RA +cdelt2 deg/pixel FLOAT 8 -999 scale factor for DEC +pc001001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc001002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +calibModNum dimensionless SMALLINT 2 NA calibration modification number +dataRelease dimensionless TINYINT 1 NA Data release Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackMeta.txt =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackMeta.txt (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackMeta.txt (revision 37801) @@ -0,0 +1,37 @@ +stackMetaID | ipptopsps : gpc1 db, matches cmf IMAGEID +batchID | ipptopsps : internal +surveyID | ipptopsps : internal, from survey name +filterID | ipptopsps : internal, lookup from filter (grizy) +stackTypeID | ipptopsps : internal, *** hard-wired name *** +tessID | smf : TESS_ID (table mapping) +projectionID | smf : parse SKYCELL (skycell.NNN or skycell.MMMM.NNN) +skycellID | smf : parse SKYCELL (skycell.NNN or skycell.MMMM.NNN) +photoCalID | dvo.images : photcode +magSat | * NOT SET +analysisVer | gpc1.stackRun.software_ver [EAM: changed this name] +expTime | cmf header EXPTIME +nP2Images | [populateStackToImage] : counts inputs NOTE is this set reasonable? +completMag | * NOT SET +astroScat | * NOT SET +photoScat | * NOT SET +nAstroRef | * NOT SET +nPhotomRef | * NOT SET +psfModelID | cmf header PSFMODEL +psfFwhm_mean | cmf header FWHM_MAJ *** probably wrong value +psfFwhm_max | cmf header FW_MJ_UW *** name and definition are inconsistent +photoZero | cmf header FPA.ZP *** this is not dvo corrected) +photoColor | * NOT SET +ctype1 | cmf header +ctype2 | cmf header +crval1 | cmf header +crval2 | cmf header +crpix1 | cmf header +crpix2 | cmf header +cdelt1 | cmf header +cdelt2 | cmf header +pc001001 | cmf header +pc001002 | cmf header +pc002001 | cmf header +pc002002 | cmf header +calibModNum | * NOT SET +dataRelease | ipptopsps : user input (skychunk.dataRelease) Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackToImage =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackToImage (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackToImage (revision 37801) @@ -0,0 +1,2 @@ +stackMetaID BIGINT 8 NA stack identifier +imageID BIGINT 8 NA hashed exposure-ccdID identifier Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackType =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackType (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/StackType (revision 37801) @@ -0,0 +1,3 @@ +stackTypeID TINYINT 1 -999 stack type identifier +name VARCHAR(20) 20 stack type name +description VARCHAR(MAX) 8000 stack type description Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Survey =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Survey (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/Survey (revision 37801) @@ -0,0 +1,4 @@ +SurveyID TINYINT 1 NA index of survey type +name VARCHAR(6) 6 survey name +description VARCHAR(MAX) 8000 +survey description Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/TelescopeConfig =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/TelescopeConfig (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/TelescopeConfig (revision 37801) @@ -0,0 +1,16 @@ +TelescopeConfig +Description: Telescope configuration + +Name Unit DataType Size Default ValueDescription + +TelescopeID dimensionless SMALLINT 2 -999 Telescope identifier (i.e. PS1, ISP) +TeleConfigID dimensionless SMALLINT 2 -999 Telescope Configuration ID +MountModel string 8 NA Mount Model Name +MirrorModel string 8 NA Mirror Model Name +Observera string 8 NA Observers Name +endDate MeanJulianDays FLOAT 8 -999 ending date of this configuration in MJD +configDetails string VARCHAR(MAX) 8000 0x00 NA text/table giving the details of the telescope configuration + + + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/TessellationType =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/TessellationType (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/TessellationType (revision 37801) @@ -0,0 +1,3 @@ +tessID TINYINT 1 -999 stack type identifier +name VARCHAR(20) 20 NULL stack type name +description VARCHAR(MAX) 8000 NULL stack type description Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsGalaxyModel =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsGalaxyModel (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsGalaxyModel (revision 37801) @@ -0,0 +1,3 @@ +galaxyModelID dimensionless INT 4 NA modelID +description dimensionless VARCHAR(128) 128 model description, spectral type etc. +priors dimensionless VARCHAR(128) 128 priors applied to this model, may also split into 7 columns Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsPhotoZRecipe =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsPhotoZRecipe (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsPhotoZRecipe (revision 37801) @@ -0,0 +1,2 @@ +photozRecipeID dimensionless TINYINT 1 -999 recipe index +description dimensionless VARCHAR(MAX) 8000 string\ describing\ the\ recipe Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsSGSepRecipe =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsSGSepRecipe (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsSGSepRecipe (revision 37801) @@ -0,0 +1,2 @@ +sgSepRecipeID dimensionless INT 4 NA Star/Galaxy Seperator Recuipe ID +description dimensionless VARCHAR(128) 128 Description Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsStarModel =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsStarModel (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsStarModel (revision 37801) @@ -0,0 +1,2 @@ +starModelID dimensionless INT 4 -999 modelID +description dimensionless VARCHAR(128) 128 model description, spectral type etc Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsStellarParamsRecipe =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsStellarParamsRecipe (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/MetaData_Tables/pcsStellarParamsRecipe (revision 37801) @@ -0,0 +1,3 @@ +stellarParamsRecipeID dimensionless INT 4 NA Na +description dimensionless VARCHAR(128) 128 Description:wq + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_VIEWS/README =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_VIEWS/README (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_VIEWS/README (revision 37801) @@ -0,0 +1,5 @@ +See Notes_on_Schema 03.txt +for what tables are combined into what views. +Once there is concensus on tables, then will combine tables to make views. Meanwhile can concatente yourself from Notes. + + Index: /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_VIEWS/Views =================================================================== --- /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_VIEWS/Views (revision 37801) +++ /branches/eam_branches/ipp-20140904/ippToPsps/pspsschema/SchemaNew19.0/PSPS_VIEWS/Views (revision 37801) @@ -0,0 +1,16 @@ +PSPS VIEWS - examples (not complete, please suggest others) + +MeanObjectView = ObjectThin join MeanObject +StackObjectView = ObjectThin join StackObjectThin join StackObjectAttributes +StackApFlxObjectView = ObjectThin join StackApFlx +StackApFlxEGObjectView = ObjectThin join StackApFlxEGUnc join StackApFlxEGCon6 join StackApFlxEGCon8 +StackModelObjectView = ObjectThin join StackModelFitExp join StackModelFitDeV join StackModelFitSer join StackPetrosian +ForcedMeanObjectView = ObjectThin join ForcedMeanObject +StackObjectPrimaryView = ObjectThin join StackObjectThin join StackObjectAttributes join StackApFlx where primaryDection is true +StackObjectBestView = ObjectThin join StackObject join StackObjectAttributes join StackApFlx where bestDetection is true +DiffDetObjectView = DiffDetObject join DiffDetection + + + + +