Index: /branches/eam_branches/ipp-20150112/hardware/dotConsolerc
===================================================================
--- /branches/eam_branches/ipp-20150112/hardware/dotConsolerc (revision 38051)
+++ /branches/eam_branches/ipp-20150112/hardware/dotConsolerc (revision 38052)
@@ -457,4 +457,6 @@
ipp095:
console: cab17con.ipp.ifa.hawaii.edu
+ ipp097:
+ console: cab17con.ipp.ifa.hawaii.edu
ippx001:
console: cab23con.ipp.ifa.hawaii.edu
@@ -665,3 +667,33 @@
ippx104:
console: cab17con.ipp.ifa.hawaii.edu
+ ipps00:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps01:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps02:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps03:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps04:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps05:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps06:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps07:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps08:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps09:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps10:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps11:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps12:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps13:
+ console: cab15con.ipp.ifa.hawaii.edu
+ ipps14:
+ console: cab15con.ipp.ifa.hawaii.edu
Index: /branches/eam_branches/ipp-20150112/ippToPsps/config/ippToPspsDbSchema.sql
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/config/ippToPspsDbSchema.sql (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/config/ippToPspsDbSchema.sql (revision 38052)
@@ -199,4 +199,19 @@
`region` varchar(50) default NULL,
KEY `batch_id` (`batch_id`),
+ CONSTRAINT `forcedobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+SET character_set_client = @saved_cs_client;
+
+--
+-- Table structure for table `diffobject`
+--
+
+DROP TABLE IF EXISTS `diffobject`;
+SET @saved_cs_client = @@character_set_client;
+SET character_set_client = utf8;
+CREATE TABLE `diffobject` (
+ `batch_id` bigint(20) unsigned NOT NULL,
+ `region` varchar(50) default NULL,
+ KEY `batch_id` (`batch_id`),
CONSTRAINT `diffobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
@@ -204,21 +219,11 @@
--
--- Table structure for table `diffobject`
---
-
-DROP TABLE IF EXISTS `diffobject`;
-SET @saved_cs_client = @@character_set_client;
-SET character_set_client = utf8;
-CREATE TABLE `diffobject` (
- `batch_id` bigint(20) unsigned NOT NULL,
- `region` varchar(50) default NULL,
- KEY `batch_id` (`batch_id`),
- CONSTRAINT `diffobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-SET character_set_client = @saved_cs_client;
-
---
-- Table structure for table `pending`
--
+-- i had to do this to get uniqueness prob sorted out:
+
+--show index from pending;
+--alter table pending drop index `name`;
+--where name was broken becaue it only indedxed stage idg
DROP TABLE IF EXISTS `pending`;
Index: /branches/eam_branches/ipp-20150112/ippToPsps/config/settings.xml
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/config/settings.xml (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/config/settings.xml (revision 38052)
@@ -51,5 +51,144 @@
64
-
+
+ ipptopsps2_scratch
+ stsci00
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci01
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci02
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci03
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci04
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci05
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci06
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci07
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci08
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci09
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci10
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci11
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci12
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci13
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci14
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci15
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci16
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci17
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci18
+ dvo
+ dvo
+ 32
+
+
+ ipptopsps2_scratch
+ stsci19
+ dvo
+ dvo
+ 32
+
Index: /branches/eam_branches/ipp-20150112/ippToPsps/config/tables.FO.vot
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/config/tables.FO.vot (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/config/tables.FO.vot (revision 38052)
@@ -96,4 +96,374 @@
flags for g forced
+
+ second moment (Mxx+Myy)
+
+
+ polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy)
+
+
+ polarization, KSB-corrected 2Mxy / (Mxx + Myy)
+
+
+ stack image id (how to carry through IPP+DVO
+
+
+ number of detections in r filter
+
+
+ number of detections included in robust psf mean in r filter
+
+
+ number of detections included in robust mean of Kron mag in r filter
+
+
+ number of detections included in robust mean of Ap Flux in r filter
+
+
+ number of detections included in robust mean of ApFlx 5 in r filter
+
+
+ number of detections included in robust mean of ApFlx 6 in r filter
+
+
+ robust mean r psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile
+
+
+ estimated error of g psf AB magnitude mean(individual errors added in quadrature)
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ mean r kron AB magnitude
+
+
+ estimated error of r kron AB magnitude mean
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ mean r aperture AB magnitude
+
+
+ estimated error of r aperture AB magnitude mean
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ Flux inside r = 5 in r unconvolved stack
+
+
+ estimated error in flxR5
+
+
+ estimated standard deviation in flxR5
+
+
+ estimated filling factor at R5
+
+
+ Flux inside r = 6 in r unconvolved stack
+
+
+ estimated error in flxR6
+
+
+ estimated standard deviation in flxR6
+
+
+ estimated filling factor at R6
+
+
+ flags for r forced
+
+
+ second moment (Mxx+Myy)
+
+
+ polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy)
+
+
+ polarization, KSB-corrected 2Mxy / (Mxx + Myy)
+
+
+ stack image id (how to carry through IPP+DVO
+
+
+ number of detections in i filter
+
+
+ number of detections included in robust psf mean in i filter
+
+
+ number of detections included in robust mean of Kron mag in i filter
+
+
+ number of detections included in robust mean of Ap Flux in i filter
+
+
+ number of detections included in robust mean of ApFlx 5 in i filter
+
+
+ number of detections included in robust mean of ApFlx 6 in i filter
+
+
+ robust mean i psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile
+
+
+ estimated error of i psf AB magnitude mean(individual errors added in quadrature)
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ mean i kron AB magnitude
+
+
+ estimated error of i kron AB magnitude mean
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ mean i aperture AB magnitude
+
+
+ estimated error of i aperture AB magnitude mean
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ Flux inside r = 5 in i unconvolved stack
+
+
+ estimated error in flxR5
+
+
+ estimated standard deviation in flxR5
+
+
+ estimated filling factor at R5
+
+
+ Flux inside r = 6 in i unconvolved stack
+
+
+ estimated error in flxR6
+
+
+ estimated standard deviation in flxR6
+
+
+ estimated filling factor at R6
+
+
+ flags for i forced
+
+
+ second moment (Mxx+Myy)
+
+
+ polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy)
+
+
+ polarization, KSB-corrected 2Mxy / (Mxx + Myy)
+
+
+ stack image id (how to carry through IPP+DVO
+
+
+ number of detections in z filter
+
+
+ number of detections included in robust psf mean in z filter
+
+
+ number of detections included in robust mean of Kron mag in z filter
+
+
+ number of detections included in robust mean of Ap Flux in z filter
+
+
+ number of detections included in robust mean of ApFlx 5 in z filter
+
+
+ number of detections included in robust mean of ApFlx 6 in z filter
+
+
+ robust mean z psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile
+
+
+ estimated error of z psf AB magnitude mean(individual errors added in quadrature)
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ mean z kron AB magnitude
+
+
+ estimated error of z kron AB magnitude mean
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ mean z aperture AB magnitude
+
+
+ estimated error of z aperture AB magnitude mean
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ Flux inside r = 5 in z unconvolved stack
+
+
+ estimated error in flxR5
+
+
+ estimated standard deviation in flxR5
+
+
+ estimated filling factor at R5
+
+
+ Flux inside r = 6 in z unconvolved stack
+
+
+ estimated error in flxR6
+
+
+ estimated standard deviation in flxR6
+
+
+ estimated filling factor at R6
+
+
+ flags for z forced
+
+
+ second moment (Mxx+Myy)
+
+
+ polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy)
+
+
+ polarization, KSB-corrected 2Mxy / (Mxx + Myy)
+
+
+ stack image id (how to carry through IPP+DVO
+
+
+ number of detections in y filter
+
+
+ number of detections included in robust psf mean in y filter
+
+
+ number of detections included in robust mean of Kron mag in y filter
+
+
+ number of detections included in robust mean of Ap Flux in y filter
+
+
+ number of detections included in robust mean of ApFlx 5 in y filter
+
+
+ number of detections included in robust mean of ApFlx 6 in y filter
+
+
+ robust mean y psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile
+
+
+ estimated error of y psf AB magnitude mean(individual errors added in quadrature)
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ mean y kron AB magnitude
+
+
+ estimated error of y kron AB magnitude mean
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ mean y aperture AB magnitude
+
+
+ estimated error of y aperture AB magnitude mean
+
+
+ standard deviation of measurements included in the robust mean
+
+
+ Flux inside r = 5 in y unconvolved stack
+
+
+ estimated error in flxR5
+
+
+ estimated standard deviation in flxR5
+
+
+ estimated filling factor at R5
+
+
+ Flux inside r = 6 in y unconvolved stack
+
+
+ estimated error in flxR6
+
+
+ estimated standard deviation in flxR6
+
+
+ estimated filling factor at R6
+
+
+ flags for y forced
+
+
+ second moment (Mxx+Myy)
+
+
+ polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy)
+
+
+ polarization, KSB-corrected 2Mxy / (Mxx + Myy)
+
+
+
+
+
+
+ VOTable description of PSPS table ForcedMeanLensing
+
+
+ ODM object identifier index
+
+
+ unique psps id
+
+
+ IPP object number
+
+
+ total number of detection measurements in all filters
+
+
+ batch id
+
+
+ stack image id (how to carry through IPP+DVO
+
per-object smear polarizability X11 term
@@ -156,87 +526,6 @@
psf model shear polarizability E2 term
-
- second moment (Mxx+Myy)
-
-
- polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy)
-
-
- polarization, KSB-corrected 2Mxy / (Mxx + Myy)
-
stack image id (how to carry through IPP+DVO
-
-
- number of detections in r filter
-
-
- number of detections included in robust psf mean in r filter
-
-
- number of detections included in robust mean of Kron mag in r filter
-
-
- number of detections included in robust mean of Ap Flux in r filter
-
-
- number of detections included in robust mean of ApFlx 5 in r filter
-
-
- number of detections included in robust mean of ApFlx 6 in r filter
-
-
- robust mean r psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile
-
-
- estimated error of g psf AB magnitude mean(individual errors added in quadrature)
-
-
- standard deviation of measurements included in the robust mean
-
-
- mean r kron AB magnitude
-
-
- estimated error of r kron AB magnitude mean
-
-
- standard deviation of measurements included in the robust mean
-
-
- mean r aperture AB magnitude
-
-
- estimated error of r aperture AB magnitude mean
-
-
- standard deviation of measurements included in the robust mean
-
-
- Flux inside r = 5 in r unconvolved stack
-
-
- estimated error in flxR5
-
-
- estimated standard deviation in flxR5
-
-
- estimated filling factor at R5
-
-
- Flux inside r = 6 in r unconvolved stack
-
-
- estimated error in flxR6
-
-
- estimated standard deviation in flxR6
-
-
- estimated filling factor at R6
-
-
- flags for r forced
@@ -300,102 +589,21 @@
psf model shear polarizability E2 term
-
- second moment (Mxx+Myy)
-
-
- polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy)
-
-
- polarization, KSB-corrected 2Mxy / (Mxx + Myy)
-
stack image id (how to carry through IPP+DVO
-
- number of detections in i filter
-
-
- number of detections included in robust psf mean in i filter
-
-
- number of detections included in robust mean of Kron mag in i filter
-
-
- number of detections included in robust mean of Ap Flux in i filter
-
-
- number of detections included in robust mean of ApFlx 5 in i filter
-
-
- number of detections included in robust mean of ApFlx 6 in i filter
-
-
- robust mean i psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile
-
-
- estimated error of i psf AB magnitude mean(individual errors added in quadrature)
-
-
- standard deviation of measurements included in the robust mean
-
-
- mean i kron AB magnitude
-
-
- estimated error of i kron AB magnitude mean
-
-
- standard deviation of measurements included in the robust mean
-
-
- mean i aperture AB magnitude
-
-
- estimated error of i aperture AB magnitude mean
-
-
- standard deviation of measurements included in the robust mean
-
-
- Flux inside r = 5 in i unconvolved stack
-
-
- estimated error in flxR5
-
-
- estimated standard deviation in flxR5
-
-
- estimated filling factor at R5
-
-
- Flux inside r = 6 in i unconvolved stack
-
-
- estimated error in flxR6
-
-
- estimated standard deviation in flxR6
-
-
- estimated filling factor at R6
-
-
- flags for i forced
-
- per-object smear polarizability X11 term
+ per-object smear polarizability X11 term
- per-object smear polarizability X12 term
+ per-object smear polarizability X12 term
- per-object smear polarizability X22 term
+ per-object smear polarizability X22 term
- per-object smear polarizability E1 term
+ per-object smear polarizability E1 term
- per-object smear polarizability E2 term
+ per-object smear polarizability E2 term
@@ -444,87 +652,6 @@
psf model shear polarizability E2 term
-
- second moment (Mxx+Myy)
-
-
- polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy)
-
-
- polarization, KSB-corrected 2Mxy / (Mxx + Myy)
-
stack image id (how to carry through IPP+DVO
-
-
- number of detections in z filter
-
-
- number of detections included in robust psf mean in z filter
-
-
- number of detections included in robust mean of Kron mag in z filter
-
-
- number of detections included in robust mean of Ap Flux in z filter
-
-
- number of detections included in robust mean of ApFlx 5 in z filter
-
-
- number of detections included in robust mean of ApFlx 6 in z filter
-
-
- robust mean z psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile
-
-
- estimated error of z psf AB magnitude mean(individual errors added in quadrature)
-
-
- standard deviation of measurements included in the robust mean
-
-
- mean z kron AB magnitude
-
-
- estimated error of z kron AB magnitude mean
-
-
- standard deviation of measurements included in the robust mean
-
-
- mean z aperture AB magnitude
-
-
- estimated error of z aperture AB magnitude mean
-
-
- standard deviation of measurements included in the robust mean
-
-
- Flux inside r = 5 in z unconvolved stack
-
-
- estimated error in flxR5
-
-
- estimated standard deviation in flxR5
-
-
- estimated filling factor at R5
-
-
- Flux inside r = 6 in z unconvolved stack
-
-
- estimated error in flxR6
-
-
- estimated standard deviation in flxR6
-
-
- estimated filling factor at R6
-
-
- flags for z forced
@@ -588,87 +715,6 @@
psf model shear polarizability E2 term
-
- second moment (Mxx+Myy)
-
-
- polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy)
-
-
- polarization, KSB-corrected 2Mxy / (Mxx + Myy)
-
stack image id (how to carry through IPP+DVO
-
-
- number of detections in y filter
-
-
- number of detections included in robust psf mean in y filter
-
-
- number of detections included in robust mean of Kron mag in y filter
-
-
- number of detections included in robust mean of Ap Flux in y filter
-
-
- number of detections included in robust mean of ApFlx 5 in y filter
-
-
- number of detections included in robust mean of ApFlx 6 in y filter
-
-
- robust mean y psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile
-
-
- estimated error of y psf AB magnitude mean(individual errors added in quadrature)
-
-
- standard deviation of measurements included in the robust mean
-
-
- mean y kron AB magnitude
-
-
- estimated error of y kron AB magnitude mean
-
-
- standard deviation of measurements included in the robust mean
-
-
- mean y aperture AB magnitude
-
-
- estimated error of y aperture AB magnitude mean
-
-
- standard deviation of measurements included in the robust mean
-
-
- Flux inside r = 5 in y unconvolved stack
-
-
- estimated error in flxR5
-
-
- estimated standard deviation in flxR5
-
-
- estimated filling factor at R5
-
-
- Flux inside r = 6 in y unconvolved stack
-
-
- estimated error in flxR6
-
-
- estimated standard deviation in flxR6
-
-
- estimated filling factor at R6
-
-
- flags for y forced
@@ -731,13 +777,4 @@
psf model shear polarizability E2 term
-
-
- second moment (Mxx+Myy)
-
-
- polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy)
-
-
- polarization, KSB-corrected 2Mxy / (Mxx + Myy)
Index: /branches/eam_branches/ipp-20150112/ippToPsps/config/tables.FW.vot
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/config/tables.FW.vot (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/config/tables.FW.vot (revision 38052)
@@ -236,7 +236,4 @@
Kron radius
-
- Kron radius error
-
PSF sky level at source
@@ -297,4 +294,71 @@
Time of mid observation of positive detection in Mean Julian Days
+
+ Flux inside r = 5
+
+
+ estimated error in flxR5
+
+
+ estimated standard deviation in flxR5
+
+
+ estimated filling factor at R5
+
+
+ Flux inside r = 6
+
+
+ estimated error in flxR6
+
+
+ estimated standard deviation in flxR6
+
+
+ estimated filling factor at R6
+
+
+
+
+
+
+ VOTable description of PSPS table ForcedWarpMeasurement
+
+
+ ODM object identifier
+
+
+ unique psps id
+
+
+ ODM detection identifier
+
+
+ object id generated by IPP
+
+
+ detection id generated by IPP
+
+
+ filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ...
+
+
+ survey flag identifier
+
+
+ forced warp measurement meta identifer
+
+
+ random warp id
+
+
+ single float givijng PrimaryCell.SkyCell identifier
+
+
+ identifier to dvo spacial region
+
+
+ Time of mid observation of positive detection in Mean Julian Days
+
per-object smear polarizability X11 term
@@ -356,28 +420,4 @@
psf model shear polarizability E2 term
-
-
- Flux inside r = 5
-
-
- estimated error in flxR5
-
-
- estimated standard deviation in flxR5
-
-
- estimated filling factor at R5
-
-
- Flux inside r = 6
-
-
- estimated error in flxR6
-
-
- estimated standard deviation in flxR6
-
-
- estimated filling factor at R6
Index: /branches/eam_branches/ipp-20150112/ippToPsps/config/tables.ST.vot
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/config/tables.ST.vot (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/config/tables.ST.vot (revision 38052)
@@ -1080,5 +1080,5 @@
Flux inside r = 3
-
+
estimated error in c8flxR3
@@ -2847,17 +2847,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is flxR10
-
-
- estimated standard deviation in flxR10
-
-
- estimated filling factor at R10
-
-
@@ -2978,17 +2965,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is flxR10
-
-
- estimated standard deviation in flxR10
-
-
- estimated filling factor at R10
-
-
@@ -3109,17 +3083,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is flxR10
-
-
- estimated standard deviation in flxR10
-
-
- estimated filling factor at R10
-
-
@@ -3240,17 +3201,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is flxR10
-
-
- estimated standard deviation in flxR10
-
-
- estimated filling factor at R10
-
-
@@ -3371,17 +3319,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is flxR10
-
-
- estimated standard deviation in flxR10
-
-
- estimated filling factor at R10
-
-
@@ -3533,17 +3468,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is c6flxR10
-
-
- estimated standard deviation in c6flxR10
-
-
- estimated filling factor at R10
-
-
@@ -3664,17 +3586,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is c6flxR10
-
-
- estimated standard deviation in c6flxR10
-
-
- estimated filling factor at R10
-
-
@@ -3795,17 +3704,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is c6flxR10
-
-
- estimated standard deviation in c6flxR10
-
-
- estimated filling factor at R10
-
-
@@ -3926,17 +3822,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is c6flxR10
-
-
- estimated standard deviation in c6flxR10
-
-
- estimated filling factor at R10
-
-
@@ -4057,17 +3940,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is c6flxR10
-
-
- estimated standard deviation in c6flxR10
-
-
- estimated filling factor at R10
-
-
@@ -4219,17 +4089,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is c8flxR10
-
-
- estimated standard deviation in c8flxR10
-
-
- estimated filling factor at R10
-
-
@@ -4350,17 +4207,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is c8flxR10
-
-
- estimated standard deviation in c8flxR10
-
-
- estimated filling factor at R10
-
-
@@ -4481,17 +4325,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is c8flxR10
-
-
- estimated standard deviation in c8flxR10
-
-
- estimated filling factor at R10
-
-
@@ -4612,17 +4443,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is c8flxR10
-
-
- estimated standard deviation in c8flxR10
-
-
- estimated filling factor at R10
-
-
@@ -4743,17 +4561,4 @@
estimated filling factor at R9
-
- Flux inside r = 10
-
-
- estimated error is c8flxR10
-
-
- estimated standard deviation in c8flxR10
-
-
- estimated filling factor at R10
-
-
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/Makefile.am (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/Makefile.am (revision 38052)
@@ -14,4 +14,5 @@
diffobjectbatch.py \
dvodetections.py \
+ dvodiffobjects.py \
dvoforceddetections.py \
dvoforcedobjects.py \
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/detectionbatch.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/detectionbatch.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/detectionbatch.py (revision 38052)
@@ -401,12 +401,18 @@
# XXX EAM 2014072 : Is this still a problem?
sql = "UPDATE " + pspsTableName + " SET psfFlux = 1e20 WHERE psfFlux <= 0.0"
- self.scratchDb.execute(sql)
-
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.infoPair("failed sql",sql)
+ raise
sql = "UPDATE " + pspsTableName + " SET apFlux = 1e20 WHERE apFlux <= 0.0"
- self.scratchDb.execute(sql)
-
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.infoPair("failed sql",sql)
+ raise
sql = "UPDATE " + pspsTableName + " SET kronFlux = 1e20 WHERE kronFlux <= 0.0"
- self.scratchDb.execute(sql)
-
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.infoPair("failed sql",sql)
+ raise
# we don't delete these anymore
results['NULLINSTFLUX'] = 0;
@@ -604,6 +610,8 @@
AND b.imageID = " + str(imageID)
- self.scratchDb.execute(sql)
-
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.InfoPair("failed sql",sql)
+ raise
'''
Updates table and generates pspsuniqueids
@@ -611,8 +619,9 @@
def updatePspsUniqueIDs(self,table, ota):
- sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, objID from \
- (select @r:=0) r, "+table+" t) as foo using (objID) set \
- uniquePspsP2id = ((" +str(self.batchID)+ "*1000000000 ) \
- + (" +str(ota) + "*10000000 )+rownum)"
+ #sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, objID from \
+ #(select @r:=0) r, "+table+" t) as foo using (objID) set \
+ #uniquePspsP2id = ((" +str(self.batchID)+ "*1000000000 ) \
+ #+ (" +str(ota) + "*10000000 )+rownum)"
+ sql = "UPDATE "+table+" set uniquePspsP2id = (("+str(self.batchID)+"*1000000000 ) + (" +str(ota) + "*10000000 )+ row)"
try: self.scratchDb.execute(sql)
except:
@@ -647,11 +656,19 @@
#self.logger.info("successful populate ");
# now add DVO IDs
+ self.logger.infoPair("Adding 'row' column to ", "Detection_"+chipname)
+ #detection has a primary index, so don't add, use this one
+ self.scratchDb.addRowCountColumnNoKey("Detection_"+chipname, "row")
self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname])
#self.logger.info("updated dvoids")
results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID")
#self.logger.info("deleted nulls")
+
+ ota = x*10+y
+
+
self.logger.info("add psps unique p2 ids")
- ota = x*10+y
self.updatePspsUniqueIDs("Detection_" + chipname, ota)
+ self.logger.infoPair("Dropping 'row' column from ", "Detection_"+chipname)
+ self.scratchDb.dropColumn("Detection_"+chipname,"row")
self.updateImageID("Detection_" + chipname, x, y)
#self.logger.info("updateImageId")
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/diffbatch.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/diffbatch.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/diffbatch.py (revision 38052)
@@ -327,6 +327,12 @@
self.logger.infoPair("need to cull"," nulls objid")
results= self.scratchDb.reportAndDeleteRowsWithNULLS("DiffDetection", "diffObjID")
+ self.logger.infoPair("Adding 'row' columns to", "DiffDetection tables")
+ # diff has a primary key, so use this instead
+ self.scratchDb.addRowCountColumnNoKey("DiffDetection", "row")
self.logger.infoPair("adding","uniquePSPSDFid")
self.updatePspsUniqueIDs("DiffDetection")
+ self.logger.infoPair("Dropping row column from", "DiffDetection table")
+ self.scratchDb.dropColumn("DiffDetection", "row")
+
'''
@@ -417,6 +423,8 @@
AND b.imageID = " + str(imageID)
- self.scratchDb.execute(sql)
-
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.infoPair("failed sql",sql)
+ raise
'''
Updates table and generates pspsuniqueids
@@ -424,7 +432,8 @@
def updatePspsUniqueIDs(self,table):
- sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \
- (select @r:=0) r, "+table+" t) as foo using (diffobjID) set \
- uniquePspsDFid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
+ #sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \
+ #(select @r:=0) r, "+table+" t) as foo using (diffobjID) set \
+ #uniquePspsDFid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
+ sql = "UPDATE DiffDetection set uniquePspsDFid = (("+str(self.batchID)+"*1000000000 ) + row)"
try: self.scratchDb.execute(sql)
except:
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/diffobjectbatch.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/diffobjectbatch.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/diffobjectbatch.py (revision 38052)
@@ -14,5 +14,6 @@
from ipptopspsdb import IppToPspsDb
from scratchdb import ScratchDb
-from dvoobjects import DvoObjects
+#from dvoobjects import DvoObjects
+from dvodiffobjects import DvoDiffObjects
from sqlUtility import sqlUtility
@@ -53,7 +54,7 @@
try:
- self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
+ self.dvoDiffObjects = DvoDiffObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
except:
- self.logger.errorPair("Unable to create instance of", "DvoObjects")
+ self.logger.errorPair("Unable to create instance of", "DvoDiffObjects")
raise
@@ -72,5 +73,5 @@
self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id)
self.ippToPspsDb.insertObjectMeta(self.batchID, self.region)
- self.dvoObjects.nativeIngestRegion(self.region)
+ self.dvoDiffObjects.nativeIngestRegion(self.region)
cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
@@ -177,17 +178,29 @@
sql += ")) "
- sql += "SET DiffDetObject.n" + filter[1] + " = NCODE, "
- sql += "DiffDetObject.nDetections = DiffDetObject.nDetections + cps.NSTACK_DET"
+ sql += "SET DiffDetObject.n" + filter[1] + " = NWARP, "
+ sql += "DiffDetObject.nDetections = DiffDetObject.nDetections + cps.NWARP"
self.logger.info(sql)
- self.scratchDb.execute(sql)
+ try:
+ self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("failed sql", sql)
+ raise
+ return False
+
# XXX this does not seem like a good thing to leave in SQL
- self.logger.infoPair("Calculating nDetections from", "n[filters]")
- for filter in filters:
+ #self.logger.infoPair("Calculating nDetections from", "n[filters]")
+ #for filter in filters:
# now do a sum of n[filters], but do not include the ones with -999
- sql = "UPDATE DiffDetObject "
- sql += "SET nDetections = nDetections + n" + filter[1]
- sql += " WHERE n" + filter[1] + " != -999"
- self.scratchDb.execute(sql)
+ # sql = "UPDATE DiffDetObject "
+ # sql += "SET nDetections = nDetections + n" + filter[1]
+ # sql += " WHERE n" + filter[1] + " != -999"
+ # try:
+ # self.scratchDb.execute(sql)
+ # except:
+ # self.logger.errorPair("failed sql", sql)
+ # raise
+
+
'''
@@ -213,5 +226,5 @@
except:
self.logger.errorPair("failed sql", sql)
- return False
+ raise
## use mean ra and dec (dec < 0)
@@ -231,5 +244,5 @@
except:
self.logger.errorPair("failed sql", sql)
- return False
+ raise
'''
@@ -251,5 +264,5 @@
except:
self.logger.errorPair("Couldn't squash out of range stuff", sql)
- return False
+ raise
self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON")
@@ -260,5 +273,5 @@
except:
self.logger.errorPair("Couldn't squash out of range stuff", sql)
- return False
+ raise
@@ -278,5 +291,5 @@
sqlLine.group("batchID", "'" + str(self.batchID) + "'")
sqlLine.group("dvoRegionID", "CAT_ID")
- sqlLine.group("skycellID", "SKYCELL_ID")
+# sqlLine.group("skycellID", "SKYCELL_ID")
sqlLine.group("dataRelease", "'" + str(self.skychunk.dataRelease) + "'")
sqlLine.group("objInfoFlag", "FLAGS")
@@ -294,5 +307,5 @@
except:
self.logger.errorPair("Couldn't populate DiffDetObject table", sql)
- return False
+ raise
# add row count columns so we can perform joins to get colors
@@ -307,7 +320,8 @@
self.logger.infoPair("update DiffDetObject from ","cps table")
-
+ #this fills ng/etc
self.updateDiffObjectFromCps(cpsTableName)
-
+ #this fills qfperfect
+ self.updateMeanObjectFromCps(cpsTableName)
# XXX EAM 20140724 : is this necessary??
#objects can have out of range ra dec in dvo - need to find and kill them at the end
@@ -350,5 +364,5 @@
except:
self.logger.errorPair("Couldn't cull outsiders from DiffDetObject table", sql)
- return False
+ raise
self.logger.infoPair("Dropping row column from", "DiffDetObject table")
@@ -368,11 +382,12 @@
def updatePspsUniqueIDs(self,table):
- sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \
- (select @r:=0) r, "+table+" t) as foo using (diffobjID) set \
- uniquePspsDOid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
+# sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \
+# (select @r:=0) r, "+table+" t) as foo using (diffobjID) set \
+# uniquePspsDOid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
+ sql = "UPDATE DiffDetObject set uniquePspsDOid = (("+str(self.batchID)+"*1000000000 ) + row)"
try: self.scratchDb.execute(sql)
except:
self.logger.errorPair('failed sql',sql)
- return
+ raise
@@ -407,7 +422,12 @@
self.scratchDb.execute("ALTER TABLE DiffDetObject CHANGE dec_ `dec` double")
+ self.logger.infoPair("Adding 'row' columns to", "Diff Det Object")
+ self.scratchDb.addRowCountColumn("DiffDetObject", "row")
+ self.logger.infoPair("Adding unique", "Diff Det Object ")
self.updatePspsUniqueIDs("DiffDetObject")
-
- self.dvoObjects.purgeRegion(self.region)
+ self.logger.infoPair("Dropping row column from", "DiffDetObject table")
+ self.scratchDb.dropColumn("DiffDetObject", "row")
+
+ self.dvoDiffObjects.purgeRegion(self.region)
#this is abuse of something but this is how I get the object batches to crash to further investigate them
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/dvo.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/dvo.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/dvo.py (revision 38052)
@@ -654,5 +654,6 @@
useP2 = 0
useST = 0
-
+ useFW = 0
+ useDF = 0
for batchType in self.skychunk.batchTypes:
self.logger.infoPair("batchType", batchType)
@@ -661,5 +662,10 @@
if batchType == "P2":
useP2 = 1
-
+ if batchType == "DF":
+ useDF = 1
+ if batchType == "FW":
+ useFW = 1
+
+
if (useP2 == 0 and useST == 1):
# grab only stacks
@@ -680,4 +686,17 @@
cmd += " -photcode-start 10000 -photcode-end 10577"
+ if (useFW == 1):
+ if (self.config.camera == "simtest"):
+ print "need photcodes for non gpc1 camera"
+ else:
+ cmd += " -photcode-start 12000 -photcode-end 12500"
+ if (useFW == 1):
+ if (self.config.camera == "simtest"):
+ print "need photcodes for non gpc1 camera"
+ else:
+ print "need photcode for warp for gpc1 camera, default is no selection, should work, could be slow"
+
+
+
self.logger.infoPair("Running dvopsps", cmd)
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/dvodiffobjects.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/dvodiffobjects.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/dvodiffobjects.py (revision 38052)
@@ -15,5 +15,5 @@
A class for ingesting DVO detections into MySQL
'''
-class DvoObjects(Dvo):
+class DvoDiffObjects(Dvo):
'''
@@ -22,5 +22,5 @@
def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb):
- super(DvoObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb)
+ super(DvoDiffObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb)
# declare DVO file types of interest
@@ -39,5 +39,5 @@
# TODO path to DVO prog hardcoded temporarily
- cmd = "dvopsps objects"
+ cmd = "dvopsps diffobj"
cmd += " -dbhost " + self.scratchDb.dbHost
cmd += " -dbname " + self.scratchDb.dbName
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/dvoforcedobjects.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/dvoforcedobjects.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/dvoforcedobjects.py (revision 38052)
@@ -40,5 +40,5 @@
# TODO path to DVO prog hardcoded temporarily
# TODO this needs to be modified for forced objects
- cmd = "dvopsps objects"
+ cmd = "dvopsps forced_warp_objects"
cmd += " -dbhost " + self.scratchDb.dbHost
cmd += " -dbname " + self.scratchDb.dbName
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/forcedobjectbatch.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/forcedobjectbatch.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/forcedobjectbatch.py (revision 38052)
@@ -14,5 +14,5 @@
from ipptopspsdb import IppToPspsDb
from scratchdb import ScratchDb
-from dvoforcedobjects import DvoForcedObjects # need one specific for cpy ?
+from dvoforcedobjects import DvoForcedObjects # need one specific for cpy.
from sqlUtility import sqlUtility
@@ -25,5 +25,5 @@
'''
-class ForcedMeanObjectBatch(Batch):
+class ForcedObjectBatch(Batch):
'''
@@ -40,5 +40,5 @@
batchID):
- super(ForcedMeanObjectBatch, self).__init__(
+ super(ForcedObjectBatch, self).__init__(
logger,
config,
@@ -71,5 +71,5 @@
self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id)
- self.ippToPspsDb.insertForcedMeanObjectMeta(self.batchID, self.region)
+ self.ippToPspsDb.insertForcedObjectMeta(self.batchID, self.region)
self.dvoForcedObjects.nativeIngestRegion(self.region)
@@ -86,4 +86,13 @@
return False
+ self.logger.infoPair("checking if we have anything in"," cpy/cpt")
+ rowcountcpt = self.scratchDb.getRowCount(cptTableName);
+ rowcountcpy = self.scratchDb.getRowCount(cpyTableName);
+ if (rowcountcpt == 0):
+ self.logger.infoPair("0 rows for ","cpt table")
+ return False
+ if (rowcountcpy == 0):
+ self.logger.infoPair("o rows for ","cpy table")
+ return False
return True
@@ -104,4 +113,19 @@
return True
+
+
+ '''
+ Updates table and generates pspsuniqueids
+ '''
+
+ def updatePspsUniqueIDs(self):
+ sql = "UPDATE ForcedMeanObject set uniquePspsFOid = (("+str(self.batchID)+"*1000000000 ) + row)"
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair('failed sql',sql)
+ raise
+
+
+
'''
@@ -146,16 +170,16 @@
SET \
ForcedMeanObject." + filter[1] + "nTotal = NWARP \
- ,ForcedMeanObject." + filter[1] + "nIncPSFMag = NUSED_WRP \
- ,ForcedMeanObject." + filter[1] + "nIncKronMag = NUSED_KRON_WRP \
- ,ForcedMeanObject." + filter[1] + "nIncApMag = NUSED_AP_WRP \
- ,ForcedMeanObject." + filter[1] + "PSFMag = MAG_PSF_WRP \
- ,ForcedMeanObject." + filter[1] + "PSFMagErr = MAG_PSF_WRP_ERR \
- ,ForcedMeanObject." + filter[1] + "PSFMagStd = MAG_PSF_WRP_STD \
- ,ForcedMeanObject." + filter[1] + "KronMag = MAG_KRON_WRP \
- ,ForcedMeanObject." + filter[1] + "KronMagErr = MAG_KRON_WRP_ERR \
- ,ForcedMeanObject." + filter[1] + "KronMagStd = MAG_KRON_WRP_STD \
- ,ForcedMeanObject." + filter[1] + "ApMag = MAG_AP_WRP \
- ,ForcedMeanObject." + filter[1] + "ApMagErr = MAG_AP_WRP_ERR \
- ,ForcedMeanObject." + filter[1] + "ApMagStd = MAG_AP_WRP_STD \
+ ,ForcedMeanObject." + filter[1] + "nIncPSFFlux = NUSED_WRP \
+ ,ForcedMeanObject." + filter[1] + "nIncKronFlux = NUSED_KRON_WRP \
+ ,ForcedMeanObject." + filter[1] + "nIncApFlux = NUSED_AP_WRP \
+ ,ForcedMeanObject." + filter[1] + "PSFFlux = FLUX_PSF_WRP \
+ ,ForcedMeanObject." + filter[1] + "PSFFluxErr = FLUX_PSF_WRP_ERR \
+ ,ForcedMeanObject." + filter[1] + "PSFFluxStd = FLUX_PSF_WRP_STDEV \
+ ,ForcedMeanObject." + filter[1] + "KronFlux = FLUX_KRON_WRP \
+ ,ForcedMeanObject." + filter[1] + "KronFluxErr = FLUX_KRON_WRP_ERR \
+ ,ForcedMeanObject." + filter[1] + "KronFluxStd = FLUX_KRON_WRP_STDEV \
+ ,ForcedMeanObject." + filter[1] + "ApFlux = FLUX_AP_WRP \
+ ,ForcedMeanObject." + filter[1] + "ApFluxErr = FLUX_AP_WRP_ERR \
+ ,ForcedMeanObject." + filter[1] + "ApFluxStd = FLUX_AP_WRP_STDEV \
,ForcedMeanObject." + filter[1] + "Flags = FLAGS "
@@ -163,5 +187,5 @@
try: self.scratchDb.execute(sql)
except:
- self.logger.errorPair("failed update MeanObject", sql)
+ self.logger.errorPair("failed update ForcedMeanObject", sql)
raise
@@ -175,18 +199,26 @@
SET " + filter[1] + "PSFMagErr = null \
WHERE " + filter[1] + "PSFMagErr > " + str(cut)
- self.scratchDb.execute(sql)
-
+# taking out the cut since this is for mag, not flux
+
+ # try: self.scratchDb.execute(sql)
+ # except:
+ # self.logger.errorPair("failed setting null meanmagerr","ForcedMeanObject")
+ # raise
self.logger.infoPair("Calculating nDetections from", "n[filters]")
for filter in filters:
# now do a sum of n[filters], but do not include the ones with -999
- sql = "UPDATE ObjectThin "
- sql += "SET nDetections = nDetections + n" + filter[1]
- sql += " WHERE n" + filter[1] + " != -999"
- self.scratchDb.execute(sql)
+ sql = "UPDATE ForcedMeanObject "
+ sql += "SET nDetections = nDetections + " + filter[1] +"nTotal "
+ sql += " WHERE " + filter[1] + "nTotal != -999"
+ #self.scratchDb.execute(sql)
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("failed update MeanObject nDetections", sql)
+ raise
'''
Inserts stuff for all mags from the cpy files
'''
- def updateForcedMeanObjectFromCpy(self, cpyTable):
+ def updateForcedMeanTablesFromCpy(self, cpyTable):
# list of all filters PSPS is interested in
@@ -225,5 +257,5 @@
" + cpyTable + " AS cpy ON (cpy.row = (ForcedMeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
SET \
- ForcedMeanObject." + filter[1] + "nIncR5 = NMEAS \
+ ForcedMeanObject." + filter[1] + "nIncR5 = NMEAS \
,ForcedMeanObject." + filter[1] + "nIncR6 = NMEAS \
,ForcedMeanObject." + filter[1] + "FmeanflxR5 = FLUX_AP_R5 \
@@ -235,32 +267,40 @@
,ForcedMeanObject." + filter[1] + "FmeanflxR6Std = FLUX_STD_AP_R6 \
,ForcedMeanObject." + filter[1] + "FmeanflxR6Fill = FLUX_FIL_AP_R6 \
- ,ForcedMeanObject." + filter[1] + "LensObjSmearX11 = X11_SM_OBJ \
- ,ForcedMeanObject." + filter[1] + "LensObjSmearX12 = X12_SM_OBJ \
- ,ForcedMeanObject." + filter[1] + "LensObjSmearX22 = X22_SM_OBJ \
- ,ForcedMeanObject." + filter[1] + "LensObjSmearE1 = E1_SM_OBJ \
- ,ForcedMeanObject." + filter[1] + "LensObjSmearE2 = E2_SM_OBJ \
- ,ForcedMeanObject." + filter[1] + "LensObjShearX11 = X11_SH_OBJ \
- ,ForcedMeanObject." + filter[1] + "LensObjShearX12 = X12_SH_OBJ \
- ,ForcedMeanObject." + filter[1] + "LensObjShearX22 = X22_SH_OBJ \
- ,ForcedMeanObject." + filter[1] + "LensObjShearE1 = E1_SH_OBJ \
- ,ForcedMeanObject." + filter[1] + "LensObjShearE2 = E2_SH_OBJ \
- ,ForcedMeanObject." + filter[1] + "LensPSFSmearX11 = X11_SM_PSF \
- ,ForcedMeanObject." + filter[1] + "LensPSFSmearX12 = X12_SM_PSF \
- ,ForcedMeanObject." + filter[1] + "LensPSFSmearX22 = X22_SM_PSF \
- ,ForcedMeanObject." + filter[1] + "LensPSFSmearE1 = E1_SM_PSF \
- ,ForcedMeanObject." + filter[1] + "LensPSFSmearE2 = E2_SM_PSF \
- ,ForcedMeanObject." + filter[1] + "LensPSFShearX11 = X11_SH_PSF \
- ,ForcedMeanObject." + filter[1] + "LensPSFShearX12 = X12_SH_PSF \
- ,ForcedMeanObject." + filter[1] + "LensPSFShearX22 = X22_SH_PSF \
- ,ForcedMeanObject." + filter[1] + "LensPSFShearE1 = E1_SH_PSF \
- ,ForcedMeanObject." + filter[1] + "LensPSFShearE2 = E2_SH_PSF \
,ForcedMeanObject." + filter[1] + "Gamma = GAMMA \
,ForcedMeanObject." + filter[1] + "E1 = E1 \
,ForcedMeanObject." + filter[1] + "E2 = E2 "
-
try: self.scratchDb.execute(sql)
except:
- self.logger.errorPair("failed update on ForcedMeanObject using cpy table", sql)
+ self.logger.errorPair("failed update on ForcedMeanObject using cpy table",sql)
+ raise
+
+ sql = "UPDATE ForcedMeanLensing JOIN \
+ " + cpyTable + " AS cpy ON (cpy.row = (ForcedMeanLensing.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
+ SET \
+ ForcedMeanLensing." + filter[1] + "LensObjSmearX11 = X11_SM_OBJ \
+ ,ForcedMeanLensing." + filter[1] + "LensObjSmearX12 = X12_SM_OBJ \
+ ,ForcedMeanLensing." + filter[1] + "LensObjSmearX22 = X22_SM_OBJ \
+ ,ForcedMeanLensing." + filter[1] + "LensObjSmearE1 = E1_SM_OBJ \
+ ,ForcedMeanLensing." + filter[1] + "LensObjSmearE2 = E2_SM_OBJ \
+ ,ForcedMeanLensing." + filter[1] + "LensObjShearX11 = X11_SH_OBJ \
+ ,ForcedMeanLensing." + filter[1] + "LensObjShearX12 = X12_SH_OBJ \
+ ,ForcedMeanLensing." + filter[1] + "LensObjShearX22 = X22_SH_OBJ \
+ ,ForcedMeanLensing." + filter[1] + "LensObjShearE1 = E1_SH_OBJ \
+ ,ForcedMeanLensing." + filter[1] + "LensObjShearE2 = E2_SH_OBJ \
+ ,ForcedMeanLensing." + filter[1] + "LensPSFSmearX11 = X11_SM_PSF \
+ ,ForcedMeanLensing." + filter[1] + "LensPSFSmearX12 = X12_SM_PSF \
+ ,ForcedMeanLensing." + filter[1] + "LensPSFSmearX22 = X22_SM_PSF \
+ ,ForcedMeanLensing." + filter[1] + "LensPSFSmearE1 = E1_SM_PSF \
+ ,ForcedMeanLensing." + filter[1] + "LensPSFSmearE2 = E2_SM_PSF \
+ ,ForcedMeanLensing." + filter[1] + "LensPSFShearX11 = X11_SH_PSF \
+ ,ForcedMeanLensing." + filter[1] + "LensPSFShearX12 = X12_SH_PSF \
+ ,ForcedMeanLensing." + filter[1] + "LensPSFShearX22 = X22_SH_PSF \
+ ,ForcedMeanLensing." + filter[1] + "LensPSFShearE1 = E1_SH_PSF \
+ ,ForcedMeanLensing." + filter[1] + "LensPSFShearE2 = E2_SH_PSF "
+
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("failed update on ForcedMeanLensing using cpy table", sql)
raise
@@ -295,32 +335,65 @@
except:
self.logger.errorPair("Couldn't populate Object table from cpt", sql)
- return False
+ raise
# add row count columns so we can perform joins to get colors
- self.logger.infoPair("Adding 'row' columns to", "Object and cps tables")
+ self.logger.infoPair("Adding 'row' columns to", "Object ,cps, cpy tables")
self.scratchDb.addRowCountColumn("ForcedMeanObject", "row")
self.scratchDb.addRowCountColumn(cpsTableName, "row")
+ self.scratchDb.addRowCountColumn(cpyTableName, "row")
+
+ self.logger.infoPair("generating unique ids for ","ForcedMeanObject")
+ self.updatePspsUniqueIDs()
self.logger.infoPair("update ForcedMeanObject from ","cps table")
-
self.updateForcedMeanObjectFromCps(cpsTableName)
- self.logger.infoPair("update ForcedMeanObject from ","cpy table")
-
- self.updateForcedMeanObjectFromCpy(cpyTableName)
+ sqlLine = sqlUtility("INSERT INTO ForcedMeanLensing (")
+
+ sqlLine.group("objID", "objID")
+ sqlLine.group("uniquePspsFOid","uniquePspsFOid")
+ sqlLine.group("ippObjID", "ippObjID")
+ sqlLine.group("batchID", "batchID")
+ sqlLine.group("nDetections", "nDetections")
+ sqlLine.group("gStackID", "gStackID")
+ sqlLine.group("rStackID", "rStackID")
+ sqlLine.group("iStackID", "iStackID")
+ sqlLine.group("zStackID", "zStackID")
+ sqlLine.group("yStackID", "yStackID")
+
+ sql = sqlLine.makeRaw(") SELECT ", " FROM ForcedMeanObject")
+
+ try:
+ self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("Couldn't populate ForcedMeanLensing table from ForcedMeanObject", sql)
+ raise
+
+ self.logger.infoPair("adding row to ","ForcedMeanLensing")
+ self.scratchDb.addRowCountColumn("ForcedMeanLensing", "row")
+
+ self.logger.infoPair("updating ForcedMeanObject and ForcedMeanLensing from ","cpy tables")
+ self.updateForcedMeanTablesFromCpy(cpyTableName)
self.logger.infoPair("Dropping row column from", "ForcedMeanObject table")
self.scratchDb.dropColumn("ForcedMeanObject", "row")
-
+ self.logger.infoPair("Dropping row column from", "ForcedMeanLensing table")
+ self.scratchDb.dropColumn("ForcedMeanLensing", "row")
##self.logger.infoPair("Purging from scratch Db", self.region + " region")
##Don't do this till after MeanObject
##self.dvoForcedObjects.purgeRegion(self.region)
-
+ self.logger.infoPair("setting minmaxobjid for ", "ForcedMeanObject table")
self.setMinMaxObjID(["ForcedMeanObject"])
-
- return True
+ self.logger.infoPair("checking if we have detctions", "FOrcedMeanObject table")
+ rowcount = self.scratchDb.getRowCount("ForcedMeanObject")
+ if rowcount == 0:
+ self.logger.infoPair("skipping this one","ForcedMeanObject ")
+ self.skipBatch = False
+ return True
+ else:
+ return True
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/forcedwarpbatch.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/forcedwarpbatch.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/forcedwarpbatch.py (revision 38052)
@@ -6,4 +6,5 @@
import time
import stilts
+import re
from java.lang import *
@@ -54,24 +55,68 @@
forcedWarpID,
batchID,
- "P2",
- gpc1Db.getForcedWarpStageSmf(forcedWarpID)
-
+ "FW",
+ None)
+ # gpc1Db.getForcedWarpStageSmf(forcedWarpID)
+ # there is a funny problem with order here: we need to open the file first to read the header to get
+ # to skycell ID
+ #forcedWarpID = self.forcedWarpID
+ self.logger.infoPair("getting the fits file for add_id",forcedWarpID)
+ self.fits = gpc1Db.getForcedWarpStageCmf(skychunk.dvoLabel, forcedWarpID)
+ self.forcedWarpID = forcedWarpID
+ self.header = self.fits.getPrimaryHeader()
+ self.warpSkyFileID = self.getKeyValue(self.header,"IMAGEID")
+ self.filterName = self.getKeyValue(self.header,"FPA.FILTERID")
+ self.logger.infoPair("found filterName",self.filterName)
+ self.logger.infoPair("found warp_skyfile_id",self.warpSkyFileID)
# get camera meta data
- meta = self.gpc1Db.getForcedWarpStageMeta(self.id)
+
+ meta = self.gpc1Db.getForcedWarpStageMeta(forcedWarpID, self.warpSkyFileID)
+
if not meta:
self.logger.errorPair("Could not get", "forced warp metadata")
raise
- self.expID = meta[0];
- self.expName = meta[1];
- self.distGroup = meta[2];
- self.analysisVer = meta[3];
- self.bias = meta[4];
- self.biasScat = meta[5];
+ self.expID = meta[0]
+ self.expTime = meta[1]
+ self.analysisVer = meta[2]
+ self.skycellName = meta[3]
+ self.tessName = meta[4]
+
+ self.logger.infoPair("found exp_id",str(self.expID))
+ self.logger.infoPair("found exp_time",str(self.expTime))
+ self.logger.infoPair("found skycell_id",self.skycellName)
+ self.logger.infoPair("found tess_id",self.tessName)
+
+
+ #tessName = self.getKeyValue(self.header, 'TESS_ID')
+ #self.tessID = self.scratchDb.getTessID(tessName)
if not self.analysisVer: self.analysisVer = -999
+ if self.analysisVer is None: self.analysisVer = "-999"
+ self.logger.infoPair("found analysisVer",self.analysisVer)
+
+ if self.tessName == 'RINGS.V3':
+
+ # skycell is, eg "skycell.1133.081"
+ # 0123456789012345
+ self.projectionID = self.skycellName[8:12]
+ #self.skycellID = self.skycellName[13:]
+ self.skycellID = self.skycellName[8:]
+ else:
+ # skycell is, eg "skycell.081"
+ # 01234567890
+ self.projectionID = 0
+ self.skycellID = self.skycellName[8:11]
+ self.skycellID=self.skycellName[8:]
+ if self.skycellID == "":
+ self.skycellID = -1
+
+ self.logger.infoPair("projection cell found ",self.projectionID)
+ self.logger.infoPair("skycell found", self.skycellID)
+
+
# create an output filename, which is {expID}.FITS
- self.outputFitsFile = "%08d.FITS" % self.expID
+ self.outputFitsFile = "%08d.FITS" % forcedWarpID
self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
self.dropTableVerbose("ForcedWarpMeasurement")
@@ -79,44 +124,35 @@
self.dropTableVerbose("ForcedWarpMeta")
self.dropTableVerbose("ForcedWarpToImage")
+ self.dropTableVerbose("ForcedWarpLensing")
if not self.config.retry:
- self.scratchDb.dropTable("ForcedWarpMeasurement")
- self.scratchDb.dropTable("ForcedWarpExtended")
- self.scratchDb.dropTable("ForcedWarpMeta")
- self.scratchDb.dropTable("ForcedWarpToImage")
+ self.scratchDb.dropTable("ForcedWarpMeasurement")
+ self.scratchDb.dropTable("ForcedWarpExtended")
+ self.scratchDb.dropTable("ForcedWarpMeta")
+ self.scratchDb.dropTable("ForcedWarpToImage")
+ self.scratchDb.dropTable("ForcedWarpLensing")
self.historyModNum = "0"
# get a few primary header values. if in test mode, then use defaults
- if self.safeDictionaryAccessWithDefault(self.header, 'MJD-OBS', "1") == "NULL":
- self.logger.errorPair("Could not get", "MJD-OBS")
- raise
-
-
- if self.safeDictionaryAccessWithDefault(self.header, 'EXPTIME', "1") == "NULL":
- self.logger.errorPair("Could not get", "EXPTIME")
- raise
-
- if self.safeDictionaryAccessWithDefault(self.header, 'FILTERID', "g.0000") == "NULL":
- self.logger.errorPair("Could not get", "FILTERID")
- raise
-
+ self.logger.infoPair("calculating ObsTime","ok")
# MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2
- self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0)
-
+ self.obsTime = float(self.header['MJD-OBS']) + (float(self.expTime) / 172800.0)
+ self.logger.infoPair("obstime",self.obsTime)
# set up some defauts
self.calibModNum = 0
self.totalNumPhotoRef = 0
-
- self.filter = self.header['FILTERID'][0:1]
- self.filterID = self.scratchDb.getFilterID(self.filter)
+ self.filterID = self.scratchDb.getFilterID(self.filterName[0])
+ self.logger.infoPair("filterID is ",self.filterID)
# insert what we know about this stack batch into the stack table
- self.ippToPspsDb.insertDetectionMeta(self.batchID, self.expID, self.filter)
+ self.logger.infoPair("inserting basics into ","ForcedWarpMeta")
+
+ self.ippToPspsDb.insertForcedWarpMeta(int(self.batchID), int(self.expID), self.filterName[0])
# dump stuff to log
- self.logger.infoPair("Cam ID", "%d" % self.id)
- self.logger.infoPair("Exp ID", "%d" % self.expID)
- self.logger.infoPair("Exp name", "%s" % self.expName)
- self.logger.infoPair("Distribution group", "%s" % self.distGroup)
+ self.logger.infoPair("add ID", "%d" % self.id)
+ self.logger.infoPair("Exp ID", self.expID)
+
+
@@ -129,6 +165,6 @@
self.logger.infoPair("Processing table", "ForcedWarpMeta")
- photoCalID = str(self.scratchDb.getPhotoCalID(forcedWarpID))
-
+ header = self.header
+# photoCalID = str(self.scratchDb.getPhotoCalID(forcedWarpID))
sqlLine = sqlUtility("INSERT INTO ForcedWarpMeta (")
@@ -136,18 +172,20 @@
#surveyID (below)
#filterdID (below)
+ sqlLine.group("forcedWarpID", str(self.forcedWarpID))
sqlLine.group("skyCellID", str(self.skycellID))
- sqlLine.group("photoCalID", photoCalID)
+# sqlLine.group("photoCalID", photoCalID)
sqlLine.group("magSat", self.getKeyFloat(header, "%.8f", 'FSATUR'))
- sqlLine.group("analVer", self.analysisVer);
- sqlLine.group("expTime", self.getKeyFloat(self.header, "%.8f", 'EXPREQ'));
+ sqlLine.group("analVer", str(self.analysisVer))
+ sqlLine.group("expTime",self.expTime)
sqlLine.group("completMag", self.getKeyFloat(header, "%.8f", 'FLIMIT'))
- sqlLine.group("astroScat", self.getKeyFloat(header, "%.8f", 'CERROR'))
- sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR'));
- sqlLine.group("nAstroRef", self.getKeyValue(header, 'NASTRO'))
- sqlLine.group("nPhotoRef", self.getKeyValue(header, 'NASTRO'))
+# sqlLine.group("astroScat", self.getKeyFloat(header, "%.8f", 'CERROR'))
+# sqlLine.group("photoScat", self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR'));
+#no nastro
+# sqlLine.group("nAstroRef", self.getKeyValue(header, 'NASTRO'))
+# sqlLine.group("nPhotoRef", self.getKeyValue(header, 'NASTRO'))
sqlLine.group("psfModelID", self.getKeyValue(header, 'PSFMODEL'))
sqlLine.group("psfFwhm_mean", self.getKeyFloat(header, "%.8f", 'FWHM_MAJ'))
sqlLine.group("psfFwhm_max", self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ'))
- sqlLine.group("photoZero", self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS'))
+# sqlLine.group("photoZero", self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS'))
#photoColor -- how do I set this? it's also not set in stack meta
sqlLine.group("ctype1", self.getKeyValue(self.header, 'CTYPE1'));
@@ -174,8 +212,9 @@
self.scratchDb.updateAllRows("ForcedWarpMeta", "batchID", str(self.batchID))
self.scratchDb.updateAllRows("ForcedWarpMeta", "surveyID", str(self.surveyID))
- self.scratchDb.updateFilterID("ForcedWarpMeta", self.filter)
+# self.scratchDb.updateFilterID("ForcedWarpMeta", self.filterID)
+ self.scratchDb.updateFilterID("ForcedWarpMeta",self.filterName[0])
self.scratchDb.updateAllRows("ForcedWarpMeta", "calibModNum", str(self.calibModNum))
self.scratchDb.updateAllRows("ForcedWarpMeta", "dataRelease", str(self.skychunk.dataRelease))
- self.tablesToExport.append("ForcedWarpMeta")
+ #self.tablesToExport.append("ForcedWarpMeta")
'''
@@ -186,8 +225,6 @@
self.logger.infoPair("Procesing table", "ForcedWarpToImage")
- for filter in self.filters:
-
- forcedwarpID = self.forcedwarpID
- if forcedwarpID > 0:
+ forcedwarpID = self.forcedWarpID
+ if forcedwarpID > 0:
imageIDs = self.gpc1Db.getImageIDsForThisForcedWarpID(forcedwarpID)
@@ -198,9 +235,11 @@
VALUES (\
" + str(forcedwarpID) + ", " + imageID + ")"
- self.scratchDb.execute(sql)
-
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.infoPair("can't execute sql",sql)
+ raise
# now update StackMeta with correct number of inputs
- self.tablesToExport.append("ForcedWarpToImage")
+ #self.tablesToExport.append("ForcedWarpToImage")
@@ -211,46 +250,232 @@
Populates the ForcedWarpMeasurement table
'''
- def populateForcedWarpMeasurementTable(self, results):
+ def populateForcedWarpMeasurement(self):
pspsTableName = "ForcedWarpMeasurement"
- ippTableName = "ForcedWarpMeasurement_psf"
+ ippTableName = "SkyChip_psf"
self.logger.infoPair("Procesing table", "ForcedWarpMeasurement")
+ BEFORE = self.scratchDb.getRowCount(ippTableName)
+ extTimeString = str(self.expTime)
+
+#missing are zp, telluricExt, airmass (not easy to get ?)
+
+ sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (")
+ sqlLine.group("ippDetectID", "IPP_IDET")
+ sqlLine.group("forcedWarpID", str(self.forcedWarpID));
+ sqlLine.group("randomWarpID", "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)")
+ sqlLine.group("filterID", str(self.filterID))
+ sqlLine.group("surveyID", str(self.surveyID))
+ sqlLine.group("skycellID", str(self.skycellID))
+ sqlLine.group("expTime", str(self.expTime))
+ sqlLine.group("obsTime", str(self.obsTime))
+ sqlLine.group("psfFlux", "PSF_INST_FLUX / "+extTimeString)
+ sqlLine.group("psfFluxErr", "PSF_INST_FLUX_SIG / "+extTimeString)
+ sqlLine.group("psfMajorFWHM", "PSF_FWHM_MAJ")
+ sqlLine.group("psfMinorFWHM", "PSF_FWHM_MIN")
+ sqlLine.group("psfTheta", "PSF_THETA")
+ sqlLine.group("psfCore", "PSF_CORE")
+ sqlLine.group("psfQf", "PSF_QF")
+ sqlLine.group("psfQfPerfect", "PSF_QF_PERFECT")
+ sqlLine.group("psfChiSq", "PSF_CHISQ")
+ sqlLine.group("psfLikelihood", "psfLikelihood(EXT_NSIGMA)")
+ sqlLine.group("momentXX", "MOMENTS_XX")
+ sqlLine.group("momentXY", "MOMENTS_XY")
+ sqlLine.group("momentYY", "MOMENTS_YY")
+ sqlLine.group("momentR1", "MOMENTS_R1")
+ sqlLine.group("momentRH", "MOMENTS_RH")
+ sqlLine.group("momentM3C", "MOMENTS_M3C")
+ sqlLine.group("momentM3S", "MOMENTS_M3S")
+ sqlLine.group("momentM4C", "MOMENTS_M4C")
+ sqlLine.group("momentM4S", "MOMENTS_M4S")
+ sqlLine.group("apFlux","AP_FLUX / " + extTimeString)
+ sqlLine.group("apFluxErr","AP_FLUX_SIG / " + extTimeString)
+ sqlLine.group("apFillF","AP_NPIX / (3.14159265359 * POW(AP_MAG_RADIUS - 0.5, 2))")
+ sqlLine.group("kronFlux","KRON_FLUX / " + extTimeString)
+ sqlLine.group("kronFluxErr","KRON_FLUX_ERR / " + extTimeString )
+ sqlLine.group("kronRad","MOMENTS_R1 * 2.5")
+ sqlLine.group("sky","SKY /" +extTimeString)
+ sqlLine.group("skyErr","SKY_SIGMA / "+extTimeString)
+ sqlLine.group("infoFlag","FLAGS")
+ sqlLine.group("infoFlag2","FLAGS2")
+ sqlLine.group("dataRelease",str(self.skychunk.dataRelease))
+
+ sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName)
+
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("failed sql: ", sql)
+ raise
+
+
+
+
# insert the per-object data (IDs, random stack ID, dataRelease, etc)
- self.selectDvoObjIDs()
-
- self.generateRandomIDs()
-
+ #self.selectDvoObjIDs()
+ self.logger.infoPair("inserting dvo info: find imageID","from externID")
+ imageID = self.scratchDb.getImageIDFromExternID(self.warpSkyFileID)
+ self.logger.infoPair("updating","forcedWarpMeasurement")
+ sqlLine = sqlUtility("UPDATE ForcedWarpMeasurement as a, " + self.scratchDb.dvoDetectionTable + " as b SET ")
+ sqlLine.group("a.objID","b.objID")
+ sqlLine.group("a.detectID","b.detectID")
+ sqlLine.group("a.ippObjID","b.ippObjID")
+ sqlLine.group("a.dvoRegionID","b.catID")
+ sqlLine.group("a.zp","b.zp")
+ sqlLine.group("a.telluricExt","b.telluricExt")
+ sqlLine.group("a.airmass","b.airmass")
+ sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " +str( imageID))
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("failed sql:",sql)
+ raise
+ self.logger.infoPair("need to cull"," nulls objid")
+ results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpMeasurement", "objID")
+
+
+
+# self.generateRandomIDs()
+
+ self.logger.infoPair("Adding 'row' columns to", "ForcedWarpMeasurement")
+ self.scratchDb.addRowCountColumn("ForcedWarpMeasurement", "row")
+ self.logger.infoPair("Adding unique ids to", "ForcedWarpMeasurement")
self.generateUniquePspsIDs()
+ self.logger.infoPair("Dropping row column from", "ForcedWarpMeasurement table")
+ self.scratchDb.dropColumn("ForcedWarpMeasurement", "row")
# add indexes ForcedWarpMeasurement
self.scratchDb.createIndex("ForcedWarpMeasurement", "objID")
-
+ self.scratchDb.createIndex("ForcedWarpMeasurement", "ippDetectID")
+ '''
+ Populates the ForcedWarpExtended table
+ '''
+ def populateForcedWarpExtended(self):
+
+ pspsTableName = "ForcedWarpExtended"
+ ippTableName = "SkyChip_xrad"
+
+ #self.tablesToExport.append("ForcedWarpExtended")
+
+ #isert stuff from SkyChip_psf
+ sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (")
+ sqlLine.group("ippDetectID", "IPP_IDET")
+ sqlLine.group("flxR5", "APER_FLUX_5")
+ sqlLine.group("flxR5Err", "APER_FLUX_ERR_5" )
+ sqlLine.group("flxR5Std", "APER_FLUX_STDEV_5")
+ sqlLine.group("flxR5Fill", "APER_FILL_5")
+ sqlLine.group("flxR6", "APER_FLUX_6")
+ sqlLine.group("flxR6Err", "APER_FLUX_ERR_6" )
+ sqlLine.group("flxR6Std", "APER_FLUX_STDEV_6")
+ sqlLine.group("flxR6Fill", "APER_FILL_6")
+
+ sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName)
+
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("failed sql: ", sql)
+ raise
+
+ #insert the things from FOrcedWarpMeasurement
+ self.logger.infoPair("inserting objID,etc from","ForcedWarpMeasurement")
+ sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, ForcedWarpMeasurement as b SET ")
+ sqlLine.group("a.objID","b.objID")
+ sqlLine.group("a.uniquePspsFWid","b.uniquePspsFWid")
+ sqlLine.group("a.detectID","b.detectID")
+ sqlLine.group("a.ippObjID","b.ippObjID")
+ sqlLine.group("a.filterID","b.filterID")
+ sqlLine.group("a.surveyID","b.surveyID")
+ sqlLine.group("a.forcedWarpID","b.forcedWarpID")
+ sqlLine.group("a.randomWarpID","b.randomWarpID")
+ sqlLine.group("a.skycellID","b.skycellID")
+ sqlLine.group("a.dvoRegionID","b.dvoRegionID")
+ sqlLine.group("a.obsTime","b.obsTime")
+ sql = sqlLine.makeEquals("Where a.ippDetectID = b.ippDetectID")
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("failed sql: ", sql)
+ raise
+
+ # add in the psps unique ids from stuff
+ self.logger.infoPair("adding information from Skychip_xrad into","ForcedWarpExtended")
+
+ # cull bad d
+ self.logger.infoPair("need to cull"," nulls objid")
+ results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpExtended", "objID")
+
+ self.scratchDb.createIndex("ForcedWarpExtended", "objID")
-
-
-
-
-
- self.tablesToExport.append("ForcedWarpMeasurement")
-
-
- '''
- Populates the ForcedWarpExtended table
- '''
- def populateForcedWarpExtendedTable(self, results):
-
- pspsTableName = "ForcedWarpExtended"
- ippTableName = "ForcedWarpExtended_psf"
-
-
- self.tablesToExport.append("ForcedWarpExtended")
-
- # get dvo ids,
- # add in the psps unique ids from stuff
+ '''
+ Populates the ForcedWarpExtended table
+ '''
+ def populateForcedWarpLensing(self):
+
+ pspsTableName = "ForcedWarpLensing"
+ ippTableName = "SkyChip_psf"
+
+ #self.tablesToExport.append("ForcedWarpExtended")
+
+ #isert stuff from SkyChip_psf
+ sqlLine = sqlUtility("INSERT INTO " + pspsTableName + " (")
+ sqlLine.group("ippDetectID", "IPP_IDET")
+ sqlLine.group("lensObjSmearX11", "LENS_X11_SM_OBJ")
+ sqlLine.group("lensObjSmearX12", "LENS_X12_SM_OBJ")
+ sqlLine.group("lensObjSmearX22", "LENS_X22_SM_OBJ")
+ sqlLine.group("lensObjSmearE1", "LENS_E1_SM_OBJ")
+ sqlLine.group("lensObjSmearE2", "LENS_E2_SM_OBJ")
+ sqlLine.group("lensObjShearX11", "LENS_X11_SH_OBJ")
+ sqlLine.group("lensObjShearX12", "LENS_X12_SH_OBJ")
+ sqlLine.group("lensObjShearX22", "LENS_X22_SH_OBJ")
+ sqlLine.group("lensObjShearE1", "LENS_E1_SH_OBJ")
+ sqlLine.group("lensObjShearE2", "LENS_E2_SH_OBJ")
+ sqlLine.group("lensPSFSmearX11", "LENS_X11_SM_PSF")
+ sqlLine.group("lensPSFSmearX12", "LENS_X12_SM_PSF")
+ sqlLine.group("lensPSFSmearX22", "LENS_X22_SM_PSF")
+ sqlLine.group("lensPSFSmearE1", "LENS_E1_SM_PSF")
+ sqlLine.group("lensPSFSmearE2", "LENS_E2_SM_PSF")
+ sqlLine.group("lensPSFShearX11", "LENS_X11_SH_PSF")
+ sqlLine.group("lensPSFShearX12", "LENS_X12_SH_PSF")
+ sqlLine.group("lensPSFShearX22", "LENS_X22_SH_PSF")
+ sqlLine.group("lensPSFShearE1", "LENS_E1_SH_PSF")
+ sqlLine.group("lensPSFShearE2", "LENS_E2_SH_PSF")
+ sql = sqlLine.makeRaw(") SELECT ", " FROM " + ippTableName)
+
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("failed sql: ", sql)
+ raise
+
+ #insert the things from FOrcedWarpMeasurement
+ self.logger.infoPair("inserting objID,etc from","ForcedWarpMeasurement")
+ sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, ForcedWarpMeasurement as b SET ")
+ sqlLine.group("a.objID","b.objID")
+ sqlLine.group("a.uniquePspsFWid","b.uniquePspsFWid")
+ sqlLine.group("a.detectID","b.detectID")
+ sqlLine.group("a.ippObjID","b.ippObjID")
+ sqlLine.group("a.filterID","b.filterID")
+ sqlLine.group("a.surveyID","b.surveyID")
+ sqlLine.group("a.forcedWarpID","b.forcedWarpID")
+ sqlLine.group("a.randomWarpID","b.randomWarpID")
+ sqlLine.group("a.skycellID","b.skycellID")
+ sqlLine.group("a.dvoRegionID","b.dvoRegionID")
+ sqlLine.group("a.obsTime","b.obsTime")
+ sql = sqlLine.makeEquals("Where a.ippDetectID = b.ippDetectID")
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("failed sql: ", sql)
+ raise
+
+ # add in the psps unique ids from stuff
+ self.logger.infoPair("adding information from Skychip_psf into","ForcedWarpLensing")
+
+ # cull bad d
+ self.logger.infoPair("need to cull"," nulls objid")
+ results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpLensing", "objID")
+
+ self.scratchDb.createIndex("ForcedWarpLensing", "objID")
+
+
'''
@@ -264,10 +489,27 @@
#this should be done
+ self.logger.infoPair("Populating","ForcedWarpMeta")
self.populateForcedWarpMeta()
-
- self.populateForcedWarpMeasurement()
+ self.tablesToExport.append("ForcedWarpMeta")
+
+ self.logger.infoPair("Populating","ForcedWarpMeasurement")
+ self.populateForcedWarpMeasurement()
+ self.tablesToExport.append("ForcedWarpMeasurement")
+
+ self.logger.infoPair("Populating","ForcedWarpExtended")
self.populateForcedWarpExtended()
+ self.tablesToExport.append("ForcedWarpExtended")
+
+ self.logger.infoPair("Populating","ForcedWarpLensing")
+ self.populateForcedWarpLensing()
+ self.tablesToExport.append("ForcedWarpLensing")
+
+
+ self.logger.infoPair("Populating","ForcedWarpToImage")
self.populateForcedWarpToImage()
-
+ self.tablesToExport.append("ForcedWarpToImage")
+
+
+ self.logger.infoPair("setting min/max objid from","ForcedWarpMeasurement")
self.setMinMaxObjID(["ForcedWarpMeasurement"])
@@ -281,5 +523,5 @@
'''
def importIppTables(self, columns="*", tableRE=""):
-
+ count = 0
if self.config.retry: return True
self.logger.infoPair("Importing FW tables with table match expression: ", tableRE)
@@ -302,4 +544,5 @@
# drop any previous tables before import
+
self.scratchDb.dropTable(table.name)
@@ -318,21 +561,14 @@
self.logger.infoPair("Done. Imported", "%d tables" % count)
self.indexIppTables()
-
return True
-
-
-
-
-
-
-
def generateRandomIDs(self):
- sql = "UPDATE ForcedWarpMeasurement set randomWarpID = FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)"
- try: self.scratchDb.execute(sql)
+ sql = "UPDATE ForcedWarpMeasurement set randomWarpID = FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)"
+ try:
+ self.scratchDb.execute(sql)
except:
self.logger.errorPair('failed sql',sql)
- return
+ raise
@@ -341,19 +577,58 @@
'''
- def generatePspsUniqueIDs(self):
- sql = "UPDATE ForcedWarpMeasurement join (select @r:=@r+1 rownum, objID from \
- (select @r:=0) r, ForcedWarpMeasurement t) as foo using (objID) set \
- uniquePspsFWid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
+ def generateUniquePspsIDs(self):
+ sql = "UPDATE ForcedWarpMeasurement set uniquePspsFWid = (("+str(self.batchID)+"*1000000000 ) + row)"
try: self.scratchDb.execute(sql)
except:
self.logger.errorPair('failed sql',sql)
- return
-
-
-
-
-
-
+ raise
+
+ '''
+ Defines the tables to be exported to psps
+ '''
def exportPspsTablesToFits(self, regex="(.*)"):
- return super(DetectionBatch, self).exportPspsTablesToFits("(ForcedWarp.*)")
+ return super(ForcedWarpBatch, self).exportPspsTablesToFits("(ForcedWarp.*)")
+
+
+ '''
+ Drop a table and report the drop
+ '''
+ def dropTableVerbose(self, table):
+ self.logger.infoPair("dropping table:",table)
+ self.scratchDb.dropTable(table)
+
+
+ '''
+ Applies indexes to the PSPS tables
+ '''
+ def alterPspsTables(self):
+
+ self.logger.debug("Altering PSPS tables (currently does nothing in stackbatch)")
+
+ return True
+
+ '''
+ Applies indexes to the IPP tables
+ '''
+ def indexIppTables(self):
+
+ self.logger.infoPair("Creating indexes on", "IPP tables")
+ self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET")
+ self.scratchDb.createIndex("SkyChip_xfit", "IPP_IDET")
+ self.scratchDb.createIndex("SkyChip_xrad", "IPP_IDET")
+ self.scratchDb.createIndex("SkyChip_xsrc", "IPP_IDET")
+ self.scratchDb.createIndex("SkyChip_xgal", "IPP_IDET")
+
+ return True
+
+ '''
+ Updates table and generates pspsuniqueids
+ '''
+
+ def updatePspsUniqueIDs(self,table):
+ sql = "UPDATE ForcedWarpMeasurement set uniquePspsFWid = (("+str(self.batchID)+"*1000000000 ) + row)"
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair('failed sql',sql)
+ raise
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/gpc1db.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/gpc1db.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/gpc1db.py (revision 38052)
@@ -145,7 +145,7 @@
elif batchType == "FW":
- stage = "forcedwarp"
- sql = "SELECT DISTINCT sky_id, radeg, decdeg FROM fullForceInput \
- JOIN addRun ON(fullForceInput.ff_id = addRun.stage_id, fullForceInput.warp_id = addRun.stage_extra1) \
+ stage = "fullforce"
+ sql = "SELECT DISTINCT add_id, radeg, decdeg FROM fullForceInput \
+ JOIN addRun ON(fullForceInput.ff_id = addRun.stage_id AND fullForceInput.warp_id = addRun.stage_extra1) \
JOIN minidvodbRun USING(minidvodb_name) \
JOIN minidvodbProcessed USING(minidvodb_id) \
@@ -219,4 +219,37 @@
+ '''
+ Gets a list of PSPS image IDs for this stack ID
+ '''
+ def getImageIDsForThisForcedWarpID(self, forcedWarpID):
+
+ self.logger.debug("Querying GPC1 for image IDs for forcedWarp ID: " + str(forcedWarpID))
+
+ # JOIN stackInputSkyfile shoule use stack_id as well as warp_id, right? (maybe not -- the join to warp is first)
+ sql = "SELECT DISTINCT CONCAT(exp_id, SUBSTR(class_id, 3, 4)) FROM ( \
+ SELECT DISTINCT exp_id,class_id \
+ FROM addRun join \
+ fullForceInput on (fullForceInput.ff_id = addRun.stage_id and \
+ fullForceInput.warp_id = addRun.stage_extra1) \
+ JOIN fullForceResult using (ff_id, warp_id) \
+ JOIN warpRun USING(warp_id) \
+ JOIN warpSkyCellMap using (warp_id) \
+ JOIN fakeRun USING(fake_id) \
+ JOIN camRun USING(cam_id) \
+ JOIN chipRun USING(chip_id) \
+ WHERE add_id = " + str(forcedWarpID) + ") AS a"
+
+ try:
+ rs = self.executeQuery(sql)
+ except:
+ self.logger.exception("Can't query for imageIDs using :" + sql )
+
+ imageIDs = []
+ while (rs.next()):
+ imageIDs.append(rs.getString(1))
+ rs.close()
+
+ return imageIDs
+
@@ -251,4 +284,49 @@
return meta
+
+ '''
+ Gets some forcedwarp stage meta data for this sky_id # TODO this SQL could surely be improved
+ '''
+ def getForcedWarpStageMeta(self, forcedWarpID, warpSkyfileID):
+ self.logger.debug("Querying GPC1 for forced warp meta data")
+
+ meta = []
+
+ sql = "SELECT \
+ exp_id, \
+ exp_time, \
+ fullForceResult.software_ver, \
+ skycell_id, \
+ warpRun.tess_id \
+ FROM fullForceInput join addRun on \
+ (fullForceInput.ff_id = addRun.stage_id and fullForceInput.warp_id = addRun.stage_extra1) \
+ JOIN fullForceResult using (ff_id,warp_id) \
+ JOIN warpRun using (warp_id) \
+ JOIN warpImfile using (warp_id) \
+ JOIN fakeRun using (fake_id) \
+ JOIN camRun using (cam_id) \
+ JOIN chipRun using (chip_id) \
+ JOIN rawExp using (exp_id) \
+ WHERE add_id = " + str(forcedWarpID) + " \
+ AND warp_skyfile_id = " + str(warpSkyfileID)
+
+ try:
+ rs = self.executeQuery(sql)
+ rs.first()
+ except:
+ self.logger.errorPair("Can't query for", "forced warp meta data")
+
+ try:
+ meta.append(rs.getString(1))
+ meta.append(rs.getString(2))
+ meta.append(rs.getString(3))
+ meta.append(rs.getString(4))
+ meta.append(rs.getString(5))
+
+ except:
+ self.logger.errorPair("getForcedWarpStageMeta()", "empty meta data")
+
+ return meta
+
@@ -300,4 +378,43 @@
'''
+ get forcedwarp stage cmf
+ '''
+ def getForcedWarpStageCmf(self, dvoDb, forcedWarpID):
+ sql = "SELECT DISTINCT fullForceResult.path_base \
+ from fullForceInput join addRun on \
+ (fullForceInput.ff_id = addRun.stage_id AND \
+ fullForceInput.warp_id = addRun.stage_extra1) \
+ join fullForceResult using (ff_id, warp_id) \
+ where add_id = %d" % forcedWarpID
+ try:
+ rs = self.executeQuery(sql)
+ rs.first()
+ except:
+ self.logger.infoPair("failed sql", sql)
+ self.logger.errorPair("can't query for fullforce cmf", "add_id = %d" %forcedWarpID)
+ return None
+
+ try:
+ pathBase = rs.getString(1)
+ except:
+ self.logger.errorPair("no full force cmfs found for add_id = %d" %forcedWarpID)
+ return None
+
+ files = []
+ if pathBase.startswith("neb"):
+
+ f=os.popen("neb-ls -p " + pathBase + ".cmf")
+ for i in f.readlines():
+
+ files.append(i.rstrip())
+
+
+ if len(files) < 1: return None
+ fits = Fits(self.logger, self.config, files[0])
+ return fits
+
+
+
+ '''
get diff stage cmf name
'''
@@ -327,5 +444,5 @@
pathBase = rs.getString(1)
except:
- self.logger.errorPair("no diff cms found for diff_skyfile_id = %d" %diff_skyfile_id)
+ self.logger.errorPair("no diff cmfs found for diff_skyfile_id = %d" %diff_skyfile_id)
return None
print pathBase
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/ipptopspsdb.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/ipptopspsdb.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/ipptopspsdb.py (revision 38052)
@@ -1342,4 +1342,5 @@
(" + str(box_id) + ", '" + batchType + "', " + str(id) + ")"
+ print sql
try: self.execute(sql)
except:
@@ -1551,6 +1552,7 @@
# XXX EAM : need to do something special at the poles...
- if (boxDim['DEC'] > 85): raise
-
+ #if (boxDim['DEC'] > 85): raise
+ #heather commented out for oB
+
# a given box centered at RA,DEC with (linear) size of SIDE has the following bounds
dR = 0.5*boxDim['SIDE'] / math.cos(math.radians(boxDim['DEC']))
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/loader.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/loader.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/loader.py (revision 38052)
@@ -27,6 +27,6 @@
from diffobjectbatch import DiffObjectBatch
-#from forcedwarpbatch import ForcedWarpBatch
-#from forcedobjectbatch import ForcedObjectBatch
+from forcedwarpbatch import ForcedWarpBatch
+from forcedobjectbatch import ForcedObjectBatch
#
#
@@ -168,6 +168,6 @@
self.logger.infoPair("Box dimensions: ", "ra / dec / size : %.1f / %.1f / %.1f -- sky bounds : (%.2f - %2.f, %2.f - %2.f)" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE'], boxDim['minRA'], boxDim['maxRA'], boxDim['minDEC'], boxDim['maxDEC']))
self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids))
-
- if batchType != "OB":
+ #previously said if not OB, but we don't need these for Do or FO
+ if (batchType == "P2" or batchType == "ST" or batchType =="DF" or batchType == "FW"):
if self.config.resetDvo or not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost):
if not self.dvoDetections.nativeIngestDetections(boxId, boxDim):
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/mysql.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/mysql.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/mysql.py (revision 38052)
@@ -258,5 +258,6 @@
self.execute(sql)
except: pass
- #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'")
+
+ # self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'")
'''
Changes the database engine for this table to InnoDb
@@ -477,4 +478,13 @@
self.execute("SET @i = 0")
self.execute("UPDATE " + table + " SET " + colName + "=(@i:=@i+1)")
- self.execute("ALTER TABLE " + table + " ADD PRIMARY KEY (" + colName + ")")
-
+ self.execute("ALTER IGNORE TABLE " + table + " ADD PRIMARY KEY (" + colName + ")")
+
+ '''
+ Adds a column to this table which contains a row count
+ '''
+ def addRowCountColumnNoKey(self, table, colName):
+
+ self.execute("ALTER TABLE " + table + " ADD COLUMN " + colName + " INTEGER FIRST")
+ self.execute("SET @i = 0")
+ self.execute("UPDATE " + table + " SET " + colName + "=(@i:=@i+1)")
+
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/objectbatch.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/objectbatch.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/objectbatch.py (revision 38052)
@@ -92,7 +92,9 @@
def updatePspsUniqueIDs(self):
- sql = "UPDATE ObjectThin join (select @r:=@r+1 rownum, objID from \
- (select @r:=0) r, ObjectThin t) as foo using (objID) set \
- uniquePspsOBid = ((" +str(self.batchID) +"*1000000000 ) + rownum)";
+ # sql = "UPDATE ObjectThin join (select @r:=@r+1 rownum, objID from \
+ # (select @r:=0) r, ObjectThin t) as foo using (objID) set \
+ # uniquePspsOBid = ((" +str(self.batchID) +"*1000000000 ) + rownum)";
+
+ sql = "UPDATE ObjectThin set uniquePspsOBid = (("+str(self.batchID)+"*1000000000 ) + row)"
try: self.scratchDb.execute(sql)
except:
@@ -184,6 +186,8 @@
SET " + filter[1] + "MeanPSFMagErr = null \
WHERE " + filter[1] + "MeanPSFMagErr > " + str(cut)
- self.scratchDb.execute(sql)
-
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.infoPair("failed sql",sql)
+ raise
'''
Inserts stuff for all mags
@@ -222,6 +226,8 @@
sql += "ObjectThin.nStackDetections = ObjectThin.nStackDetections + cps.NSTACK_DET"
self.logger.info(sql)
- self.scratchDb.execute(sql)
-
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.InfoPair("failed sql",sql)
+ raise
# XXX this does not seem like a good thing to leave in SQL
self.logger.infoPair("Calculating nDetections from", "n[filters]")
@@ -231,6 +237,8 @@
sql += "SET nDetections = nDetections + n" + filter[1]
sql += " WHERE n" + filter[1] + " != -999"
- self.scratchDb.execute(sql)
-
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.infoPair("failed sql",sql)
+ raise
'''
give objectName to objectThin
@@ -255,4 +263,5 @@
except:
self.logger.errorPair("failed sql", sql)
+ raise
return False
@@ -273,4 +282,5 @@
except:
self.logger.errorPair("failed sql", sql)
+ raise
return False
self.logger.infoPair("updating objName", "using means")
@@ -289,4 +299,5 @@
except:
self.logger.errorPair("failed sql", sql)
+ raise
return False
@@ -307,4 +318,5 @@
except:
self.logger.errorPair("failed sql", sql)
+ raise
return False
@@ -327,5 +339,6 @@
except:
self.logger.errorPair("Couldn't squash out of range stuff", sql)
- return False
+ raise
+
self.logger.infoPair("setting to null > 1e-38 and < 1e-38 in", "cps FLUX_KRON")
@@ -336,5 +349,5 @@
except:
self.logger.errorPair("Couldn't squash out of range stuff", sql)
- return False
+ raise
@@ -418,4 +431,5 @@
rs = self.scratchDb.executeQuery(sql)
+
rs.first()
nToDelete = rs.getInt(1)
@@ -435,15 +449,17 @@
except:
self.logger.errorPair("Couldn't cull outsiders from ObjectThin table", sql)
- return False
-
+ raise
+
+ ##Don't do this till after MeanObject
+ ##self.dvoObjects.purgeRegion(self.region)
+ self.logger.infoPair("updatePspsUniqueIDs","start")
+ self.updatePspsUniqueIDs()
+ self.logger.infoPair("updatePspsUniqueIDs","end")
self.logger.infoPair("Dropping row column from", "ObjectThin table")
self.scratchDb.dropColumn("ObjectThin", "row")
##self.logger.infoPair("Purging from scratch Db", self.region + " region")
-
+ self.logger.infoPair("Dropped row column", "objectThin")
##Don't do this till after MeanObject
- ##self.dvoObjects.purgeRegion(self.region)
-
- self.updatePspsUniqueIDs()
-
+
self.setMinMaxObjID(["ObjectThin"])
@@ -463,5 +479,5 @@
except:
self.logger.errorPair("Couldn't populate MeanObject table", sql)
- return False
+ raise
self.logger.infoPair("Adding 'row' columns to", "MeanObject table")
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/queue.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/queue.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/queue.py (revision 38052)
@@ -29,5 +29,5 @@
super(Queue, self).__init__(argv)
- # create various objects
+ # create various objects
self.gpc1Db = Gpc1Db(self.logger, self.config)
@@ -132,6 +132,7 @@
#dec = self.skychunk.minDec
- if (dec > 85): raise
-
+ #if (dec > 85): raise
+ # this may be a problem for poles # HAF XXX
+ if (dec > 90): raise
# print "dec, dD: ", dec, dD ,self.skychunk.maxDec
@@ -170,7 +171,7 @@
dec = dec + 2*dD
- if (dec > 85): raise
-
- #dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
+ #if (dec > 85): raise
+ if (dec > 90): raise
+ #dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
dR = 0.5*self.skychunk.boxSize
ra = self.skychunk.minRa + dR
Index: /branches/eam_branches/ipp-20150112/ippToPsps/jython/stackbatch.py
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/jython/stackbatch.py (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/jython/stackbatch.py (revision 38052)
@@ -293,5 +293,11 @@
self.generateRandomIDs()
+ self.logger.infoPair("Adding 'row' columns to", "StackObjectThin")
+ self.scratchDb.addRowCountColumn("StackObjectThin", "row")
+ self.logger.infoPair("Adding unique psps ids to", "StackObjectThin")
self.generateStackDetectRowIDsAndPspsUniqueIDs()
+ self.logger.infoPair("Dropping row column from", "StackObjectThin table")
+ self.scratchDb.dropColumn("StackObjectThin", "row")
+
# add indexes StackObjectThin
@@ -372,4 +378,14 @@
self.logger.errorPair('failed sql: ', sql)
raise
+
+
+ # we do not yet set bestdetection, this is a placeholder until we do
+ sql = "UPDATE StackObjectThin set bestDetection = 0"
+ try:
+ self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair('failed sql: ', sql)
+ raise
+
self.tablesToExport.append("StackObjectThin")
@@ -756,5 +772,5 @@
minRadius = 1
- maxRadius = 10
+ maxRadius = 9
if tablename == 'StackApFlx':
minRadius = 3
@@ -765,5 +781,5 @@
sqlLine = sqlUtility("UPDATE " + tablename + " SET")
- for radius in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:
+ for radius in [1, 2, 3, 4, 5, 6, 7, 8, 9]:
if radius < minRadius: continue
if radius > maxRadius: continue
@@ -821,6 +837,8 @@
sql = sqlLine.makeEquals("WHERE a." + filter + "ippDetectID = b.IPP_IDET AND " + psfCondition)
- self.scratchDb.execute(sql)
-
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.infoPair("failed sql", sql)
+ raise
# instrumental flux, AB mags, Jy:
# flux_inst = cts / sec
@@ -843,6 +861,8 @@
sqlLine.group(field + "Std", field + "Std" + " * POW(10, 3.56 - 0.4* b.zp)")
sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID AND b.imageID = " + str(self.imageIDs[filter]) )
- self.scratchDb.execute(sql)
-
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.infoPair("failed sql",sql)
+ raise
'''
Populates the StackToImage table
@@ -950,8 +970,10 @@
def generateStackDetectRowIDsAndPspsUniqueIDs(self):
- sql = "UPDATE StackObjectThin join (select @r:=@r+1 rownum, objID from \
- (select @r:=0) r, StackObjectThin t) as foo using (objID) set \
- stackDetectRowID = ((" + str(self.batchID) + " << 32 ) + rownum), \
- uniquePspsSTid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)";
+# sql = "UPDATE StackObjectThin join (select @r:=@r+1 rownum, objID from \
+# (select @r:=0) r, StackObjectThin t) as foo using (objID) set \
+# stackDetectRowID = ((" + str(self.batchID) + " << 32 ) + rownum), \
+# uniquePspsSTid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)";
+ sql = "UPDATE StackObjectThin set uniquePspsSTid = (("+str(self.batchID)+"*1000000000 ) + row)"
+
try: self.scratchDb.execute(sql)
except:
@@ -1010,5 +1032,5 @@
self.scratchDb.addColumn(table, filter + "haveData", "tinyint")
- fields = "objID, ippObjID, randomStackObjID, primaryDetection, bestDetection, uniquePspsSTID, "
+ fields = "objID, ippObjID, randomStackObjID, primaryDetection, bestDetection, uniquePspsSTID, StackDetectRowID, "
for i in range(len(self.filters)):
@@ -1016,10 +1038,11 @@
fields += filter + "ippDetectID, "
fields += filter + "stackDetectID, "
+# fields += filter + "StackDetectRowID, "
fields += filter + "stackMetaID " # careful here: do not add comma to last entry
if (i < len(self.filters) - 1):
fields += ", "
- if (table == "StackObjectAttributes"):
- fields += ", stackDetectRowID "
+ #if (table == "StackObjectAttributes"):
+ # fields += ", stackDetectRowID "
sql = "INSERT INTO " + table + " (" + fields + ") "
sql += "SELECT " + fields + " FROM " + mainTable
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject.txt
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject.txt (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject.txt (revision 38052)
@@ -1,21 +1,21 @@
diffObjName = ipptopsps
-diffObjID = dvo / addstar
+diffObjID = dvo:cpt.extID
uniquePspsDOid = ipptopsps
-ippObjID = dvo / addstar
+ippObjID = dvo:cpt.objID | cpt.catID <<32
surveyID = ipptopsps
-skyCellID = smf header
+skyCellID = smf header ?? how does this work for objects
htmID = PSPS
zoneID = PSPS
randomID = ipptopsps
batchID = ipptopsps
-dvoRegionID = dvo catID
-objInfoFlag = average.flags
-qualityFlag = average.flags >> 24 & 0xff (needed??)
+dvoRegionID = dvo:cpt:catID
+objInfoFlag = dvo:cpt.flags
+qualityFlag = dvo:cpt.flags >> 24 & 0xff (needed??)
consistencyFlag = ?
-ra = dvo / relastro
-dec = dvo / relastro
-raErr = dvo / relastro
-decErr = dvo / relastro
+ra = dvo:cpt.RA_MEAN
+dec = dvo:cpt.DEC_MEAN
+raErr = dvo:cpt.RA_ERR
+decErr = dvo:cpt.DEC_ERR
cx = PSPS
cy = PSPS
@@ -25,14 +25,14 @@
l = PSPS
b = PSPS
-gQfPerfect = dvo / relphot / secfilt. [XXX Not calculated]
-rQfPerfect = dvo / relphot / secfilt. [XXX Not calculated]
-iQfPerfect = dvo / relphot / secfilt. [XXX Not calculated]
-zQfPerfect = dvo / relphot / secfilt. [XXX Not calculated]
-yQfPerfect = dvo / relphot / secfilt. [XXX Not calculated]
+gQfPerfect = dvo:secfilt.PSF_QF_PERF_MAX
+rQfPerfect = dvo:secfilt.PSF_QF_PERF_MAX
+iQfPerfect = dvo:secfilt.PSF_QF_PERF_MAX
+zQfPerfect = dvo:secfilt.PSF_QF_PERF_MAX
+yQfPerfect = dvo:secfilt.PSF_QF_PERF_MAX
dataRelease = ipptopsps
-nDetections = dvo / relphot
-ng = dvo / relphot
-nr = dvo / relphot
-ni = dvo / relphot
-nz = dvo / relphot
-ny = dvo / relphot
+nDetections = ipptopsps:sum of ng+nr+ni+nz+ny
+ng = dvo:secfilt.nwarp
+nr = dvo:secfilt.nwarp
+ni = dvo:secfilt.nwarp
+nz = dvo:secfilt.nwarp
+ny = dvo:secfilt.nwarp
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanLensing
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanLensing (revision 38052)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanLensing (revision 38052)
@@ -0,0 +1,32 @@
+
+objID dimensionless BIGINT 8 NA ODM object identifier index
+uniquePspsFOid dimensionless BIGINT 8 NA unique id supplied by ipptopsps
+ippObjID dimensionless BIGINT 8 NA IPP object identifier
+nDetections dimensionless SMALLINT 2 -999 total number of detection measurements in all filters
+batchID dimensionless BIGINT 8 NA batch id
+----------------------------------------------------- replicate to bottom for each filter
+gStackID dimensionless INT 4 stack image id (how to carry through IPP + DVO)
+gLensObjSmearX11 arcsec^-2 REAL 4 -999 per-object smear polarizability X11 term
+gLensObjSmearX12 arcsec^-2 REAL 4 -999 per-object smear polarizability X12 term
+gLensObjSmearX22 arcsec^-2 REAL 4 -999 per-object smear polarizability X22 term
+gLensObjSmearE1 arcsec^-2 REAL 4 -999 per-object smear polarizability E1 term
+gLensObjSmearE2 arcsec^-2 REAL 4 -999 per-object smear polarizability E2 term
+gLensObjShearX11 dimensionless REAL 4 -999 per-object shear polarizability X11 term
+gLensObjShearX12 dimensionless REAL 4 -999 per-object shear polarizability X12 term
+gLensObjShearX22 dimensionless REAL 4 -999 per-object shear polarizability X22 term
+gLensObjShearE1 dimensionless REAL 4 -999 per-object shear polarizability E1 term
+gLensObjShearE2 dimensionless REAL 4 -999 per-object shear polarizability E2 term
+gLensPSFSmearX11 arcsec^-2 REAL 4 -999 psf model smear polarizability X11 term
+gLensPSFSmearX12 arcsec^-2 REAL 4 -999 psf model smear polarizability X12 term
+gLensPSFSmearX22 arcsec^-2 REAL 4 -999 psf model smear polarizability X22 term
+gLensPSFSmearE1 arcsec^-2 REAL 4 -999 psf model smear polarizability E1 term
+gLensPSFSmearE2 arcsec^-2 REAL 4 -999 psf model smear polarizability E2 term
+gLensPSFShearX11 dimensionless REAL 4 -999 psf model shear polarizability X11 term
+gLensPSFShearX12 dimensionless REAL 4 -999 psf model shear polarizability X12 term
+gLensPSFShearX22 dimensionless REAL 4 -999 psf model shear polarizability X22 term
+gLensPSFShearE1 dimensionless REAL 4 -999 psf model shear polarizability E1 term
+gLensPSFShearE2 dimensionless REAL 4 -999 psf model shear polarizability E2 term
+
+-- duplicated for other filters
+-- [184 byte/obj/filter + 22 byte/obj]
+
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanLensing.txt
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanLensing.txt (revision 38052)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanLensing.txt (revision 38052)
@@ -0,0 +1,37 @@
+
+objID = cpt.extID
+uniquePspsFOid = ipptopsps
+ippObjID = cpt.objID | cpt.catID << 32
+nDetections = sql sum of cps.NCODE (xnTotal)
+batchID = ipptopsps
+---------------- =
+gStackID = ipptopsps / gpc1 db
+gLensObjSmearX11 = cpy.X11_SM_OBJ
+gLensObjSmearX12 = cpy.X12_SM_OBJ
+gLensObjSmearX22 = cpy.X22_SM_OBJ
+gLensObjSmearE1 = cpy.E1_SM_OBJ
+gLensObjSmearE2 = cpy.E2_SM_OBJ
+gLensObjShearX11 = cpy.X11_SH_OBJ
+gLensObjShearX12 = cpy.X12_SH_OBJ
+gLensObjShearX22 = cpy.X22_SH_OBJ
+gLensObjShearE1 = cpy.E1_SH_OBJ
+gLensObjShearE2 = cpy.E2_SH_OBJ
+gLensPSFSmearX11 = cpy.X11_SM_PSF
+gLensPSFSmearX12 = cpy.X12_SM_PSF
+gLensPSFSmearX22 = cpy.X22_SM_PSF
+gLensPSFSmearE1 = cpy.E1_SM_PSF
+gLensPSFSmearE2 = cpy.E2_SM_PSF
+gLensPSFShearX11 = cpy.X11_SH_PSF
+gLensPSFShearX12 = cpy.X12_SH_PSF
+gLensPSFShearX22 = cpy.X22_SH_PSF
+gLensPSFShearE1 = cpy.E1_SH_PSF
+gLensPSFShearE2 = cpy.E2_SH_PSF
+
+[NOTES:
+
+ * PSF Flux, Kron Flux, Ap Flux stats are all
+ calculated from values in the existing Measure table
+
+ * R5 & R6 ap flux values & lensing parameters need
+ a new table
+
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject (revision 38052)
@@ -34,25 +34,4 @@
gFlags dimensionless BIGINT 8 NA flags for g forced
------------------------------------------------------
-gLensObjSmearX11 arcsec^-2 REAL 4 -999 per-object smear polarizability X11 term
-gLensObjSmearX12 arcsec^-2 REAL 4 -999 per-object smear polarizability X12 term
-gLensObjSmearX22 arcsec^-2 REAL 4 -999 per-object smear polarizability X22 term
-gLensObjSmearE1 arcsec^-2 REAL 4 -999 per-object smear polarizability E1 term
-gLensObjSmearE2 arcsec^-2 REAL 4 -999 per-object smear polarizability E2 term
-gLensObjShearX11 dimensionless REAL 4 -999 per-object shear polarizability X11 term
-gLensObjShearX12 dimensionless REAL 4 -999 per-object shear polarizability X12 term
-gLensObjShearX22 dimensionless REAL 4 -999 per-object shear polarizability X22 term
-gLensObjShearE1 dimensionless REAL 4 -999 per-object shear polarizability E1 term
-gLensObjShearE2 dimensionless REAL 4 -999 per-object shear polarizability E2 term
-gLensPSFSmearX11 arcsec^-2 REAL 4 -999 psf model smear polarizability X11 term
-gLensPSFSmearX12 arcsec^-2 REAL 4 -999 psf model smear polarizability X12 term
-gLensPSFSmearX22 arcsec^-2 REAL 4 -999 psf model smear polarizability X22 term
-gLensPSFSmearE1 arcsec^-2 REAL 4 -999 psf model smear polarizability E1 term
-gLensPSFSmearE2 arcsec^-2 REAL 4 -999 psf model smear polarizability E2 term
-gLensPSFShearX11 dimensionless REAL 4 -999 psf model shear polarizability X11 term
-gLensPSFShearX12 dimensionless REAL 4 -999 psf model shear polarizability X12 term
-gLensPSFShearX22 dimensionless REAL 4 -999 psf model shear polarizability X22 term
-gLensPSFShearE1 dimensionless REAL 4 -999 psf model shear polarizability E1 term
-gLensPSFShearE2 dimensionless REAL 4 -999 psf model shear polarizability E2 term
-------------------------------------------------------
gGamma arcsec^2 REAL 4 -999 second moment (Mxx + Myy)
gE1 dimensionless REAL 4 -999 polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy)
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject.txt
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject.txt (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject.txt (revision 38052)
@@ -34,25 +34,4 @@
gFlags = cps.FLAGS
---------------- =
-gLensObjSmearX11 = cpy.X11_SM_OBJ
-gLensObjSmearX12 = cpy.X12_SM_OBJ
-gLensObjSmearX22 = cpy.X22_SM_OBJ
-gLensObjSmearE1 = cpy.E1_SM_OBJ
-gLensObjSmearE2 = cpy.E2_SM_OBJ
-gLensObjShearX11 = cpy.X11_SH_OBJ
-gLensObjShearX12 = cpy.X12_SH_OBJ
-gLensObjShearX22 = cpy.X22_SH_OBJ
-gLensObjShearE1 = cpy.E1_SH_OBJ
-gLensObjShearE2 = cpy.E2_SH_OBJ
-gLensPSFSmearX11 = cpy.X11_SM_PSF
-gLensPSFSmearX12 = cpy.X12_SM_PSF
-gLensPSFSmearX22 = cpy.X22_SM_PSF
-gLensPSFSmearE1 = cpy.E1_SM_PSF
-gLensPSFSmearE2 = cpy.E2_SM_PSF
-gLensPSFShearX11 = cpy.X11_SH_PSF
-gLensPSFShearX12 = cpy.X12_SH_PSF
-gLensPSFShearX22 = cpy.X22_SH_PSF
-gLensPSFShearE1 = cpy.E1_SH_PSF
-gLensPSFShearE2 = cpy.E2_SH_PSF
----------------- =
gGamma = cpy.GAMMA
gE1 = cpy.E1
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended (revision 38052)
@@ -13,25 +13,4 @@
obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation in Mean Julian Days
------------------------------------------------------
-lensObjSmearX11 arcsec^-2 REAL 4 -999 per-object smear polarizability X11 term
-lensObjSmearX12 arcsec^-2 REAL 4 -999 per-object smear polarizability X12 term
-lensObjSmearX22 arcsec^-2 REAL 4 -999 per-object smear polarizability X22 term
-lensObjSmearE1 arcsec^-2 REAL 4 -999 per-object smear polarizability E1 term
-lensObjSmearE2 arcsec^-2 REAL 4 -999 per-object smear polarizability E2 term
-lensObjShearX11 dimensionless REAL 4 -999 per-object shear polarizability X11 term
-lensObjShearX12 dimensionless REAL 4 -999 per-object shear polarizability X12 term
-lensObjShearX22 dimensionless REAL 4 -999 per-object shear polarizability X22 term
-lensObjShearE1 dimensionless REAL 4 -999 per-object shear polarizability E1 term
-lensObjShearE2 dimensionless REAL 4 -999 per-object shear polarizability E2 term
-lensPSFSmearX11 arcsec^-2 REAL 4 -999 psf model smear polarizability X11 term
-lensPSFSmearX12 arcsec^-2 REAL 4 -999 psf model smear polarizability X12 term
-lensPSFSmearX22 arcsec^-2 REAL 4 -999 psf model smear polarizability X22 term
-lensPSFSmearE1 arcsec^-2 REAL 4 -999 psf model smear polarizability E1 term
-lensPSFSmearE2 arcsec^-2 REAL 4 -999 psf model smear polarizability E2 term
-lensPSFShearX11 dimensionless REAL 4 -999 psf model shear polarizability X11 term
-lensPSFShearX12 dimensionless REAL 4 -999 psf model shear polarizability X12 term
-lensPSFShearX22 dimensionless REAL 4 -999 psf model shear polarizability X22 term
-lensPSFShearE1 dimensionless REAL 4 -999 psf model shear polarizability E1 term
-lensPSFShearE2 dimensionless REAL 4 -999 psf model shear polarizability E2 term
-------------------------------------------------------
flxR5 adu/seconds REAL 4 -999 Flux inside r = 5
flxR5Err adu/seconds REAL 4 -999 estimated error is flxR5
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended.txt
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended.txt (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpExtended.txt (revision 38052)
@@ -13,25 +13,4 @@
obsTime = cmf
---------------- =
-lensObjSmearX11 = cmf.X11_SM_OBJ
-lensObjSmearX12 = cmf.X12_SM_OBJ
-lensObjSmearX22 = cmf.X22_SM_OBJ
-lensObjSmearE1 = cmf.E1_SM_OBJ
-lensObjSmearE2 = cmf.E2_SM_OBJ
-lensObjShearX11 = cmf.X11_SH_OBJ
-lensObjShearX12 = cmf.X12_SH_OBJ
-lensObjShearX22 = cmf.X22_SH_OBJ
-lensObjShearE1 = cmf.E1_SH_OBJ
-lensObjShearE2 = cmf.E2_SH_OBJ
-lensPSFSmearX11 = cmf.X11_SM_PSF
-lensPSFSmearX12 = cmf.X12_SM_PSF
-lensPSFSmearX22 = cmf.X22_SM_PSF
-lensPSFSmearE1 = cmf.E1_SM_PSF
-lensPSFSmearE2 = cmf.E2_SM_PSF
-lensPSFShearX11 = cmf.X11_SH_PSF
-lensPSFShearX12 = cmf.X12_SH_PSF
-lensPSFShearX22 = cmf.X22_SH_PSF
-lensPSFShearE1 = cmf.E1_SH_PSF
-lensPSFShearE2 = cmf.E2_SH_PSF
----------------- =
flxR5 = cmf.FLX_AP_R5
flxR5Err = cmf.FLX_ERR_AP_R5
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpLensing
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpLensing (revision 38052)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpLensing (revision 38052)
@@ -0,0 +1,38 @@
+
+objID dimensionless BIGINT 8 NA ODM object identifier
+uniquePspsFWid dimentionless BIGINT 8 NA unique id supplied by ipptopsps
+detectID dimensionless BIGINT 8 NA ODM detection identifier
+ippObjID dimensionless BIGINT 8 NA ippobject id generated by IPP
+ippDetectID dimensionless BIGINT 8 NA detection id generated by IPP
+filterID dimensionless TINYINT 1 NA filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ...
+surveyID dimensionless TINYINT 1 NA survey flag identifier
+forcedWarpID dimensionless BIGINT 8 NA IPP warp_id
+randomWarpID dimensionless BIGINT 8 NA Random warp_id
+skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier
+dvoRegionID dimensionless INT 4 -999 identifier to dvo spatial region
+obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation in Mean Julian Days
+------------------------------------------------------
+lensObjSmearX11 arcsec^-2 REAL 4 -999 per-object smear polarizability X11 term
+lensObjSmearX12 arcsec^-2 REAL 4 -999 per-object smear polarizability X12 term
+lensObjSmearX22 arcsec^-2 REAL 4 -999 per-object smear polarizability X22 term
+lensObjSmearE1 arcsec^-2 REAL 4 -999 per-object smear polarizability E1 term
+lensObjSmearE2 arcsec^-2 REAL 4 -999 per-object smear polarizability E2 term
+lensObjShearX11 dimensionless REAL 4 -999 per-object shear polarizability X11 term
+lensObjShearX12 dimensionless REAL 4 -999 per-object shear polarizability X12 term
+lensObjShearX22 dimensionless REAL 4 -999 per-object shear polarizability X22 term
+lensObjShearE1 dimensionless REAL 4 -999 per-object shear polarizability E1 term
+lensObjShearE2 dimensionless REAL 4 -999 per-object shear polarizability E2 term
+lensPSFSmearX11 arcsec^-2 REAL 4 -999 psf model smear polarizability X11 term
+lensPSFSmearX12 arcsec^-2 REAL 4 -999 psf model smear polarizability X12 term
+lensPSFSmearX22 arcsec^-2 REAL 4 -999 psf model smear polarizability X22 term
+lensPSFSmearE1 arcsec^-2 REAL 4 -999 psf model smear polarizability E1 term
+lensPSFSmearE2 arcsec^-2 REAL 4 -999 psf model smear polarizability E2 term
+lensPSFShearX11 dimensionless REAL 4 -999 psf model shear polarizability X11 term
+lensPSFShearX12 dimensionless REAL 4 -999 psf model shear polarizability X12 term
+lensPSFShearX22 dimensionless REAL 4 -999 psf model shear polarizability X22 term
+lensPSFShearE1 dimensionless REAL 4 -999 psf model shear polarizability E1 term
+lensPSFShearE2 dimensionless REAL 4 -999 psf model shear polarizability E2 term
+------------------------------------------------------
+
+-- [295 byte/det]
+-- this table is only for non-Galatic-plane data
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpLensing.txt
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpLensing.txt (revision 38052)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpLensing.txt (revision 38052)
@@ -0,0 +1,35 @@
+
+objID = cpt.extID
+uniquePspsFWid = ipptopsps
+detectID = cpm.extID (???)
+ippObjID = cpt.obj_ID | cpx.cat_ID << 32
+ippDetectID = cmf.IPP_IDET
+filterID = ipptopsps / gpc1
+surveyID = ipptopsps
+forcedWarpID = ipptopsps / gpc1
+randomWarpID = ipptopsps
+skyCellID = ipptopsps / gpc1
+dvoRegionID = ipptopsps / gpc1
+obsTime = cmf
+---------------- =
+lensObjSmearX11 = cmf.X11_SM_OBJ
+lensObjSmearX12 = cmf.X12_SM_OBJ
+lensObjSmearX22 = cmf.X22_SM_OBJ
+lensObjSmearE1 = cmf.E1_SM_OBJ
+lensObjSmearE2 = cmf.E2_SM_OBJ
+lensObjShearX11 = cmf.X11_SH_OBJ
+lensObjShearX12 = cmf.X12_SH_OBJ
+lensObjShearX22 = cmf.X22_SH_OBJ
+lensObjShearE1 = cmf.E1_SH_OBJ
+lensObjShearE2 = cmf.E2_SH_OBJ
+lensPSFSmearX11 = cmf.X11_SM_PSF
+lensPSFSmearX12 = cmf.X12_SM_PSF
+lensPSFSmearX22 = cmf.X22_SM_PSF
+lensPSFSmearE1 = cmf.E1_SM_PSF
+lensPSFSmearE2 = cmf.E2_SM_PSF
+lensPSFShearX11 = cmf.X11_SH_PSF
+lensPSFShearX12 = cmf.X12_SH_PSF
+lensPSFShearX22 = cmf.X22_SH_PSF
+lensPSFShearE1 = cmf.E1_SH_PSF
+lensPSFShearE2 = cmf.E2_SH_PSF
+---------------- =
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement (revision 38052)
@@ -44,5 +44,4 @@
kronFluxErr adu/seconds REAL 4 -999 Kron flux error
kronRad arcsec REAL 4 -999 Kron radius
-kronRadErr arcsed REAL 4 -999 Kron radius error
------------------------------------------------------
sky adu/sec/pixel REAL 4 -999 PSF sky level at source
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6 (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6 (revision 38052)
@@ -49,8 +49,4 @@
gc6flxR9Std Jy REAL 4 -999 estimated standard deviation in c1flxR9
gc6flxR9Fill dimensionless REAL 4 -999 estimated filling factor at R9
-gc6flxR10 Jy REAL 4 -999 Flux inside r = 10
-gc6flxR10Err Jy REAL 4 -999 estimated error is c1flxR10
-gc6flxR10Std Jy REAL 4 -999 estimated standard deviation in c1flxR10
-gc6flxR10Fill dimensionless REAL 4 -999 estimated filling factor at R10
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6.txt
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6.txt (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon6.txt (revision 38052)
@@ -49,6 +49,3 @@
gc6flxR9Std - cmf : APER_FLUX_STDEV_9 : ** badly defined in cmf
gc6flxR9Fill + cmf : APER_FILL_9
-gc6flxR10 - cmf : APER_FLUX_10 ** does not exist in cmf
-gc6flxR10Err - cmf : APER_FLUX_ERR_10 ** does not exist in cmf
-gc6flxR10Std - cmf : APER_FLUX_STDEV_10 ** does not exist in cmf
-gc6flxR10Fill - cmf : APER_FILL_10 ** does not exist in cmf
+
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8 (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8 (revision 38052)
@@ -49,8 +49,4 @@
gc8flxR9Std Jy REAL 4 -999 estimated standard deviation in c2flxR9
gc8flxR9Fill dimensionless REAL 4 -999 estimated filling factor at R9
-gc8flxR10 Jy REAL 4 -999 Flux inside r = 10
-gc8flxR10Err Jy REAL 4 -999 estimated error is c2flxR10
-gc8flxR10Std Jy REAL 4 -999 estimated standard deviation in c2flxR10
-gc8flxR10Fill dimensionless REAL 4 -999 estimated filling factor at R10
-- repeat for other filters with appropriate leading prefex.
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8.txt
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8.txt (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGCon8.txt (revision 38052)
@@ -49,6 +49,3 @@
gc8flxR9Std - cmf : APER_FLUX_STDEV_9 : ** badly defined in cmf
gc8flxR9Fill + cmf : APER_FILL_9
-gc8flxR10 - cmf : APER_FLUX_10 ** does not exist in cmf
-gc8flxR10Err - cmf : APER_FLUX_ERR_10 ** does not exist in cmf
-gc8flxR10Std - cmf : APER_FLUX_STDEV_10 ** does not exist in cmf
-gc8flxR10Fill - cmf : APER_FILL_10 ** does not exist in cmf
+
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc (revision 38052)
@@ -49,6 +49,2 @@
gflxR9Std Jy REAL 4 -999 estimated standard deviation in flxR9
gflxR9Fill dimensionless REAL 4 -999 estimated filling factor at R9
-gflxR10 Jy REAL 4 -999 Flux inside r = 10
-gflxR10Err Jy REAL 4 -999 estimated error is flxR10
-gflxR10Std Jy REAL 4 -999 estimated standard deviation in flxR10
-gflxR10Fill dimensionless REAL 4 -999 estimated filling factor at R10
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc.txt
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc.txt (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlxEGUnc.txt (revision 38052)
@@ -49,6 +49,3 @@
gflxR9Std - cmf : APER_FLUX_STDEV_9 : ** badly defined in cmf
gflxR9Fill + cmf : APER_FILL_9
-gflxR10 - cmf : APER_FLUX_10 ** does not exist in cmf
-gflxR10Err - cmf : APER_FLUX_ERR_10 ** does not exist in cmf
-gflxR10Std - cmf : APER_FLUX_STDEV_10 ** does not exist in cmf
-gflxR10Fill - cmf : APER_FILL_10 ** does not exist in cmf
+
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitDeV
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitDeV (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitDeV (revision 38052)
@@ -26,3 +26,3 @@
gdecDeVOffErr arcsec REAL 4 -999 estimated error in dec offset
gDeVCf dimensionless REAL 4 -999 DeVaucouleurs fit coverage factor
-
+gDevChisq
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitSer
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitSer (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFitSer (revision 38052)
@@ -28,4 +28,4 @@
gSerDecErr dimensionless REAL 4 -999 estimated error in decSerOff
gserCf dimensionless REAL 4 -999 Sersic fit coverage factor
+gserChisq
-
Index: /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_VIEWS/Views
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_VIEWS/Views (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/pspsschema/SchemaNew19.0/PSPS_VIEWS/Views (revision 38052)
@@ -1,16 +1,28 @@
PSPS VIEWS - examples (not complete, please suggest others)
-MeanObjectView = ObjectThin join MeanObject
-StackObjectView = ObjectThin join StackObjectThin join StackObjectAttributes
-StackApFlxObjectView = ObjectThin join StackApFlx
-StackApFlxEGObjectView = ObjectThin join StackApFlxEGUnc join StackApFlxEGCon6 join StackApFlxEGCon8
-StackModelObjectView = ObjectThin join StackModelFitExp join StackModelFitDeV join StackModelFitSer join StackPetrosian
-ForcedMeanObjectView = ObjectThin join ForcedMeanObject
-StackObjectPrimaryView = ObjectThin join StackObjectThin join StackObjectAttributes join StackApFlx where primaryDection is true
-StackObjectBestView = ObjectThin join StackObject join StackObjectAttributes join StackApFlx where bestDetection is true
-DiffDetObjectView = DiffDetObject join DiffDetection
+*note: drop batchID it is not needed
+
+MeanObjectView = ObjectThin join MeanObject
+StackObjectView = ObjectThin join StackObjectThin join StackObjectAttributes
+StackApFlxObjectView = ObjectThin join StackApFlx
+StackApFlxExGalUncObjectView = ObjectThin join StackApFlxEGUnc
+StackApFlxExGalCon6ObjectView = ObjectThin join StackApFlxEGCon6
+StackApFlxExGalCon8ObjectView = ObjectThin join StackApFlxEGCon8
+StackModelObjectView = ObjectThin join StackModelFitExp join StackModelFitDeV join StackModelFitSer join StackPetrosian
+StackModelFitExpObjectView = ObjectThin join StackModelFitExp
+StackModelFitDeVObjectView = ObjectThin join StackModelFitDeV
+StackModelFitSerObjectView = ObjectThin join StackModelFitSer
+StackModelFitPetObjectView = ObjectThin join StackPetrosian
+StackObjectPrimaryView = ObjectThin join StackObjectThin join StackObjectAttributes join StackApFlx where primaryDection is true
+* StackObjectBestView = ObjectThin join StackObject join StackObjectAttributes join StackApFlx where bestDetection is true
+ForcedMeanObjectView = ObjectThin join ForcedMeanObject
+DetectionObjectView = ObjectThin join MeanObject join Detection
+DiffDetObjectView = DiffDetObject join DiffDetection
+ForcedDetObjectView = ObjectThin join ForcedWarpDetection
+
+
+*note1: StackObjectBestView - skip - we don't currently set bestDetection
+*note2: drop batchID it is not needed
-
-
Index: /branches/eam_branches/ipp-20150112/ippToPsps/src/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20150112/ippToPsps/src/Makefile.am (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ippToPsps/src/Makefile.am (revision 38052)
@@ -1,21 +1,7 @@
-if HAVE_SVNVERSION
-IPPTOPSPS_VERSION=`$(SVNVERSION) ..`
-else
-IPPTOPSPS_VERSION="UNKNOWN"
-endif
-
-if HAVE_SVN
-IPPTOPSPS_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
-IPPTOPSPS_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
-else
-IPPTOPSPS_BRANCH="UNKNOWN"
-IPPTOPSPS_SOURCE="UNKNOWN"
-endif
# Force recompilation of ippToPspsVersion.c, since it gets the version information
Version.c: VersionDefinitions.h
VersionDefinitions.h: VersionDefinitions.h.in FORCE
- -$(RM) VersionDefinitions.h
- $(SED) -e "s|@IPPTOPSPS_VERSION@|\"$(IPPTOPSPS_VERSION)\"|" -e "s|@IPPTOPSPS_BRANCH@|\"$(IPPTOPSPS_BRANCH)\"|" -e "s|@IPPTOPSPS_SOURCE@|\"$(IPPTOPSPS_SOURCE)\"|" VersionDefinitions.h.in > VersionDefinitions.h
+ pslib-setsvnversion.pl IPPTOPSPS VersionDefinitions.h.in VersionDefinitions.h
FORCE: ;
Index: /branches/eam_branches/ipp-20150112/ppViz/src/ppCoord/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20150112/ppViz/src/ppCoord/Makefile.am (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ppViz/src/ppCoord/Makefile.am (revision 38052)
@@ -1,23 +1,8 @@
bin_PROGRAMS = ppCoord
-
-if HAVE_SVNVERSION
-PPCOORD_VERSION=`$(SVNVERSION) ..`
-else
-PPCOORD_VERSION="UNKNOWN"
-endif
-
-if HAVE_SVN
-PPCOORD_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
-PPCOORD_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
-else
-PPCOORD_BRANCH="UNKNOWN"
-PPCOORD_SOURCE="UNKNOWN"
-endif
# Force recompilation of ppCoordVersion.c, since it gets the version information
ppCoordVersion.c: ppCoordVersionDefinitions.h
ppCoordVersionDefinitions.h: ppCoordVersionDefinitions.h.in FORCE
- -$(RM) ppCoordVersionDefinitions.h
- $(SED) -e "s|@PPCOORD_VERSION@|\"$(PPCOORD_VERSION)\"|" -e "s|@PPCOORD_BRANCH@|\"$(PPCOORD_BRANCH)\"|" -e "s|@PPCOORD_SOURCE@|\"$(PPCOORD_SOURCE)\"|" ppCoordVersionDefinitions.h.in > ppCoordVersionDefinitions.h
+ pslib-setsvnversion.pl PPCOORD ppCoordVersionDefinitions.h.in ppCoordVersionDefinitions.h
FORCE: ;
Index: /branches/eam_branches/ipp-20150112/ppViz/src/ppVizPSF/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20150112/ppViz/src/ppVizPSF/Makefile.am (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ppViz/src/ppVizPSF/Makefile.am (revision 38052)
@@ -1,23 +1,8 @@
bin_PROGRAMS = ppVizPSF
-
-if HAVE_SVNVERSION
-PPVIZPSF_VERSION=`$(SVNVERSION) ..`
-else
-PPVIZPSF_VERSION="UNKNOWN"
-endif
-
-if HAVE_SVN
-PPVIZPSF_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
-PPVIZPSF_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
-else
-PPVIZPSF_BRANCH="UNKNOWN"
-PPVIZPSF_SOURCE="UNKNOWN"
-endif
# Force recompilation of ppVizPSFVersion.c, since it gets the version information
ppVizPSFVersion.c: ppVizPSFVersionDefinitions.h
ppVizPSFVersionDefinitions.h: ppVizPSFVersionDefinitions.h.in FORCE
- -$(RM) ppVizPSFVersionDefinitions.h
- $(SED) -e "s|@PPVIZPSF_VERSION@|\"$(PPVIZPSF_VERSION)\"|" -e "s|@PPVIZPSF_BRANCH@|\"$(PPVIZPSF_BRANCH)\"|" -e "s|@PPVIZPSF_SOURCE@|\"$(PPVIZPSF_SOURCE)\"|" ppVizPSFVersionDefinitions.h.in > ppVizPSFVersionDefinitions.h
+ pslib-setsvnversion.pl PPVIZPSF ppVizPSFVersionDefinitions.h.in ppVizPSFVersionDefinitions.h
FORCE: ;
Index: /branches/eam_branches/ipp-20150112/ppViz/src/ppVizPattern/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20150112/ppViz/src/ppVizPattern/Makefile.am (revision 38051)
+++ /branches/eam_branches/ipp-20150112/ppViz/src/ppVizPattern/Makefile.am (revision 38052)
@@ -1,23 +1,8 @@
bin_PROGRAMS = ppVizPattern
-
-if HAVE_SVNVERSION
-PPVIZPATTERN_VERSION=`$(SVNVERSION) ..`
-else
-PPVIZPATTERN_VERSION="UNKNOWN"
-endif
-
-if HAVE_SVN
-PPVIZPATTERN_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
-PPVIZPATTERN_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
-else
-PPVIZPATTERN_BRANCH="UNKNOWN"
-PPVIZPATTERN_SOURCE="UNKNOWN"
-endif
# Force recompilation of ppVizPatternVersion.c, since it gets the version information
ppVizPatternVersion.c: ppVizPatternVersionDefinitions.h
ppVizPatternVersionDefinitions.h: ppVizPatternVersionDefinitions.h.in FORCE
- -$(RM) ppVizPatternVersionDefinitions.h
- $(SED) -e "s|@PPVIZPATTERN_VERSION@|\"$(PPVIZPATTERN_VERSION)\"|" -e "s|@PPVIZPATTERN_BRANCH@|\"$(PPVIZPATTERN_BRANCH)\"|" -e "s|@PPVIZPATTERN_SOURCE@|\"$(PPVIZPATTERN_SOURCE)\"|" ppVizPatternVersionDefinitions.h.in > ppVizPatternVersionDefinitions.h
+ pslib-setsvnversion.pl PPVIZPATTERN ppVizPatternVersionDefinitions.h.in ppVizPatternVersionDefinitions.h
FORCE: ;
Index: /branches/eam_branches/ipp-20150112/psconfig/psconfig.csh.in
===================================================================
--- /branches/eam_branches/ipp-20150112/psconfig/psconfig.csh.in (revision 38051)
+++ /branches/eam_branches/ipp-20150112/psconfig/psconfig.csh.in (revision 38052)
@@ -323,5 +323,5 @@
case linux:
case linrh:
- gcc --version | grep -e 4.6 -e 4.5 | grep Ubuntu >& /dev/null
+ gcc --version | grep -e 4.8 -e 4.6 -e 4.5| grep Ubuntu >& /dev/null
if ($status == 0) then
set xtraflags = "-Wl,--no-as-needed"
@@ -335,5 +335,5 @@
case lin64:
case linrh64:
- gcc --version | grep -e 4.6 -e 4.5 | grep Ubuntu >& /dev/null
+ gcc --version | grep -e 4.8 -e 4.6 -e 4.5 | grep Ubuntu >& /dev/null
if ($status == 0) then
set xtraflags = "-Wl,--no-as-needed"
Index: /branches/eam_branches/ipp-20150112/psconfig/tagsets/ipp-3.0.dist
===================================================================
--- /branches/eam_branches/ipp-20150112/psconfig/tagsets/ipp-3.0.dist (revision 38051)
+++ /branches/eam_branches/ipp-20150112/psconfig/tagsets/ipp-3.0.dist (revision 38052)
@@ -31,10 +31,7 @@
NNYYN pclient ipp-2-9 -0
+# psLib-based code needs the nebclient library to interact with nebulous
YNNYY Nebulous/nebclient ipp-2-9 -0
- YNNYY Nebulous ipp-2-9 -0
- YNNYY Nebulous-Server ipp-2-9 -0
- YYYYY PS-IPP-Metadata-Config ipp-2-9 -0
- YYYYY PS-IPP-Config ipp-2-9 -0
-
+
YYYYY psLib ipp-2-9 -0
YYYYY psModules ipp-2-9 -0
@@ -54,4 +51,9 @@
YYYYY ppSub ipp-2-9 -0
YYYYY ppSim ipp-2-9 -0
+
+ YNNYY Nebulous ipp-2-9 -0
+ YNNYY Nebulous-Server ipp-2-9 -0
+ YYYYY PS-IPP-Metadata-Config ipp-2-9 -0
+ YYYYY PS-IPP-Config ipp-2-9 -0
YNNYY glueforge ipp-2-9 -0
@@ -78,5 +80,5 @@
YYYYY extsrc/gpcsw ipp-2-9 -0
- YYYYY magic ipp-2-9 -0
+# YYYYY magic ipp-2-9 -0
YYYYY magic/censorObjects ipp-2-9 -0
YYYYY magic/remove ipp-2-9 -0
Index: /branches/eam_branches/ipp-20150112/pstamp/scripts/psmkreq
===================================================================
--- /branches/eam_branches/ipp-20150112/pstamp/scripts/psmkreq (revision 38051)
+++ /branches/eam_branches/ipp-20150112/pstamp/scripts/psmkreq (revision 38052)
@@ -1,3 +1,3 @@
-#!/bin/env perl
+#!/usr/bin/env perl
###
### psmkreq
Index: /branches/eam_branches/ipp-20150112/pstamp/scripts/pstampparse.pl
===================================================================
--- /branches/eam_branches/ipp-20150112/pstamp/scripts/pstampparse.pl (revision 38051)
+++ /branches/eam_branches/ipp-20150112/pstamp/scripts/pstampparse.pl (revision 38052)
@@ -18,4 +18,5 @@
use POSIX;
use Time::HiRes qw(gettimeofday);
+use Time::Local;
my $verbose;
@@ -117,4 +118,14 @@
my_die("REQ_NAME not found in $request_file_name\n", $PS_EXIT_PROG_ERROR) if (!$req_name);
my_die("wrong EXTVER $extver found in $request_file_name\n", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
+
+if ($extver eq "1") {
+ print STDERR "WARNING WARNING WARNING\n";
+ print STDERR "Very soon version 1 postage stamp requests will be rejected. Please update your request tables to version 2 format.\n";
+ print STDERR "Note that EMAIL will not be optional\n";
+
+} elsif (!$email) {
+ my_die("ERROR: Required parameter EMAIL not found in request file header.\n", $PSTAMP_INVALID_REQUEST)
+}
+
if ($extver >= 2) {
@@ -260,4 +271,11 @@
my $num_jobs = 0;
+my $accessLevel = getUsersAccessLevel($email, $label);
+if ($accessLevel < 0) {
+ print STDERR "Data access forbidden.\n";
+ insertFakeJobForRow(undef, 0, $PSTAMP_NOT_AUTHORIZED);
+ exit 0;
+}
+
foreach my $row (@$rows) {
@@ -271,4 +289,5 @@
$row->{job_num} = 0;
$row->{error_code} = 0;
+ $row->{accessLevel} = $accessLevel;
$num_jobs += processRow($action, $row);
@@ -537,102 +556,4 @@
}
-sub processRows {
- my $action = shift;
- my $rowList = shift;
- my $num_jobs = 0;
-
- if ($action eq 'LIST') {
- # LIST is not allowed by caller. Can't get here.
- return list_targets($rowList);
- }
-
- # all rows in the list are compatible
- my $row = $rowList->[0];
-
- my $project = $row->{PROJECT};
-
- # note: resolve_project avoids running pstamptool every time by remembering the
- # last project resolved
- my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
- if (!$proj_hash) {
- foreach my $r (@$rowList) {
- insertFakeJobForRow($r, 1, $PSTAMP_UNKNOWN_PRODUCT);
- $num_jobs++;
- }
- return $num_jobs;
- }
- my $req_type = $row->{REQ_TYPE};
- my $stage = $row->{IMG_TYPE};
- my $id = $row->{ID};
- my $component = $row->{COMPONENT};
- my $tess_id = $row->{TESS_ID};
-
- my $filter = $row->{REQFILT};
- my $mjd_min = $row->{MJD_MIN};
- my $mjd_max = $row->{MJD_MAX};
- my $data_group = $row->{DATA_GROUP};
-
- my $rownum = $row->{ROWNUM};
- my $job_type = $row->{JOB_TYPE};
- my $option_mask= $row->{OPTION_MASK};
-
- my $image_db = $proj_hash->{dbname};
- my $camera = $proj_hash->{camera};
- my $need_magic = $proj_hash->{need_magic};
- # Since user can get unmagicked data "by coordinate" requests can go back in time
- # to dredge unusable data from the "dark days"...
- if ($req_type eq 'bycoord' and $mjd_min eq 0) {
- # ... so unless the user sets mjd_min clamp it to 2009-04-01
- # XXX: This value should live in the pstampProject table not be hardcoded here
- $mjd_min = 54922;
- }
-
- $need_magic = 0 if $stage eq 'stack';
-
- # XXX: magic is dead
- $need_magic = 0;
-
- my $numRows = scalar @$rowList;
-
- if (($req_type eq 'byskycell') or ($req_type eq 'bycoord')) {
- # avoid error from print below if $id isn't needed
- $id = "" if !$id;
- }
- print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n";
-
- # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
- # request specification. An array reference is returned.
- my $start_locate = gettimeofday();
-
- # XXX: perhaps we should get rid of most of this argument list.
- # Now that we are passing down compatible rows all of the
- # information required is contained there
-
- my $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
- $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
-
- my $dtime_locate = gettimeofday() - $start_locate;
- print "Time to locate_images for row $rownum $dtime_locate\n";
-
- # handle this
- # XXX: what did I mean by that comment?
- $row->{need_magic} = $need_magic;
-
- $num_jobs += queueJobs($action, $rowList, $imageList);
-
- # if a row slipped through with no jobs add a faulted one
- foreach my $row (@$rowList) {
- if ($row->{job_num} == 0) {
- print "row $row->{ROWNUM} produced no jobs\n";
- print STDERR "row $row->{ROWNUM} produced no jobs\n";
- my $error_code = $row->{error_code};
- $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code;
- insertFakeJobForRow($row, ++$row->{job_num}, $error_code);
- }
- }
-
- return $num_jobs;
-}
-
sub queueJobForImage
{
@@ -648,4 +569,8 @@
$option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_updates_allowed;
+
+ # determine if there is a cut off date for data access for this user
+ my $mjd_max;
+ my $adjustedDateCuts = adjustDateCuts($row, \$mjd_max);
my $roi_string;
@@ -692,15 +617,6 @@
my $imagefile = $image->{image};
- if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
- # XXX: should we add a faulted job so the client can know what happened if no images come back?
- # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback
- # to users, but speeds up processing significantly
- print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
-
- # for now assume yes.
-
- insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
- return 1;
- } elsif ($stage eq "stack") {
+
+ if ($stage eq "stack") {
# unconvolved stack images weren't available prior to some point in time.
# XXX: handle this more correctly by examining the stack run's config dump file.
@@ -713,4 +629,44 @@
insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_AVAILABLE);
return 1;
+ }
+ } else {
+ if ($need_magic and !$image->{magicked}) {
+ # XXX: should we add a faulted job so the client can know what happened if no images come back?
+ # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback
+ # to users, but speeds up processing significantly
+ print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
+
+ # for now assume yes.
+
+ insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
+ return 1;
+ }
+
+ if ($adjustedDateCuts) {
+ my $mjd = 0;
+ my $exp_id;
+ if ($stage eq 'diff') {
+ # diff stage doesn't have a dateobs so we need to hack it.
+ # Extract dateobs from the exp_name_1
+ if ($row->{PROJECT} eq 'gpc1') {
+ my $exp_name = $image->{exp_name_1};
+ if ($exp_name ne 'NULL') {
+ $mjd = 50000 + (substr($exp_name, 1, 4));
+ $exp_id = $image->{exp_id_1};
+ } else {
+ # it's stack-stack diff just let it go with mjd = 0
+ # XXX: deal with this
+ }
+ }
+ } else {
+ $exp_id = $image->{exp_id};
+ $mjd = dateobsToMJD($image->{dateobs});
+ }
+
+ if ($mjd > $mjd_max) {
+ print STDERR "User is not authorized for stamps from this exposure.\n";
+ insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_AUTHORIZED, $exp_id);
+ return 1;
+ }
}
}
@@ -943,4 +899,5 @@
my $job_num = shift;
my $fault = shift;
+ my $exp_id = shift;
my ($job_type, $rownum);
@@ -965,4 +922,5 @@
my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type"
. " -rownum $rownum -state stop -fault $fault";
+ $command .= " -exp_id $exp_id" if $exp_id and $exp_id ne 'NULL';
if (!$no_update) {
@@ -1220,4 +1178,83 @@
}
+# Prototype function for setting access level.
+# XXX: Implement something more flexible using the database.
+sub getUsersAccessLevel {
+ my $email = shift;
+ my $label = shift;
+
+ my $level = -1;
+
+ if (!$email) {
+ # No email provided. Eventually this will be forbidden.
+ # For now in order to temporarily continue to support the version 1 request format
+ # set the access rights based on the assigned label.
+ if ($label eq 'IFA' or $label eq 'QUB' or $label =~ 'MOPS') {
+ $level = 0;
+ } else {
+ # access rights PS1 data only
+ $level = 1;
+ }
+ } else {
+ # we've got an email check the domain
+ my ($user, $domain) = split '@', $email;
+ if ($domain) {
+ $domain = lc($domain);
+ if ($domain eq 'ifa.hawaii.edu' or $domain eq 'qub.ac.uk') {
+ $level = 0;
+ } else {
+ # XXX check for special users
+ # access rights PS1 data only
+ $level = 1;
+ }
+ } else {
+ print STDERR "Error: $email is not a valid email address\n";
+ }
+ }
+
+ return $level;
+}
+
+
+# determine the latest exposure date that a user may have access to
+# based on the user's accessLevel.
+sub adjustDateCuts {
+ my $row = shift;
+ my $r_mjd_max = shift;
+ my $adjusted = 0;
+
+ my $MJD_PS1_BEGIN = 54922; # 2009-04-01
+ my $MJD_PS1_END = 56838; # 2014-06-30 XXX get the actual date
+
+ if ($row->{accessLevel} == 1) {
+ # User has access to PS1 data only
+ my $mjd_max_user = $row->{MJD_MAX};
+ if (!$mjd_max_user or $mjd_max_user > $MJD_PS1_END) {
+ $$r_mjd_max = $MJD_PS1_END;
+ $adjusted = 1;
+ }
+ }
+ return $adjusted;
+}
+
+sub dateobsToMJD {
+ my $dateobs = shift;
+
+ my ($date, $time) = split "T", $dateobs;
+
+ my ($year, $mon, $day) = split "-", $date;
+
+ my ($hr, $min, $sec) = split ":", $time;
+ my $fraction = $sec - int($sec);
+ $sec = int($sec);
+
+ my $ticks = timegm($sec, $min, $hr, $day, $mon-1, $year-1900);
+ $ticks += $fraction;
+
+ # I'm not allowing for leap seconds here but for our purposes here that is ok
+ my $mjd = 40587.0 + ($ticks / 86400.);
+ return $mjd;
+}
+
sub my_die
{