Index: branches/eam_branches/ipp-20110404/ippToPsps/config/detection/map.xml =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/config/detection/map.xml (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/config/detection/map.xml (revision 31439) @@ -146,4 +146,10 @@ --> --> + + + + + + Index: branches/eam_branches/ipp-20110404/ippToPsps/config/detection/tables.xml =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/config/detection/tables.xml (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/config/detection/tables.xml (revision 31439) @@ -1,5 +1,5 @@ - + @@ -146,4 +146,10 @@ + + + + + + Index: branches/eam_branches/ipp-20110404/ippToPsps/config/init/data.xml =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/config/init/data.xml (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/config/init/data.xml (revision 31439) @@ -4,6 +4,6 @@
- - + +
Index: branches/eam_branches/ipp-20110404/ippToPsps/config/init/tables.vot =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/config/init/tables.vot (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/config/init/tables.vot (revision 31439) @@ -0,0 +1,430 @@ + + + + + + VOTable description of PSPS table Filter + + + filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... + + + filter name: g,r,i,z,y, etc. + + + No comment + + + + + + + + + + + +
1g
2r
3i
4z
5y
6z
+ + VOTable description of PSPS table FitModel + + + indentifier of alternate model fit + + + model name + + + string describing the model + + + description of model parameter 1 + + + description of model parameter 2 + + + description of model parameter 3 + + + description of model parameter 4 + + + description of model parameter 5 + + + description of model parameter 6 + + + description of model parameter 7 + + + description of model parameter 8 + + + description of model parameter 9 + + + description of model parameter 10 + + + description of model parameter 11 + + + description of model parameter 12 + + + description of model parameter 13 + + + description of model parameter 14 + + + description of model parameter 15 + + + description of model parameter 16 + + + description of model parameter 17 + + + description of model parameter 18 + + + description of model parameter 19 + + + description of model parameter 20 + + + description of model parameter 21 + + + description of model parameter 22 + + + description of model parameter 23 + + + description of model parameter 24 + + + description of model parameter 25 + + + description of model parameter 26 + + + description of model parameter 27 + + + description of model parameter 28 + + + description of model parameter 29 + + + description of model parameter 30 + + + description of model parameter 31 + + + description of model parameter 32 + + + description of model parameter 33 + + + description of model parameter 34 + + + description of model parameter 35 + + + description of model parameter 36 + + + description of model parameter 37 + + + description of model parameter 38 + + + description of model parameter 39 + + + + + + + + + + +
0PS_MODEL_PSF_AVGPsf avg fit
1PS_MODEL_PSFPsf fit
2PS_MODEL_DEVdeVaucouleur Fit
3PS_MODEL_EXPExponential Fit
4PS_MODEL_SERSICSersic Fit
+ + VOTable description of PSPS table PhotozRecipe + + + recipe index + + + string describing the recipe + + + + +
+ + VOTable description of PSPS table Survey + + + index of survey type + + + survey name + + + survey description + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
03PIPS1 3PI Survey
1MD01PS1 MD01 XMM-LSS-DXS 022224-043000
2MD02PS1 MD02 CDFS/GOODS/GEMS 033224-274800
3MD03PS1 MD03 IFA/Lynx 084300+444000
4MD04PS1 MD04 COSMOS 100000+021200
5MD05PS1 MD05 Lockman-DXS 110000+570000
6MD06PS1 MD06 NGC 4258 121857+471814
7MD07PS1 MD07 VISTA-Video1 140000+050000
8MD08PS1 MD08 EliasN1-DXS 160000+570000
9MD09PS1 MD09 Vimos4-DXS-SSA 220000+003000
10MD10PS1 MD10 Deep2 233000+000000
11M31PS1 M31 Andromeda 004242+411600
12STS1PS1 STS1 Outskirts of Bulge 195048+170339
26STS2PS1 STS2 Hyades 004000+150000
27STS3PS1 STS3 Praesepe 083000+200000
13CAL01PS1 Cal North Pole 000000+900000
14CAL02PS1 Cal SA92 Dense Landolt Std Field 005808+004232
15CAL03PS1 Cal Perseus Cluster / Abell 426 031900+413300
16CAL04PS1 Cal SA95 Dense Landolt Std Field 065207+001710
17CAL05PS1 Cal Taurus dark cloud (diverse extinction) 043000+250000
18CAL06PS1 Cal SA98 Densle Landolt Std Field 065207+001710
19CAL07PS1 Cal M81-M82-HolmIX (Local Group) 095640+692200
20CAL08PS1 Cal Virgo Cluster (Center) 123000+130000
21CAL09PS1 Cal Hercules Cluster (Center) 160508+1745+28
22CAL10PS1 Cal M17-M24 Region 181400+000544
23CAL11PS1 Cal SA110 Dense Landolt Std Field 184207+000544
24CAL12PS1 Cal Kepler Mission Astrometric field 192240+443000
25CAL13PS1 Cal 3C454.1 (Northern field) 225033+712919
28SSSPS1 Solar System Sweet Spot Survey
+ + VOTable description of PSPS table CameraConfig + + + camera identification number + + + configuration identification number + + + ending date of this configuration in MJD (unit = day) + + + text/table giving the details of the camera configuration + + + + +
+ + VOTable description of PSPS table PhotoCal + + + numerical code for this reduction + + + filter id + + + Photometry reduction code name + + + AB magnitude + + + photometric zero point (excluding long term variation) + + + photometric extinction term (excluding long term variation) + + + photometric color term (excluding long term variation) + + + photometric color dependent extinction (excluding long term variation) + + + color adopted for magnitude calculation + + + error in calibrating color + + + starting date of this configuration in MJD (unit = day) + + + + +
+ + VOTable description of PSPS table SkyCell + + + indentifier of the sky cell + + + indentifier of the projection cell + + + fgetPANobjID(skyCell_center_ra, skyCell_center_dec, zHeight) + + + region identifier in the Region table + + + + +
+ + VOTable description of PSPS table ProjectionCell + + + projection cell unique identifier + + + fGetPanObjID(crval1,crval2) + + + reference to the primary cell region definition in the Region table + + + reference to the projectio cell region definition in the Region table) + + + name of astrometric projection in RA + + + name of astrometric projection in DEC + + + RA corresponding to reference pixel (unit = deg) + + + DEC corresponding to reference pixel (unit = deg) + + + reference pixel value for RA + + + reference pixel value for DEC + + + elements of rotation/Dcale matrix + + + elements of rotation/Dcale matrix + + + elements of rotation/Dcale matrix + + + elements of rotation/Dcale matrix + + + + +
+ + VOTable description of PSPS table Region + + + Unique Region identifier + + + Custom Identifier + + + Region type + + + Comment + + + Area covered by the region + + + Binary object managed by the spatial library + + + + +
+ + VOTable description of PSPS table StackType + + + stack type identifier + + + stack type name + + + stack type description + + + + + + + +
0NIGHTLY_STACKNightly stack
1DEEP_STACKDeep stack
+ + VOTable description of PSPS table ImageFlags + + + Name of flag + + + Flag value + + + Description of the flag + + + + +
+ + VOTable description of PSPS table DetectionFlags + + + Name of flag + + + Flag value + + + Description of the flag + + + + +
+ + Index: branches/eam_branches/ipp-20110404/ippToPsps/config/init/tables.xml =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/config/init/tables.xml (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/config/init/tables.xml (revision 31439) @@ -9,4 +9,5 @@ + @@ -56,6 +57,6 @@
+ -
@@ -110,5 +111,6 @@
- + +
Index: branches/eam_branches/ipp-20110404/ippToPsps/config/stack/map.xml =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/config/stack/map.xml (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/config/stack/map.xml (revision 31439) @@ -9,5 +9,5 @@ - + @@ -20,5 +20,5 @@ - + @@ -44,12 +44,12 @@ - + - + - + @@ -74,5 +74,5 @@
- +
@@ -81,8 +81,8 @@ - + - + @@ -237,15 +237,15 @@ - + - - + + - - - + + + - + @@ -255,32 +255,32 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -297,32 +297,32 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -341,45 +341,45 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
- +
Index: branches/eam_branches/ipp-20110404/ippToPsps/config/stack/tables.vot =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/config/stack/tables.vot (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/config/stack/tables.vot (revision 31439) @@ -6,37 +6,106 @@ VOTable description of PSPS table StackMeta - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + stack identifier + + + skycell region identifier + + + survey flag identifier + + + photometry code numerical id + + + filter identifier + + + stack type identifier + + + saturation magnitude level + + + analysis version index + + + exposure time (unit = s) + + + number of P2 images contributing to this cell + + + 95% completion level in mag + + + astrometric scatter for chip + + + photometric scatter for chip + + + number of astrometric reference sources + + + number of photometric reference sources + + + PSF model identifier + + + mean PSF FWHM in the stack + + + maximum PSF FWHM in the stack + + + local derived photometric zero point + + + local derived photometric color term + + + name of astrometric projection in RA + + + name of astrometric projection in DEC + + + RA corresponding to reference pixel + + + DEC corresponding to reference pixel + + + reference pixel value for RA + + + reference pixel value for DEC + + + scale factor for RA + + + scale factor for DEC + + + elements of rotation/Dcale matrix + + + elements of rotation/Dcale matrix + + + elements of rotation/Dcale matrix + + + elements of rotation/Dcale matrix + + + calibration modification number + + + Data release + @@ -46,39 +115,148 @@ VOTable description of PSPS table StackDetection - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + ODM object identifier + + + ODM detection identifier + + + IPP object identifier + + + detection ID generated by IPP + + + filter identifier + + + stack type identifier + + + survey flag identifier + + + identifies best stack detection for Stacks overlapping the same region of the sky. + + + stack identifier + + + skycell identifier + + + projection cell identifier + + + Time of mid observation (unit = day) + + + measured x on CCD from PSF fit + + + measured y on CCD from PSF fit + + + estimated error in x + + + estimated error in y + + + PSF instrumental flux + + + estimated error in instrumental flux + + + ratio of peak flux to total flux + + + PSF sky level at source (adu) + + + estimated error in sky + + + star/galaxy separator + + + PSF width in major axis + + + PSF width in minor axis + + + PSF orientation angle + + + PSF likelihood + + + PSF coverage factor + + + moment XX (unit = arcsec) + + + moment XY (unit = arcsec) + + + moment YY (unit = arcsec) + + + moment M3C (unit = arcsec) + + + moment M3S (unit = arcsec) + + + moment M4C (unit = arcsec) + + + moment M4S (unit = arcsec) + + + moment R1 (unit = arcsec) + + + moment RH (unit = arcsec) + + + Aperture magnitude (unit = magnitude) + + + Aperture magnitude error (unit = magnitude) + + + Kron flux (unit = magnitude) + + + Kron flux error (unit = magnitude) + + + Kron radius (unit = arcsecs) + + + Kron radius error (unit = arcsecs) + + + indicator of strange propeties + + + number of frames contributing to source + + + indicates whether this detection/orphan is still a detection/orphan + + + date object association assigned + + + modification number in the O-D association history + + + Data release when this detection was originally taken. + @@ -88,158 +266,463 @@ VOTable description of PSPS table StackApFlx - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + ODM object identifier + + + ODM detection identifier + + + IPP object identifier + + + detection ID generated by IPP + + + filter identifier + + + stack type identifier + + + survey flag identifier + + + identifies best stack detection for Stacks overlapping the same region of the sky. + + + stack identifier + + + isophotal magnitude + + + estimated error in isophotal magnitude + + + isophotal Major Axis + + + estimated error in isophotal Major Axis + + + isophotal Minor Axis + + + estimated error in isophotal Minor Axis + + + isophotal major axis gradient + + + isophotal minor axis gradient + + + isophotal position angle + + + estimated error in isophotal position angle + + + isophotal position angle gradient + + + Petrosian radius + + + estimated error inPetrosian radius + + + Petrosian magntiude + + + estimated error in petMag + + + Petrosian radius at 50% light + + + estimated error inPetrosian radius at 50% light + + + Petrosian radius at 90% light + + + estimated error in Petrosian radius at 90% light + + + Petrosian fit coverage factor + + + Flux inside r = 1 + + + estimated error is flxR1 + + + estimated standard deviation in flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is flxR2 + + + estimated standard deviation in flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is flxR3 + + + estimated standard deviation in flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is flxR4 + + + estimated standard deviation in flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is flxR7 + + + estimated standard deviation in flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is flxR8 + + + estimated standard deviation in flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is flxR9 + + + estimated standard deviation in flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is flxR10 + + + estimated standard deviation in flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c1flxR1 + + + estimated standard deviation in c1flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c1flxR2 + + + estimated standard deviation in c1flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c1flxR3 + + + estimated standard deviation in c1flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c1flxR4 + + + estimated standard deviation in c1flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c1flxR5 + + + estimated standard deviation in c1flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c1flxR6 + + + estimated standard deviation in c1flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c1flxR7 + + + estimated standard deviation in c1flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c1flxR8 + + + estimated standard deviation in c1flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c1flxR9 + + + estimated standard deviation in c1flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c1flxR10 + + + estimated standard deviation in c1flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c2flxR1 + + + estimated standard deviation in c2flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c2flxR2 + + + estimated standard deviation in c2flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c2flxR3 + + + estimated standard deviation in c2flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c2flxR4 + + + estimated standard deviation in c2flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c2flxR5 + + + estimated standard deviation in c2flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c2flxR6 + + + estimated standard deviation in c2flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c2flxR7 + + + estimated standard deviation in c2flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c2flxR8 + + + estimated standard deviation in c2flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c2flxR9 + + + estimated standard deviation in c2flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c2flxR10 + + + estimated standard deviation in c2flxR10 + + + estimated filling factor at R10 + + + Abraham concentration index + + + Abraham asymmetry index + + + indicates whether this detection/orphan is still a detection/orphan + + + Data release when this detection was taken + @@ -249,152 +732,445 @@ VOTable description of PSPS table StackModelFit - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + ODM object identifier + + + ODM detection identifier + + + IPP object identifier + + + detection ID generated by IPP + + + filter identifier + + + stack type identifier + + + survey flag identifier + + + identifies best stack detection for Stacks overlapping the same region of the sky. + + + stack identifier + + + deVaucouleurs radius + + + estimated error in deVaucouleurs radius + + + deVaucouleurs magntiude + + + estimated error in deV_mag + + + deVaucoulerus axis ratio + + + estimated error in deVaucoulerus axis ratio + + + estmated phi of deVaucouleurs axis + + + estmated error of phi of deVaucouleurs axis + + + Offset in RA of deVaucouleurs fit from PSF RA + + + Offset in DEC of deVaucouleurs fit from PSF DEC + + + estimated error in ra offset + + + estimated error in dec offset + + + deVaucouleurs fit coverage factor + + + deVaucouleurs fit likelihood factor + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + covariance for deVaucouleurs fit + + + Exponential fit radius + + + estimated error in Exponential fit radius + + + Exponential fit magntiude + + + estimated error in expMag + + + Exponential fit axis ratio + + + estimated error in Exponential fit axis ratio + + + estmated phi of Exponential axis + + + estmated error of phi of Exponential axis + + + Offset in RA of Exponential fit from PSF RA + + + Offset in DEC of Exponential fit from PSF DEC + + + estimated error in raExpOff + + + estimated error in decExpOff + + + Exponential fit coverage factor + + + Exponential fit likelihood factor + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + covariance for Exponential fit + + + Sersic radius + + + estimated error in Sersic radius + + + Sersic magntiude + + + estimated error in serMag + + + Sersic axis ratio + + + estimated error in Sersic axis ratio + + + Sersic index + + + estimated error in Sersic index + + + estmated phi of Sersic axis + + + estmated error of phi of Sersic axis + + + Offset in RA of Sersic fit from PSF RA + + + Offset in DEC of Sersic fit from PSF DEC + + + estimated error in raSerOff + + + estimated error in decSerOff + + + Sersic fit coverage factor + + + Sersic fit likelihood factor + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + covariance for Sersic fit + + + indicates whether this detection/orphan is still a detection/orphan + + + Data release when this detection was taken + @@ -404,6 +1180,60 @@ VOTable description of PSPS table StackToImage - - + + stack identifier + + + hashed exposure-ccdID identifier + + + + + + + VOTable description of PSPS table SkinnyObject + + + ODM object identifier index + + + IPP object number + + + projection cell identifier at discovery time + + + Data release to propagate to the object + + + surveyID </coluumn> + + + + +
+ + VOTable description of PSPS table ObjectCalColor + + + ODM object identifier + + + ipp object identifier + + + filter identifier + + + color adopted for magnitude calculation (unit = mag) + + + error in calibrating color (unit = mag) + + + calibration modification number + + + Data release when this color calibration was established + Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/batch.py =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/batch.py (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/batch.py (revision 31439) @@ -4,43 +4,84 @@ import datetime import re +import sys +import os +import md5 +import shutil +import logging +from subprocess import call, PIPE, Popen + +from datastore import Datastore +from scratchdb import ScratchDb +from gpc1db import Gpc1Db +from ipptopspsdb import IppToPspsDb from java.lang import * from java.sql import * - - +from xml.etree.ElementTree import ElementTree, Element, tostring + +''' +Base class of all batch types. +''' class Batch(object): - driverName="com.mysql.jdbc.Driver" - ''' Constructor - ''' - def __init__(self, batchType, inputFitsPath, outputFitsPath, dbHost, dbName, dbUser, dbPass, survey=""): + + >>> batch = Batch(1,2,3,4,5,6,7) + >>> print batch.pspsVoTableFilePath + "../config/2/tables.vot" + ''' + def __init__(self, logger, batchType, inputFitsPath="", survey="", useFullTables=False): + + # set up logging + self.logger = logger + self.logger.info("-------------------------------------------------------------------------------") + self.logger.debug("Batch class constructor") # set up class variables + self.batchType = batchType; self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot" self.inputFitsPath = inputFitsPath - self.outputFitsPath = outputFitsPath - self.dbHost = dbHost - self.dbName = dbName - self.dbUser = dbUser - self.dbPass = dbPass self.survey = survey - - # set up JDBC connection - self.url = "jdbc:mysql://"+self.dbHost+"/"+self.dbName+"?user="+self.dbUser+"&password="+self.dbPass - self.con = DriverManager.getConnection(self.url) - self.stmt = self.con.createStatement() - - # get survey ID from init table - sql = "SELECT surveyID from Survey WHERE name = '" + survey + "'" - try: - rs = self.stmt.executeQuery(sql) - rs.first() - self.surveyID = rs.getInt(1) - except: - self.log("No survey ID found for this survey: '" + survey + "'") + self.useFullTables = useFullTables + + # TODO + self.tablesToExport = [] + + # open config + doc = ElementTree(file="config.xml") + + # create Gpc1Db object + self.gpc1Db = Gpc1Db(self.logger) + self.ippToPspsDb = IppToPspsDb(logger) + self.scratchDb = ScratchDb(logger, self.useFullTables) + + if self.survey != "": + self.surveyID = self.scratchDb.getSurveyID(self.survey) + + # get dvo info from config + dvoName = doc.find("dvo_" + self.survey + "/name").text + self.dvoLocation = doc.find("dvo_" + self.survey + "/location").text + else: + dvoName = "" + self.dvoLocation = "" self.surveyID = -1; - + + # get datastore info from config + self.datastore = Datastore(self.logger) + + # create a new batch + self.batchID = self.ippToPspsDb.createNewBatch( + self.getPspsBatchType(), + survey, + dvoName, + self.datastore.product) + + # get local storage location from config + self.batchName = "B%08d" % self.batchID + self.subDir = doc.find("localOutPath").text + "/" + self.getPspsBatchType() + "/" + dvoName + self.localOutPath = self.subDir + "/" + self.batchName + if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath) + # store today's date now = datetime.datetime.now(); @@ -48,5 +89,11 @@ if self.inputFitsPath != "": - self.parseFitsHeader() + file = open(self.inputFitsPath) + self.header = self.parseFitsHeader(file) + self.logger.info("Read primary and found " + str(len(self.header)) + " header cards") + # TODO close file? + + # create DVO tables if accessing DVO directly + if not self.useFullTables: self.scratchDb.createDvoTables() ''' @@ -55,70 +102,207 @@ def __del__(self): - self.log("Batch destructor") - self.stmt.close() - self.con.close() - - ''' - Prints a log message with the current time - ''' - def log(self, msg): - - print datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " | " + msg - - ''' - Updates a table with surveyID - ''' - def updateSurveyID(self, table): - - sql = "UPDATE " + table + " SET surveyID=%d" % self.surveyID - self.stmt.execute(sql) - - ''' - Updates a table with filterID grabbed from Filter init table - ''' - def updateFilterID(self, table): - - sql = "UPDATE "+table+" AS a, Filter AS b SET a.filterID=b.filterID WHERE b.filterType = '" + self.filter + "'" - self.stmt.execute(sql) + self.logger.debug("Batch destructor") + + + ''' + Returns the value from this dictinary or else NULL + ''' + def safeDictionaryAccess(self, header, key): + + if key in header: return header[key] + else: return "NULL" + + ''' + Finds and reads a header extension + ''' + def findAndReadFITSHeader(self, name, file): + + found = False + + while True: + + index = file.tell() + + record = file.read(80) + if not record: break; + + # quit when we reach 'END' + if record.startswith("XTENSION= 'IMAGE"): + + header = self.parseFitsHeader(file) + if header['EXTNAME'] == name: + found = True + file.seek(index + 2880, 0) + break + + file.seek(index + 2880, 0) + + if found != True: self.logger.error("...could not find extension '" + name + "'") + else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards") + + return header + + + ''' + Writes the batch manifest file + ''' + def writeBatchManifest(self): + + outPath = self.localOutPath + "/BatchManifest.xml" + tmpPath = "./tmp.xml" + self.logger.info("Creating batch manifest file here: " + outPath) + root = Element('manifest') + + # batch information + root.attrib['name'] = self.batchName + root.attrib['type'] = self.getPspsBatchType() + root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + if self.survey != "": + root.attrib['survey'] = self.getBatchFriendlySurveyType() + try: self.minObjID + except: pass + else: root.attrib['minObjId'] = str(self.minObjID) + try: self.maxObjID + except: pass + else: root.attrib['maxObjId'] = str(self.maxObjID) + + # get md5sum + p = Popen("md5sum " + self.outputFitsPath, shell=True, stdout=PIPE) + p.wait() + out = p.stdout.read() + md5sum = out[0:out.rfind(" ")] + + # get file size + fileSize = os.path.getsize(self.outputFitsPath) + + # file information + child = Element('file') + root.append(child) + child.attrib['name'] = self.outputFitsFile + child.attrib['bytes'] = str(fileSize) + child.attrib['md5'] = md5sum + + # now create doc and write to file + file = open(tmpPath, 'w') + ElementTree(root).write(file) + file.close() + + # clunky way to prettify XML + p = Popen("xmllint --format " + tmpPath + " > " + outPath, shell=True, stdout=PIPE) + p.wait() + os.remove(tmpPath) + + + ''' + tar and zips batch directory + ''' + def createTarball(self): + + # set up filenams and paths + tarFile = self.batchName + ".tar" + tarPath = self.subDir + "/" + tarFile + + self.tarballFile = tarFile + ".gz" + tarballPath = self.subDir + "/" + self.tarballFile + + # tar directory + p = Popen("tar -cvf " + tarPath + "\ + -C " + self.subDir + " \ + " + self.batchName, shell=True, stdout=PIPE) + p.wait() + + # zip tar archive + p = Popen("gzip -c " + tarPath + " > " + tarballPath, shell=True, stdout=PIPE) + p.wait() + + # delete tar file and original directory + os.remove(tarPath) + shutil.rmtree(self.localOutPath) + + ''' + Publishes this batch to the datastore + ''' + def publishToDatastore(self): + + if self.datastore.publish(self.batchName, self.subDir, self.tarballFile, "tgz"): + self.ippToPspsDb.updateLoadedToDatastore(self.batchID, 1) + + ''' + Gets PSPS-friendly survey type + ''' + def getBatchFriendlySurveyType(self): + + return "SCR" # TODO + + try: + self.survey + except: + return "NA" + + if self.survey == "3PI": return "3PI" + elif self.survey == "MD04": return "MD4" + else: + self.logger.error("Don't know this survey: '" + self.survey + "'") + return "NA" + + ''' + Gets PSPS friendly batch type + ''' + def getPspsBatchType(self): + + if self.batchType == "init": return "IN" + elif self.batchType == "detection": return "P2" + elif self.batchType == "stack": return "ST" + else: self.logger.error("Don't know this batch type: " + self.survey) + + ''' + Sets min and max obj ID using the provided table, or list of tables + ''' + def setMinMaxObjID(self, tables): + + first = True + for table in tables: + + sql = "SELECT MIN(objID), MAX(objID) FROM " + table + rs = self.scratchDb.stmt.executeQuery(sql) + rs.first() + + if first: + self.minObjID = rs.getLong(1) + self.maxObjID = rs.getLong(2) + else: + if rs.getLong(1) < self.minObjID: self.minObjID = rs.getLong(1) + if rs.getLong(2) > self.maxObjID: self.maxObjID = rs.getLong(2) + + first = False + + self.ippToPspsDb.updateMinMaxObjID(self.batchID, self.minObjID, self.maxObjID) ''' Reads FITS header and stores all fields in a dictionary object ''' - def parseFitsHeader(self): - - fitsFile = open(self.inputFitsPath) - - self.header = {} + def parseFitsHeader(self, fitsFile): + + header = {} while (True): + record = fitsFile.read(80) # quit when we reach 'END' - if record.startswith("END"): break - - # ignore comments - if record.startswith("COMMENT"): continue - match = re.match('(.*)=(.*)', record) + if re.match('END\s+', record): break + + # this regex will get param/value pairs for all header cards, ignoring comments and parsing out 'HIERACH' prefixes + match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-_\.:\s@#]+)\'*\\/*', record) if match: - # remove HIERARCH prefix - param = match.group(1).replace("HIERARCH", "") - param = param.strip() - - value = match.group(2) - # remove trailing comment after / char, if there is one - index = value.find("/") - if index != -1: value = value[0:index] - - # remove ' chars around content - value = value.replace("'", "") - - # remove leading and trailing whitespace - value = value.strip() - - # store in out dictionary object - self.header[param] = value + param = match.group(2) + value = match.group(3).strip() + if value == "NaN": value = "NULL" + header[param] = value #print param + "|" + value + "|" + + return header ''' @@ -129,8 +313,9 @@ self.pspsTables = stilts.treads(self.pspsVoTableFilePath) for table in self.pspsTables: - self.log("Creating PSPS table: " + table.name) - table.write(self.url + '#' + table.name) - - self.indexPspsTables() + self.logger.info("Creating PSPS table: " + table.name) + table.write(self.scratchDb.url + '#' + table.name) + self.tablesToExport.append(table.name) + + self.alterPspsTables(); ''' @@ -138,34 +323,21 @@ ''' def indexIppTables(self): - self.log("indexIppTables not implemented") - - - ''' - Adds an index to the supplied table and column - ''' - def createIndex(self, table, column): - - self.log("Creating index on column '"+column+"' for table '"+table+"'") - - sql = "CREATE INDEX "+table+"_index ON "+table+" ("+column+")" - try: - self.stmt.execute(sql) - except: - self.log("Index already in place on '" + column + "' for table '" + table + "'") - - - ''' - Subclass should implement this to index PSPS tables + self.logger.warn("indexIppTables not implemented") + + + ''' + Alter PSPS tables ''' - def indexPspsTables(self): - self.log("indexPspsTables not implemented") + def alterPspsTables(self): + self.logger.warn("alterPspsTables not implemented") ''' Imports IPP tables from FITS file - Accepts a regular expression filter so not all tabls need to be imported + Accepts a regular expression filter so not all tables need to be imported ''' def importIppTables(self, filter): + self.logger.info("Attempting to import tables from input FITS file") tables = stilts.treads(self.inputFitsPath) @@ -175,34 +347,74 @@ match = re.match(filter, table.name) if not match: continue - self.log("Creating IPP table " + table.name) + self.logger.info(" Reading IPP table " + table.name + " from FITS file") table = stilts.tpipe(table, cmd='explodeall') + + # drop any previous tables before import + self.scratchDb.dropTable(table.name) + + # IPP FITS files are littered with infinities, so remove these + self.logger.info(" Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + try: - table.write(self.url + '#' + table.name) + table.write(self.scratchDb.url + '#' + table.name) except: - self.log("ERROR problem writing table '" + table.name + "' to the database") - + self.logger.exception(" Problem writing table '" + table.name + "' to the database") count = count + 1 - self.log("Imported %d tables from '%s' " % (count, self.inputFitsPath)) + self.logger.info("Done. Imported %d tables" % count) self.indexIppTables() ''' - Exports PSPS tables from the database to FITS format + Exports PSPS tables from the database to FITS format. Optional regex if you want to alter table names prior to export ''' - def exportPspsTablesToFits(self): - - self.log("Exporting all PSPS tables to FITS") + def exportPspsTablesToFits(self, regex="(.*)"): + + self.logger.info("Replacing NULLs with -999 then exporting all PSPS tables to FITS") _tables = [] - self.log(" Selecting database tables") + self.logger.info(" Selecting database tables") + for table in self.tablesToExport: + + # check for an empty table + if self.scratchDb.getRowCount(table) < 1: continue + + # get everything from table + _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table) + + # replace nulls and empty fields with weird PSPS -999 pseudo-null + _table = stilts.tpipe(_table, cmd='replaceval "" -999 *') + + match = re.match(regex, table) + newTableName = match.group(1) + + # change table names + _table = stilts.tpipe(_table, cmd='tablename ' + newTableName) + _tables.append(_table) + + self.logger.info(" Writing to FITS file '" + self.outputFitsPath + "'...") + stilts.twrites(_tables, self.outputFitsPath, fmt='fits') + self.logger.info(" ...done") + self.ippToPspsDb.updateProcessed(self.batchID, 1) + + ''' + Searches all PSPS tables and reports the columns that are either partially or completely populated with NULLs + ''' + def reportNullsInAllPspsTables(self, showPartials): + for table in self.pspsTables: - _table = stilts.tread(self.url + '#SELECT * FROM ' + table.name) - _table = stilts.tpipe(_table, cmd='tablename ' + table.name) - _tables.append(_table) - - self.log(" Writing to FITS file " + self.outputFitsPath) - stilts.twrites(_tables, self.outputFitsPath, fmt='fits') - + self.scratchDb.reportNulls(table.name, showPartials) + + ''' + Searches all PSPS tables and replaces all NULLs with the provided substitute + ''' + def replaceAllPspsNulls(self, sub): + + self.logger.info("Replacing all NULL values in PSPS tables with '" + sub + "'...") + for table in self.pspsTables: + self.scratchDb.replaceNulls(table.name, sub) + self.logger.info("...done") ''' @@ -210,4 +422,31 @@ ''' def populatePspsTables(self): - self.log("Not implemented yet") - + self.logger.warn("Not implemented yet") + + ''' + Calls DVO program to 'query' DVO database and populate results to local MySQL Db table + ''' + def getIDsFromDVO(self): + + # TODO path to DVO prog hardcoded temporarily + cmd = "../src/dvograbber " + self.dvoLocation + self.logger.info("Running: '" + cmd + "'...") + p = Popen(cmd, shell=True, stdout=PIPE) + p.wait() + # out = p.stdout.read() + self.logger.info("...done") + + if self.scratchDb.getRowCount("dvoDetection") < 1: + self.logger.error("No DVO IDs found") + return False + + return True + + ''' + Checks whether this batch has already been processed and published. To be implemented by all subclasses + ''' + def alreadyProcessed(self): + self.logger.info("Not implemented") + + + Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/config.xml =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/config.xml (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/config.xml (revision 31439) @@ -0,0 +1,59 @@ + + + + + + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + + PSPS_test + IPP_PSPS + + + + + ThreePi.V1 + /data/ipp004.0/gpc1/catdirs/ThreePi.V1 + + + + MD04.merge + + /data/ipp005.0/gpc1/catdirs/MD04.merges/MD04.merge + + + + + + + + /data/ipp005.0/rhenders + + + Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/datastore.py =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/datastore.py (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/datastore.py (revision 31439) @@ -0,0 +1,85 @@ +#!/usr/bin/env jython + +from subprocess import call, PIPE, Popen +import tempfile +import logging +import sys +from xml.etree.ElementTree import ElementTree, Element, tostring + + +''' +Class encapsulating the IPP datastore +''' +class Datastore(object): + + ''' + Constructor + + ''' + def __init__(self, logger): + + # setup logger + self.logger = logger + self.logger.debug("Datastore constructor") + + # open config + doc = ElementTree(file="config.xml") + self.product = doc.find("datastore/product").text + self.type = doc.find("datastore/type").text + + self.logger.debug("Using product: '" + self.product + "' and type: '" + self.type + "'") + + ''' + Publishes an item to the datastore + ''' + def publish(self, name, path, file, fileType): + + self.logger.info("Attempting to publish '" + path +"/" + file + "' to datastore product: '" + self.product + "', type: '" + self.type + "'") + tempFile = tempfile.NamedTemporaryFile(mode='w+b') + tempFile.write(file + "|||" + fileType) + tempFile.seek(0) + + # build dsreg command command + command = "dsreg --add " + name + "\ + --copy --datapath " + path + "\ + --type " + self.type + "\ + --product " + self.product + "\ + --list " + tempFile.name + + p = Popen(command, shell=True, stdout=PIPE) + p.wait() + + if p.returncode != 0: + self.logger.error("Datastore publish failed") + ret = False + else: + self.logger.info("Datastore publish successful") + ret = True + + + tempFile.close() + return ret + + + ''' + Removes an item to the datastore + ''' + def remove(self, name): + + command = "dsreg \ + --del " + name + " \ + --rm \ + --product " + self.product + + p = Popen(command, shell=True, stdout=PIPE) + p.wait() + + if p.returncode != 0: + self.logger.error("Datastore removal of " + name + " failed") + ret = False + else: + ret = True + self.logger.info("Datastore removal of " + name + " successful") + + return ret + Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/detectionbatch.py =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/detectionbatch.py (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/detectionbatch.py (revision 31439) @@ -1,9 +1,17 @@ #!/usr/bin/env jython +import os.path +import sys import stilts from java.lang import * from java.sql import * from batch import Batch - +from gpc1db import Gpc1Db + +import logging.config + +''' +DetectionBatch class +''' class DetectionBatch(Batch): @@ -11,61 +19,124 @@ Constructor ''' - def __init__(self): + def __init__(self, logger, camID, inputFile, test=False, useFullTables=False): super(DetectionBatch, self).__init__( + logger, "detection", - "detdemo.fits", - "detection.fits", - "localhost", - "ipptopsps_scratch", - "ipp", - "ipp", - "3PI") # TODO - - - - ''' - Updates a table with stackMetaID - ''' - def updateStackMetaID(self, table): - - sql = "UPDATE " + table + " SET stackMetaID=" + self.header['STK_ID'] - self.stmt.execute(sql) - - ''' - Updates a table with stackTypeID grabbed from FitModel init table - ''' - def updateStackTypeID(self, table): - - sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '"+self.header['STK_TYPE']+"'" - self.stmt.execute(sql) + inputFile, + "MD04", + useFullTables) # TODO + #"3PI") # TODO + + self.logger.info("DetectionBatch constructor. Creating batch from: '" + inputFile + "'") + + meta = self.gpc1Db.getCameraStageMeta(camID) + + self.expID = meta[0]; + self.expName = meta[1]; + self.distGroup = meta[2]; + + self.logger.info("Processing exposure with ID: %d, name: %s and distribution group: %s" % (self.expID, self.expName, self.distGroup)) + + # create an output filename, which is {expID}.FITS + self.outputFitsFile = "%08d.FITS" % self.expID + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + + # if test mode + if test: + self.startX = 3 + self.endX = 4 + self.startY = 3 + self.endY = 4 + else: + self.startX = 0 + self.endX = 8 + self.startY = 0 + self.endY = 8 + + #self.startX = 1 + #self.endX = 2 + #self.startY = 7 + #self.endY = 8 + + self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0) + + # set up some defauts + self.calibModNum = 0 + self.dataRelease = 0 + + self.totalNumPhotoRef = 0 + + # get filterID using init table + self.filter = self.header['FILTERID'][0:1] + + # insert what we know about this stack batch into the stack table + self.ippToPspsDb.insertDetectionMeta(self.batchID, self.expID, self.filter) + ''' Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header ''' - def populateImageMeta(self): - self.log("Procesing ImageMeta table") - ''' - sql = "INSERT INTO ImageMeta (\ - stackMetaID \ - ,skycellID \ - ,photoZero \ - ,nP2Images \ - ,ctype1 \ - ,ctype2 \ - ,crval1 \ - ,crval2 \ - ,crpix1 \ - ,crpix2 \ - ,cdelt1 \ - ,cdelt2 \ - ,pc001001 \ - ,pc001002 \ - ,pc002001 \ - ,pc002002 \ + def populateFrameMeta(self): + self.logger.info("Procesing FrameMeta table") + + sql = "INSERT INTO FrameMeta (\ + frameID \ + ,frameName \ + ,cameraID \ + ,cameraConfigID \ + ,telescopeID \ + ,analysisVer \ + ,p1Recip \ + ,p2Recip \ + ,p3Recip \ + ,photoScat \ + ,expStart \ + ,expTime \ + ,airmass \ + ,raBore \ + ,decBore \ + ,ctype1 \ + ,ctype2 \ + ,crval1 \ + ,crval2 \ + ,crpix1 \ + ,crpix2 \ + ,cdelt1 \ + ,cdelt2 \ + ,pc001001 \ + ,pc001002 \ + ,pc002001 \ + ,pc002002 \ + ,polyOrder \ + ,pca1x3y0 \ + ,pca1x2y1 \ + ,pca1x1y2 \ + ,pca1x0y3 \ + ,pca1x2y0 \ + ,pca1x1y1 \ + ,pca1x0y2 \ + ,pca2x3y0 \ + ,pca2x2y1 \ + ,pca2x1y2 \ + ,pca2x0y3 \ + ,pca2x2y0 \ + ,pca2x1y1 \ + ,pca2x0y2 \ ) VALUES ( \ - " + self.header['STK_ID'] + " \ - ," + self.skycell + " \ - ," + self.header['FPA.ZP'] + " \ - ," + self.header['NINPUTS'] + " \ + " + str(self.expID) + " \ + ,'" + self.expName + "' \ + ,1 \ + ,1 \ + ,1 \ + ,' ' \ + ,' ' \ + ,' ' \ + ,' ' \ + ," + self.header['ZPT_ERR'] + " \ + ," + self.header['MJD-OBS'] + " \ + ," + self.header['EXPREQ'] + " \ + ," + self.header['AIRMASS'] + " \ + ," + self.header['RA'] + " \ + ," + self.header['DEC'] + " \ ,'" + self.header['CTYPE1'] + "' \ ,'" + self.header['CTYPE2'] + "' \ @@ -80,75 +151,519 @@ ," + self.header['PC002001'] + " \ ," + self.header['PC002002'] + " \ + ," + self.header['NPLYTERM'] + " \ + ," + self.header['PCA1X3Y0'] + " \ + ," + self.header['PCA1X2Y1'] + " \ + ," + self.header['PCA1X1Y2'] + " \ + ," + self.header['PCA1X0Y3'] + " \ + ," + self.header['PCA1X2Y0'] + " \ + ," + self.header['PCA1X1Y1'] + " \ + ," + self.header['PCA1X0Y2'] + " \ + ," + self.header['PCA2X3Y0'] + " \ + ," + self.header['PCA2X2Y1'] + " \ + ," + self.header['PCA2X1Y2'] + " \ + ," + self.header['PCA2X0Y3'] + " \ + ," + self.header['PCA2X2Y0'] + " \ + ," + self.header['PCA2X1Y1'] + " \ + ," + self.header['PCA2X0Y2'] + " \ )" - self.stmt.execute(sql) - - self.updateSurveyID("ImageMeta") - self.updateFilterID("ImageMeta") - self.updateStackTypeID("ImageMeta") - ''' - - ''' - Populates the Detection tables - ''' - def populateDetectionTables(self): - - self.log("Procesing Detection tables") - for x in range(3,4): - for y in range(3,4): - - self.log("Populating detections for OTA %d%d" % (x, y)) + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("FrameMeta", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("FrameMeta", self.filter) + self.scratchDb.updateAllRows("FrameMeta", "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.dataRelease)) + + ''' + Populates the ImageMeta table for this OTA + ''' + def populateImageMetaTable(self, ota, header): + + tableName = "ImageMeta_" + ota + + # drop then re-create table + self.scratchDb.dropTable(tableName) + sql = "CREATE TABLE " + tableName + " LIKE ImageMeta" + try: self.scratchDb.stmt.execute(sql) + except: pass + + # insert all detections into table + sql = "INSERT INTO " + tableName + " ( \ + frameID \ + ,ccdID \ + ,sky \ + ,skyScat \ + ,magSat \ + ,completMag \ + ,astroScat \ + ,photoScat \ + ,numAstroRef \ + ,numPhotoRef \ + ,nx \ + ,ny \ + ,psfFwhm \ + ,psfWidMajor \ + ,psfWidMinor \ + ,psfTheta \ + ,momentFwhm \ + ,momentWidMajor \ + ,momentWidMinor \ + ,apResid \ + ,dapResid \ + ,detectorID \ + ,qaFlags \ + ,detrend1 \ + ,detrend2 \ + ,detrend3 \ + ,detrend4 \ + ,detrend5 \ + ,detrend6 \ + ,detrend7 \ + ,detrend8 \ + ,photoZero \ + ,ctype1 \ + ,ctype2 \ + ,crval1 \ + ,crval2 \ + ,crpix1 \ + ,crpix2 \ + ,cdelt1 \ + ,cdelt2 \ + ,pc001001 \ + ,pc001002 \ + ,pc002001 \ + ,pc002002 \ + ,polyOrder \ + ,pca1x3y0 \ + ,pca1x2y1 \ + ,pca1x1y2 \ + ,pca1x0y3 \ + ,pca1x2y0 \ + ,pca1x1y1 \ + ,pca1x0y2 \ + ,pca2x3y0 \ + ,pca2x2y1 \ + ,pca2x1y2 \ + ,pca2x0y3 \ + ,pca2x2y0 \ + ,pca2x1y1 \ + ,pca2x0y2 \ + ) VALUES ( \ + " + str(self.expID) + " \ + ," + ota[2:4] + " \ + ," + self.safeDictionaryAccess(header, 'MSKY_MN') + " \ + ," + self.safeDictionaryAccess(header, 'MSKY_SIG') + " \ + ," + self.safeDictionaryAccess(header, 'FSATUR') + " \ + ," + self.safeDictionaryAccess(header, 'FLIMIT') + " \ + ," + self.safeDictionaryAccess(header, 'CERROR') + " \ + ," + self.safeDictionaryAccess(self.header, 'ZPT_OBS') + " \ + ," + self.safeDictionaryAccess(header, 'NASTRO') + " \ + ," + self.safeDictionaryAccess(header, 'NASTRO') + " \ + ," + self.safeDictionaryAccess(header, 'CNAXIS1') + " \ + ," + self.safeDictionaryAccess(header, 'CNAXIS2') + " \ + ," + str((float(self.safeDictionaryAccess(header, 'FWHM_MAJ')) + float(self.safeDictionaryAccess(header, 'FWHM_MIN')))/2.0) + " \ + ," + self.safeDictionaryAccess(header, 'FWHM_MAJ') + " \ + ," + self.safeDictionaryAccess(header, 'FWHM_MIN') + " \ + ," + self.safeDictionaryAccess(header, 'ANGLE') + " \ + ," + str((float(self.safeDictionaryAccess(header, 'IQ_FW1')) + float(self.safeDictionaryAccess(header, 'IQ_FW2')))/2.0) + " \ + ," + self.safeDictionaryAccess(header, 'IQ_FW1') + " \ + ," + self.safeDictionaryAccess(header, 'IQ_FW2') + " \ + ," + self.safeDictionaryAccess(header, 'APMIFIT') + " \ + ," + self.safeDictionaryAccess(header, 'DAPMIFIT') + " \ + ,'" + self.safeDictionaryAccess(header, 'DETECTOR') + "' \ + ," + str(self.scratchDb.getDvoImageFlags(header['SOURCEID'], header['IMAGEID'])) + " \ + ,'" + self.safeDictionaryAccess(header, 'DETREND.MASK') + "' \ + ,'" + self.safeDictionaryAccess(header, 'DETREND.DARK') + "' \ + ,'" + self.safeDictionaryAccess(header, 'DETREND.FLAT') + "' \ + ,' ' \ + ,' ' \ + ,' ' \ + ,' ' \ + ,' ' \ + ," + self.safeDictionaryAccess(self.header, 'ZPT_OBS') + " \ + ,'" + self.safeDictionaryAccess(header, 'CTYPE1') + "' \ + ,'" + self.safeDictionaryAccess(header, 'CTYPE2') + "' \ + ," + self.safeDictionaryAccess(header, 'CRVAL1') + " \ + ," + self.safeDictionaryAccess(header, 'CRVAL2') + " \ + ," + self.safeDictionaryAccess(header, 'CRPIX1') + " \ + ," + self.safeDictionaryAccess(header, 'CRPIX2') + " \ + ," + self.safeDictionaryAccess(header, 'CDELT1') + " \ + ," + self.safeDictionaryAccess(header, 'CDELT2') + " \ + ," + self.safeDictionaryAccess(header, 'PC001001') + " \ + ," + self.safeDictionaryAccess(header, 'PC001002') + " \ + ," + self.safeDictionaryAccess(header, 'PC002001') + " \ + ," + self.safeDictionaryAccess(header, 'PC002002') + " \ + ," + self.safeDictionaryAccess(header, 'NPLYTERM') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X3Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X2Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X1Y2') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X0Y3') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X2Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X1Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X0Y2') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X3Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X2Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X1Y2') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X0Y3') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X2Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X1Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X0Y2') + " \ + )" + + self.scratchDb.stmt.execute(sql) + self.scratchDb.updateFilterID(tableName, self.filter) + self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO']) + self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0") + + ''' + Populates the Detection table for this OTA + ''' + def populateDetectionTable(self, ota): + + tableName = "Detection_" + ota + + # drop then re-create table + self.scratchDb.dropTable(tableName) + sql = "CREATE TABLE " + tableName + " LIKE Detection" + try: self.scratchDb.stmt.execute(sql) + except: pass + + # insert all detections into table + sql = "INSERT INTO " + tableName + " ( \ + ippDetectID \ + ,xPos \ + ,yPos \ + ,xPosErr \ + ,yPosErr \ + ,instFlux \ + ,instFluxErr \ + ,peakADU \ + ,psfWidMajor \ + ,psfWidMinor \ + ,psfTheta \ + ,psfCf \ + ,momentXX \ + ,momentXY \ + ,momentYY \ + ,apMag \ + ,infoFlag \ + ,sky \ + ,skyErr \ + ,sgSep \ + ) \ + SELECT \ + IPP_IDET \ + ,X_PSF \ + ,Y_PSF \ + ,X_PSF_SIG \ + ,Y_PSF_SIG \ + ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+" \ + ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+")) / 1.085736) \ + ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \ + ,PSF_MAJOR \ + ,PSF_MINOR \ + ,PSF_THETA \ + ,PSF_QF \ + ,MOMENTS_XX \ + ,MOMENTS_XY \ + ,MOMENTS_YY \ + ,AP_MAG \ + ,FLAGS\ + ,SKY \ + ,SKY_SIGMA \ + ,EXT_NSIGMA \ + FROM " + ota + "_psf" + + self.scratchDb.stmt.execute(sql) + + # set obsTime + sql = "UPDATE " + tableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr) + self.scratchDb.stmt.execute(sql) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows(tableName, "historyModNum", "0") + + self.scratchDb.updateAllRows(tableName, "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID(tableName, self.filter) + + # now delete bad flux + self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "instFlux") + self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "peakADU") + + ''' + Populates the SkinnyObject table for this OTA + ''' + def populateSkinnyObjectTable(self, ota): + + tableName = "SkinnyObject_" + ota + + # drop then re-create table + self.scratchDb.dropTable(tableName) + sql = "CREATE TABLE " + tableName + " LIKE SkinnyObject" + try: self.scratchDb.stmt.execute(sql) + except: pass + + # insert all detections into table + sql = "INSERT INTO " + tableName + " ( \ + objID \ + ,ippObjID \ + ,surveyID \ + ) \ + SELECT \ + objID \ + ,ippObjID \ + ,surveyID \ + FROM Detection_" + ota + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + + ''' + Populates the Detection table for this OTA + ''' + def populateObjectCalColorTable(self, ota): + + tableName = "ObjectCalColor_" + ota + + # drop then re-create table + self.scratchDb.dropTable(tableName) + sql = "CREATE TABLE " + tableName + " LIKE ObjectCalColor" + try: self.scratchDb.stmt.execute(sql) + except: pass + + # insert all detections into table + sql = "INSERT INTO " + tableName + " ( \ + objID \ + ,ippObjID \ + ,filterID \ + ) \ + SELECT \ + objID \ + ,ippObjID \ + ,filterID \ + FROM Detection_" + ota + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + + + ''' + Applies indexes and other constraints to the PSPS tables + ''' + def alterPspsTables(self): + + self.logger.info("Altering PSPS tables") + self.logger.info("Creating indexes on PSPS tables") + self.scratchDb.makeColumnUnique("Detection", "objID") + self.scratchDb.createIndex("Detection", "ippDetectID") + + ''' + Applies indexes to the IPP tables + ''' + def indexIppTables(self): + + self.logger.info("Creating indexes on IPP tables") + + for x in range(self.startX, self.endX): + for y in range(self.startY, self.endY): + + # dodge the corners + if x==0 and y==0: continue + if x==0 and y==7: continue + if x==7 and y==0: continue + if x==7 and y==7: continue + + extension = "XY%d%d_psf" % (x, y) + self.scratchDb.createIndex(extension, "IPP_IDET") + + ''' + Updates provided table with DVO IDs from DVO table + ''' + def updateDvoIDs(self, table, sourceID, externID): + + imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID) + self.logger.info("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID) + sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \ + a.ippObjID = b.ippObjID, \ + a.detectID = b.detectID, \ + a.objID = b.objID, \ + a.infoFlag = b.flags << 32 | a.infoFlag \ + WHERE a.ippDetectID = b.ippDetectID \ + AND b.sourceID = " + str(sourceID) + " \ + AND b.imageID = " + str(imageID) + + self.scratchDb.stmt.execute(sql) + + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): + + self.populateFrameMeta() + + # dictionary objects to hold sourceIDs and imageIDs for later + sourceIDs = {} + imageIDs = {} + + file = open(self.inputFitsPath, 'r') + + # loop through all OTAs and populate ImageMeta extensions + for x in range(self.startX, self.endX): + for y in range(self.startY, self.endY): - sql = "INSERT INTO Detection ( \ - ippDetectID \ - ) \ - SELECT \ - IPP_IDET \ - FROM XY%d%d_psf" % (x, y) - - self.stmt.execute(sql) - - - ''' - Applies indexes to the PSPS tables - ''' - def indexPspsTables(self): - - self.log("Creating indexes on PSPS tables") -# self.createIndex("StackDetection", "ippDetectID") - # self.createIndex("StackApFlx", "ippDetectID") - # self.createIndex("StackModelFit", "ippDetectID") - - ''' - Applies indexes to the IPP tables - ''' - def indexIppTables(self): - - self.log("Creating indexes on IPP tables") - - for x in range(0,7): - for y in range(0,7): - - extension = "XY%d%d_psf" % (x, y) - self.createIndex(extension, "IPP_IDET") - - ''' - Does the processing, i.e. pulling stuff from IPP tables into PSPS tables - ''' - def populatePspsTables(self): - - # get filterID using init table - self.filter = self.header['FILTERID'] - self.filter = self.filter[0:1] - print "filter = ", self.filter - - #self.populateStackMeta() - self.populateDetectionTables() - #self.populateStackModelFit() - #self.populateStackApFlx() - + # dodge the corners + if x==0 and y==0: continue + if x==0 and y==7: continue + if x==7 and y==0: continue + if x==7 and y==7: continue + + ota = "XY%d%d" % (x, y) + + # load corresponding header into memory + header = self.findAndReadFITSHeader(ota + ".hdr", file) + + # store sourceID/imageID combo in Db so DVO can look up later + if not self.useFullTables: + self.scratchDb.insertNewDvoImage(header['SOURCEID'], header['IMAGEID']) + + # check we have valid sourceID/imageID pair from the header + if 'SOURCEID' not in header or 'IMAGEID' not in header: + self.logger.error("Can't read SOURCEID/IMAGEID pair from " + ota + " header") + continue + + # store these for later + sourceIDs[ota] = header['SOURCEID'] + imageIDs[ota] = header['IMAGEID'] + + # populate ImageMeta + self.populateImageMetaTable(ota, header) + self.updateImageID("ImageMeta_" + ota, x, y) + + # now run DVO code to get all IDs + if not self.useFullTables: self.getIDsFromDVO() + + # loop through all OTAs again to update with DVO IDs + self.tablesToExport = [] + self.tablesToExport.append("FrameMeta") + tables = [] + otaCount = 0 + for x in range(self.startX, self.endX): + for y in range(self.startY, self.endY): + + # dodge the corners + if x==0 and y==0: continue + if x==0 and y==7: continue + if x==7 and y==0: continue + if x==7 and y==7: continue + + ota = "XY%d%d" % (x, y) + if ota not in sourceIDs: continue + + self.logger.info("******************* Dealing with OTA " + ota + " *******************") + + # populate remainder of tables + self.populateDetectionTable(ota) + + # now add DVO IDs + self.updateDvoIDs("Detection_" + ota, sourceIDs[ota], imageIDs[ota]) + self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID") + self.updateImageID("Detection_" + ota, x, y) + + # check we have something in this Detection table + if self.scratchDb.getRowCount("Detection_" + ota) < 1: + self.logger.info("Empty tables for ota " + ota + ". Skipping.") + continue; + + # update ImageMeta with count of detections for this OTA and photoCodeID + sql = "UPDATE ImageMeta_" + ota + " SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota])) + self.scratchDb.stmt.execute(sql) + + self.populateSkinnyObjectTable(ota) + self.populateObjectCalColorTable(ota) + + # add these to list of tables to export later + self.tablesToExport.append("ImageMeta_" + ota) + self.tablesToExport.append("Detection_" + ota) + self.tablesToExport.append("SkinnyObject_" + ota) + self.tablesToExport.append("ObjectCalColor_" + ota) + tables.append("Detection_" + ota) + + otaCount = otaCount + 1 + + # if we only have one table export, i.e. FrameMeta, then get out of here + if len(self.tablesToExport) == 1: + + self.logger.error("No tables to export") + return False + + self.setMinMaxObjID(tables) + + # update FrameMeta with count OTAs in this file and total number of photometric reference sources + sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef) + self.scratchDb.stmt.execute(sql) + + return True + + ''' + Updates imageID {EXP_ID}{OTA} in the provided table + ''' + def updateImageID(self, tableName, x, y): + + sql = "UPDATE " + tableName + " SET imageID = %d%d%d" % (self.expID, x, y) + self.scratchDb.stmt.execute(sql) + + ''' + Checks whether this batch has already been processed and published + ''' + def alreadyProcessed(self): + + return self.ippToPspsDb.alreadyProcessed("detection", "exp_id", self.expID) + + + ''' + Overriding this method + ''' + def reportNullsInAllPspsTables(self, showPartials): + + # loops round all imported tables, but we want to check one OTA's worth + for table in self.pspsTables: + if table.name == "FrameMeta": self.scratchDb.reportNulls(table.name, showPartials) + else: self.scratchDb.reportNulls(table.name + "_XY33", showPartials) + + +logging.config.fileConfig("logging.conf") +logger = logging.getLogger("detectionbatch") +logger.info("Starting") + +gpc1Db = Gpc1Db(logger) +camIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.V2", "cam") +logger.info("Found %d exposures" % len(camIDs)) + +i = 0 +for camID in camIDs: + + logger.info("-------------------------------------------------- cam ID: %d" % camID) + + file = gpc1Db.getCameraStageSmf(camID) + if not os.path.isfile(file): + logger.error("Cannot read file at '" + file) + continue + + detectionBatch = DetectionBatch(logger, camID, file, False, True) + + if not detectionBatch.alreadyProcessed(): + + detectionBatch.createEmptyPspsTables() + detectionBatch.importIppTables(".*.psf") + if detectionBatch.populatePspsTables(): + detectionBatch.exportPspsTablesToFits("([a-zA-Z]+)") + detectionBatch.writeBatchManifest() + #detectionBatch.reportNullsInAllPspsTables(False) + #detectionBatch.createTarball() + #detectionBatch.publishToDatastore() -detectionBatch = DetectionBatch() -detectionBatch.createEmptyPspsTables() -detectionBatch.importIppTables("XY33\.psf") -detectionBatch.populatePspsTables() -detectionBatch.exportPspsTablesToFits() + i = i+1 + # if i > 0: sys.exit() + Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/dvoToMySQL.py =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/dvoToMySQL.py (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/dvoToMySQL.py (revision 31439) @@ -0,0 +1,209 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob +from subprocess import call, PIPE, Popen + +from gpc1db import Gpc1Db +from scratchdb import ScratchDb + +from java.lang import * +from java.sql import * +from xml.etree.ElementTree import ElementTree, Element, tostring + + +''' +Class for pulling IDs from a DVO database and shoving in a MySQL database +''' +class DvoToMySql(object): + + ''' + Constructor + + ''' + def __init__(self, logger, pathToDvo): + + # set up logging + self.logger = logger + self.pathToDvo = pathToDvo + self.logger.debug("DvoToMySql class constructor") + + self.logger.debug("Important DVO database at " + self.pathToDvo) + + # open config + doc = ElementTree(file="config.xml") + + # create database objects + self.scratchDb = ScratchDb(logger) + self.gpc1Db = Gpc1Db(self.logger) + + # create DVO tables + #self.scratchDb.createDvoTables() + + # import Images.dat table + sql = "DELETE FROM dvoMetaFull" + self.scratchDb.stmt.execute(sql) + + imagesTableName = self.importFits(self.pathToDvo, + "", + "Images.dat", + "IMAGE_ID SOURCE_ID CCDNUM EXTERN_ID FLAGS PHOTCODE NSTAR") + self.scratchDb.createIndex(imagesTableName, "IMAGE_ID") + + # insert into dvoMetaFull + self.logger.info("Inserting all image meta data into database") + sql = "INSERT INTO dvoMetaNew ( \ + sourceID, \ + imageID, \ + externID, \ + flags, \ + photcode \ + ) SELECT \ + SOURCE_ID, \ + IMAGE_ID, \ + EXTERN_ID, \ + FLAGS, \ + PHOTCODE \ + FROM " + imagesTableName + self.scratchDb.stmt.execute(sql) + + subdirs = ['n0000'] + + for subdir in subdirs: + + files = glob.glob(pathToDvo + "/" + subdir + "/*.cpm") + + #files = ['0247.06', '0244.06', '0244.10'] + + for file in files: + + # get just filename, without extension + file = os.path.basename(os.path.splitext(file)[0]) + self.logger.info("---------------------------------------------: " + file) + + if self.scratchDb.alreadyImportedThisDvoTable(file): continue + + # import cpm table and index + cpmTableName = self.importFits(self.pathToDvo, + subdir, + file + ".cpm", + "IMAGE_ID DET_ID OBJ_ID CAT_ID EXT_ID DB_FLAGS") + self.scratchDb.createIndex(cpmTableName, "CAT_ID") + self.scratchDb.createIndex(cpmTableName, "OBJ_ID") + + # import cpt table and index + cptTableName = self.importFits(self.pathToDvo, + subdir, + file + ".cpt", + "OBJ_ID CAT_ID EXT_ID") + self.scratchDb.createIndex(cptTableName, "CAT_ID") + self.scratchDb.createIndex(cptTableName, "OBJ_ID") + + # shove SOURCE_IDs into measurement table + self.logger.info("Adding SOURCE_IDs into measurements table") + sql = "ALTER TABLE "+cpmTableName+" ADD COLUMN (SOURCE_ID SMALLINT)" + self.scratchDb.stmt.execute(sql) + sql = "UPDATE "+cpmTableName+" AS a, "+imagesTableName+" AS b \ + SET a.SOURCE_ID = b.SOURCE_ID \ + WHERE a.IMAGE_ID = b.IMAGE_ID" + self.scratchDb.stmt.execute(sql) + + # shove PSPS objID in measurement table + self.logger.info("Adding PSPS objID into measurements table") + sql = "ALTER TABLE "+cpmTableName+" ADD COLUMN (PSPS_OBJ_ID BIGINT)" + self.scratchDb.stmt.execute(sql) + sql = "UPDATE "+cpmTableName+" AS a, "+cptTableName+" AS b \ + SET a.PSPS_OBJ_ID = b.EXT_ID \ + WHERE a.CAT_ID = b.CAT_ID \ + AND a.OBJ_ID = b.OBJ_ID" + self.scratchDb.stmt.execute(sql) + + self.logger.info("Putting everything into dvoDetectionFull table") + sql = "INSERT IGNORE INTO dvoDetectionFull (\ + sourceID \ + ,imageID \ + ,ippDetectID \ + ,detectID \ + ,ippObjID \ + ,objID \ + ,flags \ + ) SELECT \ + SOURCE_ID \ + ,IMAGE_ID \ + ,DET_ID \ + ,EXT_ID \ + ,CAT_ID * 1000000000 + OBJ_ID \ + ,PSPS_OBJ_ID \ + ,DB_FLAGS \ + FROM " + cpmTableName + self.scratchDb.stmt.execute(sql) + + # now drop what we don't need + self.logger.info("Dropping tables") + self.scratchDb.dropTable(cpmTableName) + self.scratchDb.dropTable(cptTableName) + + self.scratchDb.setImportedThisDvoTable(file) + + self.scratchDb.dropTable(imagesTableName) + + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("DvoToMySql destructor") + + ''' + Imports a FITS file. A regex filter lets you choose which tables to pull from the file + ''' + def importFits(self, path, subdir, file, columns): + + fullPath = path + "/" + subdir + "/" + file + + if len(subdir) < 1: tableName = file + else: tableName = subdir + "_" + file + + tableName = tableName.replace('.', '_') + + self.logger.info("Attempting to import tables from '" + fullPath + "' to '" + tableName + "'") + + tables = stilts.treads(fullPath) + + count = 0 + for table in tables: + + self.logger.info("Reading IPP table " + table.name + " from FITS file") + table = stilts.tpipe(table, cmd='explodeall') + + # IPP FITS files are littered with infinities, so remove these + self.logger.info("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + + #try: + self.logger.info("Writing FITS table to database") + table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName) + #except: + # self.logger.exception(" Problem writing table '" + table.name + "' to the database") + count = count + 1 + + self.logger.info("Done. Imported %d tables" % count) + + return tableName + +logging.config.fileConfig("logging.conf") +logger = logging.getLogger("dvotomysql") +logger.info("Starting") + +dvoToMySql = DvoToMySql(logger, "/data/ipp005.0/gpc1/catdirs/MD04.merges/MD04.merge") +#dvoToMySql = DvoToMySql(logger, "/export/ippc00.1/rhenders/MD04.merge") + +logger.info("Program complete") + Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/gpc1db.py =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/gpc1db.py (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/gpc1db.py (revision 31439) @@ -0,0 +1,212 @@ +#!/usr/bin/env jython + +import re +import sys +import glob +import os +import logging + +from mysql import MySql +from java.sql import * + + +''' +Class for GPC1 database connectivity +''' +class Gpc1Db(MySql): + + ''' + Constructor + ''' + def __init__(self, logger): + super(Gpc1Db, self).__init__(logger,"gpc1database") + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("Gpc1Db destructor") + + ''' + Gets a list of ids in this DVO database for this stage, could be cam or staticsky (so far) + ''' + def getIDsInThisDVODbForThisStage(self, dvoDb, stage): + + sql = "SELECT DISTINCT stage_id \ + FROM addRun \ + WHERE stage = '" + stage + "' \ + AND dvodb = '" + dvoDb + "'" + + try: + rs = self.stmt.executeQuery(sql) + except: + self.logger.exception("Can't query for ids in DVO") + + ids = [] + while (rs.next()): + ids.append(rs.getInt(1)) + + rs.close() + + self.logger.debug("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage)) + + return ids + + ''' + Gets a list of PSPS image IDs for this stack ID + ''' + def getImageIDsForThisStackID(self, stackID): + + self.logger.debug("Querying GPC1 for image IDs for stack ID: " + str(stackID)) + + sql = "SELECT DISTINCT CONCAT(exp_id, SUBSTR(class_id, 3, 4)) FROM ( \ + SELECT DISTINCT exp_id,class_id \ + FROM warpSkyCellMap \ + JOIN warpRun USING(warp_id) \ + JOIN stackInputSkyfile USING(warp_id) \ + JOIN stackRun USING(stack_id,skycell_id) \ + JOIN fakeRun USING(fake_id) \ + JOIN camRun USING(cam_id) \ + JOIN chipRun USING(chip_id) \ + WHERE stackRun.stack_id = " + str(stackID) + ") AS a" + + try: + rs = self.stmt.executeQuery(sql) + except: + self.logger.exception("Can't query for imageIDs") + + imageIDs = [] + while (rs.next()): + imageIDs.append(rs.getString(1)) + rs.close() + + return imageIDs + + ''' + Gets some stack-stage meta data for this sky_id # TODO this SQL could surely be improved + ''' + def getStackStageMeta(self, skyID, filter): + + self.logger.debug("Querying GPC1 for stack meta data") + + meta = [] + sql = "SELECT \ + stackRun.stack_id,\ + stackRun.skycell_id \ + FROM \ + staticskyInput, staticskyRun, stackRun, staticskyResult \ + WHERE staticskyRun.sky_id = staticskyInput.sky_id \ + AND staticskyInput.stack_id = stackRun.stack_id \ + AND staticskyInput.sky_id = staticskyResult.sky_id \ + and staticskyInput.sky_id = %d \ + and filter = '%s'" % (skyID, filter) + + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + meta.append(rs.getInt(1)) + meta.append(rs.getString(2)) + except: + self.logger.exception("Can't query for stack meta") + + return meta + ''' + Gets some camera-stage meta data for this cam_id + ''' + def getCameraStageMeta(self, camID): + + self.logger.debug("Querying GPC1 for camera meta data") + + meta = [] + sql = "SELECT exp_id, exp_name, camRun.dist_group \ + FROM camRun \ + JOIN chipRun USING(chip_id) \ + JOIN rawExp USING(exp_id) WHERE camRun.cam_id = %d" % camID + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + meta.append(rs.getInt(1)) + meta.append(rs.getString(2)) + meta.append(rs.getString(3)) + except: + self.logger.exception("Can't query for camera meta") + + return meta + + ''' + Gets a camera-stage smf for this cam_id + ''' + def getCameraStageSmf(self, camID): + + self.logger.debug("Querying GPC1 for camera smf files with cam_id = " + str(camID)) + + sql = "SELECT path_base \ + FROM camProcessedExp \ + JOIN camRun USING(cam_id) \ + WHERE camRun.cam_id = %d" % camID + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + except: + self.logger.exception("Can't query for camera smfs") + + # get path to base dir of cmf files + path = rs.getString(1) + path = path[0:path.rfind("/")] + + # list all cmf files if a neb path + files = [] + if path.startswith("neb"): + + f=os.popen("neb-ls -p "+path+"/%smf") + for i in f.readlines(): + files.append(i.rstrip()) + + # or not a neb path + else: + files = glob.glob(path + "/*.cmf") + + return files[0] # TODO just returning first file - check + + + ''' + Gets all cmf files for this sky_id. handles both absolute paths and neb paths + ''' + def getStackStageCmfs(self, skyID): + + self.logger.debug("Querying GPC1 for stack cmf files") + + sql = "SELECT path_base, num_inputs \ + FROM staticskyResult \ + WHERE sky_id = %d" % skyID + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + except: + self.logger.exception("Can't query for stack cmfs") + + # get path to base dir of cmf files + path = rs.getString(1) + #path = path[0:path.rfind("/")] + + # list all cmf files if a neb path + files = [] + if path.startswith("neb"): + + f=os.popen("neb-ls -p "+path+"%cmf") + print "neb-ls -p "+path+"%cmf" + for i in f.readlines(): + files.append(i.rstrip()) + print i.rstrip() + + # or not a neb path + else: + files = glob.glob(path + "*.cmf") + + return files + Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/initbatch.py =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/initbatch.py (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/initbatch.py (revision 31439) @@ -2,4 +2,6 @@ import stilts +import logging +import logging.config from java.lang import * from java.sql import * @@ -14,15 +16,14 @@ Constructor ''' - def __init__(self): - super(InitBatch, self).__init__( - "init", - "", - "init.fits", - "localhost", - "ipptopsps_scratch", - "ipp", - "ipp") + def __init__(self, logger): + super(InitBatch, self).__init__(logger, "init") -initBatch = InitBatch() + self.outputFitsFile = "00000000.FITS"; + self.outputFitsPath = self.localOutPath + "/" + self.outputFitsFile + +logging.config.fileConfig("logging.conf") +logger = logging.getLogger("initbatch") +initBatch = InitBatch(logger) initBatch.createEmptyPspsTables() initBatch.exportPspsTablesToFits() +initBatch.writeBatchManifest() Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/ipptopspsdb.py =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/ipptopspsdb.py (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/ipptopspsdb.py (revision 31439) @@ -0,0 +1,164 @@ +#!/usr/bin/env jython + +import re +import sys +import os +import logging + +from mysql import MySql +from java.sql import * + +''' +Class for ippToPsps database connectivity +''' +class IppToPspsDb(MySql): + + ''' + Constructor + ''' + def __init__(self, logger): + super(IppToPspsDb, self).__init__(logger,"ipptopspsdatabase") + + ''' + Creates a new batch + ''' + def createNewBatch(self, batchType, survey, dvoDb, datastoreProduct): + + sql = "INSERT INTO batch ( \ + batch_type, \ + survey, \ + dvo_db, \ + datastore_product \ + ) VALUES ( \ + '" + batchType + "', \ + '" + survey + "', \ + '" + dvoDb + "', \ + '" + datastoreProduct + "' \ + )" + + self.stmt.execute(sql) + + sql = "SELECT MAX(batch_id) FROM batch" + + batchID = -1; + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + batchID = rs.getInt(1) + except: + self.logger.exception("Unable to get batch ID") + + self.logger.info("Created new batch in ippToPsps database with batchID = %d" % batchID) + + return batchID; + + ''' + Updates min/max object ID on this table and batch + ''' + def updateMinMaxObjID(self, batchID, minObjID, maxObjID): + + sql = "UPDATE batch SET \ + min_obj_id = " + str(minObjID) + ", \ + max_obj_id = " + str(maxObjID) + " \ + WHERE batch_id = " + str(batchID) + + self.stmt.execute(sql) + + ''' + Updates batch processed field + ''' + def updateProcessed(self, batchID, processed): + + sql = "UPDATE batch \ + SET processed = " + str(processed) + " \ + WHERE batch_id = " + str(batchID) + + self.stmt.execute(sql) + + ''' + Updates batch loaded_to_datastore field + ''' + def updateLoadedToDatastore(self, batchID, loadedToDatastore): + + sql = "UPDATE batch \ + SET loaded_to_datastore = " + str(loadedToDatastore) + " \ + WHERE batch_id = " + str(batchID) + + self.stmt.execute(sql) + + ''' + Have we already processed and published this batch? + ''' + def alreadyProcessed(self, table, col, value): + + sql = "SELECT COUNT(*) FROM \ + " + table + " \ + JOIN batch USING(batch_id) \ + WHERE " + col + " = " + str(value) + " \ + AND processed \ + AND loaded_to_datastore" + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + if rs.getInt(1) > 0: + self.logger.error("Batch with "+col+" = "+str(value)+" has already been processed and published to datastore") + return True + else: + return False + except: + self.logger.exception("Unable to check whether this batch has already been processed") + + + + ''' + Inserts some detection metadata for this batch ID + ''' + def insertDetectionMeta(self, batchID, expID, filter): + + sql = "INSERT INTO detection ( \ + batch_id \ + ,exp_id \ + ,filter \ + ) VALUES ( \ + " + str(batchID) + " \ + ," + str(expID) + " \ + ,'" + filter + "' \ + )" + + self.stmt.execute(sql) + + ''' + Inserts some stack metadata for this batch ID + ''' + def insertStackMeta(self, batchID, skyID, stackID, filter, stackType): + + sql = "INSERT INTO stack ( \ + batch_id \ + ,sky_id \ + ,stack_id \ + ,filter \ + ,stack_type \ + ) VALUES ( \ + " + str(batchID) + " \ + ," + str(skyID) + " \ + ," + str(stackID) + " \ + ,'" + filter + "' \ + ,'" + stackType + "' \ + )" + + self.stmt.execute(sql) + + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("IppToPspsDb Desstructor") + self.stmt.close() + self.con.close() + + + Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/logging.conf =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/logging.conf (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/logging.conf (revision 31439) @@ -0,0 +1,28 @@ +[loggers] +keys=root,simpleExample + +[handlers] +keys=consoleHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler + +[logger_simpleExample] +level=DEBUG +handlers=consoleHandler +qualname=simpleExample +propagate=0 + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=simpleFormatter +args=(sys.stdout,) + +[formatter_simpleFormatter] +format=%(asctime)s | %(levelname)7s | %(message)s +datefmt=%Y-%m-%d %H:%M:%S Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/mysql.py =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/mysql.py (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/mysql.py (revision 31439) @@ -0,0 +1,200 @@ +#!/usr/bin/env jython + +import re +import sys +import os +import logging + +from java.lang import * +from java.sql import * +from xml.etree.ElementTree import ElementTree + +''' +Base class of all MySql database classes. +''' +class MySql(object): + + driverName="com.mysql.jdbc.Driver" + + ''' + Constructor + + ''' + def __init__(self, logger, dbType): + + # set up logging + self.logger = logger + self.logger.debug("MySql class constructor") + + # open config and grab database parameters + doc = ElementTree(file="config.xml") + self.dbName = doc.find(dbType +"/name").text + self.dbHost = doc.find(dbType +"/host").text + self.dbUser = doc.find(dbType +"/user").text + self.dbPass = doc.find(dbType +"/password").text + + # set up JDBC connection + self.url = "jdbc:mysql://"+self.dbHost+"/"+self.dbName+"?user="+self.dbUser+"&password="+self.dbPass + self.con = DriverManager.getConnection(self.url) + self.stmt = self.con.createStatement() + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("MySql destructor") + self.stmt.close() + self.con.close() + + ''' + Updates all rows of this column of this table with this value + ''' + def updateAllRows(self, table, column, value): + + sql = "UPDATE " + table + " SET " + column + " = " + value + self.stmt.execute(sql) + + ''' + Drops a table + ''' + def dropTable(self, table): + + sql = "DROP TABLE " + table + try: self.stmt.execute(sql) + except: return + + ''' + Alters a column to be unique + ''' + def makeColumnUnique(self, table, column): + + self.logger.debug("Making '"+column+"' unique on table '"+table+"'") + + sql = "ALTER TABLE " + table + " ADD UNIQUE (" + column + ")" + try: + self.stmt.execute(sql) + except: pass + #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'") + ''' + Adds an index to the supplied table and column + ''' + def createIndex(self, table, column): + + self.logger.debug("Creating index on column '"+column+"' for table '"+table+"'") + + sql = "CREATE INDEX "+table+"_"+column+"_index ON "+table+" ("+column+")" + try: + self.stmt.execute(sql) + except: pass + #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'") + + ''' + Returns a list of column names for this table + ''' + def getColumnNames(self, tableName): + + sql = "SHOW COLUMNS FROM " + tableName + rs = self.stmt.executeQuery(sql) + columns = [] + while (rs.next()): columns.append(rs.getString(1)) + rs.close() + + return columns + + ''' + Replaces all NULL values in the provided table and column ith the provided substitute + ''' + def replaceNullsInThisColumn(self, tableName, column, sub): + + sql = "UPDATE " + tableName + " SET " + column + " = " + sub + " WHERE " + column + " IS NULL" + self.stmt.execute(sql) + + ''' + Replaces all NULL values in the provided table with the provided substitute + ''' + def replaceNulls(self, tableName, sub): + + # get list of columns + columns = self.getColumnNames(tableName) + + # now loop through all columns and replace all NULLs with sub + for column in columns: + + sql = "UPDATE " + tableName + " SET " + column + " = " + sub + " WHERE " + column + " IS NULL" + self.stmt.execute(sql) + + ''' + Reports the number of rows with this columns set as this numeric value and then deletes them + ''' + def reportAndDeleteRowsWithThisValue(self, tableName, columnName, value): + + sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + columnName + " = " + value + rs = self.stmt.executeQuery(sql) + rs.first() + nBadFlux = rs.getInt(1) + self.logger.info("%d NULL %s values in table %s. Deleting." % (nBadFlux, columnName, tableName)) + + sql="DELETE from " + tableName + " WHERE " + columnName + " = " + value + self.stmt.execute(sql) + + ''' + Reports the number of rows with this column NULL and then deletes them + ''' + def reportAndDeleteRowsWithNULLS(self, tableName, columnName): + + sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + columnName + " IS NULL" + rs = self.stmt.executeQuery(sql) + rs.first() + nBadFlux = rs.getInt(1) + self.logger.info("%d NULL %s values in table %s. Deleting." % (nBadFlux, columnName, tableName)) + + sql="DELETE from " + tableName + " WHERE " + columnName + " IS NULL" + self.stmt.execute(sql) + + ''' + Searches a table and reports the columns that are either partially or completely populated with NULLs + ''' + def reportNulls(self, tableName, showPartials): + + # first, count rows + sql = "SELECT COUNT(*) FROM " + tableName + rs = self.stmt.executeQuery(sql) + rs.first() + numRows = rs.getInt(1) + + # get list of columns + columns = self.getColumnNames(tableName) + + print "+----------------------+---------------+" + print "| %25s |" % tableName + print "+----------------------+---------------+" + + # now see which columns are full of NULLS, with are partially NULL + for column in columns: + + sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + column + " IS NULL" + rs = self.stmt.executeQuery(sql) + rs.first() + if rs.getInt(1) == numRows: + print "| %20s | all NULL |" % column + elif showPartials and rs.getInt(1) > 0: + print "| %20s | partial NULL |" % column + rs.close() + print "+----------------------+---------------+" + + ''' + Returns a row count for this table + ''' + def getRowCount(self, table): + + sql = "SELECT COUNT(*) FROM " + table + try: + rs = self.stmt.executeQuery(sql) + rs.first() + return rs.getInt(1) + except: + self.logger.exception("Could not count rows for table: '" + table + "'") + return -1 + + Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/removeFromDatastore.py =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/removeFromDatastore.py (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/removeFromDatastore.py (revision 31439) @@ -0,0 +1,17 @@ +#!/usr/bin/env jython + +from datastore import Datastore +import logging +import sys +import getopt + + +logging.config.fileConfig("logging.conf") +logger = logging.getLogger("datastore") + + + + +datastore = Datastore(logger) +datastore.remove(str(sys.argv[1])) + Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/run.sh =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/run.sh (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/run.sh (revision 31439) @@ -1,1 +1,1 @@ -~/jre1.6.0_24/bin/java -Xbootclasspath/a:../jars/stilts.jar:../jars/mysql-connector-java-5.1.14-bin.jar -jar ~/jython2.5.2/jython.jar $1 +~/jre1.6.0_24/bin/java -Xbootclasspath/a:../jars/stilts.jar:../jars/mysql-connector-java-5.1.14-bin.jar -jar ~/jython2.5.2/jython.jar $1 $2 $3 $4 $5 $6 $7 $8 $9 Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/scratchdb.py =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/scratchdb.py (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/scratchdb.py (revision 31439) @@ -0,0 +1,197 @@ +#!/usr/bin/env jython + +import re +import sys +import os +import logging + +from java.lang import * +from java.sql import * + +from mysql import MySql + +''' +Class for local scratch database connectivity +''' +class ScratchDb(MySql): + + ''' + Constructor + ''' + def __init__(self, logger, useFull=False): + super(ScratchDb, self).__init__(logger,"localdatabase") + + if useFull: + self.dvoMeta = "dvoMetaNew" + self.dvoDetection = "dvoDetectionFull" + else: + self.dvoMeta = "dvoMeta" + self.dvoDetection = "dvoDetection" + + self.logger.debug("ScratchDb constructor, using DVO tables: " + self.dvoMeta + " and " + self.dvoDetection) + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("ScratchDb destructor") + + ''' + Gets survey ID for this survey name + ''' + def getSurveyID(self, name): + + sql = "SELECT surveyID FROM Survey WHERE name = '" + name + "'" + try: + rs = self.stmt.executeQuery(sql) + rs.first() + return rs.getInt(1) + except: + self.logger.exception("No survey ID found for this survey: '" + self.survey + "'") + return -1 + + ''' + Gets DVO image flags + ''' + def getDvoImageFlags(self, sourceID, externID): + + flags = 0 + + sql = "SELECT flags FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + try: + rs = self.stmt.executeQuery(sql) + rs.first() + flags = rs.getInt(1) + except: + self.logger.exception("Unable to get flags from dvo table") + + return flags + + ''' + Gets imageID from extern ID + ''' + def getImageIDFromExternID(self, sourceID, externID): + + imageID = -1 + + sql = "SELECT imageID FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + try: + rs = self.stmt.executeQuery(sql) + rs.first() + imageID = rs.getInt(1) + except: + self.logger.exception("Unable to get imageID from dvo meta table") + + return imageID + + ''' + Gets photcode (aka photoCalID from dvo table) + ''' + def getPhotoCalID(self, sourceID, externID): + + photcode = -1 + + sql = "SELECT photcode FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + try: + rs = self.stmt.executeQuery(sql) + rs.first() + photcode = rs.getInt(1) + except: + self.logger.exception("Unable to get photcode from dvo table with: " + sql) + + return photcode + + ''' + Updates a table with filterID grabbed from Filter init table + ''' + def updateFilterID(self, table, filter): + + sql = "UPDATE "+table+" AS a, Filter AS b SET a.filterID=b.filterID WHERE b.filterType = '" + filter + "'" + self.stmt.execute(sql) + + ''' + Inserts a new sourceID/imageID combo into dvoMeta + ''' + def insertNewDvoImage(self, sourceID, imageID): + + sql = "INSERT INTO dvoMeta ( \ + sourceID, \ + imageID \ + ) VALUES (\ + " + str(sourceID) + ", \ + " + str(imageID) + " \ + )" + self.stmt.execute(sql) + + ''' + Updates dvoDone table with this DVO table + ''' + def setImportedThisDvoTable(self, name): + + sql = "INSERT INTO dvoDone (name) VALUES ('" + name + "')" + self.stmt.execute(sql) + + ''' + Have we already imported this DVO table? + ''' + def alreadyImportedThisDvoTable(self, name): + + sql = "SELECT COUNT(*) FROM dvoDone WHERE name = '" + name + "'" + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + if rs.getInt(1) > 0: + self.logger.error("DVO tables " + name + " have already been imported") + return True + else: + return False + except: + self.logger.exception("Unable to check whether this DVO table has been imported") + + + ''' + Creates a table for for ID matching + ''' + def createDvoTables(self): + + self.logger.info("Creating DVO meta and detection tables") + + sql = "DROP TABLE dvoMeta" + try: self.stmt.execute(sql) + except: pass + + sql = "DROP TABLE dvoDetection" + try: self.stmt.execute(sql) + except: pass + + sql = "CREATE TABLE dvoMeta ( \ + sourceID INT, \ + imageID INT, \ + flags INT, \ + photcode INT, \ + PRIMARY KEY (sourceID, imageID) \ + )" + + try: self.stmt.execute(sql) + except: + self.logger.error("Unable to create DVO meta-data database table") + + sql = "CREATE TABLE dvoDetection ( \ + sourceID INT, \ + imageID INT, \ + ippDetectID BIGINT, \ + detectID BIGINT, \ + ippObjID BIGINT, \ + objID BIGINT, \ + flags INT, \ + PRIMARY KEY (sourceID, imageID, ippDetectID) \ + )" + #INDEX (sourceID), \ + #INDEX (imageID), \ + #INDEX (ippDetectID) \ + + try: self.stmt.execute(sql) + except: + self.logger.error("Unable to create DVO detection database table") Index: branches/eam_branches/ipp-20110404/ippToPsps/jython/stackbatch.py =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/jython/stackbatch.py (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/jython/stackbatch.py (revision 31439) @@ -1,9 +1,18 @@ #!/usr/bin/env jython + +import os.path +import sys import stilts from java.lang import * from java.sql import * + +from gpc1db import Gpc1Db from batch import Batch - +import logging.config + +''' +StackBatch class +''' class StackBatch(Batch): @@ -11,17 +20,66 @@ Constructor ''' - def __init__(self): + def __init__(self, logger, skyID, inputFile, stackType, useFullTables=False): super(StackBatch, self).__init__( + logger, "stack", - #"/data/ipp053.0/eugene/md04.20110320/staticsky/MD04.V2/skycell.087/MD04.V2.skycell.087.stk.280.000.cmf", - "demo.fits", - "stack.fits", - "localhost", - "ipptopsps_scratch", - "ipp", - "ipp", - "MD04") # TODO - - + inputFile, + "MD04", + useFullTables) # TODO + + self.logger.info("StackBatch constructor. Creating batch from: '" + inputFile + "'") + + self.skyID = skyID + + # get filterID using init table + self.filter = self.header['FPA.FILTER'] + self.filter = self.filter[0:1] + + self.stackType = stackType + meta = self.gpc1Db.getStackStageMeta(self.skyID, self.header['FPA.FILTER']) + if len(meta) < 1: return + self.stackID = meta[0]; + self.skycell = meta[1]; + + # determine skycell from header value + #self.skycell = "skycell.34" #= self.header['SKYCELL'] + self.skycell = self.skycell[8:] + + self.logger.info("Processing stack with ID: %d, type: %s and skycell: %s filter: %s" % (self.stackID, self.stackType, self.skycell, self.filter)) + + + # delete PSPS tables + self.scratchDb.dropTable("StackMeta") + self.scratchDb.dropTable("StackDetection") + self.scratchDb.dropTable("StackModelFit") + self.scratchDb.dropTable("StackApFlx") + self.scratchDb.dropTable("StackToImage") + self.scratchDb.dropTable("SkinnyObject") + self.scratchDb.dropTable("ObjectCalColor") + + # delete IPP tables + #self.scratchDb.dropTable("SkyChip_psf") + #self.scratchDb.dropTable("SkyChip_xsrc") + #self.scratchDb.dropTable("SkyChip_xfit") + #self.scratchDb.dropTable("SkyChip_xrad") + + self.logger.info("Stack type: " + self.safeDictionaryAccess(self.header, self.stackType)) + # obs time makes no sense except for nightly stacks + #if self.header['STK_TYPE'] != "NIGHTLY_STACK": self.header['MJD-OBS'] = "-999" + + # create an output filename, which is {filterID}{skycellID}.FITS + self.outputFitsFile = "%s%07d.FITS" % (self.filter, int(self.skycell)) + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + + # set some constants + self.dataRelease = "1" + self.historyModNum = "0" + + # insert what we know about this stack batch into the stack table + self.ippToPspsDb.insertStackMeta(self.batchID, self.skyID, self.stackID, self.filter, self.stackType) + + # insert sourceID/imageID combo so DVO can look it up + if not self.useFullTables: + self.scratchDb.insertNewDvoImage(self.header['SOURCEID'], self.header['IMAGEID']) ''' @@ -30,6 +88,6 @@ def updateStackMetaID(self, table): - sql = "UPDATE " + table + " SET stackMetaID=" + self.header['STK_ID'] - self.stmt.execute(sql) + sql = "UPDATE " + table + " SET stackMetaID=" + str(self.stackID) + self.scratchDb.stmt.execute(sql) ''' @@ -38,6 +96,6 @@ def updateStackTypeID(self, table): - sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '"+self.header['STK_TYPE']+"'" - self.stmt.execute(sql) + sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '" + self.stackType + "'" + self.scratchDb.stmt.execute(sql) @@ -90,5 +148,5 @@ WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition - self.stmt.execute(sql) + self.scratchDb.stmt.execute(sql) ''' @@ -138,5 +196,5 @@ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" - self.stmt.execute(sql) + self.scratchDb.stmt.execute(sql) # sersic fit has an extra parameter @@ -150,8 +208,10 @@ "+modelLong+"Covar68=b.EXT_COVAR_05_07, \ "+modelLong+"Covar78=b.EXT_COVAR_06_07, \ - "+modelLong+"Covar88=b.EXT_COVAR_07_07 \ + "+modelLong+"Covar88=b.EXT_COVAR_07_07, \ + "+"serNu=b.EXT_PAR_07, \ + "+"serNuErr=SQRT(EXT_COVAR_07_07) \ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" - self.stmt.execute(sql) + self.scratchDb.stmt.execute(sql) @@ -160,11 +220,13 @@ ''' def populateStackMeta(self): - self.log("Procesing StackMeta table") + self.logger.info("Procesing StackMeta table") sql = "INSERT INTO StackMeta (\ stackMetaID \ ,skycellID \ + ,photoCalID \ ,photoZero \ - ,nP2Images \ + ,expTime \ + ,psfModelID \ ,ctype1 \ ,ctype2 \ @@ -180,8 +242,10 @@ ,pc002002 \ ) VALUES ( \ - " + self.header['STK_ID'] + " \ + " + str(self.stackID) + " \ ," + self.skycell + " \ + ," + str(self.scratchDb.getPhotoCalID(self.header['SOURCEID'], self.header['IMAGEID'])) + " \ ," + self.header['FPA.ZP'] + " \ - ," + self.header['NINPUTS'] + " \ + ," + self.header['EXPTIME'] + " \ + ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \ ,'" + self.header['CTYPE1'] + "' \ ,'" + self.header['CTYPE2'] + "' \ @@ -197,8 +261,9 @@ ," + self.header['PC002002'] + " \ )" - self.stmt.execute(sql) - - self.updateSurveyID("StackMeta") - self.updateFilterID("StackMeta") + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackMeta", self.filter) + self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.dataRelease)) self.updateStackTypeID("StackMeta") @@ -207,9 +272,11 @@ ''' def populateStackDetection(self): - self.log("Procesing StackDetection table") + self.logger.info("Procesing StackDetection table") # insert all the detections sql = "INSERT INTO StackDetection (\ ippDetectID \ + ,skyCellID \ + ,obsTime \ ,xPos \ ,yPos \ @@ -225,15 +292,35 @@ ,psfWidMinor \ ,psfTheta \ + ,infoFlag \ ,psfCf \ + ,momentXX \ + ,momentXY \ + ,momentYY \ + ,momentM3C \ + ,momentM3S \ + ,momentM4C \ + ,momentM4S \ + ,momentR1 \ + ,momentRH \ + ,apMag \ + ,apMagErr \ + ,kronFlux \ + ,kronFluxErr \ + ,kronRad \ + ,kronRadErr \ ,nFrames \ + ,assocDate \ + ,historyModNum \ ) \ SELECT \ IPP_IDET \ + ," + self.skycell + " \ + ," + self.header['MJD-OBS'] + " \ ,X_PSF \ ,Y_PSF \ ,X_PSF_SIG \ ,Y_PSF_SIG \ - ,PSF_INST_FLUX \ - ,PSF_INST_FLUX_SIG \ + ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+" \ + ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+")) / 1.085736) \ ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \ ,SKY \ @@ -243,21 +330,41 @@ ,PSF_MINOR \ ,PSF_THETA \ + ,FLAGS << 32 | FLAGS2 \ ,PSF_QF \ + ,MOMENTS_XX \ + ,MOMENTS_XY \ + ,MOMENTS_YY \ + ,MOMENTS_M3C \ + ,MOMENTS_M3S \ + ,MOMENTS_M4C \ + ,MOMENTS_M4S \ + ,MOMENTS_R1 \ + ,MOMENTS_RH \ + ,AP_MAG \ + , NULL \ + ,KRON_FLUX \ + ,KRON_FLUX_ERR \ + , NULL \ + , NULL \ ,N_FRAMES \ + , '" + self.dateStr + "' \ + ," + self.historyModNum + " \ FROM SkyChip_psf" - self.stmt.execute(sql) - - if self.header['STK_TYPE'] != "NIGHTLY_STACK": - sql = "UPDATE StackDetection SET obsTime = " + self.header['MJD-OBS'] - self.stmt.execute(sql) - - - sql = "UPDATE StackDetection SET skycellID = " + self.skycell + ", assocDate = '"+self.dateStr+"'" - self.stmt.execute(sql) - self.updateSurveyID("StackDetection") - self.updateFilterID("StackDetection") + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackDetection", self.filter) + self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackDetection", "primaryF", "0") + self.scratchDb.updateAllRows("StackDetection", "activeFlag", "0") self.updateStackMetaID("StackDetection") self.updateStackTypeID("StackDetection") + self.updateDvoIDs("StackDetection") + + # now delete bad flux + self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux") + self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID") + ''' @@ -265,5 +372,5 @@ ''' def populateStackApFlx(self): - self.log("Procesing StackApFlx table") + self.logger.info("Procesing StackApFlx table") sql = "INSERT INTO StackApFlx \ @@ -272,12 +379,18 @@ DISTINCT IPP_IDET \ FROM SkyChip_xrad" - self.stmt.execute(sql) - - self.log(" Adding 1st convolved fluxes") + + try: + self.scratchDb.stmt.execute(sql) + except: return + + # TODO temporarily loading 1st convolved fluxes into unconvolved fields + self.logger.info(" Adding un-convolved fluxes") + self.updateApFlxs("", "< 7.0") + self.logger.info(" Adding 1st convolved fluxes") self.updateApFlxs("c1", "< 7.0") - self.log(" Adding 2nd convolved fluxes") + self.logger.info(" Adding 2nd convolved fluxes") self.updateApFlxs("c2", "> 7.0") - self.log(" Adding petrosian stuff for extended sources") + self.logger.info(" Adding petrosians for extended sources") sql = "UPDATE StackApFlx AS a, SkyChip_xsrc AS b SET \ petRadius=b.PETRO_RADIUS \ @@ -290,19 +403,14 @@ ,petR90Err=b.PETRO_RADIUS_90_ERR \ WHERE a.ippDetectID=b.IPP_IDET" - self.stmt.execute(sql) - - self.updateSurveyID("StackApFlx") - self.updateFilterID("StackApFlx") + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackApFlx", self.filter) + self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackApFlx", "primaryF", "0") + self.scratchDb.updateAllRows("StackApFlx", "activeFlag", "0") self.updateStackMetaID("StackApFlx") self.updateStackTypeID("StackApFlx") - - #rs = stmt.executeQuery(sql) - # list = [] - - #while (rs.next()): - - # print rs.getString(1) - - #rs.close() + self.updateDvoIDs("StackApFlx") ''' @@ -310,22 +418,86 @@ ''' def populateStackModelFit(self): - self.log("Procesing StackModelFit table") + self.logger.info("Procesing StackModelFit table") # insert all the detections sql = "INSERT INTO StackModelFit (ippDetectID) SELECT DISTINCT IPP_IDET from SkyChip_xfit" - self.stmt.execute(sql) + try: + self.scratchDb.stmt.execute(sql) + except: + return + # populate model parameters - self.log(" Adding deVaucouleurs fit") + self.logger.info(" Adding deVaucouleurs fit") self.updateModelFit("deV", "PS_MODEL_DEV") - self.log(" Adding exponential fit") + self.logger.info(" Adding exponential fit") self.updateModelFit("exp", "PS_MODEL_EXP") - self.log(" Adding sersic fit") + self.logger.info(" Adding sersic fit") self.updateModelFit("ser", "PS_MODEL_SERSIC") - self.updateSurveyID("StackModelFit") - self.updateFilterID("StackModelFit") + self.scratchDb.updateAllRows("StackModelFit", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackModelFit", self.filter) + self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackModelFit", "primaryF", "0") + self.scratchDb.updateAllRows("StackModelFit", "activeFlag", "0") self.updateStackMetaID("StackModelFit") self.updateStackTypeID("StackModelFit") + self.updateDvoIDs("StackModelFit") + + ''' + Populates the StackToImage table + ''' + def populateStackToImage(self): + self.logger.info("Procesing StackToImage table") + + imageIDs = self.gpc1Db.getImageIDsForThisStackID(self.stackID) + + for imageID in imageIDs: + sql = "INSERT INTO StackToImage (stackMetaID, imageID) \ + VALUES (\ + " + str(self.stackID) + ", " + imageID + ")" + self.scratchDb.stmt.execute(sql) + + # now update StackMeta with correct number of inputs + sql = "UPDATE StackMeta SET nP2Images = (SELECT COUNT(*) FROM StackToImage)" + self.scratchDb.stmt.execute(sql) + + ''' + Populates the SkinnyObject table + ''' + def populateSkinnyObject(self): + self.logger.info("Procesing SkinnyObject table") + + sql = "INSERT INTO SkinnyObject (\ + objID \ + ,ippObjID \ + ) \ + SELECT \ + objID \ + ,ippObjID \ + FROM StackDetection" + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID)) + self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.dataRelease)) + + ''' + Populates the ObjectCalColor table + ''' + def populateObjectCalColor(self): + self.logger.info("Procesing ObjectCalColor table") + + sql = "INSERT INTO ObjectCalColor (\ + objID \ + ,ippObjID \ + ) \ + SELECT \ + objID \ + ,ippObjID \ + FROM StackDetection" + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateFilterID("ObjectCalColor", self.filter) + self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.dataRelease)) @@ -333,10 +505,11 @@ Applies indexes to the PSPS tables ''' - def indexPspsTables(self): - - self.log("Creating indexes on PSPS tables") - self.createIndex("StackDetection", "ippDetectID") - self.createIndex("StackApFlx", "ippDetectID") - self.createIndex("StackModelFit", "ippDetectID") + def alterPspsTables(self): + + self.logger.info("Altering PSPS tables") + self.scratchDb.makeColumnUnique("StackDetection", "objID") + self.scratchDb.createIndex("StackDetection", "ippDetectID") + self.scratchDb.createIndex("StackApFlx", "ippDetectID") + self.scratchDb.createIndex("StackModelFit", "ippDetectID") ''' @@ -345,9 +518,27 @@ def indexIppTables(self): - self.log("Creating indexes on IPP tables") - self.createIndex("SkyChip_psf", "IPP_IDET") - self.createIndex("SkyChip_xfit", "IPP_IDET") - self.createIndex("SkyChip_xrad", "IPP_IDET") - self.createIndex("SkyChip_xsrc", "IPP_IDET") + self.logger.info("Creating indexes on IPP tables") + self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xfit", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xrad", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xsrc", "IPP_IDET") + + ''' + Updates provided table with DVO IDs from DVO table + ''' + def updateDvoIDs(self, table): + + imageID = self.scratchDb.getImageIDFromExternID(self.header['SOURCEID'], self.header['IMAGEID']) + + self.logger.info("Updating table '" + table + "' with DVO IDs...") + sql = "UPDATE IGNORE " + table + " AS a, dvoDetectionFull AS b SET \ + a.ippObjID = b.ippObjID, \ + a.stackDetectID = b.detectID, \ + a.objID = b.objID \ + WHERE a.ippDetectID = b.ippDetectID \ + AND b.sourceID = " + self.header['SOURCEID'] + "\ + AND b.imageID = " + str(imageID) + self.scratchDb.stmt.execute(sql) + ''' @@ -356,11 +547,7 @@ def populatePspsTables(self): - # determine skycell from header value - self.skycell = self.header['SKYCELL'] - self.skycell = self.skycell[8:] - - # get filterID using init table - self.filter = self.header['FPA.FILTER'] - self.filter = self.filter[0:1] + if not self.useFullTables: + if not self.getIDsFromDVO(): + return False self.populateStackMeta() @@ -368,9 +555,60 @@ self.populateStackModelFit() self.populateStackApFlx() - - -stackBatch = StackBatch() -stackBatch.createEmptyPspsTables() -stackBatch.importIppTables(".*psf") -stackBatch.populatePspsTables() -stackBatch.exportPspsTablesToFits() + self.populateStackToImage() + self.populateSkinnyObject() + self.populateObjectCalColor() + + self.setMinMaxObjID(["StackDetection"]) + + return True + + ''' + Checks whether this batch has already been processed and published + ''' + def alreadyProcessed(self): + + return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID) + +logging.config.fileConfig("logging.conf") +logger = logging.getLogger("stackbatch") +logger.info("Starting") +gpc1Db = Gpc1Db(logger) +stackType = "NIGHTLY_STACK" +skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky") +#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky") +#stackType = "DEEP_STACK" +#skyIDs = [689] +#skyIDs = [299] +#skyIDs = [302] +#skyIDs = [8508] +i = 0 +for skyID in skyIDs: + + logger.info("-------------------------------------------------- sky ID: %d" % skyID) + + cmfFiles = gpc1Db.getStackStageCmfs(skyID) + + for file in cmfFiles: + + if not os.path.isfile(file): + logger.error("Cannot read file at '" + file) + continue + + stackBatch = StackBatch(logger, skyID, file, stackType, True) + + if not stackBatch.alreadyProcessed(): + + stackBatch.createEmptyPspsTables() + stackBatch.importIppTables("") + if stackBatch.populatePspsTables(): + + #stackBatch.reportNullsInAllPspsTables(False) + stackBatch.exportPspsTablesToFits() + stackBatch.writeBatchManifest() + #stackBatch.createTarball() + #stackBatch.publishToDatastore() + + i = i + 1 + #if i > 0: sys.exit() + +logger.info("Finished") Index: branches/eam_branches/ipp-20110404/ippToPsps/perl/pspsSchema2xml.pl =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/perl/pspsSchema2xml.pl (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/perl/pspsSchema2xml.pl (revision 31439) @@ -307,5 +307,5 @@ # parse line - if ($line =~ m/\s*([a-zA-Z0-9()-\[\]]+)\s+(.*)\s*--\/(.*)/) { + if ($line =~ m/\s*([a-zA-Z0-9()-_\[\]]+)\s+(.*)\s*--\/(.*)/) { $name = $1; @@ -320,5 +320,5 @@ } # no comment case - elsif ($line =~ m/\s*([a-zA-Z0-9()-\[\]]+)\s+(.*)/) { + elsif ($line =~ m/\s*([a-zA-Z0-9()-_\[\]]+)\s+(.*)/) { $name = $1; @@ -394,4 +394,5 @@ "arraysize" => $arraySize, "datatype" => $votType); + $votWriter->dataElement('DESCRIPTION', $comment); $votWriter->endTag(); Index: branches/eam_branches/ipp-20110404/ippToPsps/src/Dvo.c =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/src/Dvo.c (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/src/Dvo.c (revision 31439) @@ -0,0 +1,220 @@ +/** @file Dvo.c + * + * @ingroup ippToPsps + * + * @author IfA + * Copyright 2011 Institute for Astronomy, University of Hawaii + */ + +#include "Dvo.h" + +/** + Destructor. + */ +static void destroy(Dvo* this) { + + if (this == NULL) return; + + this->logger->print(this->logger, MSG_DEBUG, "Dvo", "Destructor\n"); + + if (this->dvoConfig) dvoConfigFree(this->dvoConfig); + if (this->mysql) mysql_close(this->mysql); + + free(this); +} + +/** + Gets detections for this imageid + */ +static bool getDetections(Dvo* this, int32_t sourceId, int32_t imageId) { + + SkyList* skyList = NULL; + Image* image = NULL; + this->logger->print(this->logger, MSG_INFO, "Dvo", "Getting skylist from DVO database...\n"); + skyList = dvoSkyListByExternID(this->dvoConfig, sourceId, imageId, &image); + if (!skyList) { + + this->logger->print(this->logger, MSG_ERROR, + "Dvo", "Could not find skylist for sourceId=%d and image ID = %d\n", + sourceId, imageId); + + return false; + } + this->logger->print(this->logger, MSG_INFO, "Dvo", "...done\n"); + + // now get detections + this->logger->print(this->logger, MSG_INFO, "Dvo", "Getting detections...\n"); + dvoDetection* dvoDetections = NULL; + int32_t maxDetectionId = -1; + int32_t numDetections = + dvoGetDetections(skyList, image->imageID, &dvoDetections, &maxDetectionId); + + this->logger->print(this->logger, MSG_INFO, "Dvo", + "Found %d detections with a max detection ID of %d\n", + numDetections, maxDetectionId); + + if (dvoDetections) dvoFree(dvoDetections); + SkyListFree(skyList); + + this->logger->print(this->logger, MSG_INFO, "Dvo", "Deleting everything from dvo table\n"); + mysql_query(this->mysql, "DELETE FROM dvo"); + + this->logger->print(this->logger, MSG_INFO, "Dvo", "Inserting IDs into dvo table...\n"); + + char sql[500]; + for (int i=0; imysql, sql); + } + this->logger->print(this->logger, MSG_INFO, "Dvo", "...done\n"); + +#if 0 + mysql_query(this->mysql, "SELECT * FROM Filter"); + MYSQL_RES* result = mysql_store_result(this->mysql); + + int num_fields = mysql_num_fields(result); + + MYSQL_ROW row; + + while ((row = mysql_fetch_row(result))) { + + for(int i = 0; i < num_fields; i++) { + + printf("%s ", row[i] ? row[i] : "NULL"); + } + printf("\n"); + } + + mysql_free_result(result); +#endif + return true; +} + +/** + Constructor. + */ +Dvo* new_Dvo(Logger* logger, const char* dvoDatabase) { + + Dvo* this = (Dvo*)calloc(1, sizeof(Dvo)); + this->logger = logger; + this->logger->print(this->logger, MSG_DEBUG, "Dvo", "Constructor\n"); + + const char *configFile = "jython/config.xml"; + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using config file here '%s'\n", configFile); + xmlDoc* doc = xmlReadFile(configFile, NULL, 0); + + // database parameters to be retrieved from config file + char host[20]; + char user[20]; + char passwd[20]; + char dbname[20]; + + + if (doc == NULL) { + + this->logger->print(this->logger, MSG_ERROR, "Dvo", "Could not load config file\n"); + } + else { + + xmlNode* root_element = xmlDocGetRootElement(doc); + + + /* We don't care what the top level element name is */ + xmlNodePtr cur = root_element->xmlChildrenNode; + char name[50]; + unsigned char* value; + while (cur != NULL) { + + // get database stuff + if (strcmp((char*)cur->name, "localdatabase") == 0) { + + xmlNodePtr dbParams = cur->xmlChildrenNode; + + while (dbParams != NULL) { + + sprintf(name, "%s", dbParams->name); + value = xmlNodeListGetString(doc, dbParams->xmlChildrenNode, 1); + if (strcmp(name, "name") == 0) sprintf(dbname, "%s", value); + else if (strcmp(name, "user") == 0) sprintf(user, "%s", value); + else if (strcmp(name, "host") == 0) sprintf(host, "%s", value); + else if (strcmp(name, "password") == 0) sprintf(passwd, "%s", value); + + dbParams = dbParams->next; + } + } + + cur = cur->next; + } + + xmlFreeDoc(doc); + } + + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using DVO database = '%s'\n", dvoDatabase); + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using db name = '%s'\n", dbname); + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using db host = '%s'\n", host); + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using db user = '%s'\n", user); + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using db password = '%s'\n", passwd); + + int argc = 4; + char** argv; + argv = (char**)calloc(argc, sizeof(char *)); + for (int i=0; idvoConfig = dvoConfigRead(&argc, argv); + + for (int i=0; idestroy = destroy; + this->getDetections = getDetections; + + // set up MySQL connection + this->mysql = mysql_init(NULL); + + if (!this->mysql) + this->logger->print(this->logger, MSG_ERROR, "Dvo", "mysql_init(), out of memory\n"); + else { + + if (!mysql_real_connect(this->mysql, host, user, passwd, dbname, 0, NULL, 0)) + this->logger->print(this->logger, MSG_ERROR, "Dvo", "Failed to connect to database: %s\n", + mysql_error(this->mysql)); + + else + this->logger->print(this->logger, MSG_INFO, "Dvo", "Connected to %s@%s as user '%s'\n", + dbname, host, user); + + } + + return this; +} + + +/** + Main + */ +int main(int argc, char **argv) { + + Logger* logger = new_Logger(NULL, true); + + Dvo * dvo = new_Dvo(logger, "/data/ipp004.0/gpc1/catdirs/ThreePi.V1"); + dvo->getDetections(dvo, 33, 8345290); + dvo->destroy(dvo); + + logger->destroy(logger); + + return 0; +} + Index: branches/eam_branches/ipp-20110404/ippToPsps/src/Dvo.h =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/src/Dvo.h (revision 31439) +++ branches/eam_branches/ipp-20110404/ippToPsps/src/Dvo.h (revision 31439) @@ -0,0 +1,47 @@ +/** @file Dvo.h + * + * @ingroup ippToPsps + * + * @author IfA + * Copyright 2011 Institute for Astronomy, University of Hawaii + */ + +#ifndef IPPTOPSPS_DVO_H +#define IPPTOPSPS_DVO_H + +#include +#include + +#include +#include + +#include "dvo_util.h" +#include + +#include "Logger.h" + +/** + + Class encapsulating the DVO database + +*/ +typedef struct Dvo { + + // fields + Logger* logger; + dvoConfig* dvoConfig; + MYSQL* mysql; + + // methods + bool (*getDetections)(); + + // destructor + void (*destroy)(); + +} Dvo; + +// constructor +Dvo* new_Dvo(Logger* logger, const char* dvoDatabase); + +#endif // IPPTOPSPS_DVO_H + Index: branches/eam_branches/ipp-20110404/ippToPsps/src/Makefile.am =================================================================== --- branches/eam_branches/ipp-20110404/ippToPsps/src/Makefile.am (revision 31166) +++ branches/eam_branches/ipp-20110404/ippToPsps/src/Makefile.am (revision 31439) @@ -21,6 +21,13 @@ # all the programs we create -bin_PROGRAMS = initbatch detectionbatch stackbatch +bin_PROGRAMS = dvo initbatch detectionbatch stackbatch + +# DVO +dvo_CFLAGS = $(IPPTOPSPS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -I/usr/include/libxml2 +dvo_LDFLAGS = $(IPPTOPSPS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS) -lxml2 +dvo_SOURCES = \ + Dvo.c \ + Logger.c # initbatch