Index: /tags/ipp-ps2-20190404/doc/release.2015/Makefile.Common =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/Makefile.Common (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/Makefile.Common (revision 40759) @@ -33,5 +33,7 @@ %.tgz: - tar --transform 's%inputs/%%' -zcf $@ $(FILES) $*.bbl +# tar --transform 's%inputs/%%' -zcf $@ $(FILES) $*.bbl + tar --transform 's%inputs/%%' --transform 's%pics/%%' --transform 's%images/%%' -zcf $@ $^ + %.zip: zip -j $@ $? Index: /tags/ipp-ps2-20190404/doc/release.2015/inputs/astro.sty =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/inputs/astro.sty (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/inputs/astro.sty (revision 40759) @@ -97,4 +97,5 @@ % Specific astrophysical symbols +\newcommand\mathdegree{^{\circ}} \newcommand\degree{$^{\circ}$} \newcommand\degrees{$^{\circ}$} Index: /tags/ipp-ps2-20190404/doc/release.2015/inputs/code.sty =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/inputs/code.sty (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/inputs/code.sty (revision 40759) @@ -36,4 +36,5 @@ \def\nocode#1{#1} +% this is a good choice: \def\IPPstageEND#1{\textsc{#1}\endgroup}% \def\IPPstage{\begingroup\setupc@de\IPPstageEND}% @@ -42,11 +43,11 @@ \def\IPPdbtable{\begingroup\setupc@de\IPPdbtableEND}% -\def\IPPdbcolumnEND#1{\textbf{#1}\endgroup}% +\def\IPPdbcolumnEND#1{\textit{\textbf{#1}}\endgroup}% \def\IPPdbcolumn{\begingroup\setupc@de\IPPdbcolumnEND} -\def\IPPprogEND#1{\textit{\textbf{#1}}\endgroup}% +\def\IPPprogEND#1{\texttt{#1}\endgroup}% \def\IPPprog{\begingroup\setupc@de\IPPprogEND} -\def\IPPmiscEND#1{\textsf{#1}\endgroup}% +\def\IPPmiscEND#1{\texttt{#1}\endgroup}% \def\IPPmisc{\begingroup\setupc@de\IPPmiscEND} Index: /tags/ipp-ps2-20190404/doc/release.2015/inputs/lib.bib =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/inputs/lib.bib (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/inputs/lib.bib (revision 40759) @@ -6,4 +6,5 @@ % 2012ApJ...756..158S == ubercal +% paper I @ARTICLE{chambers2017, author = {{Chambers}, K.~C. and {Magnier}, E.~A. and {Metcalfe}, N. and et al.}, @@ -20,4 +21,59 @@ } +% paper II +@ARTICLE{magnier2017.datasystem, + author = {{Magnier}, E.~A. and {Schlafly}, E.~F. and {Finkbeiner}, D.~P. and et al.}, + title = "{IPP}", + journal = {ArXiv e-prints}, +archivePrefix = "arXiv", + eprint ={1612.05242}, + primaryClass = "astro-ph.HE", + keywords = {Astrophysics}, + year = 2017, + month = jan, + adsurl = {http://adsabs.harvard.edu/abs/2016arXiv160203842A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +% paper III +@ARTICLE{waters2017, + author = {{Waters}, C.~Z. and {Magnier}, E.~A. and {Price}, P.~A. and + {Chambers}, K.~C. and {Draper}, P. and {Flewelling}, H.~A. and + {Hodapp}, K.~W. and {Huber}, M.~E. and {Jedicke}, R. and {Kaiser}, N. and + {Kudritzki}, R.-P. and {Lupton}, R.~H. and {Metcalfe}, N. and + {Rest}, A. and {Sweeney}, W.~E. and {Tonry}, J.~L. and {Wainscoat}, R.~J. and + {Wood-Vasey}, W.~M. and {Builders}, P.}, + title = "{Pan-Starrs Pixel Processing: Detrending, Warping, Stacking}", + journal = {ArXiv e-prints}, +archivePrefix = "arXiv", + eprint = {1612.05245}, + primaryClass = "astro-ph.IM", + keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, + year = 2016, + month = dec, + adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205245W}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +% paper IV +@ARTICLE{magnier2017.analysis, + author = {{Magnier}, E.~A. and {Sweeney}, W.~E. and {Chambers}, K.~C. and + {Flewelling}, H.~A. and {Huber}, M.~E. and {Price}, P.~A. and + {Waters}, C.~Z. and {Denneau}, L. and {Draper}, P. and {Jedicke}, R. and + {Hodapp}, K.~W. and {Kudritzki}, R.-P. and {Metcalfe}, N. and + {Stubbs}, C.~W. and {Wainscoast}, R.~J.}, + title = "{Pan-STARRS Pixel Analysis : Source Detection $\backslash${\amp} Characterization}", + journal = {ArXiv e-prints}, +archivePrefix = "arXiv", + eprint = {1612.05244}, + primaryClass = "astro-ph.IM", + keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, + year = 2016, + month = dec, + adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205244M}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +% paper V @ARTICLE{magnier2017.calibration, author = {{Magnier}, E.~A. and {Schlafly}, E.~F. and {Finkbeiner}, D.~P. and @@ -40,55 +96,5 @@ } -@ARTICLE{waters2017, - author = {{Waters}, C.~Z. and {Magnier}, E.~A. and {Price}, P.~A. and - {Chambers}, K.~C. and {Draper}, P. and {Flewelling}, H.~A. and - {Hodapp}, K.~W. and {Huber}, M.~E. and {Jedicke}, R. and {Kaiser}, N. and - {Kudritzki}, R.-P. and {Lupton}, R.~H. and {Metcalfe}, N. and - {Rest}, A. and {Sweeney}, W.~E. and {Tonry}, J.~L. and {Wainscoat}, R.~J. and - {Wood-Vasey}, W.~M. and {Builders}, P.}, - title = "{Pan-Starrs Pixel Processing: Detrending, Warping, Stacking}", - journal = {ArXiv e-prints}, -archivePrefix = "arXiv", - eprint = {1612.05245}, - primaryClass = "astro-ph.IM", - keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, - year = 2016, - month = dec, - adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205245W}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} -} - -@ARTICLE{magnier2017.analysis, - author = {{Magnier}, E.~A. and {Sweeney}, W.~E. and {Chambers}, K.~C. and - {Flewelling}, H.~A. and {Huber}, M.~E. and {Price}, P.~A. and - {Waters}, C.~Z. and {Denneau}, L. and {Draper}, P. and {Jedicke}, R. and - {Hodapp}, K.~W. and {Kudritzki}, R.-P. and {Metcalfe}, N. and - {Stubbs}, C.~W. and {Wainscoast}, R.~J.}, - title = "{Pan-STARRS Pixel Analysis : Source Detection $\backslash${\amp} Characterization}", - journal = {ArXiv e-prints}, -archivePrefix = "arXiv", - eprint = {1612.05244}, - primaryClass = "astro-ph.IM", - keywords = {Astrophysics - Instrumentation and Methods for Astrophysics}, - year = 2016, - month = dec, - adsurl = {http://adsabs.harvard.edu/abs/2016arXiv161205244M}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} -} - -@ARTICLE{magnier2017.datasystem, - author = {{Magnier}, E.~A. and {Schlafly}, E.~F. and {Finkbeiner}, D.~P. and et al.}, - title = "{IPP}", - journal = {ArXiv e-prints}, -archivePrefix = "arXiv", - eprint ={1612.05242}, - primaryClass = "astro-ph.HE", - keywords = {Astrophysics}, - year = 2017, - month = jan, - adsurl = {http://adsabs.harvard.edu/abs/2016arXiv160203842A}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} -} - +% paper VI @ARTICLE{flewelling2017, author = {{Flewelling}, H.~A. and {Magnier}, E.~A. and {Chambers}, K.~C. and @@ -3080,5 +3086,5 @@ -@ARTICLE{2009A&A...494..707K, +@ARTICLE{2009AA...494..707K, author = {{Koppenhoefer}, J. and {Afonso}, C. and {Saglia}, R.~P. and {Henning}, T.}, @@ -3112,5 +3118,5 @@ } -@ARTICLE{2016A&A...587A..49O, +@ARTICLE{2016AA...587A..49O, author = {{Obermeier}, C. and {Koppenhoefer}, J. and {Saglia}, R.~P. and {Henning}, T. and {Bender}, R. and {Kodric}, M. and {Deacon}, N. and @@ -4805,5 +4811,23 @@ } - +@ARTICLE{2018AA...616A...1G, + author = {{Gaia Collaboration} and {Brown}, A.~G.~A. and {Vallenari}, A. and + {Prusti}, T. and {de Bruijne}, J.~H.~J. and {Babusiaux}, C. and + {Bailer-Jones}, C.~A.~L. and {Biermann}, M. and {Evans}, D.~W. and + {Eyer}, L. and et al.}, + title = "{Gaia Data Release 2. Summary of the contents and survey properties}", + journal = {\aap}, +archivePrefix = "arXiv", + eprint = {1804.09365}, + keywords = {catalogs, astrometry, techniques: radial velocities, stars: fundamental parameters, stars: variables: general, minor planets, asteroids: general}, + year = 2018, + month = aug, + volume = 616, + eid = {A1}, + pages = {A1}, + doi = {10.1051/0004-6361/201833051}, + adsurl = {http://adsabs.harvard.edu/abs/2018A%26A...616A...1G}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} @ARTICLE{2007A&A...473..603M, @@ -16485,2 +16509,103 @@ adsnote = {Provided by the SAO/NASA Astrophysics Data System} } + +@ARTICLE{2015arXiv150903700K, + author = {{Kovesi}, Peter}, + title = "{Good Colour Maps: How to Design Them}", + journal = {arXiv e-prints}, + keywords = {Computer Science - Graphics, I.3.3}, + year = "2015", + month = "Sep", + eid = {arXiv:1509.03700}, + pages = {arXiv:1509.03700}, +archivePrefix = {arXiv}, + eprint = {1509.03700}, + primaryClass = {cs.GR}, + adsurl = {https://ui.adsabs.harvard.edu/\#abs/2015arXiv150903700K}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@ARTICLE{2017MNRAS.468.3499D, + author = {{Deacon}, N.~R. and {Magnier}, E.~A. and {Best}, W.~M.~J. and + {Liu}, M.~C. and {Dupuy}, T.~J. and {Chambers}, K.~C. and {Draper}, P.~W. and + {Flewelling}, H. and {Metcalfe}, N. and {Tonry}, J.~L. and {Wainscoat}, R.~J. and + {Waters}, C.}, + title = "{Identification of partially resolved binaries in Pan-STARRS 1 data}", + journal = {\mnras}, +archivePrefix = "arXiv", + eprint = {1702.05491}, + primaryClass = "astro-ph.SR", + keywords = {binaries: visual, brown dwarfs}, + year = 2017, + month = jul, + volume = 468, + pages = {3499-3515}, + doi = {10.1093/mnras/stx440}, + adsurl = {http://adsabs.harvard.edu/abs/2017MNRAS.468.3499D}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@ARTICLE{1998ApJ...504..636H, + author = {{Hoekstra}, H. and {Franx}, M. and {Kuijken}, K. and {Squires}, G. + }, + title = "{Weak Lensing Analysis of CL 1358+62 Using Hubble Space Telescope Observations}", + journal = {\apj}, + keywords = {GALAXIES: CLUSTERS: INDIVIDUAL ALPHANUMERIC: CL 1358+62, GALAXIES: FUNDAMENTAL PARAMETERS, COSMOLOGY: GRAVITATIONAL LENSING, galaxies: clusters: individual (Cl 1358 + 62), Galaxies: Fundamental Parameters, Cosmology: Gravitational Lensing}, + year = 1998, + month = sep, + volume = 504, + pages = {636-660}, + doi = {10.1086/306102}, + adsurl = {http://adsabs.harvard.edu/abs/1998ApJ...504..636H}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} +@ARTICLE{2005ApJ...626.1070H, + author = {{Hoekstra}, H. and {Wu}, Y. and {Udalski}, A.}, + title = "{An Algorithm to Detect Blends with Eclipsing Binaries in Planet Transit Searches}", + journal = {\apj}, + eprint = {astro-ph/0501353}, + keywords = {Stars: Binaries: Eclipsing, Stars: Planetary Systems}, + year = 2005, + month = jun, + volume = 626, + pages = {1070-1078}, + doi = {10.1086/430299}, + adsurl = {http://adsabs.harvard.edu/abs/2005ApJ...626.1070H}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} +@ARTICLE{2013ApJS..206...18T, + author = {{Terziev}, E. and {Law}, N.~M. and {Arcavi}, I. and {Baranec}, C. and + {Bloom}, J.~S. and {Bui}, K. and {Burse}, M.~P. and {Chorida}, P. and + {Das}, H.~K. and {Dekany}, R.~G. and {Kraus}, A.~L. and {Kulkarni}, S.~R. and + {Nugent}, P. and {Ofek}, E.~O. and {Punnadi}, S. and {Ramaprakash}, A.~N. and + {Riddle}, R. and {Sullivan}, M. and {Tendulkar}, S.~P.}, + title = "{Millions of Multiples: Detecting and Characterizing Close-separation Binary Systems in Synoptic Sky Surveys}", + journal = {\apjs}, +archivePrefix = "arXiv", + eprint = {1210.4550}, + primaryClass = "astro-ph.SR", + keywords = {binaries: close, methods: data analysis, stars: statistics, surveys, techniques: image processing }, + year = 2013, + month = jun, + volume = 206, + eid = {18}, + pages = {18}, + doi = {10.1088/0067-0049/206/2/18}, + adsurl = {http://adsabs.harvard.edu/abs/2013ApJS..206...18T}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@INPROCEEDINGS{2007ASPC..376..269S, + author = {{Swaters}, R.~A. and {Valdes}, F.~G.}, + title = "{The NOAO High-Performance Pipeline System: The Mosaic Camera Pipeline}", +booktitle = {Astronomical Data Analysis Software and Systems XVI}, + year = 2007, + series = {Astronomical Society of the Pacific Conference Series}, + volume = 376, + editor = {{Shaw}, R.~A. and {Hill}, F. and {Bell}, D.~J.}, + month = oct, + pages = {269}, + adsurl = {http://adsabs.harvard.edu/abs/2007ASPC..376..269S}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/Makefile =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/Makefile (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/Makefile (revision 40759) @@ -14,13 +14,35 @@ all: pdf tgz pdf: analysis.pdf -tgz: analysis.tgz + +journal: analysis.journal.tgz +arxiv: analysis.arxiv.tgz quick: analysis.quick.pdf +BIBLIB = ../inputs/lib.bib + PDFPICS = \ pics/peaks.pdf \ +pics/FWHM.smooth.trend.ps1.pdf \ +pics/radial.profiles.pdf \ +pics/moment.class.pdf \ +pics/mag.resid.psf.pdf \ +pics/mag.resid.aper.pdf \ +pics/galaxy.exp.complete.pdf \ +pics/galaxy.dev.complete.pdf \ +pics/galaxy.exp.params.pdf \ +pics/galaxy.dev.params.pdf + +PNGPICS = \ +pics/peaks.pdf \ pics/FWHM.smooth.trend.ps1.png \ -pics/moment.class.pdf \ -pics/radial.profiles.pdf +pics/radial.profiles.png \ +pics/moment.class.png \ +pics/mag.resid.psf.png \ +pics/mag.resid.aper.png \ +pics/galaxy.exp.complete.png \ +pics/galaxy.dev.complete.png \ +pics/galaxy.exp.params.png \ +pics/galaxy.dev.params.png FILES = \ @@ -28,6 +50,4 @@ ../inputs/code.sty \ ../inputs/apj.bst \ -../inputs/lib.bib \ -$(PDFPICS) \ analysis.tex @@ -40,6 +60,42 @@ pdfpics: $(PDFPICS) -analysis.pdf: $(FILES) -analysis.tgz: $(FILES) + +analysis.pdf: $(FILES) $(BIBLIB) $(PDFPICS) + +analysis.journal.tgz: $(FILES) $(PDFPICS) analysis.bbl +analysis.arxiv.tgz: $(FILES) $(PNGPICS) analysis.bbl include ../Makefile.Common + +# generate PDF for editing +# ** set DO_BIBTEX to 1 above +# ** set \picdir to pics in latex file +# ** set \plotext to pdf or png as desired +# ** make pdf (or make quick) + +# generate PDF for arxiv +# ** set DO_BIBTEX to 0 above (make sure bbl file was generated earlier) +# ** swap from biblograph{lib} to input bbl in latex file +# ** set \plotext to png +# ** make pdf (confirm PDF file succeeds) + +# generate TGZ for arxiv +# ** follow steps above for PDF +# ** set \picdir to . in latex file +# ** make arxiv +# ** test by extracting into local directory +# ** ltx -pdf analysis + +# generate PDF for journal +# ** set DO_BIBTEX to 0 above (make sure bbl file was generated earlier) +# ** swap from biblograph{lib} to input bbl in latex file +# ** set \plotext to pdf +# ** make pdf (confirm PDF file succeeds) + +# generate TGZ for journal +# ** follow steps above for PDF +# ** set \picdir to . in latex file +# ** make journal +# ** test by extracting into local directory +# ** ltx -pdf analysis + Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/analysis.tex =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/analysis.tex (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/analysis.tex (revision 40759) @@ -29,5 +29,6 @@ %\def\picdir{/home/eugene/chipresid.20140404} -\def\picdir{pics} +%\def\picdir{pics} +\def\picdir{.} % Pick a terse version of the title here; @@ -58,5 +59,5 @@ % PS1 Builders L. Denneau,\altaffilmark{\IfA} -P. Draper,\altaffilmark{\DUR} +P.~W. Draper,\altaffilmark{\DUR} R. Jedicke,\altaffilmark{\IfA} K. W. Hodapp,\altaffilmark{\IfA} @@ -102,5 +103,5 @@ % insert additional keywords as appropriate: -\keywords{Surveys:\PSONE } +\keywords{methods: data analysis -- Surveys:\PSONE -- techniques: image processing -- techniques: photometric} \section{Introduction} @@ -171,5 +172,5 @@ source detection and photometry, including point-spread-function and extended source model fitting, and the techniques for ``forced'' -photometry measurements. The software describe here was used with a +photometry measurements. The software described here was used with a single consistent set of parameters for the complete PV3 analysis, used for both DR1 and DR2. @@ -331,5 +332,5 @@ Another variant of \ippprog{psphot} used in the PV3 analysis is called -\ippprog{psphotFullForce}. In this variant, a set of image all representing the +\ippprog{psphotFullForce}. In this variant, a set of images all representing the same pixels are processed together, with the positions of sources to be analyzed loaded from a supplied file. In this variant of the @@ -356,5 +357,5 @@ per image is combined with an error in the flat-field calibration and an error in measuring the atmospheric effects. We have set a goal for -\ippprog{psphot} of 3mmag photometric consistency for bright stars +\ippprog{psphot} of 3 mmag photometric consistency for bright stars between pairs of images obtained in photometric conditions at the same pointing, ie to remove sensitivity to flat-field errors. This goal @@ -366,9 +367,9 @@ individual measurements. The measurements from \ippprog{psphot} must be sufficiently representative of the true source position to enable -astrometric calibration at the 10mas level. The error in the +astrometric calibration at the 10 mas level. The error in the individual measurements will be folded together with the errors introduced by the optical system, the effects of seeing, and by the available reference catalogs. We have set a goal for \ippprog{psphot} -of 5mas consistency between the true source postion and the measured +of 5 mas consistency between the true source postion and the measured position given reasonable PSF variations under simulations. This level must be reached for images with 250 mas pixels, implying @@ -378,5 +379,5 @@ pixel relative to the size of a chip (since a single data value is used for X or Y). For the $4800^2$ GPC chips, this yields a limit of -about 0.25 milliarcsecond. +about 0.25 mas. % \subsection{Software System Goals} @@ -417,10 +418,10 @@ \end{itemize} -\section{\nocode{psphot} Analysis Process} +\section{Basic Analysis} \subsection{Overview} -The \ippprog{psphot} analysis is divided into several major stages, as -listed below. +The basic \ippprog{psphot} analysis is divided into several major +stages, as listed below. \begin{enumerate} @@ -441,7 +442,4 @@ properties (aperture or PSF) -\item {\bf Extended Source Analysis} Detailed measurements relevant to - galaxies and/or other extended (non-PSF) sources. - \item {\bf Aperture corrections} Measure the curve-of-growth, spatial aperture variations, and background-error corrections. @@ -450,4 +448,12 @@ difference image, variance image, etc, as selected. \end{enumerate} + +In addition to this basic sequence, additional analysis steps may be +performed. An ``extended source'' analysis mode is available to +measure photometry and morphology of galaxies and other resolved +sources. Forced photometry may be performed for both point-like and +extended sources. A special mode is available for the photometry of +sources detected in difference images. These different modes are +discussed in their own sections below. Table~\ref{tab:measurements} lists the types of @@ -495,5 +501,5 @@ Forced PSF Fluxes & N & N & Y & N & \ref{sec:psf.forced.fit} & All \\ Forced Galaxy Models & N & N & Y & N & \ref{sec:galaxy.forced.fit} & Have Stack Galaxy Models \\ - Lensing Parameters & N & Y & Y & N & & All \\ + Lensing Parameters & N & Y & Y & N & \ref{sec:lensing.params} & All \\ \hline \multicolumn{5}{l}{$^1$ Background subtraction is performed by {\tt ppSub} before calling {\tt psphot}} \\ @@ -508,5 +514,5 @@ conditions which are identified by the analysis software. As part of the output data for each detected source, two fields are provided -which encode these conditions as bit values in the two 32-bin +which encode these conditions as bit values in the two 32-bit integers. The following two tables list the individual bit values in these two fields. These informational and warning bits are described @@ -778,7 +784,5 @@ \code{SKY} and \code{SKY_SIGMA} are calculated for each source in the output catalog. For more details of the background subtraction, see -the discussion in Section~2.7 of \cite{waters2017}. - -% \note{give examples with simulations and show examples of over-subtraction} +the discussion in Section~3.11 of \cite{waters2017}. \subsection{Initial Source Detection} @@ -823,7 +827,7 @@ \[ \chi^2 = \sum_{i,j} (F_{i,j} - f(x,y))^2 / \sigma_{i,j}^2 \] -By approximating the error per pixel as the error on just the peak, +By approximating the error per pixel as the Poisson error on just the peak, and pulling that term out of the above equation, and recognizing that -the values x,y in the 3x3 grid centered on the peak pixel have values +the values $X,Y$ in the $3 \times 3$ grid centered on the peak pixel have values of only 0 or 1, we can greatly simplify the chi-square equation to a square matrix equation with the following values: @@ -856,5 +860,5 @@ \] -Inverting the 3x3 matrix terms for $C_{00}$, $C_{20}$, and $C_{02}$, +Inverting the $3 \times 3$ matrix terms for $C_{00}$, $C_{20}$, and $C_{02}$, the location of the peak is determined from the minimum of the bi-quadratic function above, and is given by: @@ -870,7 +874,8 @@ are calculated as discussed below. +% uses plots.sh in this directory \begin{figure}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{pics/peaks} + \includegraphics[width=\hsize,clip]{\picdir/peaks.pdf} \caption{\label{fig:peaks} Illustration of peak finding and culling peaks within a footprint. Insignificant peaks within the footprint of a brighter @@ -929,8 +934,8 @@ \label{sec:moments} +% /data/kukui.3/eugene/psphot.20161214/mana.sh \begin{figure}[htbp] \begin{center} -% \includegraphics[width=0.6\hsize]{{pics/FWHM.smooth.trend.ps1}.\plotext} - \includegraphics[width=0.95\hsize]{{pics/FWHM.smooth.trend.ps1}.png} + \includegraphics[width=0.95\hsize]{{\picdir/FWHM.smooth.trend.ps1}.\plotext} \caption{\label{fig:moments.window} Example of the biases encountered when measuring the second moments. A simulated image @@ -978,11 +983,15 @@ simulated data. An image was generated with a PSF model matching the radial profile of the PS1 PSF model with $\sigma_{\rm PSF}$ -corresponding to a FWHM of 1.4 arcseconds. As the window function -$\sigma_w$ is increased, the measured FWHM for the bright simulated -stars rises to meet the truth value. For small values of $\sigma_w$, -fainter stars are biased to low measured values of the FWHM. For -large values of $\sigma_w$, the faint stars are biased to higher -values and the scatter increases. We attempt to minimize the scatter -and trends with magnitude at the cost of overall bias. +corresponding to a FWHM of 1.4 arcseconds. For bright stars, as the +window function $\sigma_w$ is increased, the measured FWHM rises from +an initially under-estimated value to meet the truth value. For faint +stars, the measured value of the FWHM is initially under-estimated as +well. However, as the value of $\sigma_w$ increases, the measured +FWHM for faint stars rises, and then over-shoots the truth value, +while the scatter increases. Thus, for large values of $\sigma_w$, +the result is both a poorly estimated FWHM for the image and a trend +this the signal-to-noise of the star. We attempt to minimize the +scatter and trends with instrumental magnitude at the cost of overall +bias. In a real image, we do not know the true value of the PSF size. If we @@ -1212,7 +1221,8 @@ % buonanno : 1983A&AS...51...83B +% /data/kukui.3/eugene/psphot.20161214/mana.sh \begin{figure}[htbp] \begin{center} - \includegraphics[width=\hsize]{{pics/radial.profiles}.\plotext} + \includegraphics[width=\hsize]{{\picdir/radial.profiles}.\plotext} \caption{\label{fig:radial.profiles} Radial profiles of stellar images from PS1. These two profiles illustrate the radial trend of the PS1 PSFs for a star @@ -1310,7 +1320,8 @@ \code{PM_SOURCE_MODE_SATURATED}. +% /data/kukui.3/eugene/psphot.20161214/mana.sh \begin{figure}[htbp] \begin{center} - \includegraphics[width=\hsize]{{pics/moment.class}.\plotext} + \includegraphics[width=\hsize]{{\picdir/moment.class}.\plotext} \caption{\label{fig:moment.class} Illustration of PSF star selection using the second moments in $X_{\rm ccd}$ and $Y_{\rm ccd}$ @@ -1329,5 +1340,4 @@ % Madsen: %% http://www2.imm.dtu.dk/pubdb/views/edoc_download.php/3215/pdf/imm3215.pdf -% Press All candidate PSF sources are then fitted with the selected source @@ -1569,8 +1579,8 @@ We attempt to measure the radial profile of sources in order to find -the radius at which the flux of the source is matches the sky. In -this analysis, a series of up to 25 radial bins with power-law spacing -are defined and the flux of the source in each annulus is measured. -The ``sky radius'' is defined to be the radius at which the (robust +the radius at which the flux of the source matches the sky. In this +analysis, a series of up to 25 radial bins with power-law spacing are +defined and the flux of the source in each annulus is measured. The +``sky radius'' is defined to be the radius at which the (robust median) flux in the annulus is within 1 $\sigma$ of the local sky level. If this limit is not reached before the slope of the flux from @@ -1790,5 +1800,5 @@ flags the object with the bad bit \code{PM_SOURCE_MODE_FAIL}. It is also possible in this type of case for the fit to result in a very low -or negative value for the flux normalization parameter. Source for +or negative value for the flux normalization parameter. Sources for which the peak is less than 0.02 are also marked as failing the non-linear PSF fit (\code{PM_SOURCE_MODE_FAIL}). @@ -1803,5 +1813,5 @@ the flag bit (\code{PM_SOURCE_MODE_POOR}). -Sources which are pass the above tests are marked as having a valid +Sources which pass the above tests are marked as having a valid non-linear PSF model fit with the flag bit \code{PM_SOURCE_MODE_NONLINEAR_FIT}. Among these sources, those for @@ -1938,5 +1948,224 @@ \code{PM_SOURCE_MODE2_MATCHED} set. -\subsection{Extended Source Analysis} +\subsection{Aperture Correction and Total Aperture Fluxes} +\label{sec:aperture.correction} + +A PSF model will always fail to describe the flux of the stellar +sources at some level. For high-precision photometry, we need to be +able to correct for the difference between the PSF model fluxes and +the total flux of the sources. In the end, all astronomical +photometry is in some sense a relative measurement between two images. +Whether the goal is calibration of a science image taken at one +location to a standard star image at another location, or the goal is +simply the repetitive photometry of the same star at the same location +in the image, it is always necessary to compare the photometry between +two images. If this measurement is to be consistent, then the +measurement must represent the flux of the stars in the same way +regardless of the conditions under which the images were taken, at +least within some range of normal image conditions. So, for example, +two images with different image quality, or with different tracking +and focus errors, will have different PSF models. To the extent the +PSF model is inaccurate, the measured flux of the same source in the +two images will be different (even assuming all other atmospheric and +instrumental effects have been corrected). The amplitude of the error +will by determined by how inconsistently the models represent the +actual source flux. + +Aperture photometry attempts to avoid these problems, but introduces +other difficulties. In aperture photometry, if a large enough +aperture is chosen, the amount of flux which is lost will be a small +fraction of the total source flux. Even more importantly, as the +image conditions change, the amount lost will change by an even +smaller fraction, at least for a large aperture. This can be seen by +the fact that the dominant variations in the image quality are in the +focus, tracking and seeing. All of these errors initially affect the +cores of the stellar images, rather than the wide wings. The wide +wings are largely dominated by scattering in the optics and scattering +in the atmosphere. The amplitude and distribution of these two +scattering functions do not change significantly or quickly for a +single telescope and site. Aperture photometry can then be used to +correct the PSF photometry. + +The difficulty for aperture photometry is the need to make an accurate +measurement of the local background for each source. As the aperture +grows, errors in the measurement of the sky flux start to become +dominant. If the aperture is too small, then variations in the image +quality are dominant. The brighter is the source, the smaller is the +error introduced by the large size of the aperture. However, the +number of very bright stars is limited in any image, and of course the +brighter stars are more likely to suffer from non-linearity or +saturation. + +% /data/kukui.1/eugene/psphot.examples.20190423/compare.sh +\begin{figure*}[htbp] + \begin{center} + \includegraphics[width=\hsize,clip]{\picdir/{mag.resid.psf}.\plotext} + \caption{\label{fig:mag.resid.psf} PSF Photometry demonstration. + The bottom panel shows the difference of the measured PSF + photometry for stars in the first image of the STS sequence + compared to the next 17 images, after correction for a relative + zero point. Black dots are from stars for which both measurements + have {\tt PSF\_QF} $> 0.95$, while grey dots have lower {\tt + PSF\_QF} values. The top three panels show histograms in three + instrumental magnitude ranges for the magnitude difference divided + by the reported measurement error: $N\sigma = (m_0 - m_1) / + \sqrt{\sigma_0^2 + \sigma_1^2}$. The red curves are Gaussian fits + to these histograms, with the measured standard deviations in the + upper-right corners of the plots. The instrumental magnitude + ranges are listed in the upper-left corners of the three plots and + the boundaries are marked as vertical red lines in the lower plot. + } + \end{center} +\end{figure*} + +% /data/kukui.1/eugene/psphot.examples.20190423/compare.sh +\begin{figure*}[htbp] + \begin{center} + \includegraphics[width=\hsize,clip]{\picdir/{mag.resid.aper}.\plotext} + \caption{\label{fig:mag.resid.aper} Aperture Photometry + demonstration. The plots show identical measurements to those in + Figure~\ref{fig:mag.resid.psf}, but for aperture photometry, as discussed in + Section~\ref{sec:aperture.correction}, rather than PSF photometry.} + \end{center} +\end{figure*} + +In order to thread the needle between these effects, \ippprog{psphot} +measures the aperture photometry on a modest-sized aperture, and then +uses the PSF model to extrapolate to a large aperture. When the PSF +fluxes are calculated, the aperture flux for the modest-sized aperture +is also determined. The aperture is a circular aperture with radius +set to a fixed multiple (\code{PSF_APERTURE_SCALE}) of $\sigma_w$, the +width of the Gaussian window function determined based on the analysis +of the second moments (see Section~\ref{sec:moments}). For the PV3 +$3\pi$ analysis, the aperture window radius is $4.5 \times \sigma_w$, +while the large reference aperture radius is set to 25 pixels +(\code{PSF_REF_RADIUS} = 6\farcs4). These corrected aperture +magnitudes are saved in the output catalogs as \code{AP_MAG}, the +uncorrected aperture magnitudes are saved as \code{AP_MAG_RAW}, and +the radius used to measure the raw aperture flux is saved as +\code{AP_MAG_RADIUS}. The corresponding flux and the flux error are +saved as \code{AP_FLUX} and \code{AP_FLUX_SIG}. + +With these aperture magnitudes in hand, it is now possible to make an +average correction to the PSF magnitudes to bring the PSF and aperture +magnitudes to the same system. This correction is measured using the +same stars from which the PSF model is measured, as long as the PSF +magnitude error for the star is less than 0.03 mag. The correction is +calculated using the weighted average of the values $m_{\rm AP} - +m_{\rm PSF}$. Since the PSF may vary across the image, the correction +is determined as a function of position in the image. Like the PSF +model, the 2D variations of the aperture correction may be modeled as +a polynomial or via interpolation in a grid. For the $3\pi$ PV3 +analysis, a grid with a maximum of $6\times 6$ samples per GPC1 chip +image was used. The reported PSF magnitudes for all objects have this +aperture correction applied. + +% growth curve analysis in psphot: +% in psphotChoosePSF : call psphotMakeGrowthCurve +% in psphotMakeGrowthCurve : boolean GROWTH_FROM_SOURCES, use +%% pmGrowthCurveGenerateFromSources or +%% pmGrowthCurveGenerate (uses PSF model only) +%% GROWTH_FROM_SOURCES is set to TRUE for default recipe + +%% ApTrend: +%% in psphotApResid, called by psphotReadout near the end of the +%% analysis +%% ApTrend = f(x,y) for (apMag - psfMag) for psfMagErr <= 0.03 +%% apMag is growth curve corrected +%% psfMag is raw + +%% raw psfMag and raw apMag are measured +%% apMag = apMagRaw + growth curve correction (from apRadius to 25 pix +%% = PSF_REF_RADIUS) +%% psfMag = psfMagRaw + aperture trend ( + growth curve) + +% How important is this effect? Consider a typical bright source with a +% flux of (say) 40,000 counts in an image of background 1000 counts per +% pixel, with FWHM of 4 pixels. In principle, the flux of this source +% should be measurable with an accuracy of roughly 0.57\% +% ($\frac{\sqrt{40000 + 1000 \times 12}}{40000}$). However, the +% measurement of the sky is limited at some finite level by Poisson +% statistics. If we are required to use an aperture of (say) 25 pixels +% in radius (eg, 5 arcseconds for an 0.2 arcsec / pixel detector), and +% we have an annulus of twice this radius to measure the local sky, then +% we will have an error of XXX. +% +% \note{outline the variation of {\em ApResid} as a function of +% magnitude}. + +%%% \ippprog{psphot} measures the aperture correction ({\em ApResid}) for every PSF +%%% candidate source, then calculates the trend of this correction as a +%%% function of the magnitude. This trend is fitted with a line. The +%%% resulting function can be used to determine the effective aperture +%%% correction for an infinite flux source and the average bias inherent +%%% in the sky measurement for the image. The scatter of the +%%% PSF-candidate source measurements about this trend is a measure of how +%%% well we can measure photometry from the image by applying the specific +%%% PSF model. The slope of this trend is a measure of the bias in the +%%% local sky measurment for each source. In principal, the measured sky +%%% levels could be modified by this bias. More generally, the measured +%%% bias in a collection of images could be used to improve the model +%%% fitting or sky fitting portion of the software the remove the bias +%%% term. + +\ippprog{psphot} allows a collection of PSF model functions to be tried on all +PSF candidate sources. For each model test, the above corrected +ApResid scatter is measured. The PSF model function with the smallest +value for the ApResid scatter is then used by \ippprog{psphot} as the best PSF +model for this image. The number of models to be tested is specified +by the configuration keyword \code{PSF_MODEL_N}. The configuration +variables \code{PSF_MODEL_0}, \code{PSF_MODEL_1}, through +\code{PSF_MODEL_N - 1} specify the names of the models which should be +tested. + +\subsection{Stellar Photometry Example} + +To illustrate the quality of the stellar photometry as measured with +PSF and aperture magnitudes, we show the results of an analysis of a +set of 18 images obtained by PS1 19 February 2010. These images were +obtained for the stellar transit survey ``Pan-Planets'' +\citep{2016AA...587A..49O} and thus target a relatively dense +Galactic plane field. The observations were obtained with +approximately consistent pointing, reducing our sensitivity to +small-scale variations in the flat-field structures. + +Figures~\ref{fig:mag.resid.psf} and ~\ref{fig:mag.resid.aper} show +comparisons of the PSF and aperture photometry measured for these 18 +images. In these figures, the photometry has been measured using the +configuration for \ippprog{psphot} as used for the full PV3 +\ippstage{chip} analysis. The first image of the sequence is compared +to the remaining 17 images. A relative zero point correction is +applied, measured as the median of the photometry difference for stars +with signal-to-noise greater than 50. The combined error is reported +and used to generate the histograms shows in the figures. From these +two figures, one can observe the trade-off between PSF and aperture +photometry. For the brightest instrumental magnitudes, corresponding +to signal-to-noise ratios greater than roughly 300, aperture +magnitudes provide a more consistent measurement of the stellar +fluxes, while the PSF magnitudes are more reliable for fainter +sources. Catastophic failures or extreme outliers are also reduced +for the PSF photometry. + +We largely attribute the behavior on the bright end to systematic +errors in the photometry due to our inability to perfectly represent +the shape of the PSF. The PSF of stars at the bright end will depend +on the brightness because of the ``Brighter-Fatter'' effect +\citep[][]{2014JInst...9C3048A,2015JInst..10C5032G}, in which the +charge already present in the pixels will force the newly arriving +photoeletrons to be systematically pushed away from the accumulating +stellar image, but we do not include a brightness term in our PSF +model. Detector or electronic non-linearity may also affect the PSF +shape and thus the PSF photometry, though non-linearity will affect +the reported photometry for both PSF and aperture magnitudes. + +We believe the observed behavior at the faint end is primarily a +result of the increased crowding. Aperture photometry is more +adversely affected by close neighbors than PSF photometry. Compared +to the formal errors, the faint PSF photometry is the most reliable, +with the aperture photometry degrading rapidly as the flux of the star +decreases. + +\section{Extended Source Analysis} +\label{sec:extended.source} After the initial, fast analysis of the image relying primarily on the @@ -1973,10 +2202,10 @@ depend on the filter as follows: (\grizy) = (21.5, 21.5, 21.5, 20.5, 19.5). These values were chosen to have roughly similar -signal-to-noise in a typical stack image for neutral color objects. -The magnitude limits for the Petrosian parameters were set to 25.0 for -all filters, far below the detection limits and effectively not -limiting the analysis based on apparent magnitude. For both galaxy -model fits and Petrosian parameters, the Galactic latitude cut was -defined by $|b| > b_{\rm min}$ where $b_{\rm min} = b_0 + r_b +signal-to-noise in a typical stack image for objects with colors of +typical galaxies. The magnitude limits for the Petrosian parameters +were set to 25.0 for all filters, far below the detection limits and +effectively not limiting the analysis based on apparent magnitude. For +both galaxy model fits and Petrosian parameters, the Galactic latitude +cut was defined by $|b| > b_{\rm min}$ where $b_{\rm min} = b_0 + r_b e^{\frac{-l^2}{2 \sigma_b^2}}$. For the PV3 analysis, $b_0 = $20\degree, $r_b = $15\degree, $\sigma_b = $50\degree. This contour @@ -1984,5 +2213,5 @@ the total time spent on the galaxy modeling analysis at the expense of galaxy photometry in the plane (though Kron photometry is available -for those sources). +for those sources). % galaxy model fits performed based on limits set in psphotChooseAnalysisOptions.c @@ -2013,5 +2242,5 @@ % if |b| > 20.0 + 15.0 exp(-long^2 / (2 * 50^2)) -\subsubsection{Radial Profiles} +\subsection{Radial Profiles} \label{sec:radial.profile.v2} @@ -2082,5 +2311,5 @@ % \note{these profiles are not saved in PSPS} -\subsubsection{Petrosian Radii and Magnitudes} +\subsection{Petrosian Radii and Magnitudes} \label{sec:petrosian} @@ -2141,5 +2370,5 @@ -\subsubsection{Convolved Galaxy Model Fits} +\subsection{Convolved Galaxy Model Fits} \label{sec:galaxy.conv.fit} @@ -2347,5 +2576,5 @@ any of the parameters. -\subsubsection{Fixed Aperture Photometry} +\subsection{Fixed Aperture Photometry} \label{sec:fixed.aperture.photom} @@ -2397,141 +2626,129 @@ % last bin is first with inner radius >= skyRadius -\subsection{Aperture Correction and Total Aperture Fluxes} -\label{sec:aperture.correction} - -A PSF model will always fail to describe the flux of the stellar -sources at some level. For high-precision photometry, we need to be -able to correct for the difference between the PSF model fluxes and -the total flux of the sources. In the end, all astronomical -photometry is in some sense a relative measurement between two images. -Whether the goal is calibration of a science image taken at one -location to a standard star image at another location, or the goal is -simply the repetitive photometry of the same star at the same location -in the image, it is always necessary to compare the photometry between -two images. If this measurement is to be consistent, then the -measurement must represent the flux of the stars in the same way -regardless of the conditions under which the images were taken, at -least within some range of normal image conditions. So, for example, -two images with different image quality, or with different tracking -and focus errors, will have different PSF models. To the extent the -PSF model is inaccurate, the measured flux of the same source in the -two images will be different (even assuming all other atmospheric and -instrumental effects have been corrected). The amplitude of the error -will by determined by how inconsistently the models represent the -actual source flux. - -Aperture photometry attempts to avoid these problems, but introduces -other difficulties. In aperture photometry, if a large enough -aperture is chosen, the amount of flux which is lost will be a small -fraction of the total source flux. Even more importantly, as the -image conditions change, the amount lost will change by an even -smaller fraction, at least for a large aperture. This can be seen by -the fact that the dominant variations in the image quality are in the -focus, tracking and seeing. All of these errors initially affect the -cores of the stellar images, rather than the wide wings. The wide -wings are largely dominated by scattering in the optics and scattering -in the atmosphere. The amplitude and distribution of these two -scattering functions do not change significantly or quickly for a -single telescope and site. Aperture photometry can then be used to -correct the PSF photometry. - -The difficulty for aperture photometry is the need to make an accurate -measurement of the local background for each source. As the aperture -grows, errors in the measurement of the sky flux start to become -dominant. If the aperture is too small, then variations in the image -quality are dominant. The brighter is the source, the smaller is the -error introduced by the large size of the aperture. However, the -number of very bright stars is limited in any image, and of course the -brighter stars are more likely to suffer from non-linearity or -saturation. - -In order to thread the needle between these effects, \ippprog{psphot} -measures the aperture photometry on a modest-sized aperture, and then -uses the PSF model to extrapolate to a large aperture. When the PSF -fluxes are calculated, the aperture flux for the modest-sized aperture -is also determined. The aperture is a circular aperture with radius -set to a fixed multiple (\code{PSF_APERTURE_SCALE}) of $\sigma_w$, the -width of the Gaussian window function determined based on the analysis -of the second moments (see Section~\ref{sec:moments}). For the PV3 -$3\pi$ analysis, the aperture window radius is $4.5 \times \sigma_w$, -while the large reference aperture radius is set to 25 pixels -(\code{PSF_REF_RADIUS} = 6\farcs4). These corrected aperture -magnitudes are saved in the output catalogs as \code{AP_MAG}, the -uncorrected aperture magnitudes are saved as \code{AP_MAG_RAW}, and -the radius used to measure the raw aperture flux is saved as -\code{AP_MAG_RADIUS}. The corresponding flux and the flux error are -saved as \code{AP_FLUX} and \code{AP_FLUX_SIG}. - -With these aperture magnitudes in hand, it is now possible to make an -average correction to the PSF magnitudes to bring the PSF and aperture -magnitudes to the same system. This correction is measured using the -same stars from which the PSF model is measured, as long as the PSF -magnitude error for the star is less than 0.03 mag. The correction is -calculated using the weighted average of the values $m_{\rm AP} - -m_{\rm PSF}$. Since the PSF may vary across the image, the correction -is determined as a function of position in the image. Like the PSF -model, the 2D variations of the aperture correction may be modeled as -a polynomial or via interpolation in a grid. For the $3\pi$ PV3 -analysis, a grid with a maximum of $6\times 6$ samples per GPC1 chip -image was used. The reported PSF magnitudes for all objects have this -aperture correction applied. - -% growth curve analysis in psphot: -% in psphotChoosePSF : call psphotMakeGrowthCurve -% in psphotMakeGrowthCurve : boolean GROWTH_FROM_SOURCES, use -%% pmGrowthCurveGenerateFromSources or -%% pmGrowthCurveGenerate (uses PSF model only) -%% GROWTH_FROM_SOURCES is set to TRUE for default recipe - -%% ApTrend: -%% in psphotApResid, called by psphotReadout near the end of the -%% analysis -%% ApTrend = f(x,y) for (apMag - psfMag) for psfMagErr <= 0.03 -%% apMag is growth curve corrected -%% psfMag is raw - -%% raw psfMag and raw apMag are measured -%% apMag = apMagRaw + growth curve correction (from apRadius to 25 pix -%% = PSF_REF_RADIUS) -%% psfMag = psfMagRaw + aperture trend ( + growth curve) - -% How important is this effect? Consider a typical bright source with a -% flux of (say) 40,000 counts in an image of background 1000 counts per -% pixel, with FWHM of 4 pixels. In principle, the flux of this source -% should be measurable with an accuracy of roughly 0.57\% -% ($\frac{\sqrt{40000 + 1000 \times 12}}{40000}$). However, the -% measurement of the sky is limited at some finite level by Poisson -% statistics. If we are required to use an aperture of (say) 25 pixels -% in radius (eg, 5 arcseconds for an 0.2 arcsec / pixel detector), and -% we have an annulus of twice this radius to measure the local sky, then -% we will have an error of XXX. -% -% \note{outline the variation of {\em ApResid} as a function of -% magnitude}. - -%%% \ippprog{psphot} measures the aperture correction ({\em ApResid}) for every PSF -%%% candidate source, then calculates the trend of this correction as a -%%% function of the magnitude. This trend is fitted with a line. The -%%% resulting function can be used to determine the effective aperture -%%% correction for an infinite flux source and the average bias inherent -%%% in the sky measurement for the image. The scatter of the -%%% PSF-candidate source measurements about this trend is a measure of how -%%% well we can measure photometry from the image by applying the specific -%%% PSF model. The slope of this trend is a measure of the bias in the -%%% local sky measurment for each source. In principal, the measured sky -%%% levels could be modified by this bias. More generally, the measured -%%% bias in a collection of images could be used to improve the model -%%% fitting or sky fitting portion of the software the remove the bias -%%% term. - -\ippprog{psphot} allows a collection of PSF model functions to be tried on all -PSF candidate sources. For each model test, the above corrected -ApResid scatter is measured. The PSF model function with the smallest -value for the ApResid scatter is then used by \ippprog{psphot} as the best PSF -model for this image. The number of models to be tested is specified -by the configuration keyword \code{PSF_MODEL_N}. The configuration -variables \code{PSF_MODEL_0}, \code{PSF_MODEL_1}, through -\code{PSF_MODEL_N - 1} specify the names of the models which should be -tested. +\subsection{Galaxy Model Simulations} + +To test the galaxy model analysis, we have generated a series of +simulated images containing both stars and galaxies on which we have +run the \ippprog{psphot} PSF-convolved galaxy model fitting analysis. +The images generated for this analysis have dimensions of $4000 \times +4000$ pixels with a spatial scale of 0.25 arcseconds per pixel. The +images are generated using an effective exposure time of 30 seconds, +with zero points matching the PS1 \rps-filter, and a realistic sky +brightness of 20.86 magnitudes per square arcsecond. The stars are +injected into these images with fluxes drawn from a realistic stellar +luminosity function and random spatial locations. For each image, the +same underlying simulated stellar population was used. Galaxies are +injected into the image with positions on a regularly spaced grid with +separation of 120 pixels. A total of 1089 galaxies are injected in +each image, with all galaxies in an image having the same total +magnitude. Galaxies were injected for 51 magnitude bins, ranging from +17.0 to 22.0. Given the parameters of the image, these values span +the range from high signal-to-noise ($> 100$) to undetectable (S/N $< +1$). Note that we are not attempting to represent a realistic +astronomical distribution of galaxies, but rather attempting to +understand our ability to recover galaxies of a certain type, size, +and shape in a given image. + +The galaxies are injected using Exponential and DeVaucouleur profiles +in separate simulation runs. The major axis values are randomly +distributed between 1 and 10 pixels (0.25 - 2.5 arcseconds) while the +aspect ratios are randomly chosen in a range from 0.25 to 1.0. The +position angles are set by the sequence in the image and allowed to +vary from 0 to 180 degrees. The images are then convolved with a PSF +model using the \code{PS1_V1} profile ($\kappa = 0.2$) and noise is +added using Poisson statistics for the detected photons. + +For the figures below, we present results as a function of the (input) +instrumental magnitude of the galaxy minus the instrumental magnitude +corresponding to the stellar $5 \sigma$ detection limit. We make the +simplifying assumption that the stellar detection threshold +encapsulates enough information about the sensitivity of the images +that this magnitude difference may be used to compare the results +shown here to images with other depths. Thus this and subsequent +figures may be compared with the reported detection limits from the +PS1 $3\pi$ survey. Note for reference that the typical stellar +detection limits in the PS1 $3\pi$ stack images are (\grizy) = (23.3, +23.2, 23.1, 22.3, 21.4). The minimum Kron magnitudes for which galaxy +model fits were performed for the PV3 analysis +(Section~\ref{sec:extended.source}) thus correspond to -1.6 to -1.8 in +these plots. + +Figure~\ref{fig:galaxy.complete} shows completeness for the detection +of the Exponential and DeVaucouleur model galaxies. This analysis +does not indicate if the galaxy was detected {\em as a galaxy} (\ie, +was the extended nature of the source sufficiently clear), only if +the source was detected by the peak-finding algorithm. As expected, +the more compact galaxies are more likely to be detected; Exponential +profile galaxies, with a broader light distribution for the same +effective radius, are less likely to be detected for the same +magnitude than DeVaucouleur profile galaxies. This completeness +should be compared to our earlier work \citep{2013MNRAS.435.1825M} in +which we injected a realistic population of simulated galaxies into +real PS1 images. That work found that the 50\% completeness for the +typical galaxy was roughly 0.5 magnitude brighter than the 50\% +stellar completeness limit, somewhat fainter than the completeness +shown in Figure~\ref{fig:galaxy.complete}. However, that previous +work did not explore the depedency of the completeness on the galaxy +size or profile. The difference suggests that the galaxies in the +earlier work were generally compact. + +% /data/kukui.1/eugene/galaxies.20190425/tap_psphot_galaxies.pro : go.bigtest.ckgalaxy +\begin{figure}[htbp] + \begin{center} + \includegraphics[width=\hsize,clip]{\picdir/{galaxy.exp.complete}.\plotext} + \includegraphics[width=\hsize,clip]{\picdir/{galaxy.dev.complete}.\plotext} + \caption{\label{fig:galaxy.complete} Top: Completeness curves for + simulated galaxies with Exponential profiles. Bottom: + Completeness curves for simulated galaxies with DeVaucouleur + profiles. The curves are shown as a function of the difference + between the injected instrumental magnitude of the galaxy and the + magnitude corresponding to the $5\sigma$ detection threshold for a + PSF-like source. The black curves shows the compleness for all + galaxies. The three colored curves show the completeness for + three major axis ranges. Compact galaxies are more likely to be + detected since peaks are detected after convolution with the + PSF. } + \end{center} +\end{figure} + +Figures~\ref{fig:exp.params} and \ref{fig:dev.params} demonstrate the +recovery of galaxy parameters in these simulations for galaxy using +the Exponential and DeVaucouleur models, respectively. +Both figures show the reliability of the measured magnitudes, major +and minor axis sizes, and ellipticities. For all recovered +parameters, the standard deviation of the difference between the +measured parameter and the truth value is shown for all galaxies as a +function of magnitude, as well as for subsets in major-axis ranges. +The mean of the difference, illustrating any biases, is also given +for all galaxies. The comparison for the major and minor axis sizes +are shown as absolute measurements (in pixels) and as a fraction of +axis in question. + +Some overall patterns can be observed. First, the parameters measured +for the exponential profile galaxies are consistently more reliable +than those measured for the DeVaucouleur profiles. Second, the errors +in the estimated magnitudes are primarily driven by errors in the size +measurements: If the sizes are over-estimated, the fluxes are also +over-estimated. Finally, the magnitudes and major axes are more +accurate for the smaller galaxies, but the ellipticities are more +accurate for the larger galaxies. + +% /data/kukui.1/eugene/galaxies.20190425/tap_psphot_galaxies.pro : go.bigtest.ckgalaxy +\begin{figure*}[htbp] + \begin{center} + \includegraphics[width=\hsize,clip]{\picdir/{galaxy.exp.params}.\plotext} + \caption{\label{fig:exp.params} Parameter recovery for simulated + galaxies with Exponential profiles. } + \end{center} +\end{figure*} + +% /data/kukui.1/eugene/galaxies.20190425/tap_psphot_galaxies.pro : go.bigtest.ckgalaxy +\begin{figure*}[htbp] + \begin{center} + \includegraphics[width=\hsize,clip]{\picdir/{galaxy.dev.params}.\plotext} + \caption{\label{fig:dev.params} Parameter recovery for simulated + galaxies with DeVaucouleur profiles. } + \end{center} +\end{figure*} \section{Forced Photometry Modes} @@ -2676,4 +2893,188 @@ models for each galaxy model measured for the stack image. +\subsection{Galaxy Lensing Parameters} +\label{sec:lensing.params} + +Weak-lensing studies frequently use non-parametric measurements of the +ellipticities of galaxies to quantify the strength of gravitational +lensing, and thus directly measure mass distributions in the Universe. +The classic approach was originally described by +\cite{1995ApJ...449..460K} and applied to a set of deep HST +observations. The details of the technique were further refined by +\cite{1998ApJ...504..636H}; in the discussion below we primarily use +their notation, though we explicitly cast their integrals as sums over +discrete pixels. + +The KSB-style analysis of object ellipticities has also been used by +several authors to search for marginally-resolved binary stars +in wide-field imaging data. The use of the lensing statistics for +this application was described by \cite{2005ApJ...626.1070H} in the +context of vetting planet transit events in data from the Optical +Gravitational Lensing Experiment (OGLE). \cite{2013ApJS..206...18T} +applied the techinique to PTF data to search for binary stars and +\cite{2017MNRAS.468.3499D} used the same technique to search for +binary companions to known ultracool dwarfs using Pan-STARRS $3\pi$ +data. The work by \cite{2017MNRAS.468.3499D} used images and their +own analysis of the pixels with the program Sextractor +\citep{sextractor}. + +For the Pan-STARRS $3\pi$ PV3 analysis, we have measured the full set +of KSB lensing parameters for all objects with measured second moments +(i.e.,, excluding saturated stars, suspected cosmic rays, and other +likely defects) of the data to enable both lensing studies and binary +/ multiple star searches. Here we describe the measurements as +performed within \ippprog{psphot}, reviewing the mathematical +framework as described by \cite{1995ApJ...449..460K} and +\cite{1998ApJ...504..636H}. + +The goal of the KSB technique is to measure the intrinsic ellipticity +of objects (i.e., galaxies, in the case of weak lensing studies) as +would be observed sky on the without instrumental effects and to +determine the impact weak graviational lensing would have on the +observed shapes, after correction for the instrumental effects. The +analysis starts with the observed ellipticity of objects as represented +by the two polarization components derived from the second moments +(see Section~\ref{sec:moments}): +\begin{eqnarray} +\label{eqn:polarization} + e_1 = \frac{M_{xx} - M_{yy}}{M_{xx} + M_{yy}} \\ + e_2 = \frac{2 M_{xy}}{M_{xx} + M_{yy}}. \\ +\end{eqnarray} +These two quantities have values which range from -1 to +1, with a +circularly-symmetric object having $(e_1,e_2) = 0.0,0.0$. The two +polarization components vary sinusoidally with twice the position +angle of the object: an elongated object aligned with the $x$-pixel +axis will have positive values of $e_1$ and $e_2$ values near zero, +while the same object aligned with the $y$-pixel axis will negative +$e_1$ values. An object with a position angle on the 45\degree\ lines +between the pixel axes will have large positive or negative values of +$e_2$ and low absoluate values of $e_1$. + +Note that in our analysis of the second moments, we are applying a +Gaussian window function to down-weight the noise contributions from +pixels at high radii and low flux (see Section~\ref{sec:moments}). +This type of window function is also assumed in the KSB formalism, and +is represented in the equations below as $W$. + +The measured ellipticity of an object observed in a real instrument +will be affected by the point spread function of the instrument. To +first order, the effect on the polarization components can be +described as a combination of the circularly symmetric seeing disc, +which smears the observed shapes (driving $e_1,e_2$ to low absolute +values) and the shearing effect of the anisotropic component of the +PSF, in which the observed shape is stretched in one direction +relative to the others (driving $e_1,e_2$ to larger absolute values). + +KSB and HFK quantify the change in the observed polarization due to +the smearing effect of the PSF with +\begin{equation} + \delta e^{\rm sm}_\alpha = P^{\rm sm}_{\alpha, \beta} p_{\beta} +\end{equation} +$p_\beta$ is a measurement of the +anisotropy of the PSF (see below), and $P^{\rm sm}_{\alpha,\beta}$ is +the ``Smear Polarizability'' of the object, defined as +\begin{eqnarray} + P^{\rm sm}_{\alpha \beta} = X^{\rm sm}_{\alpha \beta} - e_\alpha e^{\rm sm}_\beta +\end{eqnarray} +where +\begin{eqnarray} +X^{\rm sm}_{1,1} &=& \frac{1}{T} \sum f \left[ W + 2W^\prime r^2 + W^{\prime \prime} (x^2 - y^2)^2 \right] \\ +X^{\rm sm}_{1,2} &=& \frac{1}{T} \sum f \left[ 2W^{\prime\prime} (x^2 - y^2) x y \right] \\ +X^{\rm sm}_{2,2} &=& \frac{1}{T} \sum f \left[ W + 2W^\prime r^2 + 4W^{\prime \prime} x^2 y^2 \right] +\end{eqnarray} +and +\begin{eqnarray} +e^{\rm sm}_1 &=& \frac{1}{T} \sum f \left[ 2W^\prime + W^{\prime \prime} (x^2 + y^2) \right] (x^2 - y^2) \\ +e^{\rm sm}_2 &=& \frac{1}{T} \sum f \left[ 2W^\prime + W^{\prime \prime} (x^2 + y^2) \right] 2 x y. +\end{eqnarray} +In these equations, $T = M_{xx} + M_{yy}$ and $W$ is the window +function applied when measuring the second moments. The terms +$W^\prime$ and $W^{\prime \prime}$ are the derivatives of the window +function with respect to $r^2 = x^2 + y^2$. Since the window function +is a circularly-symmetric Gaussian with width $\sigma_w$, the +derivatives are simply $W^\prime = -\frac{1}{2\sigma^2_w} W$ and +$W^{\prime \prime} = \frac{1}{4\sigma^4_w} W$. + +The elements of the equations above can be written in terms of the second and higher-order +moments calculated in Section~\ref{sec:moments}: +\begin{eqnarray} +X^{\rm sm}_{1,1} &=& \frac{1}{T} \left[ 1 - \frac{R_2}{\sigma^{2}} + \frac{(M_{xxxx} - 2 M_{xxyy} + M_{yyyy})}{4 \sigma^{4}} \right] \\[0.1in] +X^{\rm sm}_{1,2} &=& \frac{1}{T} \left[ \frac{(M_{xyyy} - M_{xxxy})}{2 \sigma^{4}} \right] \\[0.1in] +X^{\rm sm}_{2,2} &=& \frac{1}{T} \left[ 1 - \frac{R_2}{\sigma^{2}} + \frac{ M_{xxyy}}{\sigma^{4}} \right] +\end{eqnarray} +and +\begin{eqnarray} +e^{\rm sm}_1 &=& \frac{1}{T} \left[ \frac{M_{xx} - M_{yy}}{\sigma^{2}} + \frac{M_{xxxx} - M_{yyyy}}{4 \sigma^{4}} \right] \\[0.1in] +e^{\rm sm}_2 &=& \frac{1}{T} \left[ \frac{(M_{xxxy} + M_{xyyy})}{2\sigma^{4}} - \frac{2 M_{xy}}{\sigma^{2}} \right] +\end{eqnarray} +where $R_2 = M_{xx} + M_{yy}$. + +KSB and HFK use the observed ellipticities of stars and the smear +polarizability of the stars to estimate the anisotropy due to the PSF: +\begin{eqnarray} +p_\alpha = \frac{e^*_{\alpha}}{P^{{\rm sm},*}_{\alpha \alpha}} +\end{eqnarray} +where the terms with the $*$ represent parameters measured on stars. + +%% \begin{eqnarray} +%% p_1 &=& M_{xx} - M_{yy} \\ +%% p_2 &=& 2 M_{xy} +%% \end{eqnarray} + +Similarly, the impact of shear can be quantified by the ``Shear +Polarizabilty'' in a similar fashion: +\begin{equation} + \delta e^{\rm sh}_\alpha = P^{\rm sh}_{\alpha, \beta} p_{\beta} +\end{equation} +where now the shear polarizability $P^{\rm sh}_{\alpha \beta}$ is +defined as +\begin{eqnarray} + P^{\rm sh}_{\alpha \beta} = X^{\rm sh}_{\alpha \beta} - e_\alpha e^{\rm sh}_\beta +\end{eqnarray} +where +\begin{eqnarray} +X^{\rm sh}_{1,1} &=& \frac{1}{T} \sum f \left[ 2W(x^2 + y^2) + 2W^\prime (x^2 - y^2)^2 \right] \\ +X^{\rm sh}_{1,2} &=& \frac{1}{T} \sum f \left[ 4W^\prime(x^2 - y^2) x y \right] \\ +X^{\rm sh}_{2,2} &=& \frac{1}{T} \sum f \left[ 2W(x^2 + y^2) + 8W^\prime x^2 y^2 \right] +\end{eqnarray} +and +\begin{eqnarray} +e^{\rm sh}_1 &=& 2 e_1 + \frac{2}{T} \sum f W^\prime (x^2 + y^2) (x^2 - y^2) \\ +e^{\rm sh}_2 &=& 2 e_2 + \frac{2}{T} \sum f W^\prime (x^2 + y^2) 2 x y. +\end{eqnarray} + +Re-writing in terms of the second and higher-order moments calculated +in Section~\ref{sec:moments}, we find: +\begin{eqnarray} +X^{\rm sh}_{1,1} &=& \frac{1}{T} \left[ 2 R_2 - \frac{(M_{xxxx} - 2 M_{xxyy} + M_{yyyy})}{\sigma^{2}} \right] \\ +X^{\rm sh}_{1,2} &=& \frac{1}{T} \left[ \frac{2(M_{xyyy} - M_{xxxy})}{\sigma^{2}} \right] \\ +X^{\rm sh}_{2,2} &=& \frac{1}{T} \left[ 2 R_2 - \frac{4 M_{xxyy}}{\sigma^{2}} \right] +\end{eqnarray} +and +\begin{eqnarray} +e^{\rm sh}_1 &=& \frac{1}{T} \left[ 2 (M_{xx} - M_{yy}) + \frac{( M_{yyyy} - M_{xxxx})}{\sigma^{2}} \right] \\ +e^{\rm sh}_2 &=& \frac{1}{T} \left[ 4 M_{xy} - \frac{2 (M_{xxxy} + M_{xyyy})}{\sigma^{2}} \right] +\end{eqnarray} + +In the Pan-STARRS PV3 analysis, we have measured the elements of the +smear polarizability ($X^{\rm sm}_{\alpha \beta}$, $e^{\rm + sm}_\alpha$) and the shear polarizability ($X^{\rm sh}_{\alpha + \beta}$, $e^{\rm sh}_\alpha$) for all objects on each of the warp +images. We have also selected only the PSF stars from the images and +interpolated a smoothed version of these parameters to the location of +the objects, using the grid described above to interpolate the PSF +parameters. We also determine the interpolated PSF ellipticities +($e^*_1, e^*_2$) from the equivalent smooth grid. Thus, for every +object in the $3\pi$ survey, we are able to report the PSF and object +elements of the KSB analysis. These lensing parameters are measured +for each of the warps, and then averaged over all warps for each of +the filters. The average values are calculated by including only +measurements from the same warp detection used in the average +photometry (nominally, the primary skycell; see Paper V, Section +5.4.4) and excluding any measurements for which the \code{PSF_QF} or +\code{PSF_QF_PERFECT} is less than 0.85. + +% \note{example of using the lensing elements for binaries?} + \section{Difference Image Photometry} @@ -2721,4 +3122,5 @@ types of sources. This model is fitted in the same portion of the code which performs the unconvolved galaxy model analysis. + % \note{describe the trailed analytical model}. @@ -2800,7 +3202,18 @@ image. -% \section{Examples and Tests} - -% \section{Conclusions} +\section{Conclusions} + +The Pan-STARRS Image Processing Pipeline has used the \code{psphot} +software to detect and characterize astronomical sources in images +from both the PS\,1 and PS\,2 telescopes since 2008. This software +system has produced highly-reliable stellar photometry and astrometry +measurements as demonstrated by the high-quality data products +released as part of the Pan-STARRS Data Releases 1 and 2. This +configurable software system has been used for stellar photometry in +direct detection model, difference image mode, and forced photometry +mode, as well as galaxy photometry and morphology analysis. To date +(2019 May), over 900 billion detections of sources in more than 2 +million PS\,1 exposures have been characterized (some representing +repeated measurements of the same exposures). \acknowledgments Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/examples/.mana =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/examples/.mana (revision 40759) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/examples/.mana (revision 40759) @@ -0,0 +1,238 @@ +input moments.sh +show.pol +input moments.sh +show.pol +print e1 e2 +input moments.sh +show.pol +print theta e1 e2 +set fr = f0 +rotate fr 190 +tv -n tv fr -0.01 0.5 +rotate fr 90 +set fr = f0 +rotate fr 90 +tv -n tv fr -0.01 0.5 +set fr = f0 +rotate fr 95 +tv -n tv fr -0.01 0.5 +rotate fr 85 +tv -n tv fr -0.01 0.5 +set fr = f0 +rotate fr 85 +tv -n tv fr -0.01 0.5 +star fr {fr[][0]/2} {fr[0][]/2} 71 +echo star fr {fr[][0]/2} {fr[0][]/2} 71 +buff +input moments.sh; test.rot +tv fr -0.01 0.2 +set fr = f0 +rotate fr 5 +tv fr +input moments.sh; test.rot +tv fr -0.01 0.2 +tv f0 +set fr = f0 +rotate fr 2 +tv fr +input moments.sh; test.rot; tv f0 +input moments.sh; test.rot; tv f0 +tv fr +input moments.sh; test.rot; tv -ch 1 f0; tv -ch 2 fr +input moments.sh; test.rot; tv -ch 1 f0; tv -ch 2 fr +input moments.sh; test.rot; tv -ch 1 f0; tv -ch 2 fr +input moments.sh; test.rot; tv -ch 1 f0; tv -ch 2 fr +resize 800 800 +set fr = f0 +rotate fr 10 +tv -ch 2 fr +input moments.sh; test.rot; tv -ch 1 f0; tv -ch 2 fr +input moments.sh; test.rot; tv -ch 1 f0; tv -ch 2 fr +input moments.sh; test.rot; tv -ch 1 f0; tv -ch 2 fr +buffers +echo {fr[][0]/2} {fr[0][]/2} +$x = 360 - 720/2 +$y = 360 - 720/2 +$xo = 360 - 720/2 +$yo = 360 - 720/2 +$x = $xo*dcos(2) + $yo*dsin(2) +$y = $yo*dcos(2) - $xo*dsin(2) +echo $x +echo $y +echo $xo +echo quit +echo {dcos(2)} {dsin(2)} +input moments.sh; test.rot; tv -ch 1 f0; tv -ch 2 fr +echo {fr[][0]/2} {fr[0][]/2} +input moments.sh; test.rot; tv -ch 1 f0; tv -ch 2 fr +header +head f0 +head fr +show.pol +lim -n 2 theta e1; clear; box; plot theta e1 +lim -n 2 theta e2; clear; box; plot theta e2 +input moments.sh; show.pol +lim -n 2 theta e2; clear; box; plot theta e2 +plot -c blue theta e1 +vstat e1 +tv -n 0 -0.01 0.2 f0 +tv -n 0 f0 -0.01 0.2 +input moments.sh; show.pol +tv -n 0 f0 -0.01 0.2 +input moments.sh; show.pol +tv -n 0 f0 -0.01 0.2 +vstat e1 +vstat e2 +imsmooth.2d +imsmooth.generic +imsmooth +fft2d +input moments.sh +input moments.sh +opihi verbose on +input moments.sh +pwd +!cat moments.sh +input moments.sh +opihi verbose off +show.smear +show.smear 1.0 +buffers +input moments.sh +show.smear 1.0 +input moments.sh +show.smear 1.0 +buffers +tv -n 0 -ch 1 f0 -0.01 0.5 +tv -n 0 -ch 1 f0 -0.01 1.0 +tv -n 0 -ch 2 psf -0.01 1.0 +tv -n 0 -ch 2 psf -0.01 0.5 +tv -n 0 -ch 2 fs -0.01 1.0 +stat fs +tv -n 0 -ch 2 fs 0 2e-9 +tv im_R -0.01 0.1 +stat im_R +tv im_R -0.0005 0.001 +tv im_I +stat im_I +tv kr_R -0.0005 0.001 +tv ff_R +stat ff_R +stat kr_R +set ff_R = ff_R * 512^2 +set ff_I = ff_I * 512^2 +stat ff_R +tv ff_R 0 0.05 +tv ff_I 0 0.05 +stat ff_I +fft2d -inverse ff_R ff_I to ot_R ot_I +stat ot_R +set out_R = out_R * 512^2 +set out_R = ot_R * 512^2 +set out_I = ot_I * 512^2 +stat out_R +stat out_I +tv out_R +set fs = fs * 512^4 +tv fs +tv fs 0 152 +tv fs 0 512 +input moments.sh +show.smear 1.0 +tv -n 0 -ch 2 fs 0 1 +tv -n 0 -ch 2 psf 0 1 +tv -n 0 -ch 2 psf 0 0.5 +tv -n 0 -ch 1 f0 0 0.5 +tv -n 0 -ch 3 fs 0 0.5 +star +star f0 256 256 128 +star -q f0 256 256 128; $Mxx = ($SXg/2.355)^2; $Myy = ($SYg/2.355)^2; +?? +star -q f0 256 256 128; $Mxx = ($SXg/2.355)^2; $Myy = ($SYg/2.355)^2; echo $Zcg $Mxx $SXYg $Myy +star -q psf 256 256 128; $Mxx = ($SXg/2.355)^2; $Myy = ($SYg/2.355)^2; echo $Zcg $Mxx $SXYg $Myy +star -q fs 256 256 128; $Mxx = ($SXg/2.355)^2; $Myy = ($SYg/2.355)^2; echo $Zcg $Mxx $SXYg $Myy +echo {1256.63706807*25.1327415272} +input moments.sh +show.smear +show.smear 1 +input moments.sh +show.smear +show.smear 1 +input moments.sh +show.smear 1 +lim -n 2 theta e1_o; clear; box; plot theta e1_o -c blue; plot theta e1_s -c red +vectors +create t 0 720 5 +vectors +create t 0 360 5 +vectors +lim -n 2 t e1_o; clear; box; plot t e1_o -c blue; plot t e1_s -c red +lim -n 2 t e1_o; clear; box; plot t e1_o -c blue; plot t e1_s -c red; plot t e2_o -c blue -pt box; plot t e2_s -c red -pt box +lim -n 2 t e1_o; clear; box; plot t e1_o -c blue; plot t e1_s -c red; plot t e2_o -c blue -pt obox; plot t e2_s -c red -pt obox +input moments.sh +show.smear 1 +star psf {psf[][0]/2} {psf[0][]/2} 128 +$Mxx_p = ($SXg/2.355)^2; $Myy_p = ($SYg/2.355)^2; $Mxy_p = $SXYg +lim -n 2 theta Mxx_s; clear; box; plot t Mxx_o -c blue; plot t Mxx_s -c red +set Mxx_S = Mxx_o + $Mxx_p +lim -n 2 theta Myy_s; clear; box; plot theta Myy_o -c blue; plot theta Myy_s -c red +lim -n 2 theta 0 200; clear; box; plot theta Myy_o -c blue; plot theta Myy_s -c red +lim -n 2 theta 0 150; clear; box; plot theta Myy_o -c blue; plot theta Myy_s -c red +set dMxx = Mxx_s - Mxx_o +set dMxy = Mxy_s - Mxy_o +set dMyy = Myy_s - Myy_o +lim -n 2 theta dMxx; clear; box; plot theta dMxx +line -c red $Mxx_p 0 to $Mxx_p 360 +line -c red 0 $Mxx_p to 360 $Mxx_p +lim -n 2 theta dMxx; clear; box; plot theta dMxx; line -c red 0 $Mxx_p to 360 $Mxx_p +lim -n 2 theta dMyy; clear; box; plot theta dMyy; line -c red 0 $Myy_p to 360 $Myy_p +vstat dMyy +lim -n 2 theta 35 37; clear; box; plot theta dMyy; line -c red 0 $Myy_p to 360 $Myy_p +lim -n 2 theta dMxx; clear; box; plot theta dMxx; line -c red 0 $Mxx_p to 360 $Mxx_p +lim -n 2 theta dMxy; clear; box; plot theta dMxy; line -c red 0 $Mxy_p to 360 $Mxy_p +lim -n 1 theta Mxy_s; clear; box; plot theta Mxy_s -c red -pt cir -sz 2; plot theta Mxy_o -c blue -pt cir -sz 2; line -c red 0 $Mxy_p to 360 $Mxy_p +input moments.sh +show.smear 1 +lim -n 1 theta Mxy_s; clear; box; plot theta Mxy_s -c red -pt cir -sz 2; plot theta Mxy_o -c blue -pt cir -sz 2; line -c red 0 $Mxy_p to 360 $Mxy_p +set dMxy = Mxy_s - Mxy_o +lim -n 2 theta dMxy; clear; box; plot theta dMxy; line -c red 0 $Mxy_p to 360 $Mxy_p +input moments.sh +test.convolve +test.convolve 1 +lim theta Mxx_s; clear; box; plot theta Mxx_s +lim theta Mxx_s; clear; box; plot theta Mxx_s; plot -c red theta Mxx_o +set dMxx = Mxx_s - Mxx_o +lim -n 1 theta dMxx; clear; box; plot theta dMxx +line 0 $Mxx_p to 360 $Mxx_p -c red +set dMyy = Myy_s - Myy_o +lim theta Myy_s; clear; box; plot theta Myy_s; plot -c red theta Myy_o +lim -n 0 theta 0 110; clear; box; plot theta Myy_s; plot -c red theta Myy_o +lim -n 0 theta 0 160; clear; box; plot theta Myy_s; plot -c red theta Myy_o +lim -n 0 theta 0 150; clear; box; plot theta Myy_s; plot -c red theta Myy_o +lim -n 1 theta dMyy; clear; box; plot theta dMyy +line 0 $Myy_p to 360 $Myy_p -c red +lim -n 1 theta 30 40; clear; box; plot theta dMyy; line 0 $Myy_p to 360 $Myy_p -c red +lim -n 1 theta 35 37; clear; box; plot theta dMyy; line 0 $Myy_p to 360 $Myy_p -c red +lim -n 1 theta 35.9 36.1; clear; box; plot theta dMyy; line 0 $Myy_p to 360 $Myy_p -c red +lim -n 1 theta 35.99 36.01; clear; box; plot theta dMyy; line 0 $Myy_p to 360 $Myy_p -c red +lim -n 0 theta -10 10; clear; box; plot theta Mxy_s; plot -c red theta Mxy_o +lim -n 0 theta Mxy_s; clear; box; plot theta Mxy_s; plot -c red theta Mxy_o +lim -n 0 theta Mxy_o; clear; box; plot theta Mxy_s; plot -c red theta Mxy_o +lim -n 0 theta Mxy_o; clear; box; plot theta Mxy_s; plot -c red theta Mxy_o -pt ocir -sz 2 +input moments.sh +test.convolve +test.convolve 0 +lim -n 0 theta 0 150; clear; box; plot theta Myy_s; plot -c red theta Myy_o +set dMyy = Myy_s - Myy_o +set dMxy = Mxy_s - Mxy_o +set dMxx = Mxx_s - Mxx_o +lim -n 1 theta dMxx; clear; box; plot theta dMxx +lim -n 1 theta dMxx; clear; box; plot theta dMxx; line 0 $Mxx_p to 360 $Mxx_p -c red +lim -n 0 theta 0 150; clear; box; plot theta Mxx_s; plot -c red theta Mxx_o +lim -n 0 theta 0 120; clear; box; plot theta Mxx_s; plot -c red theta Mxx_o +lim -n 0 theta 0 150; clear; box; plot theta Myy_s; plot -c red theta Myy_o +lim -n 1 theta dMyy; clear; box; plot theta dMyy; line 0 $Myy_p to 360 $Myy_p -c red +echo $Mxx_p $Myy_p +lim -n 0 theta Mxy_s; clear; box; plot theta Mxy_s; plot -c red theta Mxy_o +lim -n 1 theta dMxy; clear; box; plot theta dMxy; line 0 $Mxy_p to 360 $Mxy_p -c red +pwd Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/examples/moments.sh =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/examples/moments.sh (revision 40759) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/examples/moments.sh (revision 40759) @@ -0,0 +1,211 @@ + +macro imconvolve + if ($0 != 4) + echo "USAGE: imconvolve (image) (kernel) (output)" + break + end + + # if kernel is not the same size as input, pad + if ($1[][0] != $2[][0]) + echo "image sizes do not match" + break + end + if ($1[0][] != $2[0][]) + echo "image sizes do not match" + break + end + + local Nx Ny + $Nx = $1[][0] + $Ny = $1[0][] + + fft2d $1 0 to im_R im_I + fft2d $2 0 to kr_R kr_I + + $Npix = $Nx*$Ny + set ff_R = $Npix*(im_R * kr_R - im_I * kr_I) + set ff_I = $Npix*(im_I * kr_R + im_R * kr_I) + + fft2d -inverse ff_R ff_I to ot_R ot_I + set tmppow = $Npix*sqrt(ot_R^2 + ot_I^2) + + extract tmppow $3 0 0 {$Nx/2} {$Ny/2} {$Nx/2} {$Ny/2} $Nx $Ny + extract tmppow $3 {$Nx/2} 0 {$Nx/2} {$Ny/2} 0 {$Ny/2} $Nx $Ny + extract tmppow $3 0 {$Ny/2} {$Nx/2} {$Ny/2} {$Nx/2} 0 $Nx $Ny + extract tmppow $3 {$Nx/2} {$Ny/2} {$Nx/2} {$Ny/2} 0 0 $Nx $Ny +end + +macro show.pol + + mcreate z 500 500 + set x = xramp(z) - 250 + set y = yramp(z) - 250 + set f0 = 10*exp(-0.5*(x^2 + (y/25)^2)) + + delete -q e1 e2 theta + for rot 0 360 5 + set fr = f0 + rotate fr $rot + + star -q fr {fr[][0]/2} {fr[0][]/2} 101 + + $Mxx = ($SXg/2.355)^2 + $Myy = ($SYg/2.355)^2 + $Mxy = $SXYg + + $T = $Mxx + $Myy + $e1 = ($Mxx - $Myy) / $T + $e2 = 2*$Mxy / $T + + concat $rot theta + concat $e1 e1 + concat $e2 e2 + end + + lim -n graph -1.5 1.5 -1.5 1.5; clear; box; plot -pt cir -sz 2 -c blue e1 e2 +end + +macro show.smear + if ($0 != 2) + echo "USAGE: show.smear (sigma)" + break + end + + mcreate z 512 512 + set x = xramp(z) - z[][0]/2 + set y = yramp(z) - z[0][]/2 + + set f0 = 10*exp(-0.5*((x/2)^2 + (y/10)^2)) + set psf_r = exp(-0.5*((x/2)^2 + (y/6)^2)) + rotate psf_r 30.0 + extract psf_r psf {psf_r[][0]/2 - 256} {psf_r[0][]/2 - 256} 512 512 0 0 512 512 + star -q psf {psf[][0]/2} {psf[0][]/2} 128 + $Mxx_p = ($SXg/2.355)^2 + $Myy_p = ($SYg/2.355)^2 + $Mxy_p = $SXYg + + delete -q e1_s e2_s e1_o e2_o theta Mxx_s Mxy_s Myy_s Mxx_o Mxy_o Myy_o + for rot 0 360 5 + set fr = f0 + rotate fr $rot + delete -q frs + extract fr frs {fr[][0]/2 - 256} {fr[0][]/2 - 256} 512 512 0 0 512 512 + + imconvolve frs psf fs + + star -q fs {fs[][0]/2} {fs[0][]/2} 128 + + $Mxx = ($SXg/2.355)^2 + $Myy = ($SYg/2.355)^2 + $Mxy = $SXYg + + $T = $Mxx + $Myy + $e1 = ($Mxx - $Myy) / $T + $e2 = 2*$Mxy / $T + + concat $rot theta + concat $e1 e1_s + concat $e2 e2_s + concat $Mxx Mxx_s + concat $Mxy Mxy_s + concat $Myy Myy_s + + # fr is the rotate version of f0 + star -q fr {fr[][0]/2} {fr[0][]/2} 128 + + $Mxx = ($SXg/2.355)^2 + $Myy = ($SYg/2.355)^2 + $Mxy = $SXYg + + $T = $Mxx + $Myy + $e1 = ($Mxx - $Myy) / $T + $e2 = 2*$Mxy / $T + + concat $e1 e1_o + concat $e2 e2_o + concat $Mxx Mxx_o + concat $Mxy Mxy_o + concat $Myy Myy_o + end + + lim -n graph -1.5 1.5 -1.5 1.5; clear; box; plot -pt cir -sz 2 -c blue e1_o e2_o; plot -pt cir -sz 2 -c red e1_s e2_s; +end + +macro test.rot + + mcreate z 720 720 + set x = xramp(z) - 360 + set y = yramp(z) - 360 + + set f1 = 10*exp(-0.5*(((x-100)/2)^2 + ((y-100)/2)^2)) + set f2 = 10*exp(-0.5*(((x+100)/2)^2 + ((y-100)/2)^2)) + set f3 = 10*exp(-0.5*(((x-100)/2)^2 + ((y+100)/2)^2)) + set f4 = 10*exp(-0.5*(((x+100)/2)^2 + ((y+100)/2)^2)) + set f0 = f1 + f2 + f3 + f4 + + set f1 = 10*exp(-0.5*((x/2)^2 + (y/2)^2)) + set f0 = dsin(5*x)*dcos(5*y) + f1 + + set fr = f0 + + rotate fr 2 +end + + +macro test.convolve + if ($0 != 2) + echo "USAGE: show.smear (sigma)" + break + end + + mcreate z 512 512 + set x = xramp(z) - z[][0]/2 + set y = yramp(z) - z[0][]/2 + + # convolve psf (oriented along yy axis) with object (rotating) + # object 1: + set f0 = exp(-0.5*((x/2)^2 + (y/10)^2)) + + set psf_r = exp(-0.5*((x/2)^2 + (y/6)^2)) + rotate psf_r 30.0 + extract psf_r psf {psf_r[][0]/2 - 256} {psf_r[0][]/2 - 256} 512 512 0 0 512 512 + + # set psf = exp(-0.5*((x/2)^2 + (y/6)^2)) + star -q psf {psf[][0]/2} {psf[0][]/2} 128 + $Mxx_p = ($SXg/2.355)^2 + $Myy_p = ($SYg/2.355)^2 + $Mxy_p = $SXYg + + delete -q e1_s e2_s e1_o e2_o theta Mxx_s Mxy_s Myy_s Mxx_o Mxy_o Myy_o + for rot 0 360 5 + set fr = f0 + rotate fr $rot + concat $rot theta + + delete -q frs + extract fr frs {fr[][0]/2 - 256} {fr[0][]/2 - 256} 512 512 0 0 512 512 + + imconvolve frs psf fs + + star -q fs {fs[][0]/2} {fs[0][]/2} 128 + + $Mxx = ($SXg/2.355)^2 + $Myy = ($SYg/2.355)^2 + $Mxy = $SXYg + + concat $Mxx Mxx_s + concat $Mxy Mxy_s + concat $Myy Myy_s + + # fr is the rotated version of f0 + star -q fr {fr[][0]/2} {fr[0][]/2} 128 + + $Mxx = ($SXg/2.355)^2 + $Myy = ($SYg/2.355)^2 + $Mxy = $SXYg + + concat $Mxx Mxx_o + concat $Mxy Mxy_o + concat $Myy Myy_o + end +end Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/stages.tex =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/stages.tex (revision 40759) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.analysis/stages.tex (revision 40759) @@ -0,0 +1,854 @@ +% \documentclass[iop,floatfix]{emulateapj} +% \documentclass[iop,floatfix,onecolumn]{emulateapj} +\documentclass[12pt,preprint]{aastex} +% \pdfoutput=1 + +\RequirePackage{color} +\RequirePackage{code} +\input{astro.sty} + +% online version may use color, but print version needs b/w +\def\plotmode{col} +%\def\plotmode{bw} + +%\def\plotext{pdf} +\def\plotext{ps} + +%\def\picdir{/home/eugene/chipresid.20140404} +\def\picdir{/data/pikake.2/eugene/chipresid.20140404} + +% Pick a terse version of the title here; +\shorttitle{PS1 Data Processing System} +\shortauthors{E.A. Magnier et al} +\begin{document} +\title{Pan-STARRS Data Processing System} + +% this is a crude trick to get the order of affiliations right. These +% names are used in the affiliations below. The user needs to (1) set +% the order and numbers to have the correct sequence in the author +% list and (2) re-order the list at the bottom (and comment-out as needed) +\def\IfA{1} +\def\CfA{2} +\def\MPIA{3} +\def\Princeton{3} +\def\USNO{4} +\def\JHU{1} + +% This example has a first author from UH: +\author{ +Eugene A. Magnier,\altaffilmark{\IfA} +IPP Team, +%PS Builder List +% W.~S. Burgett,\altaffilmark{\IfA} +% K.~C. Chambers,\altaffilmark{\IfA} +% L. Denneau,\altaffilmark{\IfA} +% P. Draper,\altaffilmark{\DUR} +% H.~A. Flewelling,\altaffilmark{\IfA} +% T. Grav,\altaffilmark{\IfA} +% J. N. Heasley,\altaffilmark{\IfA} +% K. W. Hodapp,\altaffilmark{\IfA} +% M. E. Huber,\altaffilmark{\IfA} +% R. Jedicke,\altaffilmark{\IfA} +% N. Kaiser,\altaffilmark{\IfA} +% R.-P. Kudritzki,\altaffilmark{\IfA} +% G. A. Luppino,\altaffilmark{\IfA} +% R. H. Lupton,\altaffilmark{\Princeton} +% E. A. Magnier,\altaffilmark{\IfA} +% N. Metcalfe,\altaffilmark{\DUH} +% D. G. Monet,\altaffilmark{\USNO} +% J.~S. Morgan,\altaffilmark{\IfA} +% P. M. Onaka,\altaffilmark{\IfA} +% P.~A. Price,\altaffilmark{\Princeton} +% C.~W. Stubbs,\altaffilmark{\CfA} +% W.~E. Sweeney,\altaffilmark{\IfA} +% J.~L. Tonry, \altaffilmark{\IfA} +% R. J. Wainscoat,\altaffilmark{\IfA} and +% C. Z. Waters,\altaffilmark{\IfA} +} % this bracket terminates author list + +% The ordering here should be sequential, matching the sequence in the list of authors: +\altaffiltext{\IfA}{Institute for Astronomy, University of Hawaii, 2680 Woodlawn Drive, Honolulu HI 96822} +% \altaffiltext{\CfA}{Harvard-Smithsonian Center for Astrophysics, 60 Garden Street, Cambridge, MA 02138} +% \altaffiltext{\Princeton}{Department of Astrophysical Sciences, Princeton University, Princeton, NJ 08544, USA} +% \altaffiltext{\USNO}{US Naval Observatory, Flagstaff Station, Flagstaff, AZ 86001, USA} +% \altaffiltext{\JHU}{Department of Physics and Astronomy, Johns Hopkins University, 3400 North Charles Street, Baltimore, MD 21218, USA} +% \altaffiltext{\MPIA}{Max Planck Institute for Astronomy, K\"onigstuhl 17, D-69117 Heidelberg, Germany} +\begin{abstract} + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum +bibendum nisi id tristique posuere. Duis eu mollis nulla. Maecenas est +turpis, mattis tempor urna vitae, placerat rhoncus sem. Lorem ipsum +dolor sit amet, consectetur adipiscing elit. Sed quis velit +nisl. Aliquam erat volutpat. Cras lacinia, nisl tristique auctor +molestie, dolor nulla rhoncus purus, ac accumsan nunc nunc ac +nibh. Maecenas vitae mollis mauris. Ut sollicitudin pulvinar purus, +eget luctus lorem tincidunt vitae. Vestibulum eu mattis neque. Nulla +in tortor id urna dapibus gravida a vel leo. + +\end{abstract} + +% insert additional keywords as appropriate: +\keywords{Surveys:\PSONE } + +% \section{INTRODUCTION}\label{sec:intro} + +\section{IPP Software Subsystems} + +\subsection{Processing Database} + +A critical element in the Pan-STARRS IPP infrastructure is the +processing database. This database, using the mysql database engine, +tracks information about each of the processing stages. It is used as +the point of mediation of all IPP operations. Processing stages +within the IPP perform queries of the database to identify the data to +be processed at a given stage. As the processing for a particular +stage is completed, summary information about the stage is written +back to the database. In this way, the database records this history +of the processing, and also provides the information needed to +successive processing stages to begin their own tasks. + +The processing database is colloquially referred to as the `gpc1' +database, since a single instance of the database is used to track the +processing of images and data products related to the PS1 GPC1 camera. +This same database engine also has instances for other cameras +processed by the IPP, e.g., GPC2, the test cameras TC1, TC3, the +Imaging Sky Probe (ISP), etc. + +Within the processing database, the various processing stages are +represented as a set of tables. In general, there is a top level +table which defines the conceptual list of processing items either to +be done, in progress, or completed. An associated table lists the +details of elements which have been processed. For example, one +critical stage is the Chip processing stage, discussed below, in which +the individual chips from an exposure are detrended and sources are +detected. Within the gpc1 database, there is a top-level table called +`chipRun' in which each exposure has a single entry. Associated with +this table is the `chipProcessedImfile' table, which contains one row +for each of the (up to 60) chips associated with the exposure. The +top-level tables, such as chipRun, are populated once the system has +decided that a specific item (e.g., an exposure) should be processed +at that stage. Initially, the entry is given a state of `run', +denoting that the exposure is ready to be processed. The low-level +table entries, such as the chipProcessedImfile entries, are only +populated once the element (e.g., the chip) has been processed by the +analysis system. Once all elements for a given stage, e.g., chips in +this case, are completed, then the status of the top-level table entry +(chipRun) are switched from `run' to `done'. + +If the analysis of an element (e.g., chip) completed successfully, +then the corresponding table row (e.g., chipProcessedImfile) is listed +with a fault of 0. If the analysis failed, then a non-zero fault is +recorded. An analysis which results in a fault is one in which the +failure is thought to be temporary. For example, if a computer had a +network glitch and was unable to write out some of the output files, +this would be an ephemeral failure which was not a failing of the +data, but merely the processing system. On the other hand, if the +analysis failed because of a problem with the input data, this is +noted by setting a non-zero value in a different table field, +`quality'. For example, if the chip analysis failed to discover any +stars because the image was completely saturated, the analysis can +complete successfully (fault = 0), but the `quality' field will be set +to a non-zero value. The various processing stages are able to select +only the good (quality = 0) elements of a prior stage when choosing +items for processing. For example, the Camera calibration stage will +only use data from chips with good quality data, dropping the failed +chips from the rest of the analysis. On the other hand, a fault in +one of the elements for a given stage will block any dependent stages +from processing that item. In this way, if a glitch occurs and a chip +from an exposure failed to be written to disk in the Chip stage, the +system will not partially process the exposure with the rest of the +chips. Since many of the faults which occur are ephemeral, the +processing stages are set up to occasional clear and re-try the +faulted entries. Thus, automatic processing is able to keep the data +flowing even in the face of occasional network glitches or hardware +crashes. + +\subsection{Nebulous} + +\subsection{Pantasks \& Parallel Processing} + +\subsection{DVO} + +The Pan-STARRS IPP uses an internal database system, distinct from the +publically visible database system, to determine the association +between multiple detections of the same astronomical object and as +part of the astrometric and photometric calibration process. This +database system, called the ``Desktop Virtual Observatory'' (DVO) was +developed originally for the LONEOS project, and used as part of the +CFHT Elixir system (Magnier \& Cuillandre REF). The capabilities of +this databasing system have been somewhat expanded for the Pan-STARRS +context. + +One of the main purposes of the DVO system is to define the +relationship between individual detections of an astronomical object +and the definition of that object. Before describing the database +schema, we will discuss the process by which detections are associated +with objects. New detections are generally added to the database in a +group associated with, for example, an image from the GPC1 camera. As +new detections are loaded, they are compared to the objects already +stored in the database. If an object is already found in the database +within the match radius, the new detection is associated to that +object. If more than one object exists within the database, the +detection is associated with the closest object. + +Detections in DVO have a special piece of metadata called the +\code{photcode} which identifies the source of the measurement. A +\code{photcode} has a name which in general consists of the name of +the camera or telescope (e.g., GPC1 or 2MASS), the name (or short-hand +name) of the filter used for the measurement (e.g., $g$), and an +identifier for the detector, if not unique (e.g., XY01 for GPC1). +Along with each name, there is a numerical value for the photcode. A +table within the DVO system, \code{Photcode}, lists the photcoes and +defines a number of additional pieces of information for each +photcode. These include the nominal zero point and airmass slope, as +well as color trends to transform a measurement in the specific +photcode to a common system. There are 3 classes of photcodes defined +within the DVO system. Those photcodes associated with detections +from an image loaded into the database system are called \code{DEP} +photcodes. There are also photcodes associated with the average +photometry values, called SEC photcodes. There are also those +measurements which come from external data sources for which DVO does +not have any information to determine a calibration (e.g., +instrumental magnitudes and detector coordinates). These are +measurements are reference values and are assigned REF photcodes. + +In the implementation of DVO used for the PV3 calibration analysis, +the database tables are stored on disk using binary FITS tables. Each +type of database table is stored as a separate file, or a collection +of files for table which are spatially partitioned. The binary FITS +tables may be optionally compressed using the (to date) experimental +FITS binary table compression strategy outlined by REF. In this +compression scheme, using a strategy similar to that used for FITS +image compression (REF), the data stored in the binary table is +compressed and stored in the 'HEAP' section of the FITS table. In +brief, each column in the FITS table is compressed as one (or more) +blocks. The standard fields which describe the data column format +(e.g., TFORM1) are replaced with columns which describe the location +and size of the compressed data in the HEAP section; the information +about the uncompressed data is moved to a field with 'Z' prepended +(e.g., ZFORM1) and an additional field is added to define the +compression algorithm (e.g., ZCTYP1). The column names (e.g., TTYPE1) +and units (e.g., TUNIT1) are retained in their original form. The +compression algorithm can treat the entire column as a single block of +data, or it may be broken into a number of chunks, each compressed in +turn (this must be the same for all columns). Additional header +information is added to describe the block sizes and infomation needed +to describe the HEAP data section. The compression algorithms +currently defined consist of the GZIP, RICE, PLIO, and HCOMPRESS +(REFS). For GZIP, the compression algorithm may transpose the byte +order before compression: for floating point data of a similiar +dynamic range, this choice may allow for better compression as each +byte in the 4 or 8 byte floating point value is more likely to be +similar to the same byte in other rows than to the other bytes of the +same row value. This option is called \code{GZIP_2} in the FITS +standard, as opposed to the standard order, \code{GZIP_1}. The DVO +system can be set to specify the compression options for each column +in the tables. In practice, we have chosen a default in which +floating point numbers used \code{GZIP_2}, character strings use +\code{GZIP_1}, integers use \code{RICE}. + +\subsubsection{Sky Partition} + +DVO includes two major classes of database tables: those containing +information directly about astronomical objects in the sky and those +containing other supporting information. The object-related tables +are partitioned on the basis of position in the sky: objects within a +region bounded by lines of constant RA,DEC are contained in a specific +file. The boundaries and the associated partition names are stored in +one of the supporting tables, \code{SkyTable}. This table contains +the definitions of the boundaries for each sky region (\code{R_MIN}, +\code{R_MAX}, \code{D_MIN}, \code{D_MAX}), the name of the sky region, +an ID (\code{INDEX}, equal to the sequence number of the region in the +table), and index entries to enable navigation within the table. The +regions are defined in a hierarchical sense, with a series of levels +each containing a finer mesh of regions covering the sky. + +In the default used by the PV3 DVO, the partitioning scheme is based +on the one used by the Hubble Space Telescope Guide Star Catalog +files. Level 0 is a single region covering the full sky. Level 1 +divides the sky in Declination into bands 7.5\degree\ high. Level 2 +subdivides these Declination bands in the RA direction, with spacing +related to the stellar density. Level 3 divides these RA chunks into +4 - 8 smaller partitions. This level exactly matches the HST GSC +layout, and uses the same naming convention to identify the +partitions: n0000/0000, etc. \note{more on the names?}. Level 4 +further divides these regions by a factor of 16. In the +\code{SkyTable}, a region at one level has a pointer to its parent +region (the one which contains it) and a sequence pointing to its +children (regions it contains). The \code{SkyTable} enables fast +lookups of the on-disk partitions which map to a specific coordinate +on the sky. In general, a single DVO will have the full sky +represented with tables at a single level, though it is possible for +mixed levels to be used, this mode is not well tested. For the PV3 +master database, the partitioning at the 5th level results in \approx +150,000 regions to cover the full sky, of which \approx 110,000 are +used for the PV3 $3\pi$ data. The densest portions of the bulge +contain at most \approx 300k astronomical objects in the database +files, with an associated maximum of 30M measurements in these files. +With the compression scheme described above, this makes the largest +database files \approx 3GB, which can be loaded into memory in 30 +seconds on our partition machines. + +The DVO software system allows the tables which are partitioned across +the sky to also be distributed across multiple computers, which we +call partition hosts. A single file defines the names of these +partition hosts and the location of the database partition on the +disks of that machine. The \code{SkyTable} contains elements to +define by ID the parition host to which a partitioned set of tables +has been assigned. Operations which query the database, or perform +other operations on the database, are aware of the partitioning scheme +and will launch their operations as remote processes on the machines +which contain the data they need. For example, a query for data from +a small region will launch sub-query operations on the machines which +contain the data overlapping the region of interest. These remote +query operations will select the database information which matches +the query request (i.e., applying restrictions as defined) and return +to the master process the results. The results from the various +partition hosts are then merged into a single result by the master +process. This parallelization is critical to querying and +manipulating the enormous database on a reasonable timescale. + +\subsection{Tables which describe objects} + +Two tables carry the most important information about the astronomical +objects in the database: Average and SecFilt. These two tables +specify the main average properties of the astronomical object. The +Average table includes the astrometric information ($\alpha, \delta, +\mu \alpha, \mu \delta, \pi$) and associated errors, data quality +flags for each object, links to the other tables, and a number of IDs, +with one row for each astronomical object. \note{go into complete + detail here on the IDs?}. The SecFilt table\footnote{The name + SecFilt is a bit of a historical misnomer: originally, DVO was + designed for a monochromatic survey and data for a single + photometric band was maintained in the Average table. Later, DVO + was adapted to a multifilter system and additional filters were + added to the SecFilt (Secondary Filter) table. Eventually, the + schema was normalized and all photometric data placed in SecFilt, + with the Primary filter concept being dropped, but the name has + since stuck.} contains average photometric information for a +collection of filters. A given DVO instance has a specified set of +filters for which average photometry is stored in the SecFilt table. +The number and choice of filters for the SecFilt may be modified by +the database administrator fairly easily, but the process of updating +the database is somewhat expensive (\approx 24 hours for the current +PV3 instance). Thus the choice is semi-static for a given DVO +instance. In the case of the PV3 DVO instance, 9 average bandpasses +are defined: {\it g, r, i, z, y, J, H, K, w}. The SecFilt table +contains one row for each filter for each object, thus the PV3 DVO +contains 9 times as many rows as the Average table. The order of the +table is fixed in relation to the Average table: row $i$ of Average +defines the object with photometry contained in rows $9i \rightarrow 9i + +8$ ($i$ is zero counting). + +The individual measurements of the astronomical objects are carried in +the table \code{Measure}. This table lists the values measured by +\code{psphot} for each chip, warp, or stack image. This includes the +instrumental magnitudes for the PSF, aperture, and Kron photometry; +raw position (Xccd, Yccd) and second moments (Mxx, Myy, Mxy), along +with shape parameters of the PSF model at the position of the object +(FWx, FWy, theta). This table also includes metadata such as the +exposure time, the date \& time of the observation, airmass, azimuth, +and information specifying the filter \note{describe the photcodes}. +The \code{Measure} table also carried the calibration magnitude offsts +($M_{\rm cal}$ and $M_{\rm flat}$ discussed below) and the +astrometrically calibrated position. Astrometric offsets for several +systematic corrections discussed below are also defined for each +measurement. Since stacks and forced warp photometry may have +non-significant values, the table is somewhat de-normalized in that it +also carried instrumental flux values for the PSF, aperture, and Kron +photometry. + +In the \code{Measure} table, there are three fields which provide two +independent links from the specific measurement to the associated +object: \code{Measure.catID} specifies the spatial partition to which +the measurement belongs; \code{Measure.objID} specifies to which entry +in the \code{Average} table the measurement belongs. These two 32 bit +fields can thus be combined into a single 64 bit unique ID for all +objects in the database. In addition, the field \code{Measure.averef} +specifies the row number in the \code{Average} table of the associated +object. The \code{Measure} table may be unsorted, in which case it is +slow to find the measurements associated with a specific object (a +full table scan is required). After the table is sorted, the +\code{Measure} rows for a given object are grouped together. In the +case, the fields \code{Average.measureOffset} and +\code{Average.Nmeasure} define an index for the code to jump to the +list of measurements for a single object. The field +\code{Measure.imageID} defines the link from the measurement to the +image which supplied the measurement. + +\note{some discussion of the db construction, addstar, dvomerge, etc?} + +For the warp images, we also measure the weak lensing KSB parameters +related to the shear and smear tensors. These measurements are stored +in the \code{Lensing} table, along with the radial aperture fluxes for +radii numbers 5, 6, \& 7 (XX, XX, XX arcsec). This table contains one +row for every warp row. Similarly to the \code{Measure} table, the fields +\code{objID}, \code{catID}, and \code{averef} define links from the +\code{Lensing} table to the \code{Average} table. In a similar +fashion, the fields \code{Average.lensingOffset} and +\code{Average.Nlensing} are the index into the sorted \code{Lensing} +table entries. \note{discuss failure of the Lensing to Measure + indexing} + +The values stored in the \code{Lensing} table are used to calculate +average values for each of these types of measurements in each +filter. The \code{Lensobj} table stores the averaged KSB and radial +aperture photometry for each of the 5 filters \grizy. This table +contains one entry per object per filter. The table is not generally +stored unsorted as it is calculated after the full database is +populated. The link from \code{Average} to \code{Lensobj} is defined +by the fields \code{Average.offsetLensobj} and +\code{Average.Nlensobj}. Each \code{Lensobj} row also includes the +photcode (filter) for which the average lensing (and radial aperture) +properties have been calculated. + +The \code{Galphot} table stores the results of the forced galaxy +fitting analysis for each object that has been measured. This table +contains one row per filter and model type (Sersic, Exponential, +DeVaucouleur) if forced galaxy models have been calculate for the +object. \note{need to expand on this somewhat} + +The \code{Starpar} table carries measurements provide by Greg Green \& +Eddie Schlafly from their analysis of the SED of objects in the PS1 +$3\pi$ data, using the \note{PV1?} version of the analysis (Green et +al REF). In this work, the goal was a 3D model of the dust in the +Galaxy based on Pan-STARRS (\note{and WISE \& 2MASS?}) photometry. As +part of this analysis, the authors fit the SEDs of all \note{stellar?} +sources with stellar models including free parameters of extinction, +distance modulus, metallicity, and absolute r-band magnitude. While +these photometric distance modulus measurements are not extremely +precise (see below), they provide a constraint on the distance is used +in our analysis of the astrometry (see Section~\ref{sec:astrometry}). + +\subsection{Other Tables} + +Data from GPC1 (and other cameras processed by the IPP) are loaded +into DVO in units \code{smf} files generated by the Camera calibration +stage. As described above, these files contain all measurements from +a complete exposure, with measurements from each chip grouped into +separate FITS tables. When these measurements are loaded into the +\code{Measure} and similar tables, a subset of the information from +the chip header is used to populated a row in the DVO \code{Image} +table. This table contains one row for each chip known to DVO, with +information such as the filter (\code{photcode}), the exposure time, +the airmass, the astrometric calibration terms, the photometric +zero point, etc. For GPC1 and other mosaic cameras, an additional row +is defined to carry the projection and camera distortion elements of +the astrometry model. As chips are loaded into this table, they are +assigned an internal ID (a running sequence in the table). Images may +also be assigned an external ID: in the case of the GPC1 images, this +ID is defined by the processing mysql database and is guaranteed to be +unique within the processing system. + +Other tables are used to track information used by the calibration +system. This includes the complete set of flat-field corrections +determined by the photometry calibration analysis (see +Section~\ref{sec:relphot}) and the astrometric flat-field corrections +determined by the astrometry calibration analysis (see Section~\ref{sec:relastro}) + +\section{IPP Data Processing Stages} + +\subsection{Download from Summit} + +As exposures are taken by the PS1 telescope \& camera system, the 60 +OTA CCDs are read out by the camera software system and each chip is +written to disk on computers at the summit in the PS1 facility. The +chip images are written to a collection of machines in the PS1 +facility called the `pixel servers'. After the images are written to +disk, a summary listing of the information about the exposure and the +chip images are written to an http server system called the +`datastore'. The datastore exposes, via http, a list of the exposures +obtained since the start of the PS1 operations. Requests to this +server may restrict to the latest by time. Each row in the listing +includes basic information about the exposure: an exposure identifier +(e.g., o5432g0123o; see~\ref{GPC1.names} for details), the date and +time of the exposure, \note{etc}. The row also provides a link to a +listing of the chips associated with that exposure. This listing +includes a link to the individual chip FITS files as well as an md5 +CHECKSUM. Systems which are allowed access may download chip FITS +files via http requests to the provided links. + +During night-time operations, while the telescope is observing the sky +and the camera subsystem is saving images to the pixel servers and +adding their information to the datastore list, the IPP subsystem +called `summitcopy' monitors the datastore in order to discover new +exposures ready for download. Once a new exposure has been listed on +the datastore, summitcopy adds an entry of the exposure to a table in +the processing database (summitExp). This tells the summitcopy to +look for the list of chips, which are then added to another table +(summitImfile). The summitcopy system then attempts to download the +chips from the summit pixel servers with an http request. As the chip +files are downloaded, their md5 checksum values are calculated and +compared with the value reported by the camera subsystem / datastore. +Download failures are rare and marked as a non-zero fault, allowing for a +manual recovery, rather than automatically rejecting the failed +chips. + +\subsection{Image Registration} + +Once chips for an exposure have all been downloaded, the exposure is +ready to be registered. In this context, `registration' refers to the +process of adding them to the database listing of known, raw exposures +(not to be confused with 'registration' in the sense of pixel +re-alignment). The result of the registration analysis is an entry +for each exposure in the rawExp table, and one for each chip in the +rawImfile table. These tables are critical for downstream processing +to identify what exposures are available for processing in any other +stage. In the registration stage, a large amount of descriptive +metadata for each chip is added to the rawImfile table, some of which +is extracted from the chip FITS file headers (e.g., RA, DEC, FILTER) +and some of which is determined by a quick analysis of the pixels +(e.g., mean pixel values, standard deviation). The chip-level +information is merged into a set of exposure-level metadata and added +to the rawExp table entry. The exposure-level metadata may be the +same as any one of the chip, in a case where the values are duplicated +across the chip files (e.g., the name of the telescope or the date \& +time of the exposure), or it may be a calculation based on the values +from each chip (e.g., average of the average pixel values). + +Unlike much of the rest of the IPP stage, the raw exposures may only +have a single entry in the registration tables of the processing +database tables (rawExp and rawImfile). + +\subsection{Chip Processing} + +The science analysis of an exposure begins with the processing of the +individual chips, the Chip Processing stage. This analysis step has +two main goals: the removal of the instrumental signature from the +pixel values (detrending) and the detection of the sources in the +objects. In the Chip stage, the individual chips are processed +independently in parallel within the data processing cluster. Within +the processing computer cluster, most of the data storage resources +are in the form of computers with large raids as well as substantial +processing capability. The processing system attempts to locate one +copy of specific raw chips on pre-defined computers for each chip. +The processing system is aware of this data localization and attempts +to target the processing of a particular chip to the machine on which +the data for that chip is stored. The output products are then +primarily saved back to the same machine. This `targetted' processing +was an early design choice to minimize the system wide network load +during processing. In practice, as computer disks filled up at +different rates, the data has not been localized to a very high +degree. The targeted processing has probably reduced the network load +somewhat but it has not been as critical of a requirement as +originally expected. + +The Chip processing stage consists of: reading the raw image into +memory, appyling the detrending steps (see~\note{Waters et al}), +stiching the individual OTA cells into a single chip image, detection +and characterization of the sources in the image (see~\note{Magnier et + al}), and output of the various data products. These include the +detrended chip image, variance image, and mask image, as well as the FITS +catalog of detected sources. The PSF model and background model are +also saved, along with a processing log. A selection of summary +metadata describing the processing results are saved and written to +the processing database along with the completion status of the +process. Finally, binned chip images are generated (on two scales, +binned by 16 and 256 pixels) for use in the visualization system of +the processing monitor tool. + +\subsection{Camera Calibration} + +After sources have been detected and measured for each of the chip, +the next stage is to perform a basic calibration of the full exposure. +This stage starts with the collection of FITS tables containing the +instrumental measurements of the detected sources, primarily the +positions ($x_{\rm ccd}, y_{\rm ccd}$) and the instrumental PSF +magnitudes. The data for all chips of an exposure are loaded by the +analysis program. The header information is used to determine the +coordinates of the telescope boresite (RA, DEC, Position angle). +These three coordinates are used, along with a model of the camera +layout, to generate an initial guess for the astrometry of each chip. +Reference star coordinates and magnitudes are loaded from a reference +catalog for a region corresponding to the boundaries of the exposure, +padded by a large fraction of the exposure diameter in case of a +modest pointing error. The guess astrometry is used to match the +reference catalog to the observed stellar positions in the focal plane +coordinate system. Once an acceptable match is found, the astrometric +calibration of the individual chips is performed, including a a fit to +a single model for the distortion introduced by the camera optics. +After the astrometic analysis is completed, the photometric +calibration is determined using the final match to the reference +catalog. At this stage, pre-determined color terms may be included to +convert the reference photometry to an appropriate photometric +system. For PS1, this is used to generate synthetic w-band photometry +for areas where no PS1-based calibrated w-band photometry is +available. For more details, see \note{Magnier et al}. + +In addition to the astrometric and photometric calibrations, the +Camera stage also generates the dynamic masks for the images. The dynamic +masks include masking for optical ghosts, glints, saturated stars, +diffraction spikes, and electronic crosstalk. The mask images +generated by the Chip stage are updated with these dynamic masks and a +new set of files are saved for the downstream analysis stages. + +The Camera stage also merges the binned chip images +(see~\ref{sec:chip}) into single jpeg images of the entire focal +plane. These jpeg images can then be displayed by the process +monitoring system to visualize the data processing. + +\subsection{Warp} + +Once astrometric and photometric calibrations have been performed, +images are geometrically transformed into a set of common pixel-grid +images with simple projections from the sky. These images, called +skycells, can then be used in subsequent stacking and difference image +analysis without concern about the astrometric transformation of an +exposure. This processing is called `warping'; the warp analysis +stage is run on all exposures before they are processed further. For +details on the warping algorithm, see \note{Waters et al paper}. + +The output products from the Warp stage consist of the skycell images +containing the signal, the variance, and the mask information. These +images have been shipped to STScI and \note{are available / will be + available} from the image extraction tools \note{in DR2}. + +\subsection{Stack} + +The skycell images generated by the Warp process are added together to +make deeper, higher signal-to-noise images in the Stack stage. The +stacks also fill in coverage gaps between different exposures, +resulting in an image of the sky with more uniform coverage than a +single exposure. See~\note{Waters paper} for details on the stack +combination algorithm. + +In the IPP processing, stacks may be made with various options for the +input images. During nightly science processing, the 8 exposures per +filter for each Medium Deep field are combined into a set of stacks +for that field. These so-called `nightly stacks' are used by the +transient survey projects to detect the faint supernovae, among other +transient events. For the PV3 $3\pi$ analysis, all filter images from +the $3\pi$ survey observation were stacked together to generate a +single set of images with $\sim 10 - 20\times$ the exposure of the +individual survey exposures. The signal, variance, and mask images +resulting from these deep stacks are part of the DR1 release and are +available from the image extraction tools. + +For the PV3 processing of the Medium Deep fields, stacks have been +generated for the nightly groups and for the full depth using all +exposures (deep stacks). In addition, a 'best seeing' set of stack +have been produced \note{using image quality cuts to be described}. +We have also generated out-of-season stacks for the Medium Deep +fields, in which all image not from a particular observing season for +a field are combined into a stack. These later stacks are useful as +deep templates when studying long-term transient events in the Medium +Deep fields as they are not (or less) contaminated by the flux of the +transients from a given season. + +\subsection{Stack Photometry} + +The stack images are generated in the Stack stage of the IPP, but the +source detection and extraction analysis of those images is deferred +until a separate stage, the Stack Photometry stage. This separation +is maintained because the stack photometry analysis is performed on +all 5 filter stack images at the same time. By deferring the +analysis, the processing system may decouple the generation of the +pixels from the source detection. This makes the sequencing of +analysis somewhat easier and less subject to blocks due to a failure +in the stacking analysis. + +The stack photometry algorithms are described in detail in +\note{Magnier et al}. In short, sources are detected in all 5 filter +images down to the $5\sigma$ significance. The collection of detected +sources is merged into a single master list. If a source is detected +in at least two bands, or only in $y$-band, then a PSF model is fitted +to the pixels of the other bands in which the source was not detected. +This forced photometry results in lower significance measurements of +the flux at the positions of objects which are thought to be real +sources, by virtue of triggering a detection in at least two bands. +The relaxed limit for $y$-band is included to allow for searches of +$y$-dropout objects: it is known that faint, high-redshift quasars may +be detected in $y$-band only. The casual user of the PV3 dataset +should be wary of sources detected only in $y$-band as these are +likely to have a higher false-positive rate than the other stack +sources. + +The stack photometry output files consist of a set of FITS tables in a +single file, with one file for each filter. Within one of these +files, the tables include: the measurements of sources based on the +PSF model; aperture like parameters such as the Petrosian flux and +radius; the convolved Galaxy model fits; the radial aperture +measurements. \note{is this list complete?} + +The stack photometry output catalogs are re-calibrated for both +photometry and astrometry in a process very similar to the Camera +calibration stage. In the case of the stack calibration, however, +each skycell is processed independently. The same reference catalog +is used for the Camera and Stack calibration stages. + +\subsection{Forced Warp Photometry} + +Traditionally, projects which use multiple exposures to increase the +depth and sensitivity of the observations have generated something +equivalent to the stack images produced by the IPP analysis. In +theory, the photometry of the stack images produces the `best' +photometry catalog, with best sensitivity and the best data quality at +all magnitudes (c.f, CFHT Legacy survey, COSMOS, etc). In practice, +the stack images have some significant limitations due to the +difficulty of modelling the PSF variations. This difficulty is +particularly severe for the Pan-STARRS $3\pi$ survey stacks due to the +combination of the substantial mask fraction of the individual +exposures, the large instrinsic image quality variations within a +single exposure, and the wide range of image quality conditions under +which data were obtained and used to generate the $3\pi$ PV3 stacks. + +For any specific stack, the point spread function at a particular +location is the result of the combination of the point spread +functions for those individual exposures which went into the stack at +that point. Because of the high mask fraction, the exposures which +contributed to pixels at one location may be somewhat different just a +few tens of pixels away. Because of the intrinsic variations in the +PSF across an exposure and because of the variations from exposure to +exposure, the distribution of point spread functions of the images +used at one position may be quite different from those at a nearby +location. In the end, the stack images have a effective point spread +function which is not just variable, but changing significantly on +small scales in a highly textured fashion. + +Any measurement which relies on a good knowledge of the PSF at the +location of an object either needs to determine the PSF variations +present in the stack, or the measurement will be somewhat degraded. +The highly textured PSF variations make this a very challenging +problem: not would such a PSF model require an unusually fine-grained +PSF model, there would likely not be enough PSF stars in an given +stack to determine the model at the resolution required. The IPP +photometry analysis code uses a PSF model with 2D variations using a +grid of at most $6\times 6$ samples per skycell, a number reasonably +well-matched to the density of stars at most moderate Galactic +latitudes. This scale is far too large to track the fine-grained +changes apparent in the stack images. + +Thus PSF photometry as well as convolved Galaxy models in the stack +are degraded by the PSF variations. Aperture-like measurements are in +general not as affected by the PSF variations, as long as the aperture +in question is large compared to the FWHM of the PSF. + +%% The IPP team initially explored the option of convolving each input +%% warp to a single target PSF chosen to match the worst of the input +%% images for a given stack. + +The PV3 $3\pi$ analysis solves this problem by using the sources +detected in the Stack images and performing forced photometry on the +individual warp images used to generate the stack. This analysis is +performed on all warps for a single filter as a single job, though +this is more of a bookkeepping aid as it is not necessary for the +analysis of the different warps to know about the results of the other +warps. + +In the forced warp photometry, the positions of sources are loaded +from the stack outputs. PSF stars are pre-identified and a PSF model +generated for each warp based on those stars, using the same stars for +all warps to the extent possible (PSF stars which are excessively +masked on a particular image are not used to model the PSF). The PSF +model is fitted to all of the known source positions in the warp +images. Aperture magnitudes, Kron magnitudes, and moments are also +measured at this stage for each warp. Note that the flux measurement +for a faint, but significant, source from the stack image may be at a +low significance ($< 5\sigma$) in any individual warp image; the flux +may even be negative for specific warps. When combined together, +these low-significance measurements will result in a signficant +measurement as the signal-to-noise increases by $\sqrt{N}$. + +\subsection{Forced Galaxy Models} + +The convolved galaxy models are also re-measured on the warp images by +the forced photometry analysis stage. In this analysis, the galaxy +models determined by the stack photometry analysis are used to seed +the analysis in the individual warps. The purpose of this analysis is +the same as the forced PSF photometry: the PSF of the stack is poorly +determined due to the masking and PSF variations in the inputs. +Without a good PSF model, the PSF-convolved galaxy models are of +limited accuracy. + +In the forced galaxy model analysis, we assume that the galaxy +position and position angle, along with the Sersic index if +appropriate, have been sufficiently well determined in the stack +analysis. In this case, the goal is to determine the best values for +the major and minor axis of the elliptical contour and at the same +time the best normalization corresponding to the best elliptical shape +(and thus the best galaxy magnitude value). + +For each warp image, the Stack value for the major and minor axis are +used as the center of a $7\times 7$ grid search of the major and minor +axis parameter values. The grid spacing is defined as a function of +the signal-to-noise of the galaxy in the stack image so that bright +galaxies are measured with a much finer grid spacing that faint +galaxies \note{need to quantify this}. For each grid point, the major +and minor axis values at that point are determined for the model. The +model is then generated and convolved with the PSF model for the warp +image at that point. The resulting model is then compared to the warp +pixel data values and the best fit normalization value is defined. +The normalization and the $\chi^2$ value for each grid point is +recorded. + +For a given galaxy, the result is a collection of $\chi^2$ values for +each of the grid points spanning all warp images. A single $\chi^2$ +grid can then be made from all warps by combining each grid point +across the warps. The combined $\chi^2$ for a single grid point is +simply the sum of all $\chi^2$ values at that point. If, for a single +warp image, the galaxy model is excessively masked, then that image +will be dropped for all grid points for that galaxy. The reduced +$\chi^2$ values can be determined by tracking the total number of warp +pixels used across all warps to generate the combined $\chi^2$ values. +From the combined grid of $\chi^2$ values, the point in the grid with +the minimum $\chi^2$ is found. Quadratic interpolation is used to +determine the major, minor axis values for the interpolated minimum +$\chi^2$ value. The errors on these two parameters is then found by +determining the contour at which the \note{reduced?} $\chi^2$ +increases by 1. + +Thus the Forced Galaxy Model analysis uses the PSF information from +each warp to determine a best set of convovled galaxy models for each +object in the stack images. \note{discuss the subset of galaxy models + and objects}. + +\subsection{Difference Images} + +Two of the primary science drivers for the Pan-STARRS system are the +search hazardous asteroids and the search for Type Ia supernovae to +measure the history of the expansion of the universe. Both of these +projects require the discovery of faint, transient source in the +images. For the hazardous asteroids, and solar system studies in +general, the sources are transient because they are moving between +observations; supernovae are stationary but transient in brightness. +In both cases, the discovery of these sources can be enhanced by +subtracting a static reference image from the image taken at a certain +epoch. The quality of such a difference image can be enhanced by +convolving one or both of the images so that the PSFs in the two +images are matched. \note{discuss Alard-Lupton}. + +In the Difference Image stage, the IPP generates diffferece images for +specified pairs of images. It is possible for the difference image to +be generated from a pair of warp images, from a warp and a stack of +some variety, or from a pair of stacks. During the PS1 survey, pairs +of exposures, call TTI pairs (see~\note{Survey Strategy}), were +obtained for each pointing within a $\approx$ 1 hour period in the +same filter, and to the extent possible with the same orientation and +boresite position. The standard PS1 nightly processing generated +difference images from the resulting warp pairs (`warp-warp diffs'). + +The nightly stacks generated for the Medium Deep fields were combined +with a template reference stack image to generate `stack-stack diffs' +for these fields each night. + +For the PV3 processing, the entire collection of warps for the $3\pi$ +survey were combined with the $3\pi$ stacks to generate `warp-stack +diffs'. + +\subsection{Addstar : DVO Ingest} + +\subsection{Calibration Operations} + +\subsection{IPP to PSPS} + +\subsection{PSPS Load \& Merge} + +\section{IPP Hardware Systems} + +\subsection{Kihei Processing Cluster} + +\subsection{Los Alamos National Labs} + +\subsection{UH Cray Cluster} + +\end{document} Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.calibration/Makefile =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.calibration/Makefile (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.calibration/Makefile (revision 40759) @@ -13,7 +13,49 @@ all: pdf tgz pdf: calibration.pdf -tgz: calibration.tgz + +journal: calibration.journal.tgz +arxiv: calibration.arxiv.tgz quick: calibration.quick.pdf + +PNGPICS = \ +pics/gpc1.layout.pdf \ +pics/A1.pdf \ +pics/A4.pdf \ +pics/photflat.example.v1.png \ +pics/rings.v3.example.png \ +pics/allsky.photom.v2.png \ +pics/photom.pv3.3v4.png \ +pics/KHexample.png \ +pics/KHmap.png \ +pics/DCR.example.png \ +pics/astroflat.gri.v2.png \ +pics/astroflat.zy.v2.png \ +pics/allsky.astrom.pv3.3.png \ +pics/astroflat.repair.png \ +pics/allsky.histogram.astrom.compare.png \ +pics/gaia.photom.v1.png \ +pics/gaia.astrom.mean.png \ +pics/gaia.astrom.sigma.png + +PDFPICS = \ +pics/gpc1.layout.pdf \ +pics/A1.pdf \ +pics/A4.pdf \ +pics/photflat.example.v1.pdf \ +pics/rings.v3.example.pdf \ +pics/allsky.photom.v2.pdf \ +pics/photom.pv3.3v4.pdf \ +pics/KHexample.pdf \ +pics/KHmap.pdf \ +pics/DCR.example.pdf \ +pics/astroflat.gri.v2.pdf \ +pics/astroflat.zy.v2.pdf \ +pics/allsky.astrom.pv3.3.pdf \ +pics/astroflat.repair.pdf \ +pics/allsky.histogram.astrom.compare.pdf \ +pics/gaia.photom.v1.pdf \ +pics/gaia.astrom.mean.pdf \ +pics/gaia.astrom.sigma.pdf FILES = \ @@ -21,19 +63,20 @@ ../inputs/code.sty \ ../inputs/apj.bst \ -pics/photflat.example.sm.png \ -pics/allsky.photom.sigma.sm.png \ -pics/rings.v3.example.png \ -pics/KHexample.png \ -pics/KHmap.png \ -pics/dcr.r2.g.png \ -pics/astroflat.gri.sm.png \ -pics/astroflat.zy.sm.png \ -pics/allsky.astrom.sigma.png \ -pics/gaia.photom.png \ -pics/gaia.astrom.png \ calibration.tex +pics/%.pdf : pics/%.ps + echo $^ + echo $< + echo $@ + echo $* + ps2pdf -dEPSCrop $< $@ + +# pdfpics: $(PDFPICS) + calibration.pdf: $(FILES) -calibration.tgz: $(FILES) + +calibration.journal.tgz: $(FILES) $(PDFPICS) calibration.bbl +calibration..arxiv.tgz: $(FILES) $(PNGPICS) calibration.bbl include ../Makefile.Common + Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.calibration/calibration.tex =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.calibration/calibration.tex (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.calibration/calibration.tex (revision 40759) @@ -21,6 +21,6 @@ %% NOTE: 2019 Feb versions of the figures are generated in /data/kukui.1/eugene/cal.paper.20190217 -%\def\picdir{/home/eugene/chipresid.20140404} -\def\picdir{/data/pikake.2/eugene/chipresid.20140404} +\def\picdir{pics} +%\def\picdir{.} % Pick a terse version of the title here; @@ -41,7 +41,9 @@ \def\MPIA{6} \def\ARI{7} -\def\Princeton{8} -\def\DUR{9} -\def\CfA{10} +\def\STScI{8} +\def\JHU{9} +\def\Princeton{10} +\def\DUR{11} +\def\CfA{12} % This example has a first author from UH: @@ -54,4 +56,5 @@ S. R\"oser,\altaffilmark{\ARI} E. Schilbach,\altaffilmark{\ARI} +S. Casertano,\altaffilmark{\STScI,\JHU} K.~C. Chambers,\altaffilmark{\IfA} H.~A. Flewelling,\altaffilmark{\IfA} @@ -62,5 +65,5 @@ % PS1 Builders L. Denneau,\altaffilmark{\IfA} -P. Draper,\altaffilmark{\DUR} +P.~W. Draper,\altaffilmark{\DUR} K. W. Hodapp,\altaffilmark{\IfA} R. Jedicke,\altaffilmark{\IfA} @@ -81,4 +84,5 @@ } % this bracket terminates author list +% The ordering here should be sequential, matching the sequence in the list of authors: \altaffiltext{\IfA}{Institute for Astronomy, University of Hawaii, 2680 Woodlawn Drive, Honolulu HI 96822} \altaffiltext{\LBL}{Lawrence Berkeley National Laboratory, One Cyclotron Road, Berkeley, CA 94720, USA} @@ -88,13 +92,9 @@ \altaffiltext{\MPIA}{Max Planck Institute for Astronomy, K\"onigstuhl 17, D-69117 Heidelberg, Germany} \altaffiltext{\ARI}{Astronomisches Rechen-Institut, Zentrum f\"ur Astronomie der Universit\"at Heidelberg, M\"ochhofstrasse 12-14, D-69120 Heidelberg, Germany} +\altaffiltext{\STScI}{Space Telescope Science Institute, 3700 San Martin Drive, Baltimore, MD 21218, USA} +\altaffiltext{\JHU}{Department of Physics and Astronomy, Johns Hopkins University, 3400 North Charles Street, Baltimore, MD 21218, USA} \altaffiltext{\Princeton}{Department of Astrophysical Sciences, Princeton University, Princeton, NJ 08544, USA} \altaffiltext{\DUR}{Department of Physics, Durham University, South Road, Durham DH1 3LE, UK} \altaffiltext{\CfA}{Harvard-Smithsonian Center for Astrophysics, 60 Garden Street, Cambridge, MA 02138} - -% The ordering here should be sequential, matching the sequence in the list of authors: -% \altaffiltext{\USNO}{US Naval Observatory, Flagstaff Station, Flagstaff, AZ 86001, USA} -% \altaffiltext{\JHU}{Department of Physics and Astronomy, Johns Hopkins University, 3400 North Charles Street, Baltimore, MD 21218, USA} - -% \altaffiltext{\Strassborg}{ \begin{abstract} @@ -114,5 +114,5 @@ % insert additional keywords as appropriate: -\keywords{Surveys:\PSONE } +\keywords{astrometry -- methods: statistical -- proper motions -- Surveys:\PSONE -- techniques: photometric} \section{Introduction}\label{sec:intro} @@ -247,5 +247,5 @@ \begin{figure} \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{{pics/gpc1.layout}.pdf} + \includegraphics[width=0.9\hsize,angle=0,clip]{{\picdir/gpc1.layout}.pdf} \caption{Diagram illustrating layout of OTA devices in GPC1. The blue dots mark the locations of the amplifiers for xy00 cells in @@ -476,34 +476,4 @@ \end{eqnarray} -%% Include a description of the WCS keywords used to represent the fit elements? - -%% {\bf WCS Keywords} When this polynomial representation is written to -%% the output files, a set of WCS keywords are used to define the -%% astrometric transformation elements. It is necessary to transform the -%% simply polynomials above into an alternate form: -%% \begin{eqnarray} -%% P & = & \sum_{i,j} C^P_{i,j} (X_{\rm chip} - X_0)^i (Y_{\rm chip} - Y_0)^j \\ -%% Q & = & \sum_{i,j} C^Q_{i,j} (X_{\rm chip} - X_0)^i (Y_{\rm chip} - Y_0)^j -%% \end{eqnarray} - -%% \note{need to complete this discussion of the WCS keywords, both -%% standard and non-standard, used to represent these polynomial -%% transformations} - -%% \begin{verbatim} -%% Here is a list of the keywords -%% and the related terms from Eqns above: -%% CTYPE1,2 : RA---WRP, DEC--WRP -%% CTYPE1,2 : RA---DIS, DEC--DIS -%% CRVAL1,2 : C^{L,M}_{0,0} -%% CRPIX1,2 : X_0, Y_0 -%% PC001001 : C^{L}_{1,0} -%% PC001002 : C^{L}_{0,1} -%% PC002001 : C^{M}_{1,0} -%% PC002002 : C^{M}_{0,1} -%% PCA1XiYj : C^{L}_{i,j} -%% PCA2XiYj : C^{M}_{i,j} -%% \end{verbatim} - \subsection{Cross-Correlation Search} @@ -543,6 +513,4 @@ astrometry guess for the chip. -%% \note{option to downweight based on photometric inconsistency : not used in PS1 analysis} - \subsection{Pipeline Astrometric Calibration} @@ -586,6 +554,4 @@ representing the distortion. -%% \note{write out the math of the gradients} - Once the common distortion coming from the optics and atmosphere have been modeled, \ippprog{psastro} determines polynomial transformations @@ -598,9 +564,5 @@ order for the final iterations. -%% \note{quality of the fits as a result of this stage}. - \subsection{Pipeline Photometric Calibration} - -%% \note{define / describe the robust median} After the astrometric calibration is determined, the photometric @@ -671,7 +633,7 @@ Section~\ref{sec:synthdb}) was merged in. Next, the full Tycho database was added, followed by the AllWISE database. After the Gaia -release in August 2016 \citep{2016AA...595A...2G}, we generated a DVO -database of the Gaia positional and photometric information and merged -that into the master PV3 $3\pi$ DVO database. +Data Release 1 (DR1) in August 2016 \citep{2016AA...595A...2G}, we +generated a DVO database of the Gaia positional and photometric +information and merged that into the master PV3 $3\pi$ DVO database. The master DVO database is used to perform the full photometric and @@ -869,6 +831,4 @@ \end{table*} -%% \note{need to describe the assignment of flags, etc, for the external data sources}. - \section{Photometry Calibration} @@ -1033,6 +993,4 @@ \subsection{Relphot Analysis} -%% \note{how many exposures are not in ubercal?} - Relative photometry is used to determine the zero points of the exposures which were not included in the ubercal analysis. The @@ -1059,6 +1017,4 @@ is taken up as an additional element of the atmospheric attenuation. -%% \note{color-color terms between chips?} - We write a global $\chi^2$ equation which we attempt to minimize by finding the best mean magnitudes for all objects and the best @@ -1095,7 +1051,4 @@ rejections do not catch all cases of bad measurements. -%% \citep[\code{PSF_QF} $< 0.85$, see][]{magnier2017.analysis}; -%% \note{refer to the PSPHOT bad and poor psphot bits?} - After the initial iterations, we also perform outlier rejections based on the consistency of the measurements. For each star, we use a two @@ -1112,6 +1065,4 @@ deviation (of the measurements used for the mean) greater than 0.005 mags or 2$\times$ the median standard deviation, whichever is greater. - -%% \note{is this true?} Similarly for images, we exclude those with more than 2 magnitudes of @@ -1134,6 +1085,4 @@ dominates where they are present. -% \note{do we drop this when calculating the final mean mags?} -% \note{do I need to present the math?} \begin{equation} \mu = \frac{\sum m_i w_i \sigma_i^{-2}}{\sum w_i \sigma_i^{-2}} @@ -1175,8 +1124,10 @@ % this is PV3.0 [pre-calibrations] +% updated version at: +% /data/kukui.1/eugene/cal.paper.images.20190217/flatplots.sh photflat.example \begin{figure*}[htbp] \begin{center} \begin{minipage}{0.85\linewidth} - \includegraphics[width=\textwidth,clip]{{pics/photflat.example.v1}.png} + \includegraphics[width=\textwidth,clip]{{\picdir/photflat.example.v1}.\plotext} \end{minipage} \hspace{-3.0in} @@ -1592,9 +1543,9 @@ % generate from : -% /data/kukui.1/eugene/czw.paper.images.20181130 (see .dvo) +% /data/kukui.1/eugene/cal.paper.images.20190217/rings.sh \begin{figure*}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{{pics/rings.v3.example}.png} + \includegraphics[width=\hsize,clip]{{\picdir/rings.v3.example}.\plotext} \caption{\label{fig:rings.v3.example} Illustration of overlapping skycells and the identification of the ``primary'' detections.} @@ -1828,8 +1779,9 @@ \subsection{Photometry Calibration Quality} +% /data/kukui.1/eugene/cal.paper.images.20190217/scatter.sh : allsky.scatter.photom \begin{figure*}[htbp] \begin{center} %width=\hsize - \includegraphics[height=\vsize,clip]{{pics/allsky.photom.v1}.png} + \includegraphics[height=\vsize,clip]{{\picdir/allsky.photom.v2}.\plotext} \caption{\label{fig:allsky.photom.sigma} Consistency of photometry measurements across the sky. Each panel shows a map of the @@ -1876,7 +1828,8 @@ 18)$ millimagnitudes. +% /data/kukui.1/eugene/cal.paper.images.20190217/kronrepair.sh : full.figure \begin{figure*}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{{pics/photom.pv3.3v4}.png} + \includegraphics[width=\hsize,clip]{{\picdir/photom.pv3.3v4}.\plotext} \caption{\label{fig:photom.pv3.3v4} Sample comparison of PV3.3 and PV3.4 photometry illustrating the impact of the issues identified @@ -1919,8 +1872,10 @@ \section{Astrometry Calibration} - +\label{sec:astrometry} + +% /data/kukui.3/eugene/pv3.stats.20161202/mana.sh \begin{figure*}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{{pics/KHexample}.png} + \includegraphics[width=\hsize,clip]{{\picdir/KHexample}.\plotext} \caption{\label{fig:KHexample} Illustration of the Koppenh\"ofer Effect on OTA04. {\bf Bottom left} X-direction before correction. The solid line shows the measured @@ -1933,9 +1888,8 @@ \end{figure*} -% from: /data/kukui.3/eugene/pv3.stats.20161202/ - +% /data/kukui.3/eugene/pv3.stats.20161202/mana.sh \begin{figure}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{{pics/KHmap}.png} + \includegraphics[width=\hsize,clip]{{\picdir/KHmap}.\plotext} \caption{\label{fig:KHmap} Map of the amplitude of the Koppenh\"ofer Effect on chips across the focal plane. In the @@ -2077,7 +2031,8 @@ % /data/ipp094.0/eugene/pv3.cam.20150607/astrom.corrections/dcr.meas.20151203.0.fits +% /data/kukui.3/eugene/dcr.20141205/dvo.dcr.sh : figure8 \begin{figure}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{{pics/dcr.r2.g}.png} + \includegraphics[width=\hsize,clip]{{\picdir/DCR.example}.\plotext} \caption{\label{fig:DCRexample} Example of the DCR trend in the g-band. {\bf top:} DCR trend in the parallactic direction {\bf @@ -2118,14 +2073,17 @@ % /data/ipp105.0/eugene/astrom.20170225/astroflat.20170217/astroflat.20170217.med.cam.dX.g.fits +% last version in : +% /data/kukui.1/eugene/cal.paper.images.20190217/flatplots.sh astroflat.example \begin{figure*}[htbp] \begin{center} - \includegraphics[width=0.85\textwidth,clip]{{pics/astroflat.gri.v1}.png} + \includegraphics[width=0.85\textwidth,clip]{{\picdir/astroflat.gri.v2}.\plotext} \caption{\label{fig:astroflat.gri} High-resolution astrometric flat-field correction images for $gri$.} \end{center} \end{figure*} +% /data/kukui.1/eugene/cal.paper.images.20190217/flatplots.sh astroflat.example \begin{figure*}[htbp] \begin{center} - \includegraphics[width=0.85\textwidth,clip]{{pics/astroflat.zy.v1}.png} + \includegraphics[width=0.85\textwidth,clip]{{\picdir/astroflat.zy.v2}.\plotext} \caption{\label{fig:astroflat.zy} High-resolution astrometric flat-field correction images for $zy$.} \end{center} @@ -2325,14 +2283,14 @@ For the initial PV3 analysis, we again used the 2MASS coordinates as -an external astrometric reference. After the DR1 object parameters -were ingested into the PSPS database, the Gaia DR1 astrometry was -released \citep{2016AA...595A...4L}. This gave us the option to use -the Gaia positions for the external astrometric reference. We re-did -the astrometric analysis and generated a Gaia-based astrometry table -for the Pan-STARRS DR1. For Pan-STARRS DR2, the average object -coordinates are based on the analysis using the Gaia coordinates. The -Gaia DR1 coordinates used a fixed 2015 epoch. Coordinates were -propagated from that epoch to the epoch for each PS1 image as -described above. +an external astrometric reference. After the Pan-STARRS DR1 object +parameters were ingested into the PSPS database, the Gaia DR1 +astrometry was released \citep{2016AA...595A...4L}. This gave us the +option to use the Gaia positions for the external astrometric +reference. We re-did the astrometric analysis and generated a +Gaia-based astrometry table for the Pan-STARRS DR1. For Pan-STARRS +DR2, the average object coordinates are based on the analysis using +the Gaia DR1 coordinates. The Gaia DR1 coordinates used a fixed 2015 +epoch. Coordinates were propagated from that epoch to the epoch for +each PS1 image as described above. \subsection{Object Astrometry} @@ -2347,5 +2305,5 @@ PS1 \ippstage{chip}-stage measurements were used for the astrometry measurement (no stack or forced-warp measurements). If available, the -2MASS and Gaia astrometry for an object was also used in the +2MASS and Gaia DR1 astrometry for an object was also used in the calculation of the astrometry. Measurements which were kept for the astrometric fit for an object were marked with the bit-flags @@ -2355,5 +2313,5 @@ the bit flag \code{ID_MEAS_POOR_ASTROM}. -If 2MASS or Gaia astrometry measurements +If 2MASS or Gaia DR1 astrometry measurements were available for an object, {\em all} measurements for that object are marked with the bit-flag \code{ID_MEAS_OBJECT_HAS_2MASS} or @@ -2494,12 +2452,13 @@ \subsection{Astrometry Calibration Quality} +% /data/kukui.1/eugene/cal.paper.images.20190217/scatter.sh : allsky.scatter.astrom \begin{figure*}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{{pics/allsky.astrom.pv3.3}.png} + \includegraphics[width=\hsize,clip]{{\picdir/allsky.astrom.pv3.3}.\plotext} \caption{\label{fig:allsky.astrom.sigma} Consistency of astrometry measurements across the sky. Each panel shows a map of the standard deviation of astrometry residuals for stars in each pixel. The median value of the standard deviations across the sky - is $(\sigma_\alpha, \sigma_\delta) = (22, 23)$ milliarcseconds. + is $(\sigma_\alpha, \sigma_\delta) = (16, 16)$ milliarcseconds. These values reflect the typical single-measurement errors for bright stars. See discussion regarding the astrometric flat which @@ -2508,7 +2467,8 @@ \end{figure*} +% /data/kukui.1/eugene/cal.paper.images.20190217/flatplots.sh : astroflat.repair \begin{figure*}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{{pics/astroflat.repair}.png} + \includegraphics[width=\hsize,clip]{{\picdir/astroflat.repair}.\plotext} \caption{\label{fig:astroflat.repair} Comparison of the high-resolution astrometric flat-field images used for PV3.2 @@ -2535,7 +2495,8 @@ %% filter y : 42867074 stars +% /data/kukui.1/eugene/cal.paper.images.20190217/scatter.sh : allsky.histogram.astrom.compare \begin{figure*}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{{pics/allsky.histogram.astrom.compare}.png} + \includegraphics[width=\hsize,clip]{{\picdir/allsky.histogram.astrom.compare}.\plotext} \caption{\label{fig:allsky.astro.histogram} Illustration of the impact of the astrometric flat-field correction used for PV3.2 vs @@ -2577,5 +2538,5 @@ photometry, we attribute this to failure of the PSF fitting due to crowding. The celestial North pole regions have somewhat elevated -errors in both R.A. and DEC, with some specifc structures. Some of +errors in both R.A.\ and DEC, with some specifc structures. Some of these structures may be due to the larger typical seeing at these high airmass regions, but some are due to astrometric failures which stem @@ -2583,6 +2544,6 @@ Section~\ref{sec:pole.problems} for further details). Several features can be seen which appear to be an effect of the tie to the -Gaia astrometry: the stripes near the center of the DEC image and the -right side of the R.A. image. The mesh of circular outlines one the 2 +Gaia DR1 astrometry: the stripes near the center of the DEC image and the +right side of the R.A.\ image. The mesh of circular outlines one the 2 degree scale is due to the outer edge of the focal plane where the astrometric calibration is poorly determined. @@ -2697,4 +2658,5 @@ \subsection{Comparison to Gaia} +\label{sec:gaia.tie} After the full relative astrometry analysis was performed for the PV3 @@ -2704,12 +2666,12 @@ observations. Gaia DR1 objects which are bright enough to have proper motion and parallax solutions are in general saturated in the PS1 -observations. Thus, we are limited to using the Gaia mean positions -reported for the fainter stars. We extracted all Gaia sources not +observations. Thus, we are limited to using the Gaia DR1 mean positions +reported for the fainter stars. We extracted all Gaia DR1 sources not marked as a duplicate from the Gaia archive and generated a DVO -database from this dataset. We then merged the Gaia DVO into the PV3 +database from this dataset. We then merged the Gaia DR1 DVO into the PV3 master DVO database. We re-ran the complete relative astrometry -analysis using Gaia as an additional measurement. We applied the +analysis using Gaia DR1 as an additional measurement. We applied the analysis described above, applying the estimated distances to -determine preliminary proper motions. The Gaia mean epoch is reported +determine preliminary proper motions. The Gaia DR1 mean epoch is reported as 2015.0, so all Gaia measurements were assigned this epoch. We wanted to ensure the Gaia measurements dominated the astrometric @@ -2723,14 +2685,16 @@ even at a lower weight, helps to tile over those gaps. +% /data/kukui.3/eugene/pv3.stats.20161022/plots.sh + \begin{figure*}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{{pics/gaia.photom}.png} - \caption{\label{fig:gaia.photom} Comparison with Gaia - photometry. {\bf Left} Mean of PS1 - Gaia, {\bf Right} Standard - deviation of PS1 - Gaia. For pixels with $|b| > 30$ and $\delta > - -30$, the standard deviation of the PS1 - Gaia mean values is 7 - millimagnitudes, while the median of the standard deviations is 12 + \includegraphics[width=\hsize,clip]{{\picdir/gaia.photom.v1}.\plotext} + \caption{\label{fig:gaia.photom} Comparison with Gaia DR1 + photometry. {\bf Left} Mean of PS1 - Gaia DR1, {\bf Right} Standard + deviation of PS1 - Gaia DR1. For pixels with $|b| > 30$ and $\delta > + -30$, the standard deviation of the PS1 - Gaia DR1 mean values is 6.9 + millimagnitudes, while the median of the standard deviations is 12.4 millimagnitudes. The former is a statement about the consistency - of the Gaia and Pan-STARRS\,1 photometry, while the latter + of the Gaia DR1 and Pan-STARRS\,1 photometry, while the latter reflects the combined bright-end errors for both systems. } \end{center} @@ -2738,5 +2702,5 @@ Figure~\ref{fig:gaia.photom} shows a comparison between the Pan-STARRS -photometry in $g,r,i$ and the Gaia photometry in the $G$-band. To +photometry in $g,r,i$ and the Gaia DR1 photometry in the $G$-band. To compare the PS1 photometry to the very broadband Gaia G filter, we have determined a transformation based on a 3rd order polynomial fit @@ -2779,9 +2743,10 @@ \begin{figure*}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{{pics/gaia.astrom}.png} + \includegraphics[width=0.48\hsize,clip]{{\picdir/gaia.astrom.mean}.\plotext} + \includegraphics[width=0.48\hsize,clip]{{\picdir/gaia.astrom.sigma}.\plotext} \caption{\label{fig:gaia.astrom} Comparison with Gaia - astrometry. {\bf Left} Mean of PS1 - Gaia, {\bf Right} Standard - deviation of PS1 - Gaia. The median value of the standard - deviations is $(\sigma_\alpha, \sigma_\delta) = (4, 3)$ + astrometry. {\bf Left} Mean of PS1 - Gaia DR1, {\bf Right} Standard + deviation of PS1 - Gaia DR1. The median value of the standard + deviations is $(\sigma_\alpha, \sigma_\delta) = (4.8, 3.1)$ milliarcseconds. } \end{center} @@ -2789,16 +2754,16 @@ Figure~\ref{fig:gaia.astrom} shows a comparison between the Pan-STARRS -mean astrometry positions in $\alpha,\delta$ and the Gaia astrometry. +mean astrometry positions in $\alpha,\delta$ and the Gaia DR1 astrometry. For this comparison, we have seleted all PS1 stars with Gaia -measurements with $14 < i < 19$ and with at least 10 total +measurements with $14 < \ips < 19$ and with at least 10 total measurements. For Figure~\ref{fig:gaia.astrom}, we calculate the -difference between the position predicted by PS1 at the Gaia epoch +difference between the position predicted by PS1 at the Gaia DR1 epoch (using the proper motion and parallax fit) and the position reported by Gaia. For each pixel, we determine the histogram of these -differences in the R.A\. and DEC directions, and calculate the median +differences in the R.A.\ and DEC directions, and calculate the median and the 68\%-ile range. In Figure~\ref{fig:gaia.astrom}, these values are plotted as a color scale. -There is good consistency between the PS1 and Gaia astrometry. There +There is good consistency between the PS1 and Gaia DR1 astrometry. There are patterns from the Galactic plane (though not very strongly at the bulge). There are also clear features due to the PS1 exposure @@ -2809,14 +2774,179 @@ statisics of the per-exposure measurement residuals (Figure~\ref{fig:allsky.astrom.sigma}. The standard deviations of the -median differences are ($\sigma_\alpha, \sigma_\delta) = (4, 3)$ +median differences are ($\sigma_\alpha, \sigma_\delta) = (4.8, 3.1)$ milliarcseconds. For a future data release, we will recalibrate the Pan-STARRS $3\pi$ -astrometry using the Gaia DR2 release. The addition of Gaia-measured -proper motions will obviate the need to correct for the Galactic rotation. +astrometry using the Gaia DR2 release \citep{2018AA...616A...1G}. The +addition of Gaia-measured proper motions will obviate the need to +correct for the Galactic rotation. + +\section{Polar Astrometry Issues} +\label{sec:pole.problems} + +Internal consistency testing of the PV3 stack measurements indicated +potential problems with the astrometric registration of the exposures +in small areas near the North Pole. These issues were originally +suggested by a few high-latitude sources with significant differences +in morphology or position across bands, including strong (and +anomalous) apparent color gradients. Direct investigation of a few of +these anomalous sources demonstrated the presence of significant +misalignments between exposures; one of the worst cases is shown in +Figure~\ref{fig:pole.issue.example}. While such sources appeared to be +rare, astrometric registration errors have the potential to affect +several different source properties: morphology and photometry in +addition to astrometry. Therefore we carried out an astrometric +regsitration test for all skycells North of $\delta=+70\mathdegree$. + +\begin{figure*}[htbp] + \begin{center} + \includegraphics[width=\hsize,clip]{{\picdir/A1}.pdf} + \caption{\label{fig:pole.issue.example} Example of a stack source badly affected by polar astrometry failures. Source from multiple detections from skycell 2643.093.} + \end{center} +\end{figure*} + +This test was based primarily on the ``original detection positions'', +\ie, the positions of detections found in individual exposures as +measured after each exposure's astrometric calibration, but before +recalibration of the combined values to the Gaia reference frame +(described in Section~\ref{sec:gaia.tie}) since that step had the +opportunity to repair any astrometric failures. We started by +collecting the original detection positions (as defined above) for +each skycell. To ensure good signal-to-noise ratios and minimize +potential spurious detections, we used only the top quartile (in flux) +of detections within each chip. We grouped these detections on a +filter-by-filter basis within a radius of $ 2\farcs5 $ (10 pixels), +ensuring that each group contained only one source per exposure, and +retaining only groups with at least five detections; we then recorded +the 2-D position dispersion for each group. The mean positions for +each group were cross-correlated against the Gaia DR2 sources \citep{2018AA...616A...1G}, showing +that these were real sources and providing information on their +absolute astrometry. + +Overall, the vast majority of the detection groups thus defined have +good consistency between source positions, resulting in an astrometric +dispersion of 1 pixel or less. A few ``bad'' groups, defined as +having an internal dispersion $ > 1 $ pixel, can result from spurious +sources or other anomalies, and are generally rare (fewer than a few +percent of all groups). However, some skycells have a significant +fraction ($ > 10\%$) of bad groups. Direct inspection demonstrates +that the incidence of bad groups is related to astrometric +registration failures. + +\begin{figure*}[htbp] + \begin{center} + \includegraphics[width=\hsize,clip]{{\picdir/A4}.pdf} + \caption{\label{fig:pole.bad.histogram} Histogram of the fraction of bad groups for each skycell (red line).} + \end{center} +\end{figure*} + +Bad skycells, defined as those with more than 10\% bad groups, are +essentially limited to the North polar cap ($ \delta > +80\mathdegree$). +Of the 2500 skycells in this region, 164, or 6.6\%, have more than 10\% +bad groups; 64 of these have more than 20\% bad groups. By comparison, +essentially no skycells between $+70\mathdegree$ and $+80\mathdegree$ have +more than 10\% bad groups. Figure~\ref{fig:pole.bad.histogram} shows a histogram +of the fraction of bad groups for each skycell. + +In order to have an independent validation of the impact of this +astrometric alignment issue, we also carried out a photometric test +based on a comparison between stack and mean object photometry. In the +presence of modest registration errors, mean object photometry would +not be affected, as individual detection woulds have the correct +signal, and averaging their flux in catalog space would yield the +correct total magnitude. On the other hand, imperfect stacking would +result in a dilution of the total signal on a pixel-by-pixel basis, +and result in potentially larger estimated sizes and smaller total +flux for stack sources. Indeed, mean magnitudes are brighter than +stack magnitudes for a significant fraction of the sources in the same +skycells that are identified as bad by the relative astrometry test. +Therefore we confirm that the astrometric registration issues result +in poor stack photometry for the affected skycells. + +Further investigaion revealed that the cause of these failures was an +error in the internal reference catalog used for the PV3 analysis (see +Section~\ref{sec:synthdb}). This reference catalog used PS1 +observations to generate a catalog of \grizy\ photometry tied to the +2MASS astrometric system. The astrometry used for this catalog was +generated using the analysis discussed in Section~\ref{sec:astrometry} +to define a collection of reference stars with a coordinate system +tied to 2MASS but with the higher accuracy of the Pan-STARRS +measurements on small spatial scales. Unfortunately, in the vicinity +of the celestial north pole, this reference catalogs was contaminated +by a number of poor measurements. In this portion of the sky, the +astrometric registration of the exposures is more challanging due to +the degeneracy between boresite position errors and field rotation. +In addition, the PS1 telescope suffers from larger pointing errors +near the celestial north pole, largely for the same reason. Because +of these two factors, a number of exposures near the celestial pole +were included in the reference database with invalid astrometry, +injecting apparently good reference stars in the database with +positions displaced from the true position by 1-2 arcseconds. +Sometimes a chip processed in this region would find an astrometric +solution using only good reference stars. Sometimes the solution +would use only bad reference stars, resulting in a chip apparently +displaced from the truth position by 1-2 arcseconds. + +To correct the astrometry failures that caused the original errors in +the reference catalog, we extended the field rotation search range for +the polar exposures. We also added tests to the analysis of the +exposures to ensure they would not fail in a marginal way and +introduce poor solutions into the calibration database. We then ran a +test to confirm that we could generate good astrometry in this region +with an acceptable reference catalog. + +We first used the PV3 mean astrometry and photometry to define a new +reference catalog in the assumption that the bulk of the failures +would be eliminated by the astrometric recalibration. We reprocesed a +section of the polar cap data using this PV3-based reference catalog +and re-ran the astrometric registration test was repeated on the +reprocessed exposures. The reprocessing greatly ameliorated the +registration issue, as shown in Figure~\ref{fig:pole.bad.histogram}. +Here the red line shows the histogram of the fraction of bad groups +for each skycell {\sl before reprocessing}, while the black line +refers to the results {\sl after reprocessing}. The improvement is +apparent. After reprocessing, only 23 cells, instead of the original +164, exceed 10\% of bad groups, and even for these the fraction of bad +groups is substantially reduced. + +To further improve the astrometric calibration reliability in this +region, we have generated a new reference catalog combining the PS1 +PV3 photometry with astrometry from Gaia DR2 \citep{2018AA...616A...1G}. We are reprocessing all +images from the region North of $+70\mathdegree$ and will provide a +complete Polar Region release using the same data as used for DR2. +This updated release is expected to be available from MAST near the +end of summer 2019. + +We consider skycells with more than 10\% bad groups to have been +adversely affected by this problem. Uses of DR2 should be aware that +the affected skycells have poor astrometry and effective image +quality. However, as these images may be useful to the community, +they are available from the MAST cutout server. Users who attempt to +download these problem skycells will see a warning message and should +avoid using the skycell images for quantitative measurements without +extreme caution. Since stack measurements from these skycells are +significantly damaged, the DR2 release has set the measured stack +properties of these objects to a null value. Again, users should +exercise caution with sources from the affected skycells. \section{Conclusion} -\note{WRITE THIS} +The Pan-STARRS Data Release 2 provides astromtry and photometry of +roughly 3 billion astronomical objects across the $3\pi$ survey +region. The photometry system has been shown to be reliable across +the sky at the level of (8.0, 7.0, 9.0, 10.7, 12.4) millimags in +(\grizy). The median value of the measure standard deviations for +stars across the sky is $(\sigma_g, \sigma_r, \sigma_i, \sigma_z, +\sigma_y) = (14, 14, 15, 15, 18)$ millimags, reflecting the systematic +floor on the accuracy of individual measurements of bright stars. The +astrometric calibration is tied to the Gaia DR1 frame with a +systematic error floor of ($\sigma_\alpha, \sigma_\delta) = (4.8, +3.1)$ milliarcseconds. The median residual astrometric scatter for +bright objects across the sky is 16 milliarcseconds in both R.A.\ and +DEC. Caution should be used for 164 skycells in the celestial north +pole regions where the reference catalog was contaminated with +astrometric failures. The Pan-STARRS DR2 photometry and astrometry +will be a valuable resource for many years for the astronomical +community. \acknowledgments @@ -2836,14 +2966,17 @@ NASA Science Mission Directorate, the National Science Foundation under Grant No. AST-1238877, the University of Maryland, and Eotvos -Lorand University (ELTE) and the Los Alamos National Laboratory. -Colormaps for Figures \ref{fig:photflat}, -\ref{fig:allsky.photom.sigma}, \ref{fig:photom.pv3.3v4}, -\ref{fig:astroflat.gri}, \ref{fig:astroflat.zy}, -\ref{fig:allsky.astrom.sigma}, and \ref{fig:astroflat.repair} from -Peter Kovesi \citep[Good Colour Maps: How to Design Them.][]{2015arXiv150903700K}. +Lorand University (ELTE) and the Los Alamos National Laboratory. EAM +is also supported for portions of this work by National Science +Foundation Grant No. AST-1313455. Colormaps for Figures +\ref{fig:photflat}, \ref{fig:allsky.photom.sigma}, +\ref{fig:photom.pv3.3v4}, \ref{fig:astroflat.gri}, +\ref{fig:astroflat.zy}, \ref{fig:allsky.astrom.sigma}, and +\ref{fig:astroflat.repair} are based on the matplotlib ``magma'' +colormap with additional guidance from Peter Kovesi's work \citep[Good + Colour Maps: How to Design Them.][]{2015arXiv150903700K}. \bibliographystyle{apj} -\bibliography{lib}{} -% \input{calibration.bbl} +% \bibliography{lib}{} +\input{calibration.bbl} \end{document} Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.calibration/fluxave.sh =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.calibration/fluxave.sh (revision 40759) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.calibration/fluxave.sh (revision 40759) @@ -0,0 +1,82 @@ + +# what is the difference in the weighted average magnitue vs weighted average flux? +macro mag.or.flux + + # generate 1000 measurements of a Gaussian distributed flux at the given average S/N + # measure the weigthed average flux + # convert the fluxes to mags + # measure the weighted average mag + + if ($0 != 4) + echo "USAGE: mag.or.flux (S/N) (Nmeas) (Ntrial)" + break + end + + local Nmeas SN + $SN = $1 + $Nmeas = $2 + $Ntrial = $3 + + $Flux = 100.0 + $dFlux = $Flux / $SN + + create n 0 $Nmeas + + delete -q Fo Mo + for i 0 $Ntrial + gaussdev df $Nmeas 0.0 $dFlux + + set flux = $Flux + df + + set s1 = flux / $dFlux^2 + set s2 = zero(flux) + 1 / $dFlux^2 + + vstat -q s1 + $S1 = $TOTAL + + vstat -q s2 + $S2 = $TOTAL + + concat {$S1 / $S2} Fo + + set mag = -2.5*log(flux) + $dMag = 1.086 * ($dFlux / $Flux) + + set s1 = mag / $dMag^2 + set s2 = zero(mag) + 1 / $dMag^2 + + vstat -q s1 + $S1 = $TOTAL + + vstat -q s2 + $S2 = $TOTAL + + concat {$S1 / $S2} Mo + end + + clear -s + + section a 0 0 1 0.5 + vstat -q Fo + set N = ramp(Fo) + lim N Fo; box; plot N Fo + line -c red 0 $MEAN to $Ntrial $MEAN + line -c blue 0 {$MEAN - $SIGMA} to $Ntrial {$MEAN - $SIGMA} + line -c blue 0 {$MEAN + $SIGMA} to $Ntrial {$MEAN + $SIGMA} + # echo $MEAN +/- $SIGMA + $Fave = $MEAN + $dFave = $SIGMA + + section b 0 0.5 1 0.5 + vstat -q Mo + set N = ramp(Mo) + lim N Mo; box; plot N Mo + line -c red 0 $MEAN to $Ntrial $MEAN + line -c blue 0 {$MEAN - $SIGMA} to $Ntrial {$MEAN - $SIGMA} + line -c blue 0 {$MEAN + $SIGMA} to $Ntrial {$MEAN + $SIGMA} + # echo $MEAN +/- $SIGMA + $Mave = $MEAN + $dMave = $SIGMA + + echo $Mave vs {-2.5*log($Fave)} : {$Mave + 2.5*log($Fave)}, $dMave vs {1.086 * $dFave / $Fave} +end Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.datasystem/Makefile =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.datasystem/Makefile (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.datasystem/Makefile (revision 40759) @@ -9,8 +9,11 @@ all: pdf tgz -tgz: datasystem.tgz +journal: datasystem.journal.tgz +arxiv: datasystem.arxiv.tgz pdf: datasystem.pdf quick: datasystem.quick.pdf + +BIBLIB = ../inputs/lib.bib FILES = \ @@ -18,11 +21,18 @@ ../inputs/code.sty \ ../inputs/apj.bst \ -../inputs/lib.bib \ PS1_Data_Analysis_System_Overview.pdf \ -skypartition.png \ datasystem.tex -datasystem.pdf: $(FILES) -datasystem.tgz: $(FILES) +PNGPICS = \ +skypartition.png + +EPSPICS = \ +skypartition.eps + +datasystem.pdf: $(FILES) $(BIBLIB) $(PNGPICS) + +datasystem.journal.tgz: $(FILES) $(EPSPICS) datasystem.bbl + +datasystem.arxiv.tgz: $(FILES) $(PNGPICS) DIST_TGT = datasystem.pdf datasystem.ps Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.datasystem/datasystem.tex =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.datasystem/datasystem.tex (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.datasystem/datasystem.tex (revision 40759) @@ -1,12 +1,7 @@ -% \documentclass[preprint2]{emulateapj} % works for 2-column \documentclass[iop,floatfix]{emulateapj} -% \documentclass[iop,floatfix,onecolumn]{emulateapj} -% \documentclass[12pt,preprint]{aastex} % \documentclass[10pt,preprint]{aastex} % use for 1-column -% \documentclass[preprint]{aastex} % \pdfoutput=1 %\RequirePackage{deluxetable} -- included by aastex? -%\RequirePackage{nsfprop} % defines \subsubsubsection but breaks 2-col \RequirePackage{color} \RequirePackage{code} @@ -15,13 +10,15 @@ \usepackage[T1]{fontenc}% (2) specify encoding +% these options allow the code to swap between figure types & versions: + % online version may use color, but print version needs b/w \def\plotmode{col} %\def\plotmode{bw} -%\def\plotext{pdf} -\def\plotext{ps} - -%\def\picdir{/home/eugene/chipresid.20140404} -\def\picdir{/data/pikake.2/eugene/chipresid.20140404} +%\def\plotext{png} +\def\plotext{eps} +% \def\plotext{ps} + +\def\picdir{figures} % Pick a terse version of the title here; @@ -54,5 +51,5 @@ %PS Builder List L. Denneau,\altaffilmark{\IfA} -P. Draper,\altaffilmark{\DUR} +P.~W. Draper,\altaffilmark{\DUR} K. W. Hodapp,\altaffilmark{\IfA} R. Jedicke,\altaffilmark{\IfA} @@ -101,5 +98,5 @@ % insert additional keywords as appropriate: -\keywords{Surveys:\PSONE } +\keywords{Surveys:\PSONE; Methods: data analysis; Techniques: image processing} \section{Introduction} @@ -231,17 +228,4 @@ PV3 data release, with some details on the scale of computing needed to reduce this large number of exposures. - -% Finally, -% Section~\ref{sec:discussion} presents a discussion of some of the -% lessons learned in the creation of the IPP, and its utility in -% reducing data from other cameras and telescopes. - -%% {\color{red} {\em Note: These papers are being placed on arXiv.org to -%% provide crucial support information at the time of the public -%% release of Data Release 1 (DR1). We expect the arXiv versions to -%% be updated prior to submission to the Astrophysical Journal in -%% January 2017. Feedback and suggestions for additional information -%% from early users of the data products are welcome during the -%% submission and refereeing process.}} \section{Overview of Pan-STARRS Data Processing} @@ -277,5 +261,5 @@ ingests the calibrated measurements from the IPP, MOPS, and others and generates a high-availability database with web-based - interactions for public consumption \citep[][]{flewelling2017}. + interactions for public consumption (Paper VI). \end{itemize} @@ -301,6 +285,4 @@ emphasis on the analysis, calibration, and database ingest stages. The MOPS is described in detail by \cite{2013PASP..125..357D}. - -% the summit systems are described by \note{REF?}. \begin{figure*}[htbp] @@ -361,10 +343,10 @@ Petrosian aperture photometry, etc). The results of the stack photometry analysis are used to drive a forced-photometry analysis of -the warp images. These analysis steps are discussed in detail by -\citet[][]{magnier2017.analysis}. The data products from the camera, -stack, and forced-warp photometry analysis stages are ingested into -the internal calibration database (DVO, the Desktop Virtual -Observatory) and used for photometric and astrometric calibrations -\citet[see Section~\ref{sec:DVO} and][]{magnier2017.calibration}. +the warp images. These analysis steps are discussed in detail in +Paper IV. The data products from the camera, stack, and forced-warp +photometry analysis stages are ingested into the internal calibration +database (DVO, the Desktop Virtual Observatory) and used for +photometric and astrometric calibrations (see Section~\ref{sec:DVO} +and Paper V). \subsection{Data Access and Distribution} @@ -384,6 +366,5 @@ (PV1 \& PV2), the data were ingested into the PSPS database system and made available to the PS1SC community through a web portal based at -the IfA as well as the MAST portal \citep[see][for full - details]{flewelling2017}. +the IfA as well as the MAST portal (see Paper VI for full details). \section{IPP Data Processing Stages} @@ -401,11 +382,4 @@ \hline {\bf Stage} & {\bf Primary Table} & {\bf Secondary Table(s)} & {\bf Key} \\% & {\bf Notes} \\ -%%D \begin{deluxetable}{llll} -%%D \tablecolumns{5} -%%D \tablewidth{0pc} -%%D \tablecaption{GPC1 Database Schema Outline} -%%D \tablehead{\colhead{Stage} & \colhead{Primary Table} & \colhead{Secondary Table} & \colhead{Key}} % & \colhead{Notes}} -%%D \startdata -%\hline \ippstage{summitcopy} & \ippdbtable{pzDataStore} & & \\% & Lists locations to check for new exposures.\\ & \ippdbtable{summitExp} & \ippdbtable{summitImfile} & \ippdbcolumn{summit_id} \\% & Exposures available at the telescope.\\ @@ -445,9 +419,7 @@ & \ippdbtable{lapRun} & \ippdbtable{lapExp} & \ippdbcolumn{lap_id} \\% & \\ \ippstage{remote} & \ippdbtable{remoteRun} & \ippdbtable{remoteComponent} & \ippdbcolumn{remote_id} \\% & \\ -%%D \enddata \hline \end{tabular} \label{tab:database_schema} -%%D \end{deluxetable} \end{center} \end{table*} @@ -602,5 +574,5 @@ For GPC1, the \ippstage{registration} stage is also the stage at which the \ippprog{burntool} analysis is run. This analysis is more -completely described in \citet{waters2017}. In brief, the +completely described in Paper III. In brief, the \ippprog{burntool} program identifies bright sources on the image, and identifies persistence trails that result from the incomplete transfer @@ -653,33 +625,15 @@ not been as critical of a requirement as originally expected. -%% In the \ippstage{chip} stage, -%% the individual OTA image files are processed independently in parallel -%% within the data processing cluster. \note{move this to kihei -%% discussion?} Within the processing computer cluster, most of the -%% data storage resources are in the form of computers with large raids -%% as well as substantial processing capability. The processing system -%% attempts to locate one copy of specific raw registered data on -%% pre-defined computers that have been set as storage targets for that -%% OTA. The processing system is aware of this data localization and -%% attempts to target the processing for each OTA to the machine on which -%% the data for that detector is stored. The output products are then -%% primarily saved back to the same machine. This ``targetted'' processing -%% was an early design choice to minimize the system wide network load -%% during processing. In practice, as computer disks filled up at -%% different rates, the data has not been localized to a very high -%% degree. - The actual image processing is performed by the \ippprog{ppImage} program. This program reads the raw data into memory and applies the -detrend corrections \citep[see][]{waters2017} to each cell in the OTA -(stored as different extensions in the FITS file format), and then -mosaics the cells into a single contiguous \ippstage{chip} stage -image. This step also creates in memory additional images to hold the -mask data, which indicates which pixels may not be valid, and the -variance image, constructed as the Poissonian noise on the number of -electrons detected based on the original pixel value and the detector -gain. A background model is then fit across the image and subtracted -to remove the expected contribution from the sky -\citep[see][]{waters2017} for details. +detrend corrections (see Paper III) to each cell in the OTA (stored as +different extensions in the FITS file format), and then mosaics the +cells into a single contiguous \ippstage{chip} stage image. This step +also creates in memory additional images to hold the mask data, which +indicates which pixels may not be valid, and the variance image, +constructed as the Poissonian noise on the number of electrons +detected based on the original pixel value and the detector gain. A +background model is then fit across the image and subtracted to remove +the expected contribution from the sky (see Paper III for details). With the image calibration procedure finished, object identification @@ -689,5 +643,5 @@ this analysis, removing the need to write out and re-read the image data. The details of the detection and characterization of the -sources in the image are provided in \citet{magnier2017.analysis}. +sources in the image are provided in Paper IV. The results of the image processing are then written to disk, @@ -715,23 +669,4 @@ in which case an entry for this exposure is added to the \ippdbtable{camRun} table, and processing continues. - -%% The \ippstage{chip} processing stage consists of: reading the raw image into -%% memory, applying the detrending steps \citep[see][]{waters2017}, -%% stiching the individual OTA cells into a single chip image, detection -%% and characterization of the sources in the image -%% \citep[see][]{magnier2017b}, and output of the various data products. -%% These include the detrended chip image, variance image, and mask -%% image, as well as the FITS catalog of detected sources. The PSF model -%% and background model are also saved, along with a processing log. A -%% selection of summary metadata describing the processing results are -%% saved and written to the processing database along with the completion -%% status of the process. Finally, binned chip images are generated (on -%% two scales, binned by 16 and 256 pixels) for use in the visualization -%% system of the processing monitor tool. \note{describe elsewhere?} - -%% The database structure for the \stage{chip} stage mimics that of raw -%% data, with a \ippdbtable{chipRun} characterizing the processing of a -%% single exposure, mapping to a set of \ippdbtable{chipProcessedImfile} -%% entries for each OTA via a common \ippdbcolumn{chip_id}. \subsection{Camera Calibration} @@ -755,11 +690,10 @@ to help guarantee a solution in the case of a modest pointing error. The guess astrometry is used to match the reference catalog to the -observed stellar positions in the focal plane coordinate system -\citep[see][]{magnier2017.calibration}. Early on in the PS1SC -mission, the nightly processing (PV0) used a reference catalog based -on a combination of external catalogs (2MASS, Tycho, USNO). Later, -reference catalogs based on Pan-STARRS data was used. For the $3\pi$ PV3 analysis, -the reference catalog was based on Pan-STARRS data from the PV2 -analysis \citep[see][for more details]{magnier2017.calibration}. +observed stellar positions in the focal plane coordinate system. +Early on in the PS1SC mission, the nightly processing (PV0) used a +reference catalog based on a combination of external catalogs (2MASS, +Tycho, USNO). Later, reference catalogs based on Pan-STARRS data was +used. For the $3\pi$ PV3 analysis, the reference catalog was based on +Pan-STARRS data from the PV2 analysis (see Paper V for more details). Once an acceptable match is found, the astrometric calibration of the @@ -787,7 +721,7 @@ so a fixed color transformation is used to generate synthetic w-band photometry from the \rps\ \& \ips\ photometry. For more details, see -\cite{magnier2017.calibration}. The result of these calibrations is -stored as a single multi-extension FITS table containing the results -from each OTA as a separate extension. +Paper V. The result of these calibrations is stored as a single +multi-extension FITS table containing the results from each OTA as a +separate extension. In addition to the astrometric and photometric calibrations, the @@ -884,5 +818,5 @@ \ippstage{chip} stage images (including the variance images and the updated masks) to the \ippprog{pswarp} program. For details on the -warping algorithm, see \cite{waters2017}. The outputs of this program +warping algorithm, see Paper III. The outputs of this program are the geometrically transformed images (signal, variance, and mask) containing all input pixels warped to the common skycell pixel grid, @@ -892,8 +826,4 @@ extraction tools at the MAST archive at STScI as part of the DR2 data release. - -%% A catalog is -%% also generated containing the locations of sources from the input -%% catalog that fall within area of the \ippstage{warp}. When the \ippstage{warp} jobs have completed, an entry for the skycell @@ -928,12 +858,11 @@ generated for the nightly groups and for the full depth using all exposures, producing ``deep stacks''. In addition, a ``best seeing'' -set of stacks have been produced using image quality cuts described by -\citet[][Paper VII]{huber2017}. We have also generated out-of-season -stacks for the Medium Deep fields, in which all images {\em not} from a -particular observing season for a field are combined into a stack. -These later stacks are useful as deep templates when studying -long-term transient events in the Medium Deep fields as they are not -(or less) contaminated by the flux of the transients from a given -season. +set of stacks have been produced using image quality cuts described in +Paper VII. We have also generated out-of-season stacks for the Medium +Deep fields, in which all images {\em not} from a particular observing +season for a field are combined into a stack. These later stacks are +useful as deep templates when studying long-term transient events in +the Medium Deep fields as they are not (or less) contaminated by the +flux of the transients from a given season. When a given set of \ippstage{stack} stage processing is defined, @@ -951,5 +880,5 @@ and catalogs to the \ippprog{ppStack} program, which performs the image combinations. Input warps are combined based on a weighting -defined by the median variance for each image; see~\cite{waters2017} +defined by the median variance for each image; see~Paper III for details on the stack combination algorithm. In addition to the standard image, mask, and variance produced at other stages, @@ -987,17 +916,17 @@ The input images are passed to the \ippprog{psphotStack} program which does the analysis. The stack photometry algorithms are described in -detail in \cite{magnier2017.analysis}. In short, sources are detected -in all 5 filter images down to the $5\sigma$ significance. The -collection of detected sources is merged into a single master list. -If a source is detected in at least two bands, or only in \yps{} band, -then a PSF model is fitted to the pixels of the other bands in which -the source was not detected. This forced photometry results in lower -significance measurements of the flux at the positions of objects -which are thought to be real sources, by virtue of triggering a -detection in at least two bands. The relaxed limit for \yps{} band is -included to allow for searches of \yps{} dropout objects: it is known -that faint, high-redshift quasars may be detected in \yps{} band only. -Sources detected only in \yps{} band are therefore more likely to have -a higher false-positive rate than the other stack sources. The +detail in Paper IV. In short, sources are detected in all 5 filter +images down to the $5\sigma$ significance. The collection of detected +sources is merged into a single master list. If a source is detected +in at least two bands, or only in \yps{} band, then a PSF model is +fitted to the pixels of the other bands in which the source was not +detected. This forced photometry results in lower significance +measurements of the flux at the positions of objects which are thought +to be real sources, by virtue of triggering a detection in at least +two bands. The relaxed limit for \yps{} band is included to allow for +searches of \yps{} dropout objects: it is known that faint, +high-redshift quasars may be detected in \yps{} band only. Sources +detected only in \yps{} band are therefore more likely to have a +higher false-positive rate than the other stack sources. The parameters of the PSF model are allowed to vary with position in the skycell. The PSF model is also used to convolve the analytical galaxy @@ -1085,8 +1014,4 @@ in question is large compared to the FWHM of the PSF. -%% The IPP team initially explored the option of convolving each input -%% warp to a single target PSF chosen to match the worst of the input -%% images for a given stack. - The IPP analysis solves this problem by using the sources detected in the stack images and performing forced photometry on the @@ -1109,22 +1034,4 @@ stage image products along with the \ippstage{skycal} catalog to the \ippprog{psphotFullForce} program. - -%% In this program, the positions of sources are loaded from the input -%% catalog. PSF stars are pre-identified from the stack image and a PSF -%% model generated for each \ippstage{warp} image based on those stars, -%% using the same stars for all warps to the extent possible (PSF stars -%% which are excessively masked on a particular image are not used to -%% model the PSF). The PSF model is fitted to all of the known source -%% positions in the warp images. Aperture magnitudes, Kron magnitudes, -%% and moments are also measured at this stage for each warp. Note that -%% the flux measurement for a faint, but significant, source from the -%% stack image may be at a low significance (less than the $5\sigma$ -%% criterion used when the photometry is not run in this forced mode) in -%% any individual warp image; the flux may even be negative for specific -%% warps. When combined together, these low-significance measurements -%% will result in a signficant measurement as the signal-to-noise -%% increases by the square root of the number of measurements. The -%% individual warp measurements are combined together to generate -%% averages values within DVO. The convolved galaxy models are also re-measured on the @@ -1180,124 +1087,19 @@ images, from a \ippstage{warp} and a \ippstage{stack} of some variety, or from a pair of \ippstage{stack} stage images. During the PS1 -survey, pairs of exposures, called TTI pairs \citep[see Survey - Strategy in][]{chambers2017}, were obtained for each pointing within -a $\approx$ 1 hour period in the same filter, and to the extent -possible with the same orientation and boresite position. The -standard PS1 nightly processing generated difference images from the -resulting pairs of \ippstage{warp} images. The nightly processing -generated \ippstage{stack} images for the Medium Deep fields, and -these were combined with a template reference \ippstage{stack} image -to generate ``stack-stack diffs'' each night they were observed. For -the PV3 $3\pi$ processing, the entire collection of \ippstage{warp} -stage images for the survey were combined with images generated by the +survey, pairs of exposures, called TTI pairs (see Survey Strategy in +Paper I), were obtained for each pointing within a $\approx$ 1 hour +period in the same filter, and to the extent possible with the same +orientation and boresite position. The standard PS1 nightly +processing generated difference images from the resulting pairs of +\ippstage{warp} images. The nightly processing generated +\ippstage{stack} images for the Medium Deep fields, and these were +combined with a template reference \ippstage{stack} image to generate +``stack-stack diffs'' each night they were observed. For the PV3 +$3\pi$ processing, the entire collection of \ippstage{warp} stage +images for the survey were combined with images generated by the \ippstage{stack} processing to generate ``warp-stack diffs'', for eventual public released. -When a \ippstage{diff} processing is defined, an entry is added to the -\ippdbtable{diffRun} table, and the appropriate input images are added -to the \ippdbtable{diffInputSkyfile} table, with one entry for each -skycell that is covered by the images. For a \ippstage{diff} -generated from two \ippstage{warp} stage products, the input images -have their \ippdbcolumn{warp_id} values recorded in the -\ippdbcolumn{warp1} and \ippdbcolumn{warp2} for each skycell that -overlaps. If two \ippstage{stack} stages are to be used in the -difference, their \ippdbcolumn{stack_id} entries are recorded in the -\ippdbcolumn{stack1} and \ippdbcolumn{stack2} fields. As each -\ippstage{stack} only covers a single skycell, the \ippstage{diff} is -usually defined indirectly, using other information from the -\ippdbtable{stackRun} table to select appropriate -\ippdbcolumn{stack_id} values. Similarly, \ippstage{diff} processing -is defined for the mixed case by creating entries that populate one of -\ippdbcolumn{warp1} and \ippdbcolumn{stack1} and populating one of -\ippdbcolumn{warp2} and \ippdbcolumn{stack2}. In all cases, the -minuend of the subtraction to be performed is the ``1'' entry, and the -subtrahend is the ``2'' entry. - -Jobs are created based on the entries of -\ippdbtable{diffInputSkyfile}, with the appropriate images and -catalogs passed to the \ippprog{ppSub} program. This does the -subtraction, as well as the photometry of any sources detected in the -\ippstage{diff} image. Sources may be detected as a positive source -(flux in the minuend is higher than the subtrahend) or as a negative -source (flux in the subtrahend is higher). The algorithm used for PSF -matching is described in \citet{waters2017}. Upon completion of these -jobs, statistics about the processing are written to an entry in the -\ippdbtable{diffSkyfile} table. An \ippmisc{advance} checks for the -completion of all of the components listed in -\ippdbtable{diffInputSkyfile}, and marks the \ippdbtable{diffRun} -entry as such. - -\section{Post-Processing : Database Ingest and Calibration} -\label{sec:postprocessing} - -\subsection{DVO} -\label{sec:DVO} - -\subsubsection{Overview} - -% intro -The Pan-STARRS IPP uses an internal database system, distinct from the -publicly visible database system, to determine the association -between multiple detections of the same astronomical object and as -part of the astrometric and photometric calibration process. This -database system, called the ``Desktop Virtual Observatory'' (DVO) was -developed originally for the LONEOS project -\citep{1995DPS....27.0110B}, and used as part of the CFHT Elixir -system \citep{2004PASP..116..449M}. The capabilities of this -databasing system have been somewhat expanded for the Pan-STARRS -context. - -% overview -DVO tracks three main classes of information: 1) average properties of -astronomical objects; 2) measurements of those objects (from which the -average properties are derived); 3) properties of the images which -provided some or all of the measurements. In addition, certain -metadata tables define general features of the database. -Table~\ref{tab:DVO_schema} lists the full collection of database -tables used by DVO. - -%Figure~\ref{fig:DVO_schema} -%illustrates the schematic relationship between these types of -%measurements. - -In the most basic implementation, a collection of measurements for -detections from a set of images is loaded into DVO along with the -metadata describing the images. The latter includes properties such -as the exposure time, airmass, filter, time \& date of the exposure, -etc. Critically, the image metadata includes an astrometric -transformation relating the detection coordinate on the image to the -coordinate on the sky. As the collection of measurements are loaded -into DVO, the software constructs astronomical objects based on those -detections. If images overlap, multiple observations of the same -astronomical object are grouped together. Thus, a single DVO database -will contain a one-to-many relationship between the images and the -measurements and a many-to-one relationship between the measurements -and the derived astronomical objects. - -% -%% These tables fall into one of several classes: -%% those which store information about the average properties of -%% astronomical objects; those which store information about individual -%% measurements; those which store information about the images; those -%% which store supporting information (metadata). - -%% DVO includes two major classes of database tables: those containing -%% information about astronomical objects in the sky and those containing -%% other supporting information. The object-related tables are -%% partitioned on the basis of position in the sky: objects within a -%% region bounded by lines of constant RA,DEC are contained in a specific -%% file. The boundaries and the associated partition names are stored in -%% one of the supporting tables, \ippdbtable{SkyTable}. This table -%% contains the definitions of the boundaries for each sky region -%% (\ippdbcolumn{R_MIN}, \ippdbcolumn{R_MAX}, \ippdbcolumn{D_MIN}, -%% \ippdbcolumn{D_MAX}), the name of the sky region, an ID -%% (\ippdbcolumn{INDEX}, equal to the sequence number of the region in -%% the table), and index entries to enable navigation within the table. -%% The regions are defined in a hierarchical sense, with a series of -%% levels each containing a finer mesh of regions covering the sky. - -\subsubsection{DVO Schema} - -\begin{table*}[hb] +\begin{table*} \begin{center} \caption{DVO Database Tables\label{tab:DVO_schema}} @@ -1323,4 +1125,83 @@ \end{table*} +When a \ippstage{diff} processing is defined, an entry is added to the +\ippdbtable{diffRun} table, and the appropriate input images are added +to the \ippdbtable{diffInputSkyfile} table, with one entry for each +skycell that is covered by the images. For a \ippstage{diff} +generated from two \ippstage{warp} stage products, the input images +have their \ippdbcolumn{warp_id} values recorded in the +\ippdbcolumn{warp1} and \ippdbcolumn{warp2} for each skycell that +overlaps. If two \ippstage{stack} stages are to be used in the +difference, their \ippdbcolumn{stack_id} entries are recorded in the +\ippdbcolumn{stack1} and \ippdbcolumn{stack2} fields. As each +\ippstage{stack} only covers a single skycell, the \ippstage{diff} is +usually defined indirectly, using other information from the +\ippdbtable{stackRun} table to select appropriate +\ippdbcolumn{stack_id} values. Similarly, \ippstage{diff} processing +is defined for the mixed case by creating entries that populate one of +\ippdbcolumn{warp1} and \ippdbcolumn{stack1} and populating one of +\ippdbcolumn{warp2} and \ippdbcolumn{stack2}. In all cases, the +minuend of the subtraction to be performed is the ``1'' entry, and the +subtrahend is the ``2'' entry. + +Jobs are created based on the entries of +\ippdbtable{diffInputSkyfile}, with the appropriate images and +catalogs passed to the \ippprog{ppSub} program. This does the +subtraction, as well as the photometry of any sources detected in the +\ippstage{diff} image. Sources may be detected as a positive source +(flux in the minuend is higher than the subtrahend) or as a negative +source (flux in the subtrahend is higher). The algorithm used for PSF +matching is described in Paper III. Upon completion of these +jobs, statistics about the processing are written to an entry in the +\ippdbtable{diffSkyfile} table. An \ippmisc{advance} checks for the +completion of all of the components listed in +\ippdbtable{diffInputSkyfile}, and marks the \ippdbtable{diffRun} +entry as such. + +\section{Database Ingest and Calibration} +\label{sec:postprocessing} + +\subsection{DVO} +\label{sec:DVO} + +\subsubsection{Overview} + +% intro +The Pan-STARRS IPP uses an internal database system, distinct from the +publicly visible database system, to determine the association +between multiple detections of the same astronomical object and as +part of the astrometric and photometric calibration process. This +database system, called the ``Desktop Virtual Observatory'' (DVO) was +developed originally for the LONEOS project +\citep{1995DPS....27.0110B}, and used as part of the CFHT Elixir +system \citep{2004PASP..116..449M}. The capabilities of this +databasing system have been somewhat expanded for the Pan-STARRS +context. + +% overview +DVO tracks three main classes of information: 1) average properties of +astronomical objects; 2) measurements of those objects (from which the +average properties are derived); 3) properties of the images which +provided some or all of the measurements. In addition, certain +metadata tables define general features of the database. +Table~\ref{tab:DVO_schema} lists the full collection of database +tables used by DVO. + +In the most basic implementation, a collection of measurements for +detections from a set of images is loaded into DVO along with the +metadata describing the images. The latter includes properties such +as the exposure time, airmass, filter, time \& date of the exposure, +etc. Critically, the image metadata includes an astrometric +transformation relating the detection coordinate on the image to the +coordinate on the sky. As the collection of measurements are loaded +into DVO, the software constructs astronomical objects based on those +detections. If images overlap, multiple observations of the same +astronomical object are grouped together. Thus, a single DVO database +will contain a one-to-many relationship between the images and the +measurements and a many-to-one relationship between the measurements +and the derived astronomical objects. + +\subsubsection{DVO Schema} + \paragraph{Photcodes} @@ -1426,17 +1307,5 @@ magnitude. While these photometric distance modulus measurements are not extremely precise, they provide a constraint on the distance which -is used in our analysis of the astrometry -\citep[see][]{magnier2017.calibration}. - -%% Similarly to the \ippdbtable{Measure} table, the fields -%% \ippdbcolumn{objID}, \ippdbcolumn{catID}, and \ippdbcolumn{averef} -%% define links from the \ippdbtable{Lensing} table to the -%% \ippdbtable{Average} table. In a similar fashion, the fields -%% \ippdbtable{Average}.\ippdbcolumn{lensingOffset} and -%% \ippdbtable{Average}.\ippdbcolumn{Nlensing} are the index into the -%% sorted \ippdbtable{Lensing} table entries. \note{discuss failure of -%% the Lensing to Measure indexing} - -% \note{Average used above but defined below} +is used in our analysis of the astrometry (see Paper V). \paragraph{Object Tables} @@ -1530,12 +1399,4 @@ across the different IPP stages. -%% Data from GPC1 (and other cameras processed by the IPP) are loaded -%% into DVO in units \code{smf} files generated by the \ippstage{camera} -%% calibration stage (see section \ref{sec:camera} below). As -%% described above, these files contain all measurements from a complete -%% exposure, with measurements from each chip grouped into separate FITS -%% tables. When these measurements are loaded into the -%% \ippdbtable{Measure} and similar tables, - \paragraph{Other Tables} @@ -1544,5 +1405,5 @@ determined by the photometry calibration analysis and the astrometric flat-field corrections determined by the astrometry calibration -analysis \citep[see][]{magnier2017.calibration}. +analysis (see Paper V). \subsubsection{Sky Partition} @@ -1551,5 +1412,5 @@ \begin{figure*}[htbp] \begin{center} - \includegraphics[width=\hsize,clip]{skypartition.png} + \includegraphics[width=\hsize,clip]{skypartition.\plotext} \caption{\label{fig:sky.partition} Level 3 sky paritioning. The blue grid shows the outlines of the different regions assigned to @@ -1686,6 +1547,6 @@ allows valid joins between tables to select the different kinds of attributes of the same astronomical objects. This 64-bit integer ID -is constructed based on the coordinates of the object, as described by -\cite[][]{flewelling2017}. In short, the digits of the right +is constructed based on the coordinates of the object, as described in +Paper VI. In short, the digits of the right ascension and declination coordinates are used to define a single 64-bit integer with spatial resolution of roughly 3 milliarcseconds. @@ -1761,17 +1622,17 @@ Upon completion of the processing of each stage, the results of the photometry analysis are stored in a large number of individual catalog -files as described in \cite{magnier2017.analysis}. The data from -these files are loaded into a DVO database to define the astronomical -objects and to allow for calibration analysis. The program which -loads the data into the DVO database is called \ippprog{addstar}, and -is associated with the the \ippstage{addstar} processing stage. The -measurement catalogs generated by the \ippstage{camera}, -\ippstage{skycal}, \ippstage{fullforce}, and \ippstage{diff} stages -are loaded into DVOs in this fashion, although not every measurement -in each catalog are included in the master DVO that is constructed. -For a particular re-processing version, a single master DVO is -constructed for the positive image stages (\ippstage{camera}, -\ippstage{skycal}, \ippstage{fullforce}) and a separate one is -constructed for the difference image analysis stage results. +files as described in Paper IV. The data from these files are loaded +into a DVO database to define the astronomical objects and to allow +for calibration analysis. The program which loads the data into the +DVO database is called \ippprog{addstar}, and is associated with the +the \ippstage{addstar} processing stage. The measurement catalogs +generated by the \ippstage{camera}, \ippstage{skycal}, +\ippstage{fullforce}, and \ippstage{diff} stages are loaded into DVOs +in this fashion, although not every measurement in each catalog are +included in the master DVO that is constructed. For a particular +re-processing version, a single master DVO is constructed for the +positive image stages (\ippstage{camera}, \ippstage{skycal}, +\ippstage{fullforce}) and a separate one is constructed for the +difference image analysis stage results. The construction of the master DVO is performed in a hierarchical @@ -1818,6 +1679,6 @@ Once the master DVO database has been constructed, high-quality astrometric and photometric calibrations can be calculated. The -details of the calibration analysis are discussed in -\cite{magnier2017.calibration}. We present a brief summary here. +details of the calibration analysis are discussed in Paper V. We +present a brief summary here. Astrometric calibration consists of measuring and correcting @@ -1832,5 +1693,5 @@ a function of position in the camera (essentially an astrometric flat-field correction), as a function of the brightness of the star -(the so-called Koppenh\"offer effect, see~\citealt{magnier2017.calibration}), and as +(the so-called Koppenh\"ofer effect, see~Paper V), and as a function of airmass and color (differential chromatic refraction). Once the systematic errors have been measured, they are applied back @@ -1865,6 +1726,4 @@ the Medium Deep fields. -%% (listed in Table~\ref{tab:flat-field-seasons}) XXX add this table - After the \"ubercal analysis of the photometric periods is completed, the determined zero points, airmass corrections, and flat-field terms @@ -1884,5 +1743,5 @@ flat-field correction addresses photometric variations due to spatial variations in the PSF due to the optics and low-level effects on the -chips \citep[see][]{magnier2017.calibration}. After the systematic corrections +chips (see Paper V). After the systematic corrections have been determined and applied back to the database, a final relative photometry analysis pass is performed. @@ -1898,5 +1757,5 @@ database starts once the PS1 photometry and astrometry measurements have been calibrated within the DVO system. The construction takes -place in several stages, described in detail by \cite{flewelling2017}. +place in several stages, described in detail in Paper VI. We summarize those steps here. @@ -2148,6 +2007,4 @@ \end{figure} -%\code{ls /tmp} - \subsubsection{Pantasks scripts: ippTasks} @@ -2194,6 +2051,4 @@ \ippmisc{DONE}, and removes them from the book, as these represent jobs that have finished. - -% \note{the manipulation above takes place in the task.exit subscript} The associated \ippmisc{run} task generates jobs constructed from the @@ -2342,5 +2197,5 @@ used for the warp tessellation. A \ippdbcolumn{projection_cell} is a unit of sky defined to be a square four degrees on each side which has -a single tangent plane projection \citep[][see]{waters2017}. +a single tangent plane projection (Paper III). Once this entry is defined, it is populated with all exposures (stored in the @@ -2420,15 +2275,4 @@ data to that instance. -% The basic user commands to interact -% with Nebulous are to 1) create a new storage object and associated -% instance; 2) add a new instance to an existing storage object; 3) -% remove (cull) an instance; 4) delete a storage object; and 5) find a -% file associated with a given storage objects. Note that these user -% commands do not affect the files on disk \note{true for cull?} -% (exception: the create function will create an empty file if one does -% not exist). They only change the state of the Nebulous database; it -% is the responsibility of the user program to read and write data to a -% file and to create the copies, etc. - For the Nebulous users, the identifier of a storage object is a unique string with the form of a UNIX file path: e.g. a/b/c/file. When a @@ -2547,14 +2391,12 @@ Requests to this server may restrict to the latest by time. Each row in the listing includes basic information about the exposure: an -exposure identifier \citep[e.g., o5432g0123o; see][for - details]{chambers2017}, the date and time of the exposure, the -telescope commanded pointing, the filter and exposure time, and the -observation comment for that exposure. The row also provides a link -to a listing of the chips associated with that exposure. This listing -includes a link to the individual chip FITS files as well as an md5 -checksum. Systems which are allowed access may download the raw chip -FITS files via http requests to the provided links. - -% \note{add a discussion of gpc1 filenames?} +exposure identifier (e.g., o5432g0123o; see Paper I for details), the +date and time of the exposure, the telescope commanded pointing, the +filter and exposure time, and the observation comment for that +exposure. The row also provides a link to a listing of the chips +associated with that exposure. This listing includes a link to the +individual chip FITS files as well as an md5 checksum. Systems which +are allowed access may download the raw chip FITS files via http +requests to the provided links. The IPP also uses datastores to provide access to its own data @@ -2666,13 +2508,12 @@ isolation of source objects is included, providing the organization of detections that is used in the \ippprog{psphot} photometry analysis -\citep{magnier2017.analysis}. The PSF matching required for \ippstage{stack} -and \ippstage{diff} stage image combinations is as well. The -unification of configuration options between config files on disk and -the options specified on the command line is handled by -\ippmisc{psModules} functions, as is the construction of data -structures in memory to represent the astronomical camera based on the -header information in the input file. The functions to generate and -apply the detrend corrections to the data are also provided by this -library. +(Paper IV). The PSF matching required for \ippstage{stack} and +\ippstage{diff} stage image combinations is as well. The unification +of configuration options between config files on disk and the options +specified on the command line is handled by \ippmisc{psModules} +functions, as is the construction of data structures in memory to +represent the astronomical camera based on the header information in +the input file. The functions to generate and apply the detrend +corrections to the data are also provided by this library. \section{IPP Hardware Systems} @@ -2688,8 +2529,8 @@ by the University of Hawaii. This site was chosen based on the original development funding provided by the Air Force Research Labs -\citep[see][for more details]{chambers2017}. Once the Air Force -funding stopped being a significant driver for Pan-STARRS, the cluster was -physically moved from the MHPCC to a similar nearby computing center -located at the Maui Research and Technology Center. +(see Paper I for more details). Once the Air Force funding stopped +being a significant driver for Pan-STARRS, the cluster was physically +moved from the MHPCC to a similar nearby computing center located at +the Maui Research and Technology Center. The computing cluster is comprised of three main types of computers, @@ -2753,4 +2594,24 @@ had been generated for all tasks, the component lists were merged, and the Moab job control file was constructed. + +\begin{table} +\caption{\label{tab:SC_processing_parameters} Cost values for remote processing} +\begin{center} +\begin{tabular}{lcc} +\hline +\hline +{\bf IPP Stage} & {\bf $t_\mathrm{task}$ (s)} & {\bf $S_\mathrm{task}$} \\ +\hline + \ippstage{chip} & 150 & 2 \\ + \ippstage{camera} & 1700 & 2 \\ + \ippstage{warp} & 110 & 2 \\ + \ippstage{stack} & 1500 & 6 \\ + \ippstage{staticsky} & 7200 & 6 \\ +% \ippstage{diff} & 300 & 2 \\ + \ippstage{fullforce} & 300 & 2 \\ +\hline +\end{tabular} +\end{center} +\end{table} The control file contains the resource requests for the job, as well @@ -2772,41 +2633,4 @@ values used for the various IPP processing stages. -\begin{table*} -\caption{\label{tab:SC_processing_parameters} Cost values for remote processing} -\begin{center} -\begin{tabular}{lcc} -\hline -\hline -{\bf IPP Stage} & {\bf $t_\mathrm{task}$ (s)} & {\bf $S_\mathrm{task}$} \\ -\hline - \ippstage{chip} & 150 & 2 \\ - \ippstage{camera} & 1700 & 2 \\ - \ippstage{warp} & 110 & 2 \\ - \ippstage{stack} & 1500 & 6 \\ - \ippstage{staticsky} & 7200 & 6 \\ -% \ippstage{diff} & 300 & 2 \\ - \ippstage{fullforce} & 300 & 2 \\ -\hline -\end{tabular} -\end{center} -\end{table*} - -%%\begin{deluxetable}{lcc} -%% \tablecolumns{3} -%% \tablewidth{0pc} -%% \tablecaption{Cost values for remote processing} -%% \tablehead{\colhead{IPP Stage}&\colhead{$t_\mathrm{task}$ (s)}&\colhead{$S_\mathrm{task}$}} -%% \startdata -%% \ippstage{chip} & 150 & 2 \\ -%% \ippstage{camera} & 1700 & 2 \\ -%% \ippstage{warp} & 110 & 2 \\ -%% \ippstage{stack} & 1500 & 6 \\ -%% \ippstage{staticsky} & 7200 & 6 \\ -%%% \ippstage{diff} & 300 & 2 \\ -%% \ippstage{fullforce} & 300 & 2 -%% \enddata -%% \label{tab:SC processing parameters} -%%\end{deluxetable} - Once the preparation for the job is complete, the input and output file lists, the task list, and the job control file are transferred @@ -2905,16 +2729,7 @@ \input{datasystem.bbl} -% \appendix - -% Table \ref{tab: database schema} provides a list of a majority of the -% tables in the GPC1 database schema. Tables that have been excluded -% are either no longer used in IPP processing, or are used for rare -% reductions that were not used for the PV3 data release. The tables -% are grouped into stages, with the primary table and any secondary -% tables for that stage listed together, along with the primary key -% column that link the tables together. - \end{document} +% this is a 'deluxetable' version of table 1 \begin{center} \begin{deluxetable}{lllll} Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.datasystem/dvo.sh =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.datasystem/dvo.sh (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.datasystem/dvo.sh (revision 40759) @@ -1,9 +1,11 @@ + +# create an empty database with: +# addstar -add-id -D CATDIR catdir.test2 -D SKY_DEPTH 2 macro figure2 - resize 1000 510; region 180 0 83 ait; box -c black -tickpad 0.5 -pad 0.5 -ticks 0000 -labels 0000; skycat -all -depth 3 -c blue style -sz 0.4 -c green; ecliptic; style -c red; galactic style -sz 0.8 -c darkgreen; ecliptic; style -c red; galactic - png -name skypartition.png - +# png -name skypartition.png + ps -name skypartition.ps end Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/Makefile =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/Makefile (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/Makefile (revision 40759) @@ -2,4 +2,5 @@ DO_PDFLATEX = 1 + DO_BIBTEX = 0 @@ -9,10 +10,42 @@ all: pdf tgz -tgz: detrend.tgz pdf: detrend.pdf + +journal: detrend.journal.tgz +arxiv: detrend.arxiv.tgz quick: detrend.quick.pdf -ALLPICS = \ +JPICS = \ +images/gpc1.layout.pdf \ +images/o5677g0123o_M_OS_NL_XY23.png \ +images/o5677g0123o_to_DARK_XY23.png \ +images/B_profile_v1.pdf \ +images/o5677g0123o_VIDEODARK_VDim_Rdark_XY22.png \ +images/o5677g0123o_VIDEODARK_VDim_VDdark_XY22.png \ +images/o5220g0025o_nofringe_XY53.png \ +images/o5220g0025o_fringe_XY53.png \ +images/gpc1_mask_indexed.png \ +images/GPC1_Ghosts_with_Zoom.pdf \ +images/full_fpa_glints.png \ +images/o6802g0338o_SATSTAR_XY51.png \ +images/persistent_charge.png \ +images/o5677g0123n4o_XY11_bt_trail.pdf \ +images/pattern_row_edit.png \ +images/correlated.noise.png \ +images/N157.v1.png \ +images/N157.v2.png \ +images/warp_2046019_sci.png \ +images/warp_2046019_var.png \ +images/warp_2046019_mask.png \ +images/stack_3956997_sci.png \ +images/stack_3956997_var.png \ +images/stack_3956997_mask.png \ +images/stack_3956997_num.png \ +images/stack_3956997_exp.png \ +images/stack_3956997_expwt.png + +APICS = \ +images/gpc1.layout.pdf \ images/o5677g0123o_M_OS_NL_XY23_sm.png \ images/o5677g0123o_to_DARK_XY23_sm.png \ @@ -23,21 +56,19 @@ images/o5220g0025o_fringe_XY53_sm.png \ images/gpc1_mask_indexed.png \ -images/full_fpa_ghosts_sm.png \ +images/GPC1_Ghosts_with_Zoom.pdf \ images/full_fpa_glints_sm.png \ images/o6802g0338o_SATSTAR_XY51_sm.png \ -images/o5677g0123o_nbt_XY11.png \ -images/o5677g0124o_nbt_XY11.png \ -images/o5677g0123o_wbt_XY11.png \ -images/o5677g0124o_wbt_XY11.png \ +images/persistent_charge_sm.png \ images/o5677g0123n4o_XY11_bt_trail.pdf \ images/pattern_row_edit.png \ -images/o5379g0103o_npt_XY57_sm.png \ -images/o5379g0103o_wpt_XY57_sm.png \ +images/correlated.noise_sm.png \ +images/N157.v1_sm.png \ +images/N157.v2_sm.png \ images/warp_2046019_sci_sm.png \ images/warp_2046019_var_sm.png \ images/warp_2046019_mask.png \ images/stack_3956997_sci_sm.png \ +images/stack_3956997_var_sm.png \ images/stack_3956997_mask.png \ -images/stack_3956997_var_sm.png \ images/stack_3956997_num_sm.png \ images/stack_3956997_exp_sm.png \ @@ -48,12 +79,10 @@ ../inputs/apj.bst \ ../inputs/code.sty \ -$(ALLPICS) \ detrend.tex detrend.pdf: $(FILES) -detrend.tgz: $(FILES) + +detrend.journal.tgz: $(FILES) $(JPICS) detrend.bbl +detrend.arxiv.tgz: $(FILES) $(APICS) detrend.bbl include ../Makefile.Common - -# submission : -# tar --transform 's%inputs/%%' -zcf waters2017.tgz $(FILES) Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/detrend.bbl =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/detrend.bbl (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/detrend.bbl (revision 40759) @@ -1,3 +1,3 @@ -\begin{thebibliography}{19} +\begin{thebibliography}{20} \expandafter\ifx\csname natexlab\endcsname\relax\def\natexlab#1{#1}\fi @@ -112,4 +112,9 @@ \apj, 756, 158 +\bibitem[{{Swaters} \& {Valdes}(2007)}]{2007ASPC..376..269S} +{Swaters}, R.~A. \& {Valdes}, F.~G. 2007, in Astronomical Society of the + Pacific Conference Series, Vol. 376, Astronomical Data Analysis Software and + Systems XVI, ed. R.~A. {Shaw}, F.~{Hill}, \& D.~J. {Bell}, 269 + \bibitem[{{Tonry} \& {Onaka}(2009)}]{2009amos.confE..40T} {Tonry}, J. \& {Onaka}, P. 2009, in Advanced Maui Optical and Space Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/detrend.tex =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/detrend.tex (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/detrend.tex (revision 40759) @@ -21,11 +21,11 @@ %\def\plotmode{bw} -% arxiv needs PDF graphics, but publishers mostly was PS -%\def\plotext{pdf} -\def\plotext{ps} +% arxiv needs small graphics, but publishers want full-scale +%\def\plotopt{_sm} +\def\plotopt{} % use this to make the figure picture path flexible: -%\def\picdir{PATH} -\def\picdir{ALTPATH} +%\def\picdir{images} +\def\picdir{.} % CZW commands from my previous draft. @@ -69,5 +69,5 @@ K.~C. Chambers,\altaffilmark{\IfA} W.~S. Burgett,\altaffilmark{\IfA} -P. Draper,\altaffilmark{\DUR} +P.~W. Draper,\altaffilmark{\DUR} H.~A. Flewelling,\altaffilmark{\IfA} K. W. Hodapp,\altaffilmark{\IfA} @@ -125,11 +125,5 @@ % insert additional keywords as appropriate: -\keywords{Surveys:\PSONE } - -%% NOTE 2018.12.06 EAM : Things that still need to be done prior to submission: -%% * generate a valid bibtex entry for the diff paper, include in refs -%% * better covariance discussion (wait until analysis.tex version written) -%% * annotate the ghost, glint, and perhaps other images with arrows to illustrate features -%% * check on total size vs arxiv limits +\keywords{Surveys:\PSONE; techniques: image processing; methods: data analysis; } \section{Introduction} @@ -284,5 +278,5 @@ \begin{figure}[htpb] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{{images/gpc1.layout}.pdf} + \includegraphics[width=0.9\hsize,angle=0,clip]{{\picdir/gpc1.layout}.pdf} \caption{Diagram illustrating layout of OTA devices in GPC1. The blue dots mark the locations of the amplifiers for xy00 cells in @@ -326,8 +320,8 @@ \centering \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_M_OS_NL_XY23_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/o5677g0123o_M_OS_NL_XY23\plotopt.png} \end{minipage}% \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_to_DARK_XY23_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/o5677g0123o_to_DARK_XY23\plotopt.png} \end{minipage} \caption{{\bf Dark Correction:} An example of the dark model application to exposure o5677g0123o, OTA23 (2011-04-26, 43s \gps{} filter). The left panel shows the image data mosaicked to the OTA level, and has had the static mask applied, the overscan subtracted, and the detector non-linearity corrected. The right panel, shows the same exposure with the dark applied in addition to the processing shown on the left, removing the amplifier glows in the cell corners.} @@ -365,13 +359,4 @@ right. The OTA $Y$ labels increase upward in the mosaic. -%%\textit{Note: These papers are being placed on the arXiv.org to -%% provide crucial support information at the time of the public -%% release of Data Release 1 (DR1). We expect the arXiv versions to be -%% updated prior to submission to the Astrophysical Journal in January -%% 2017. Feedback and suggestions for additional information from early -%% users of the data products are welcome during the submission and -%% refereeing process.} - - \section{GPC1 Detrend Details} \label{sec:detrending} @@ -465,5 +450,5 @@ \begin{figure}[htpb] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/B_profile_v1.pdf} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/B_profile_v1.pdf} \caption{Example showing a profile cut across exposure o5676g0195, OTA67 (2011-04-25, 43s \gps{} filter). The entire first row of @@ -476,5 +461,5 @@ A-mode dark instead results in the third (blue) curve, which shows a significant increase in gradients across the cells. The fourth - (red) curve is the result of applying the PATTERN.CONTINUITY + (red) curve is the result of applying the \nocode{PATTERN.CONTINUITY} correction along with the B-mode dark model. Although this creates a larger gradient across the mosaicked images, it @@ -543,8 +528,8 @@ \centering \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_VIDEODARK_VDim_Rdark_XY22_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/o5677g0123o_VIDEODARK_VDim_Rdark_XY22\plotopt.png} \end{minipage}% \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_VIDEODARK_VDim_VDdark_XY22_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/o5677g0123o_VIDEODARK_VDim_VDdark_XY22\plotopt.png} \end{minipage} \caption{{\bf Video Dark:} An example of the video dark model application to exposure o5677g0123o, OTA22 (2011-04-26, 43s \gps{} filter), which has a video cell located in cell xy16. The left panel shows the image data mosaicked to the OTA level, and has had the static mask applied, the overscan subtracted, the detector non-linearity corrected, and a regular dark applied. The right panel, shows the same exposure with a video dark applied instead of the standard dark. The main impact of this change is the improved correction of the corner glows, which are over subtracted with the standard dark.} @@ -629,8 +614,8 @@ \centering \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5220g0025o_nofringe_XY53_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/o5220g0025o_nofringe_XY53\plotopt.png} \end{minipage}% \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5220g0025o_fringe_XY53_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/o5220g0025o_fringe_XY53\plotopt.png} \end{minipage} \caption{{\bf Fringing:} Example of the \yps{} filter fringe pattern @@ -813,5 +798,5 @@ \begin{figure}[b] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/gpc1_mask_indexed.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/gpc1_mask_indexed.png} \caption{Image map of the GPC1 static mask. The CTE regions are clearly visible as roughly triangular patches covering the corners of some OTAs. Some entire cells are masked, including an entire column of cells on OTA14. Calcite cells remove large areas from OTA17 AND OTA76.} \label{fig:static mask} @@ -1057,7 +1042,6 @@ \begin{figure*}[htpb] \centering -% \includegraphics[width=0.9\hsize,angle=0,clip]{images/full_fpa_ghosts.jpg} -% \includegraphics[width=0.9\hsize,angle=0,clip]{images/full_fpa_ghosts_sm.png} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/GPC1_Ghosts_with_Zoom.png} +% \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/GPC1_Ghosts_with_Zoom.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/GPC1_Ghosts_with_Zoom.pdf} \caption{{\bf Ghosts:} Example of optical ghosts in GPC1. The central $6 \times 6$ detectors from exposure o5677g0123o @@ -1075,6 +1059,5 @@ \begin{figure*}[htpb] \centering -% \includegraphics[width=0.9\hsize,angle=0,clip]{images/glint_example_o5379g0103o.jpg} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/full_fpa_glints_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/full_fpa_glints\plotopt.png} \caption{{\bf Glints:} Example of a glint on exposure o5379g0103o (2010-07-02, 45s \ips{} filter). The source star out of the field of view creates a long reflection that extends through OTA73 and OTA63.} \label{fig:optical glints} @@ -1083,5 +1066,5 @@ \begin{figure}[htpb] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o6802g0338o_SATSTAR_XY51_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/o6802g0338o_SATSTAR_XY51\plotopt.png} \caption{Example of saturated star, with diffraction spikes extending from the core on exposure o6802g0338o, OTA51 (2014-05-25, 45s \gps{} filter).} \label{fig:saturated star} @@ -1330,16 +1313,6 @@ \begin{figure}[htpb] \centering - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_nbt_XY11.png} - \end{minipage}% - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_nbt_XY11.png} - \end{minipage} - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_wbt_XY11.png} - \end{minipage}% - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_wbt_XY11.png} - \end{minipage} + %% need a small version of this for arxiv + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/persistent_charge\plotopt.png} \caption{{\bf Persistent Charge:} Example of OTA11 cell xy50 on exposures o5677g0123o (left) and o5677g0124o (right). The top panels show the image with all appropriate detrending steps, but without burntool, and the bottom show the same with burntool applied. There is some slight over subtraction in fitting the initial trail, but the impact of the trail is greatly reduced in both exposures.} \label{fig:burntool images} @@ -1378,5 +1351,5 @@ \begin{figure}[htpb] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123n4o_XY11_bt_trail.pdf} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/o5677g0123n4o_XY11_bt_trail.pdf} \caption{{\bf Burntool Correction:} Example of a profile cut along @@ -1461,5 +1434,5 @@ \tablecolumns{3} \tablewidth{0pc} - \tablecaption{Cells which have PATTERN.ROW correction applied} + \tablecaption{Cells which have \nocode{PATTERN.ROW} correction applied} \tablehead{\colhead{OTA} & \colhead{Cell columns} & \colhead{Additional cells}} \startdata @@ -1479,12 +1452,4 @@ \end{deluxetable} -% this figure does not really clarify anything -% \begin{figure}[htpb] -% \centering -% \includegraphics[width=0.9\hsize,angle=0,clip]{images/linearity_XY27_xy16.png} -% \caption{Example of the linearity correction as a fraction of observed flux for OTA27, cell xy16.} -% \label{fig: nonlinearity} -% \end{figure} - \subsection{Pattern correction} \label{sec:pattern} @@ -1510,5 +1475,5 @@ correction cannot fully remove this structure from the images, and the noisemap value only indicates the level of the average variance added -by these bias offsets. Therefore, we apply the PATTERN.ROW correction +by these bias offsets. Therefore, we apply the \ippmisc{PATTERN.ROW} correction in an attempt to mitigate the offsets and correct the image values. To force the rows to agree, a second order clipped polynomial is @@ -1527,6 +1492,10 @@ \begin{figure}[htpb] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/pattern_row_edit.png} - \caption{Diagram illustrating in red which cells on GPC1 require the PATTERN.ROW correction to be applied. The footprint of each OTA is outlined, and cell xy00 is marked with either a filled box or an outline. The labeling of the non-existent corner OTAs is provided to orient the focal plane.} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/pattern_row_edit.png} + \caption{Diagram illustrating in red which cells on GPC1 require the + \nocode{PATTERN.ROW} correction to be applied. The footprint of + each OTA is outlined, and cell xy00 is marked with either a filled + box or an outline. The labeling of the non-existent corner OTAs + is provided to orient the focal plane.} \label{fig: pattern row cells} \end{figure} @@ -1535,5 +1504,5 @@ the \gps{} filter, as the read noise is the dominant noise source in that filter. At longer wavelengths, the noise from the Poissonian -variation in the sky level increases. The PATTERN.ROW correction is +variation in the sky level increases. The \ippmisc{PATTERN.ROW} correction is still applied to data taken in the other filters, as the increase in sky noise does not fully obscure the row-by-row noise. @@ -1563,15 +1532,46 @@ \begin{figure*}[htpb] \centering - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_npt_XY57_sm.png} - \end{minipage}% - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_wpt_XY57_sm.png} - \end{minipage} - \caption{{\bf Correlated Noise:} Example of the PATTERN.ROW correction on exposure o5379g0103o OTA57 cell xy01 (\ips{} filter 45s). The left panel shows the cell with all appropriate detrending except the PATTERN.ROW, and the right shows the same cell with PATTERN.ROW applied. The correction reduces the correlated noise on the right side, which is most distant from the read out amplifier. There is a slight over subtraction along the rows near the bright star.} + %% need small version for arxiv + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/{correlated.noise\plotopt}.png} + \caption{{\bf Correlated Noise:} Example of the + \nocode{PATTERN.ROW} correction on exposure o5379g0103o OTA57 + cell xy01 (\ips{} filter 45s). The left panel shows the cell with + all appropriate detrending except the \nocode{PATTERN.ROW}, and + the right shows the same cell with \nocode{PATTERN.ROW} applied. + The correction reduces the correlated noise on the right side, + which is most distant from the read out amplifier. There is a + slight over subtraction along the rows near the bright star.} \label{fig: pattern row example} \end{figure*} \subsubsection{Pattern Continuity} + +\begin{figure*}[htpb] + \centering + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/{N157.v1\plotopt}.png} + \caption{These four panels illustrate the impact of the + \nocode{PATTERN.ROW}, \nocode{PATTERN.CONTINUITY}, and background + subtraction steps on a large galaxy. Upper-left: all detrends + except \nocode{PATTERN.ROW}, \nocode{PATTERN.CONTINUITY}, and background + subtraction applied to a single GPC1 image of NGC 157. + Upper-right: same image as upper-left with \nocode{PATTERN.ROW} applied. + Lower-right: same image as upper-right with + \nocode{PATTERN.CONTINUITY} applied. Lower-left: same image as + lower-right with background subtraction.} + \label{fig:ngc157.with.pattern} +\end{figure*} + +\begin{figure*}[htpb] + \centering + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/{N157.v2\plotopt}.png} + \caption{These two panels illustrate the impact of the + \nocode{PATTERN.CONTINUITY}, and background subtraction steps on a + large galaxy, without \nocode{PATTERN.ROW}. Left: all detrends + and \nocode{PATTERN.CONTINUITY}, but not \nocode{PATTERN.ROW} and + background subtraction, applied to a single GPC1 image of NGC 157. + Right: same image as left with background subtraction. Without + the \nocode{PATTERN.ROW} correction, the background is much less affected.} + \label{fig:ngc157.without.pattern} +\end{figure*} The background sky levels of cells on a single OTA do not always have @@ -1582,9 +1582,9 @@ along the rows of the cells that are not stable. This common feature across the columns of cells results in a ``saw tooth'' pattern -horizontally across an the mosaicked OTA, and as the background model +horizontally across the mosaicked OTA, and as the background model fits a smooth sky level, this induces over- and under subtraction at the cell boundaries. -The PATTERN.CONTINUITY correction, attempts to match the edges of a +The \ippmisc{PATTERN.CONTINUITY} correction, attempts to match the edges of a cell to those of its neighbors. For each cell, a thin box 10 pixels wide running the full length of each edge is extracted and the median @@ -1612,4 +1612,91 @@ effect of this correction on an image profile is shown in Figure \ref{fig:dark switching}. + +\subsection{Background (``Sky'') Subtraction} + +During the \IPPstage{chip}-stage processing, after the detrending +steps are done but before source detection begins, a model of the +background light is subtracted from each chip image. The decision to +subtract a background model is somewhat tricky as the trade-offs are +not clear in all possible cases. It is helpful to consider the types +of sources which contribute to the background light in astronomical +images. + +First, there is ``scattered light'', which means flux that reaches the +detector from a path that is different from the path through the +optics taken by the light from the imaged stars. In an ideal +telescope, no light could ever reach the detector without being imaged +by the optics. However, in a real telescope, especially in wide-field +systems such as the Pan-STARRS telescopes, it is impossible to +sufficiently baffle the optical path to prevent ``scattered'' +light\footnote{We put the term ``scattered'' in quotes because this + background may include light which reaches the detector directly + from the sky or other light source rather than scattering off + elements of the optical system.} from reaching the detector without +blocking the main optical path. This class of background light may +include sharp features such as the glints discussed +above(Section~\ref{sec:glints}), but in this discussion we are +primarily concerned with large-scale structures. Another type of +``scattered'' background light source would be the large out-of-focus +pupil image observed in \eg, the NOAO and CTIO wide-field imagers +\citep{2007ASPC..376..269S}. + +Second, there are direct terrestrial contributions to the background +light. This source of light follows the same path as the light from +the stars to the detector, but has an origin much closer to the +telescope. This may include glow from emission lines in the +atmosphere, light from the moon or terrestrial sources scattered off +thin (or thick!) clouds or just scattered in the clear atmosphere via +Rayleigh off dust particles and gas molecules in the atmosphere. Both +``scattered'' and direct terrestrial contributions to the background +light are not expected to be consistent for a given location on the +sky, though the pupil ghost image may well be the same for a fixed +telescope pointing and night sky brighness. + +Finally, there are astrophysical contributions to the background +light. These range from the nearby zodiacal light to the +extragalactic background. Depending on the context and the source +being measured, astrophysical background sources may even include the +diffuse flux from large galaxies. When measuring the flux of point +sources, it is necessary to subtract (or otherwise model) any +large-scale diffuse background component. When measuring a larger +object, e.g., a well-resolved galaxy, it is necessary to make a +decision what portion of the large-scale flux is a background and what +is part of the flux of the object being measured. + +When one measures the flux of an object in an image, two approaches to +the background light are possible. On the one hand, one could attempt +to include the background as part of the model-fitting parameters at +the time of the analysis. Alternatively, one could attempt to model +and subtract the background first and not include it in the per-object +model fit. For the IPP analysis, we choose the later method for two +reasons. First, in tests of the former method, we find that the +photometry of fitted objects is more inconsistent if the sky is fitted +for each object than if it is determined in a separate step +(presumably due to the extra degree of freedom in the model fitting). +Second, by subtracting a background model, we remove varying +backgrounds from the image so that the resulting pixels can later be +combined to make a deep stack. + +The details of the background model are discussed in Paper IV. +Briefly, the background subtraction is performed on each chip +independently. The image is divided into a grid of points with a +spacing of 400 pixels. A superpixel of size $800 \times 800$ pixels +is used to measure the background corresponding to each point. +Bilinear interpolation is used to estimate the background value at any +point in the full image. This approach works well to follow the +large-scale background structures from the terrestrial and scattered +sources, and to subtract the background light of large-scale +astronomical feasures for the analysis of point sources or small-scale +feasures such as small galaxies. However, this process acts as a +high-pass filter, with the result that galaxies larger than a certain +size will have a significant portion of their light subtracted. In +addition, the \ippmisc{PATTERN.ROW} and \ippmisc{PATTERN.CONTINUITY} +corrections described above (Section~\ref{sec:pattern}) also +over-subtract large galaxies, and interact badly with the background +model. Figures~\ref{fig:ngc157.with.pattern} and +\ref{fig:ngc157.without.pattern} illustrate the impact of the +background subtraction on a large galaxy both with and withouth the +\ippmisc{PATTERN.ROW} correction. \section{GPC1 Detrend Construction} @@ -1738,50 +1825,4 @@ \label{sec:warping} -\begin{figure}[htpb] - \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/warp_2046019_sci_sm.png} - \caption{Example of the warp image for skycell skycell.1146.095 - centered at ($\alpha,\delta$) = (11.934, -4.197) for exposure - o5104g0266o, (2009-09-30, 60s \rps{} filter). The data from four - OTAs contribute to this image, although they are all truncated by - the skycell boundaries. This skycell image is aligned such that - north points to the top of the image, and east to the left. The - contributing OTAs are OTA20, OTA21, OTA30, OTA31.} - \label{fig:warp image} -\end{figure} - -\begin{figure}[htpb] - \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/warp_2046019_var_sm.png} - \caption{Example of the warp variance image for skycell - skycell.1146.095 of exposure o5104g0266o, the same as in Figure - \ref{fig:warp image}. This variance map retains information about - the higher flux levels that were found in burntool corrected - persistence trails, which appear here as streaks along the - original OTA y axis. The dark glows that are corrected in the - dark model are also more visible, especially on certain cell - edges. As both of these effects are corrected in the science - image, there are no significant features visible there.} - \label{fig:warp variance} -\end{figure} - -\begin{figure}[htpb] - \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/warp_2046019_mask.png} - \caption{Example of the warp mask image for skycell skycell.1146.095 - of exposure o5104g0266o, the same as in Figure \ref{fig:warp - image}. This mask image shows the many small defects removed - from the image, along with larger advisory trails on corrected - burntool trails. The saturated cores of the bright stars are also - masked, along with the diffraction spikes found on these stars. A - ghost mask is visible just below the center as an elliptical - region. -% In addition OTA24 shows the precautionary crosstalk bleed masks -% for the two brightest stars applied to all cells within the same -% row. - \label{fig:warp mask} - } -\end{figure} - In order to perform image combination operations (stacking and differences), the individual OTA images are geometrically transformed @@ -1805,4 +1846,47 @@ pixel grid is thus subdivided into smaller sub-images called 'skycells'. + +\begin{figure}[htpb] + \centering + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/warp_2046019_sci\plotopt.png} + \caption{Example of the warp image for skycell skycell.1146.095 + centered at ($\alpha,\delta$) = (11.934, -4.197) for exposure + o5104g0266o, (2009-09-30, 60s \rps{} filter). The data from four + OTAs contribute to this image, although they are all truncated by + the skycell boundaries. This skycell image is aligned such that + north points to the top of the image, and east to the left. The + contributing OTAs are OTA20, OTA21, OTA30, OTA31.} + \label{fig:warp image} +\end{figure} + +\begin{figure}[htpb] + \centering + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/warp_2046019_var\plotopt.png} + \caption{Example of the warp variance image for skycell + skycell.1146.095 of exposure o5104g0266o, the same as in Figure + \ref{fig:warp image}. This variance map retains information about + the higher flux levels that were found in burntool corrected + persistence trails, which appear here as streaks along the + original OTA y axis. The dark glows that are corrected in the + dark model are also more visible, especially on certain cell + edges. As both of these effects are corrected in the science + image, there are no significant features visible there.} + \label{fig:warp variance} +\end{figure} + +\begin{figure}[htpb] + \centering + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/warp_2046019_mask.png} + \caption{Example of the warp mask image for skycell skycell.1146.095 + of exposure o5104g0266o, the same as in Figure \ref{fig:warp + image}. This mask image shows the many small defects removed + from the image, along with larger advisory trails on corrected + burntool trails. The saturated cores of the bright stars are also + masked, along with the diffraction spikes found on these stars. A + ghost mask is visible just below the center as an elliptical + region. + \label{fig:warp mask} + } +\end{figure} A tessellation can be defined for a limited region, with only a small @@ -1858,5 +1942,5 @@ \begin{figure}[t] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_sci_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/stack_3956997_sci\plotopt.png} \caption{Example of the stack image for skycell skycell.1146.095 centered at ($\alpha,\delta$) = (11.934, -4.197) in the \rps{} @@ -1905,5 +1989,5 @@ \begin{figure}[t] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_var_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/stack_3956997_var\plotopt.png} \caption{Example of the stack variance image for skycell skycell.1146.095 centered at ($\alpha,\delta$) = (11.934, -4.197) @@ -1946,5 +2030,5 @@ \begin{figure}[t] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_mask.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/stack_3956997_mask.png} \caption{Example of the stack mask image for skycell skycell.1146.095 centered at ($\alpha,\delta$) = (11.934, -4.197) @@ -1994,5 +2078,5 @@ \begin{figure}[t] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_num_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/stack_3956997_num\plotopt.png} \caption{Example of the stack number image for skycell skycell.1146.095 centered at ($\alpha,\delta$) = (11.934, -4.197) @@ -2041,5 +2125,5 @@ \begin{figure}[t] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_exp_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/stack_3956997_exp\plotopt.png} \caption{Example of the stack exposure time image for skycell skycell.1146.095 centered at ($\alpha,\delta$) = (11.934, -4.197) @@ -2098,5 +2182,5 @@ \begin{figure}[t] \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_expwt_sm.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{\picdir/stack_3956997_expwt\plotopt.png} \caption{Example of the stack weighted exposure image for skycell skycell.1146.095 centered at ($\alpha,\delta$) = (11.934, -4.197) @@ -2372,5 +2456,5 @@ Finally, a large number of issues arise due to the row-to-row bias -issues. The PATTERN.ROW correction is used on a limited number of +issues. The \ippmisc{PATTERN.ROW} correction is used on a limited number of cells, to minimize any possible distortion of bright stars or dense fields by the fitting process. As the row-to-row bias changes very Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/make.small.images =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/make.small.images (revision 40759) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.detrend/make.small.images (revision 40759) @@ -0,0 +1,7 @@ +#!/bin/csh -f + +foreach f (`cat big.images.txt | grep -v "^#"`) + set g = `echo $f | sed s/.png\$/_sm.png/` + echo convert -scale 50% $f $g + convert -scale 50% $f $g +end Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.diffs/Makefile =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.diffs/Makefile (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.diffs/Makefile (revision 40759) @@ -2,5 +2,5 @@ DO_PDFLATEX = 1 -DO_BIBTEX = 1 +DO_BIBTEX = 0 help: @@ -17,12 +17,26 @@ ../inputs/astro.sty \ ../inputs/apj.bst \ +../inputs/code.sty \ diffs.tex \ -references.bib \ -apj-jour.bib \ +diffs.bbl \ figures/noise_model/variance.eps \ figures/noise_model/covariance.eps \ figures/stack/stack_image.eps \ figures/all/kernel.eps \ -figures/all/sn2009kf.eps +figures/all/sn2009kf.eps \ +figures/diff/stamps_mean.eps \ +figures/diff/stamps_rms.eps \ +figures/diff/kernel_x.eps \ +figures/diff/kernel_xx.eps \ +figures/diff/kernel_xy.eps \ +figures/diff/deconv_max.eps \ +figures/noise_model/input_phot.eps \ +figures/noise_model/warp_phot.eps \ +figures/noise_model/diff_phot.eps \ +figures/noise_model/stack_phot.eps \ +figures/stack/stack_conv_m_dm.eps \ +figures/stack/stack_unconv_m_dm.eps \ +figures/stack/stack_conv_hist.eps \ +figures/stack/stack_unconv_hist.eps include ../Makefile.Common @@ -30,2 +44,4 @@ # submission : # tar --transform 's%inputs/%%' -zcf waters2017.tgz $(FILES) + +detrend.tgz: $(FILES) Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.diffs/diffs.tex =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.diffs/diffs.tex (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.diffs/diffs.tex (revision 40759) @@ -63,11 +63,30 @@ \section{Introduction} -The age of synoptic surveys has come. As optical (and other -wavelength) telescopes are surveying ever increasing areas to ever -fainter flux limits with multiple repeats, there is a growing interest -in the study of transient phenomena. This technological progress is -well matched with the current scientific emphases on large samples of -supernovae (SN), microlenses, asteroids, and other transients and -variable sources. +The past three decades have seen the increasing importance of +time-domain surveys in astronomy. These include asteroid searches +such as the Lincoln Near-Earth Asteroid Research +\citep[LINEAR][]{2000Icar..148...21S}, the Lowell Observatory +Near-Earth Object Search \citep[LONEOS,][]{1995DPS....27.0110B}, the +Catalina Sky Survey \citep{2003DPS....35.3604L}, and ATLAS +\citep{2018PASP..130f4505T}; microlensing surveys such as MACHO +\citep{1993ASPC...43..291A} and Optical Gravitational Lens Experiment +\citep[OGLE,][]{1992AcA....42..253U}; and searches for supernovae and +other transient sources such as ASAS-SN \citep{2014ApJ...788...48S}, the +Palomar Transient Factory \citep[PTF,][]{2009PASP..121.1395L}, and the Robotic Optical +Transient Search Experiment \citep[ROTSE-I,][]{2000ApJ...542..251A}. + +The Pan-STARRS Observatory \citep{chambers2017} has been a leader in +the searches for both explosive transient / supernova and potentially +hazardous asteroids. According to the statistics maintained by David +Bishop\footnote{http://www.rochesterastronomy.org/snimages/archives.html}, +since 2009, 40\% of all supernova have been discovered by +Pan-STARRS\,1. Similarly, 24\% of all Near Earth Objects (NEOs) +discovered to date have been found by +Pan-STARRS\footnote{https://cneos.jpl.nasa.gov/stats/site\_all.html}. +Since 2014, when Pan-STARRS shifted its primary mission to the search +for NEOs, this fraction has increased to 41\%. Both of these search +programs use nightly observations to hunt for features which have +changed, either between multiple images in a single night or between +the current image and an archival reference image. PSF-matched image differencing\footnote{We eschew the popular term @@ -88,5 +107,5 @@ of the convolution kernel as a linear combination of basis functions, which allows the least-squares problem to be reduced to a matrix -equation. \citet{2000A&AS..144..363A} showed how this can be expanded +equation. \citet{2000AAS..144..363A} showed how this can be expanded to allow spatial variation of the kernel across the images. Of course, the basis functions used for the kernel may be completely @@ -137,5 +156,5 @@ basis functions, $g_i(x,y) k_i(u,v)$, where the inclusion of $g_i(x,y)$ allows for spatial variation of the kernel. In order to -enforce conservation of flux \citep[following][]{2000A&AS..144..363A}, +enforce conservation of flux \citep[following][]{2000AAS..144..363A}, we specify that all of the kernel basis functions have zero sum, $\sum_{u,v} k_i(u,v) = 0\ \forall i$. This may be achieved by scaling @@ -161,10 +180,10 @@ $\chi^2$ between wide and narrow kernels. Setting $c_i \equiv 0$ and $p_i \equiv 0$ reduces the above equation to the -\citet{2000A&AS..144..363A} formalism, but with the normalisation +\citet{2000AAS..144..363A} formalism, but with the normalisation ($b_0$) included explicitly. In practise, the above sum will only be over small regions (known as ``stamps''), and if we assume that the spatial variation is not large, then we can simply use the coordinates of the stamp centres for the $g_i(x,y)$; this allows a faster -calculation \citep{2000A&AS..144..363A}. +calculation \citep{2000AAS..144..363A}. To simplify the equation, we write @@ -199,5 +218,5 @@ other special polynomial for the $f_i(x,y)$ and $g_i(x,y)$ is simple and convenient. The kernel basis function sets of -\citet{1998ApJ...503..325A} and \citet{2000A&AS..144..363A} are +\citet{1998ApJ...503..325A} and \citet{2000AAS..144..363A} are \begin{equation} g_i(x,y) k_i'(u,v) = \psi_i x^\ell y^m u^p v^q \exp((u^2+v^2)/2s_i^2) @@ -338,11 +357,13 @@ \subsection{Stamps} -The choice of stamps is key to successful PSF-matching --- the -convolution kernel is only as good as the stamps used to construct it. -We use a merged list of sources from photometry of the two input -images as the basis of our stamps list. Sources with a flag -indicating that it is anything other than a pristine astrophysical -source are excluded. At the present time, we make no effort to select -sources of a particular color or range of colors. +Since we restrict the analysis of the kernel required for PSF matching +to the small ``stamps'' centered on bright stars, the choice of stamps +is key to successful PSF-matching. The convolution kernel is only as +good as the stamps used to construct it. We use a merged list of +sources from photometry of the two input images as the basis of our +stamps list. Sources with a flag indicating that it is anything other +than a pristine astrophysical source are excluded. At the present +time, we make no effort to select sources of a particular color or +range of colors. We exclude sources with any masked pixels that would affect the @@ -490,5 +511,5 @@ progressive software packages producing \citep[e.g., SWarp:][]{2002ASPC..281..228B} and using \citep[e.g., - SExtractor:][]{1996A&AS..117..393B} weight maps to characterise the + SExtractor:][]{1996AAS..117..393B} weight maps to characterise the noise over the image. Because of simplicity and lower calculation cost relative to weights or standard deviations, we prefer to @@ -993,5 +1014,6 @@ \clearpage \bibliographystyle{apj} -\bibliography{apj-jour,references} +%\bibliography{apj-jour,references} +\input{diffs.bbl} \end{document} Index: /tags/ipp-ps2-20190404/doc/release.2015/ps1.diffs/references.bib =================================================================== --- /tags/ipp-ps2-20190404/doc/release.2015/ps1.diffs/references.bib (revision 40758) +++ /tags/ipp-ps2-20190404/doc/release.2015/ps1.diffs/references.bib (revision 40759) @@ -1,4 +1,186 @@ -@ARTICLE{2000A&AS..144..363A, +@ARTICLE{2000ApJ...542..251A, + author = {{Akerlof}, C. and {Balsano}, R. and {Barthelmy}, S. and {Bloch}, J. and + {Butterworth}, P. and {Casperson}, D. and {Cline}, T. and {Fletcher}, S. and + {Gisler}, G. and {Hills}, J. and {Kehoe}, R. and {Lee}, B. and + {Marshall}, S. and {McKay}, T. and {Pawl}, A. and {Priedhorsky}, W. and + {Seldomridge}, N. and {Szymanski}, J. and {Wren}, J.}, + title = "{Rapid Optical Follow-up Observations of SGR Events with ROTSE-I}", + journal = {\apj}, + keywords = {Gamma Rays: Bursts, Gamma Rays: Observations, pulsars: individual (SGR 1900+14, SGR 1806-20)}, + year = 2000, + month = oct, + volume = 542, + pages = {251-256}, + doi = {10.1086/309535}, + adsurl = {http://adsabs.harvard.edu/abs/2000ApJ...542..251A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@ARTICLE{2009PASP..121.1395L, + author = {{Law}, N.~M. and {Kulkarni}, S.~R. and {Dekany}, R.~G. and {Ofek}, E.~O. and + {Quimby}, R.~M. and {Nugent}, P.~E. and {Surace}, J. and {Grillmair}, C.~C. and + {Bloom}, J.~S. and {Kasliwal}, M.~M. and {Bildsten}, L. and + {Brown}, T. and {Cenko}, S.~B. and {Ciardi}, D. and {Croner}, E. and + {Djorgovski}, S.~G. and {van Eyken}, J. and {Filippenko}, A.~V. and + {Fox}, D.~B. and {Gal-Yam}, A. and {Hale}, D. and {Hamam}, N. and + {Helou}, G. and {Henning}, J. and {Howell}, D.~A. and {Jacobsen}, J. and + {Laher}, R. and {Mattingly}, S. and {McKenna}, D. and {Pickles}, A. and + {Poznanski}, D. and {Rahmer}, G. and {Rau}, A. and {Rosing}, W. and + {Shara}, M. and {Smith}, R. and {Starr}, D. and {Sullivan}, M. and + {Velur}, V. and {Walters}, R. and {Zolkower}, J.}, + title = "{The Palomar Transient Factory: System Overview, Performance, and First Results}", + journal = {\pasp}, +archivePrefix = "arXiv", + eprint = {0906.5350}, + primaryClass = "astro-ph.IM", + year = 2009, + month = dec, + volume = 121, + pages = {1395}, + doi = {10.1086/648598}, + adsurl = {http://adsabs.harvard.edu/abs/2009PASP..121.1395L}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@ARTICLE{2014ApJ...788...48S, + author = {{Shappee}, B.~J. and {Prieto}, J.~L. and {Grupe}, D. and {Kochanek}, C.~S. and + {Stanek}, K.~Z. and {De Rosa}, G. and {Mathur}, S. and {Zu}, Y. and + {Peterson}, B.~M. and {Pogge}, R.~W. and {Komossa}, S. and {Im}, M. and + {Jencson}, J. and {Holoien}, T.~W.-S. and {Basu}, U. and {Beacom}, J.~F. and + {Szczygie{\l}}, D.~M. and {Brimacombe}, J. and {Adams}, S. and + {Campillay}, A. and {Choi}, C. and {Contreras}, C. and {Dietrich}, M. and + {Dubberley}, M. and {Elphick}, M. and {Foale}, S. and {Giustini}, M. and + {Gonzalez}, C. and {Hawkins}, E. and {Howell}, D.~A. and {Hsiao}, E.~Y. and + {Koss}, M. and {Leighly}, K.~M. and {Morrell}, N. and {Mudd}, D. and + {Mullins}, D. and {Nugent}, J.~M. and {Parrent}, J. and {Phillips}, M.~M. and + {Pojmanski}, G. and {Rosing}, W. and {Ross}, R. and {Sand}, D. and + {Terndrup}, D.~M. and {Valenti}, S. and {Walker}, Z. and {Yoon}, Y. + }, + title = "{The Man behind the Curtain: X-Rays Drive the UV through NIR Variability in the 2013 Active Galactic Nucleus Outburst in NGC 2617}", + journal = {\apj}, +archivePrefix = "arXiv", + eprint = {1310.2241}, + primaryClass = "astro-ph.HE", + keywords = {galaxies: active, galaxies: nuclei, galaxies: Seyfert, line: formation, line: profiles}, + year = 2014, + month = jun, + volume = 788, + eid = {48}, + pages = {48}, + doi = {10.1088/0004-637X/788/1/48}, + adsurl = {http://adsabs.harvard.edu/abs/2014ApJ...788...48S}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@ARTICLE{1992AcA....42..253U, + author = {{Udalski}, A. and {Szymanski}, M. and {Kaluzny}, J. and {Kubiak}, M. and + {Mateo}, M.}, + title = "{The Optical Gravitational Lensing Experiment}", + journal = {\actaa}, + keywords = {Charge Coupled Devices, Galactic Bulge, Gravitational Lenses, Star Distribution, Statistical Analysis, Galactic Clusters, Galactic Halos, Variable Stars}, + year = 1992, + volume = 42, + pages = {253-284}, + adsurl = {http://adsabs.harvard.edu/abs/1992AcA....42..253U}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@INPROCEEDINGS{1993ASPC...43..291A, + author = {{Alcock}, C. and {Allsman}, R.~A. and {Axelrod}, T.~S. and {Bennett}, D.~P. and + {Cook}, K.~H. and {Park}, H.~S. and {Marshall}, S.~L. and {Stubbs}, C.~W. and + {Griest}, K. and {Perlmutter}, S. and {Sutherland}, W. and {Freeman}, K.~C. and + {Peterson}, B.~A. and {Quinn}, P.~J. and {Rodgers}, A.~W.}, + title = "{The MACHO Project - a Search for the Dark Matter in the Milky-Way}", +booktitle = {Sky Surveys. Protostars to Protogalaxies}, + year = 1993, + series = {Astronomical Society of the Pacific Conference Series}, + volume = 43, + editor = {{Soifer}, B.~T.}, + month = jan, + pages = {291}, + adsurl = {http://adsabs.harvard.edu/abs/1993ASPC...43..291A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@ARTICLE{2018PASP..130f4505T, + author = {{Tonry}, J.~L. and {Denneau}, L. and {Heinze}, A.~N. and {Stalder}, B. and + {Smith}, K.~W. and {Smartt}, S.~J. and {Stubbs}, C.~W. and {Weiland}, H.~J. and + {Rest}, A.}, + title = "{ATLAS: A High-cadence All-sky Survey System}", + journal = {\pasp}, +archivePrefix = "arXiv", + eprint = {1802.00879}, + primaryClass = "astro-ph.IM", + year = 2018, + month = jun, + volume = 130, + number = 6, + pages = {064505}, + doi = {10.1088/1538-3873/aabadf}, + adsurl = {http://adsabs.harvard.edu/abs/2018PASP..130f4505T}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@INPROCEEDINGS{2003DPS....35.3604L, + author = {{Larson}, S. and {Beshore}, E. and {Hill}, R. and {Christensen}, E. and + {McLean}, D. and {Kolar}, S. and {McNaught}, R. and {Garradd}, G. + }, + title = "{The CSS and SSS NEO surveys}", +booktitle = {AAS/Division for Planetary Sciences Meeting Abstracts \#35}, + year = 2003, + series = {Bulletin of the American Astronomical Society}, + volume = 35, + month = may, + eid = {36.04}, + pages = {982}, + adsurl = {http://adsabs.harvard.edu/abs/2003DPS....35.3604L}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@INPROCEEDINGS{1995DPS....27.0110B, + author = {{Bowell}, E. and {Koehn}, B.~W. and {Howell}, S.~B. and {Hoffman}, M. and + {Muinonen}, K.}, + title = "{The Lowell Observatory Near-Earth-Object Search: A Progress Report}", +booktitle = {AAS/Division for Planetary Sciences Meeting Abstracts \#27}, + year = 1995, + series = {Bulletin of the American Astronomical Society}, + volume = 27, + month = jun, + eid = {01.10}, + pages = {1057}, + adsurl = {http://adsabs.harvard.edu/abs/1995DPS....27.0110B}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@ARTICLE{2000Icar..148...21S, + author = {{Stokes}, G.~H. and {Evans}, J.~B. and {Viggh}, H.~E.~M. and + {Shelly}, F.~C. and {Pearce}, E.~C.}, + title = "{Lincoln Near-Earth Asteroid Program (LINEAR)}", + journal = {\icarus}, + year = 2000, + month = nov, + volume = 148, + pages = {21-28}, + doi = {10.1006/icar.2000.6493}, + adsurl = {http://adsabs.harvard.edu/abs/2000Icar..148...21S}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@ARTICLE{chambers2017, + author = {{Chambers}, K.~C. and {Magnier}, E.~A. and {Metcalfe}, N. and et al.}, + title = "{IPP}", + journal = {ArXiv e-prints}, +archivePrefix = "arXiv", + eprint ={1612.05560}, + primaryClass = "astro-ph.HE", + keywords = {Astrophysics}, + year = 2017, + month = jan, + adsurl = {http://adsabs.harvard.edu/abs/2016arXiv160203842A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + +@ARTICLE{2000AAS..144..363A, author = {{Alard}, C.}, title = "{Image subtraction using a space-varying kernel}",