Index: trunk/doc/release.2015/inputs/lib.bib =================================================================== --- trunk/doc/release.2015/inputs/lib.bib (revision 40438) +++ trunk/doc/release.2015/inputs/lib.bib (revision 40439) @@ -16291,2 +16291,13 @@ adsnote = {Provided by the SAO/NASA Astrophysics Data System} } + +@book{lanczos1956applied, + title={Applied analysis}, + author={Lanczos, C.}, + lccn={lc56012218}, + series={Prentice-Hall mathematics series}, + url={https://books.google.com/books?id=JmNKAAAAMAAJ}, + year={1956}, + publisher={Prentice-Hall} +} + Index: trunk/doc/release.2015/ps1.detrend/detrend.bbl =================================================================== --- trunk/doc/release.2015/ps1.detrend/detrend.bbl (revision 40438) +++ trunk/doc/release.2015/ps1.detrend/detrend.bbl (revision 40439) @@ -1,3 +1,3 @@ -\begin{thebibliography}{10} +\begin{thebibliography}{15} \expandafter\ifx\csname natexlab\endcsname\relax\def\natexlab#1{#1}\fi @@ -47,4 +47,41 @@ {Huber}, M., {TBD}, A., {TBD}, B., \& et~al. 2017, ArXiv e-prints +\bibitem[{Lanczos(1956)}]{lanczos1956applied} +Lanczos, C. 1956, Applied analysis, Prentice-Hall mathematics series + (Prentice-Hall) + +\bibitem[{{Lupton} {et~al.}(1999){Lupton}, {Gunn}, \& + {Szalay}}]{1999AJ....118.1406L} +{Lupton}, R.~H., {Gunn}, J.~E., \& {Szalay}, A.~S. 1999, \aj, 118, 1406 + +\bibitem[{{Magnier} \& {Cuillandre}(2004)}]{2004PASP..116..449M} +{Magnier}, E.~A. \& {Cuillandre}, J.-C. 2004, \pasp, 116, 449 + +\bibitem[{{Magnier} {et~al.}(2017){Magnier}, {Schlafly}, {Finkbeiner}, \& + et~al.}]{magnier2017.datasystem} +{Magnier}, E.~A., {Schlafly}, E.~F., {Finkbeiner}, D.~P., \& et~al. 2017, ArXiv + e-prints + +\bibitem[{{Magnier} {et~al.}(2016{\natexlab{a}}){Magnier}, {Schlafly}, + {Finkbeiner}, {Tonry}, {Goldman}, {R{\"o}ser}, {Schilbach}, {Chambers}, + {Flewelling}, {Huber}, {Price}, {Sweeney}, {Waters}, {Denneau}, {Draper}, + {Hodapp}, {Jedicke}, {Kudritzki}, {Metcalfe}, {Stubbs}, \& + {Wainscoast}}]{magnier2017.calibration} +{Magnier}, E.~A., {Schlafly}, E.~F., {Finkbeiner}, D.~P., {Tonry}, J.~L., + {Goldman}, B., {R{\"o}ser}, S., {Schilbach}, E., {Chambers}, K.~C., + {Flewelling}, H.~A., {Huber}, M.~E., {Price}, P.~A., {Sweeney}, W.~E., + {Waters}, C.~Z., {Denneau}, L., {Draper}, P., {Hodapp}, K.~W., {Jedicke}, R., + {Kudritzki}, R.-P., {Metcalfe}, N., {Stubbs}, C.~W., \& {Wainscoast}, R.~J. + 2016{\natexlab{a}}, ArXiv e-prints + +\bibitem[{{Magnier} {et~al.}(2016{\natexlab{b}}){Magnier}, {Sweeney}, + {Chambers}, {Flewelling}, {Huber}, {Price}, {Waters}, {Denneau}, {Draper}, + {Jedicke}, {Hodapp}, {Kudritzki}, {Metcalfe}, {Stubbs}, \& + {Wainscoast}}]{magnier2017.analysis} +{Magnier}, E.~A., {Sweeney}, W.~E., {Chambers}, K.~C., {Flewelling}, H.~A., + {Huber}, M.~E., {Price}, P.~A., {Waters}, C.~Z., {Denneau}, L., {Draper}, P., + {Jedicke}, R., {Hodapp}, K.~W., {Kudritzki}, R.-P., {Metcalfe}, N., {Stubbs}, + C.~W., \& {Wainscoast}, R.~J. 2016{\natexlab{b}}, ArXiv e-prints + \bibitem[{{Price} {et~al.}(2017){Price}, {TBD}, {TBD}, \& et~al.}]{price2017} {Price}, P.~A., {TBD}, A., {TBD}, B., \& et~al. 2017, ArXiv e-prints @@ -73,58 +110,3 @@ IAU General Assembly, 22, 2251124 -\bibitem[{{York} {et~al.}(2000){York}, {Adelman}, {Anderson}, {Anderson}, - {Annis}, {Bahcall}, {Bakken}, {Barkhouser}, {Bastian}, {Berman}, {Boroski}, - {Bracker}, {Briegel}, {Briggs}, {Brinkmann}, {Brunner}, {Burles}, {Carey}, - {Carr}, {Castander}, {Chen}, {Colestock}, {Connolly}, {Crocker}, {Csabai}, - {Czarapata}, {Davis}, {Doi}, {Dombeck}, {Eisenstein}, {Ellman}, {Elms}, - {Evans}, {Fan}, {Federwitz}, {Fiscelli}, {Friedman}, {Frieman}, {Fukugita}, - {Gillespie}, {Gunn}, {Gurbani}, {de Haas}, {Haldeman}, {Harris}, {Hayes}, - {Heckman}, {Hennessy}, {Hindsley}, {Holm}, {Holmgren}, {Huang}, {Hull}, - {Husby}, {Ichikawa}, {Ichikawa}, {Ivezi{\'c}}, {Kent}, {Kim}, {Kinney}, - {Klaene}, {Kleinman}, {Kleinman}, {Knapp}, {Korienek}, {Kron}, {Kunszt}, - {Lamb}, {Lee}, {Leger}, {Limmongkol}, {Lindenmeyer}, {Long}, {Loomis}, - {Loveday}, {Lucinio}, {Lupton}, {MacKinnon}, {Mannery}, {Mantsch}, {Margon}, - {McGehee}, {McKay}, {Meiksin}, {Merelli}, {Monet}, {Munn}, {Narayanan}, - {Nash}, {Neilsen}, {Neswold}, {Newberg}, {Nichol}, {Nicinski}, {Nonino}, - {Okada}, {Okamura}, {Ostriker}, {Owen}, {Pauls}, {Peoples}, {Peterson}, - {Petravick}, {Pier}, {Pope}, {Pordes}, {Prosapio}, {Rechenmacher}, {Quinn}, - {Richards}, {Richmond}, {Rivetta}, {Rockosi}, {Ruthmansdorfer}, {Sandford}, - {Schlegel}, {Schneider}, {Sekiguchi}, {Sergey}, {Shimasaku}, {Siegmund}, - {Smee}, {Smith}, {Snedden}, {Stone}, {Stoughton}, {Strauss}, {Stubbs}, - {SubbaRao}, {Szalay}, {Szapudi}, {Szokoly}, {Thakar}, {Tremonti}, {Tucker}, - {Uomoto}, {Vanden Berk}, {Vogeley}, {Waddell}, {Wang}, {Watanabe}, - {Weinberg}, {Yanny}, {Yasuda}, \& {SDSS Collaboration}}]{2000AJ....120.1579Y} -{York}, D.~G., {Adelman}, J., {Anderson}, Jr., J.~E., {Anderson}, S.~F., - {Annis}, J., {Bahcall}, N.~A., {Bakken}, J.~A., {Barkhouser}, R., {Bastian}, - S., {Berman}, E., {Boroski}, W.~N., {Bracker}, S., {Briegel}, C., {Briggs}, - J.~W., {Brinkmann}, J., {Brunner}, R., {Burles}, S., {Carey}, L., {Carr}, - M.~A., {Castander}, F.~J., {Chen}, B., {Colestock}, P.~L., {Connolly}, A.~J., - {Crocker}, J.~H., {Csabai}, I., {Czarapata}, P.~C., {Davis}, J.~E., {Doi}, - M., {Dombeck}, T., {Eisenstein}, D., {Ellman}, N., {Elms}, B.~R., {Evans}, - M.~L., {Fan}, X., {Federwitz}, G.~R., {Fiscelli}, L., {Friedman}, S., - {Frieman}, J.~A., {Fukugita}, M., {Gillespie}, B., {Gunn}, J.~E., {Gurbani}, - V.~K., {de Haas}, E., {Haldeman}, M., {Harris}, F.~H., {Hayes}, J., - {Heckman}, T.~M., {Hennessy}, G.~S., {Hindsley}, R.~B., {Holm}, S., - {Holmgren}, D.~J., {Huang}, C.-h., {Hull}, C., {Husby}, D., {Ichikawa}, - S.-I., {Ichikawa}, T., {Ivezi{\'c}}, {\v Z}., {Kent}, S., {Kim}, R.~S.~J., - {Kinney}, E., {Klaene}, M., {Kleinman}, A.~N., {Kleinman}, S., {Knapp}, - G.~R., {Korienek}, J., {Kron}, R.~G., {Kunszt}, P.~Z., {Lamb}, D.~Q., {Lee}, - B., {Leger}, R.~F., {Limmongkol}, S., {Lindenmeyer}, C., {Long}, D.~C., - {Loomis}, C., {Loveday}, J., {Lucinio}, R., {Lupton}, R.~H., {MacKinnon}, B., - {Mannery}, E.~J., {Mantsch}, P.~M., {Margon}, B., {McGehee}, P., {McKay}, - T.~A., {Meiksin}, A., {Merelli}, A., {Monet}, D.~G., {Munn}, J.~A., - {Narayanan}, V.~K., {Nash}, T., {Neilsen}, E., {Neswold}, R., {Newberg}, - H.~J., {Nichol}, R.~C., {Nicinski}, T., {Nonino}, M., {Okada}, N., {Okamura}, - S., {Ostriker}, J.~P., {Owen}, R., {Pauls}, A.~G., {Peoples}, J., {Peterson}, - R.~L., {Petravick}, D., {Pier}, J.~R., {Pope}, A., {Pordes}, R., {Prosapio}, - A., {Rechenmacher}, R., {Quinn}, T.~R., {Richards}, G.~T., {Richmond}, M.~W., - {Rivetta}, C.~H., {Rockosi}, C.~M., {Ruthmansdorfer}, K., {Sandford}, D., - {Schlegel}, D.~J., {Schneider}, D.~P., {Sekiguchi}, M., {Sergey}, G., - {Shimasaku}, K., {Siegmund}, W.~A., {Smee}, S., {Smith}, J.~A., {Snedden}, - S., {Stone}, R., {Stoughton}, C., {Strauss}, M.~A., {Stubbs}, C., {SubbaRao}, - M., {Szalay}, A.~S., {Szapudi}, I., {Szokoly}, G.~P., {Thakar}, A.~R., - {Tremonti}, C., {Tucker}, D.~L., {Uomoto}, A., {Vanden Berk}, D., {Vogeley}, - M.~S., {Waddell}, P., {Wang}, S.-i., {Watanabe}, M., {Weinberg}, D.~H., - {Yanny}, B., {Yasuda}, N., \& {SDSS Collaboration}. 2000, \aj, 120, 1579 - \end{thebibliography} Index: trunk/doc/release.2015/ps1.detrend/detrend.tex =================================================================== --- trunk/doc/release.2015/ps1.detrend/detrend.tex (revision 40438) +++ trunk/doc/release.2015/ps1.detrend/detrend.tex (revision 40439) @@ -200,7 +200,7 @@ \citep{magnier2017.datasystem}, but a short summary follows. The raw image data is stored on the processing cluster, with a database -storing the metadata of exposure parameters. These raw images can be -launched for the initial \IPPstage{chip} stage processing. This stage -performs the image detrending (described below in section +containing the metadata of exposure parameters. These raw images can +be launched for the initial \IPPstage{chip} stage processing. This +stage performs the image detrending (described below in section \ref{sec:detrending}), as well as the single epoch photometry \citep{magnier2017.analysis}, in parallel on the individual OTA device @@ -230,7 +230,7 @@ are provided in \citet{magnier2017.analysis}. -The limited version of same reduction procedure described above is -also performed in real time on new exposures as they are observed by -the telescope. This process is automatic, with new exposures being +A limited version of same reduction procedure described above is also +performed in real time on new exposures as they are observed by the +telescope. This process is automatic, with new exposures being downloaded from the summit to the main IPP processing cluster at the Maui Research and Technology Center in Kihei, and registered into the @@ -271,5 +271,5 @@ right, the OTA labels decrease in $X$ label, with the empty OTA00 located in the lower right. The OTA $Y$ labels increase upward in the -mosaic. \czw{This is somewhat of a mess?} +mosaic. \textit{Note: These papers are being placed on the arXiv.org to @@ -328,138 +328,8 @@ constructed for the signal image, the mask image, and the variance map image. The single epoch photometry is done at this stage as well. -The following subsections (\ref{sec:burntool} - \ref{sec:background}) -detail these detrending steps, presented in the order in which they -are applied to the individual OTA image data. \czw{I haven't - rearranged into regular and ``special'' yet.} - -\subsection{Burntool / Persistence effect} -\label{sec:burntool} - -Pixels that approach the saturation point on GPC1, which varies by -cell with common values around 60000 DN, introduce ``persistent -charge'' on that and subsequent images. During the read out process -of a cell with such a bright pixel, some of the charge remains in the -undepleted region of the silicon and is not shifted down the detector -column toward the amplifier. This charge remains in the starting -pixel and slowly leaks out of the undepleted region, contaminating -subsequent pixels during the read out process, resulting in a ``burn -trail'' that extends from the center of the bright source away from -the amplifier (vertically along the pixel columns toward the top of -the cell). - -%associated with it -%is not fully shifted down the detector column toward the amplifier. -%As a result, this charge remains in the starting cell, and is -%partially collected in subsequent shifts, resulting in a ``burn -%trail'' that extends from the center of the bright source away from -%the amplifier (vertically along the pixel columns toward the top of -%the cell). - -This incomplete charge shifting in nearly full wells continues as each -row is read out. This results in a remnant charge being deposited in -the pixels that the full well was shifted through. In following -exposures, this remnant charge leaks out, resulting in a trail that -extends from the initial location of the bright source on the previous -image towards the amplifier (vertically down along the pixel column). -This remnant charge can remain on the detector for up to thirty -minutes. -%, requiring the locations of these ``burns'' be retained -%between exposures. - -Both of these types of persistence trails are measured and optionally -repaired via the \IPPprog{burntool} program. This program does an -initial scan of the image, and identifies objects with pixel values -higher than a conservative threshold of 30000 DN. The trail from the -peak of that object is fit with a one-dimensional power law in each -pixel column above the threshold, based on empirical evidence that -this is the functional form of this persistence effect. This fit also -matches the expectation that a constant fraction of charge is -incompletely transferred at each shift beyond the persistence -threshold. Once the fit is done, the model can be subtracted from -the image. The location of the source is stored in a table along -with the exposure PONTIME, which denotes the number of seconds since -the detector was last powered on and provides an internally -consistent time scale. - -For subsequent exposures, the table associated with the previous image -is read in, and after correcting trails from the stars on the new -image, the positions of the bright stars from the table are used to -check for remnant trails from previous exposures on the image. These -are fit and subtracted using a one-dimensional exponential model, -again based on empirical studies. The output table retains this -remnant position for 2000 seconds after the initial PONTIME recorded. -This allows fits to be attempted well beyond the nominal lifetime of -these trails. Figure \ref{fig:burntool images} shows an example of a -cell with a persistence trail from a bright star, the post-correction -result, as well as the pre and post correction versions of the same -cell on the subsequent exposure. The profiles along the detector -columns for these two exposures are presented in Figure -\ref{fig:burntool plot}. - -Using this method of correcting the persistence trails has the -challenge that it is based on fits to the raw image data, which may -have other signal sources not determined by the persistence effect. -The presence of other stars or artifacts in the detector column can -result in a poor model to be fit, resulting in either an over- or -under-subtraction of the trail. For this reason, the image mask is -marked with a value indicating that this correction has been applied. -These pixels are not fully excluded, but they are marked as suspect, -which allows them to be excluded from consideration in subsequent -stages, such as image stacking. - -The cores of very bright stars can also be deformed by this process, -as the burntool fitting subtracts flux from only one side of the star. -As most stars that result in persistence trails already have saturated -cores, they are already ignored for the purpose of PSF determination -and are flagged as saturated by the photometry reduction. - -\begin{figure} - \centering - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_nobt.png} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} - \end{minipage}% - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_nobt.png} -% \caption{(b)} -% \end{subfigure} - \end{minipage} - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_bt.png} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} - \end{minipage}% - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_bt.png} -% \caption{(b)} -% \end{subfigure} - \end{minipage} - \caption{Example of OTA11 cell xy50 on exposures o5677g0123o (left) and o5677g0124o (right). The top panels show the image with all appropriate detrending steps, but without burntool, and the bottom show the same with burntool applied. There is some slight over subtraction in fitting the initial trail, but the impact of the trail is greatly reduced in both exposures.} - \label{fig:burntool images} -\end{figure} - - -\begin{figure} - \centering - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_bt_trail.png} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} - \end{minipage}% - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_bt_trail.png} -% \caption{(b)} -% \end{subfigure} - \end{minipage} - - \caption{Example of a profile cut along the y-axis through a bright star on exposure o5677g0123o OTA11 in cell xy50 (left panel) and on the subsequent exposure o5677g0124o (right panel). In both figures, the green points show the image corrected with all appropriate detrending steps, but without burntool applied, illustrating the amplitude of the persistence trails. The red points show the same data after the burntool correction, which reduces the impact of these features. Both exposures are in the \gps{} filter with exposure times of 43s} - \label{fig:burntool plot} -\end{figure} - - +The following subsections (\ref{sec:overscan} - \ref{sec:background}) +detail the detrending process used on GPC1 that are common to other +detectors. The GPC1 specific detrending steps are included after, +explaining these additional steps that remove the instrument signature. \subsection{Overscan} @@ -471,60 +341,5 @@ them. Each row has an overscan value subtracted, calculated by finding the median value of that row's overscan pixels and then -smoothing between rows with a three-row boxcar median. \czw{something - about this sounding like real pixels?} - -\subsection{Non-linearity Correction} -\label{sec:nonlinearity} - -The pixels of GPC1 are not uniformly linear at all flux levels. In -particular, at low flux levels, some pixels have a tendency to sag -relative to the expected linear value. This effect is most pronounced -along the edges of the detector cells, although some entire cells show -evidence of this effect. - -To correct this sag, we studied the behavior of a series of flat -frames for a ramp of exposure times with approximate logarithmically -equal spacing between 0.01s and 57.04s. As the exposure time -increases, the signal on each pixel also increases in what is expected -to be a linear manner. Each of the flat exposures in this ramp is -overscan corrected, and then the median is calculated for each cell, -as well as for the rows and columns within ten pixels of the edge of -the science region. From these median values at each exposure time -value, we can construct the expected trend by fitting a linear model -for the region considered. This fitting was limited to only the range -of fluxes between 12000 and 38000 counts, as these ranges were found -to match the linear model well. This range avoids the non-linearity -at low fluxes, as well as the possibility of high-flux non-linearity -effects. - -We store the average flux measurement and deviation from the linear -fit for each exposure time for all regions on all detector cells in -the linearity detrend look up tables. An example of this data is -shown in figure \ref{fig: nonlinearity}. When this correction is -applied to science data, these lookup tables are loaded, and a linear -interpolation is performed to determine the correction needed for the -flux in that pixel. This look up is performed for both the row and -column of each pixel, to allow the edge correction to be applied where -applicable, and the full cell correction elsewhere. The average of -these two values is then applied to the pixel value, reducing the -effects of pixel nonlinearity. - -This non-linearity effect appears to be stable in time for the -majority of the detector pixels, with little evident change over the -survey duration. However, as the non-linearity is most pronounced at -the edges of the detector cells, those are the regions where the -correction is most likely to be incomplete. Because of this fact, -most pixels in the static mask with either the DARKMASK or FLATMASK -bit set are found along these edges. As the non-linearity correction -is unable to reliably restore these pixels, they produce inconsistent -values after the dark and flat have been applied, and are therefore -rejected. - -\begin{figure} - \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/linearity_XY27_xy16.png} - \caption{Example plot of the linearity correction as a fraction of observed flux for OTA27, cell xy16.} - \label{fig: nonlinearity} -\end{figure} +smoothing between rows with a three-row boxcar median. \subsection{Dark/Bias Subtraction} @@ -548,5 +363,5 @@ Applying the dark model is simply a matter of calculating the response -to the exposure time and detector temperature for the image to be +for the exposure time and detector temperature of the image to be corrected, and subtracting the resulting dark signal from the image. Figure \ref{fig:dark image} shows the results of the dark subtraction. @@ -608,15 +423,9 @@ \begin{figure} \centering -% \begin{subfigure}[]{.45\hsize} \begin{minipage}{0.45\hsize} \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_M_OS_NL_XY23_b1.jpg} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} \end{minipage}% \begin{minipage}{0.45\hsize} \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_to_DARK_XY23_b1.jpg} -% \caption{(b)} -% \end{subfigure} \end{minipage} \caption{An example of the dark model application to exposure o5677g0123o, OTA23 (2011-04-26, 43s \gps{} filter). The left panel shows the image data mosaicked to the OTA level, and has had the static mask applied, the overscan subtracted, and the detector non-linearity corrected. The right panel, shows the same exposure with the dark applied in addition to the processing shown on the left, removing the amplifier glows in the cell corners.} @@ -644,5 +453,5 @@ To generate a correction for this change, a set of video dark models were created by running the standard dark construction process on a -series of dark frames that have had the video signal enabled for some +series of dark frames that had the video signal enabled for some cells. GPC1 can only run video signals on a subset of the OTAs at a given time. This requires two passes to enable the video signal @@ -669,15 +478,9 @@ \begin{figure} \centering -% \begin{subfigure}[]{.45\hsize} \begin{minipage}{0.45\hsize} \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_VIDEODARK_VDim_Rdark_XY22_b1.jpg} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} \end{minipage}% \begin{minipage}{0.45\hsize} \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_VIDEODARK_VDim_VDdark_XY22_b1.jpg} -% \caption{(b)} -% \end{subfigure} \end{minipage} \caption{An example of the video dark model application to exposure o5677g0123o, OTA22 (2011-04-26, 43s \gps{} filter), which has a video cell located in cell xy16. The left panel shows the image data mosaicked to the OTA level, and has had the static mask applied, the overscan subtracted, the detector non-linearity corrected, and a regular dark applied. The right panel, shows the same exposure with a video dark applied instead of the standard dark. The main impact of this change is the improved correction of the corner glows, which are over subtracted with the standard dark.} @@ -689,5 +492,5 @@ Based on a study of the positional dependence of all detected sources, -we have discovered that the cells in GPC1 do not have uniform noise +we discovered that the cells in GPC1 do not have uniform noise characteristics. Instead, there is a gradient along the pixel rows, with the noise generally higher away from the read out amplifier @@ -739,36 +542,4 @@ the additional empirical variance term in place of a single read noise value. - -%% In the detection process, we expect false positives at a rate equal to -%% the one-tailed probability beyond the detection threshold. For these -%% tests, only detections measured at the $\sigma_{thresh} = 5\sigma$ -%% level are used, to match that used in the photometry on science data. -%% This probability can be converted into a number of false detections by -%% considering a given area. As the detections must be isolated to not -%% be detected as an extended object, this area must be reduced by the -%% area a given PSF occupies. Combining this, we find that we expect a -%% probability $P = 1 - \Phi_{normal}(5) = \frac{1}{2} -%% \erfcinv\left(\frac{5}{\sqrt{2}}\right)$, and an area given $N$ -%% exposures of area $X\times Y$, $A = \frac{X \times Y \times -%% N}{A_{PSF}}$. For a typical $1"$ seeing, $A_{PSF}$ is approximately -%% 16 pixels. Using this model for the false positives, we found that -%% the added read noise was insufficient to account for the observed -%% false positive rate. Inverting this relation, we can measure -%% $\sigma_{obs}$, the true threshold level based on the number of false -%% positives observed. This $\sigma_{obs}$ is the combined to form a -%% boost factor $B = \sigma_{thresh} / \sigma_{obs}$ that amplifies the -%% noisemap to match the observed false detection rate. - -%% The row-to-row variations that contribute to the extra noise are -%% correlated with the dark model, -%% changes, the effective noise also changes. To ensure that the -%% noisemap accurately matches the true noise level, we have created -%% different noisemap models for the three major time ranges of the dark -%% model. We do not see any strong evidence that the noisemaps have the -%% A/B modes visible in the dark, and so we do not generate different -%% models for each individual dark model. The additional pixel-to-pixel -%% variance from this noisemap is added to the Poissonian variance to -%% form the science variance image generated by the \IPPstage{chip} -%% processing. \subsection{Flat} @@ -807,168 +578,4 @@ on this process are contained in \citet{magnier2017.calibration}. -\subsection{Pattern correction} -\label{sec:pattern} - -%% Due to detector specific issues that are not cleanly removed by the -%% dark model, we have a set of ``pattern'' corrections that are applied -%% to some selection of the OTAs in the camera. This is done to reduce -%% the effect that detector differences have on the measured astronomical -%% signal that are not stable enough to be corrected with a static model. -%% Because of this, the pattern corrections attempt to identify and -%% correct the detector issues based on appropriate filtering the -%% individual science exposures. - -%% In addition to the standard detrend corrections, we apply additional -%% adjustments for features that are not completely removed by the dark -%% model. - -%% The PATTERN.ROW correction is used to remove any remaining row-by-row -%% bias variation, and the PATTERN.CONTINUITY correction attempts to -%% ensure that the cells of a given OTA are consistent with the other -%% cells on that OTA. - -\subsubsection{Pattern Row} -%% Statistics so I have them written down somewhere -%% chipProcessedImfile.bg/bg_stdev by filter for XY33 (a good chip) -%% filter bg_mean stdev median Qsig bg_stdev_mean stdev median Qsig -%% g 36.37422026669 64.64175104057 32.693 6.10284 14.696938349131 78.80460307171 8.8401 0.5417843 -%% r 200.96143304525 471.87743546238 117.105 94.55608 33.854672792146 79.01642728089 13.4564 5.3771355 -%% i 447.00504994458 938.38517801037 286.810 154.71397 57.298335510188 99.38392923935 20.0217 24.2254723 -%% z 317.54933679054 390.38930252748 241.014 114.13316 48.359069000176 94.44452756094 17.9404 9.1535209 -%% y 371.09019536218 293.57439970375 288.481 133.38769 43.724342221691 135.04286534327 19.9029 7.5396461 - -As discussed above in the dark and noisemap sections, certain -detectors have significant bias offsets between adjacent rows, caused -by drifts in the bias level due to cross talk. The magnitude of these -offsets increases as the distance from the readout amplifier and -overscan region increases, resulting in horizontal streaks that are -more pronounced along the large x pixel edge of the cell. As the -level of the offset is apparently random between exposures, the dark -correction cannot fully remove this structure from the images, and the -noisemap value only indicates the level of the average variance added -by these bias offsets. Therefore, we apply the PATTERN.ROW correction -in an attempt to mitigate the offsets and correct the image values. -To force the rows to agree, a second order clipped polynomial is fit -to each row in the cell. Four fit iterations are run, and pixels -$2.5\sigma$ deviant are excluded from subsequent fits, to minimize the -effect stars and other astronomical signals have. This final trend is -then subtracted from that row. Simply doing this subtraction will -also have the effect of removing the background sky level. To prevent -this, the constant and linear terms for each row are stored, and -linear fits are made to these parameters as a function of row, -perpendicular to the initial fits. This produces a plane that is -added back to the image to restore the background offset and any -linear ramp that exists in the sky. - -These row-by-row variations have the largest impact on data taken in -the \gps{} filter, as the read noise is the dominant noise source in -that filter. At longer wavelengths, the noise from the Poissonian -variation in the sky level increases. The PATTERN.ROW correction is -still applied to data taken in the other filters, because the increase -in sky noise does not fully obscure the row-by-row noise. - -This correction was required on all cells on all OTAs prior to -2009-12-01, at which point a modification of the camera electronics -reduced the scale of the row-by-row offsets for the majority of the -OTAs. \czw{describe modification} As a result, we only apply this -correction to the cells where it is still necessary, as shown in -Figure \ref{fig: pattern row cells}. A list of these cells is in -Table \ref{tab:pattern_row_cells}. - -Although this correction largely resolves the row-by-row offset issue -in a satisfactory way, large and bright astronomical objects can bias -the fit significantly. This results in an oversubtraction of the -offset near these objects. As the offsets are calculated on the pixel -rows, this oversubtraction is not uniform around the object, but is -preferentially along the horizontal x axis of the object. Most -astronomical objects are not significantly distorted by this, with -this only becoming on issue for only bright objects comparable to the -size of the cell (598 pixels = 150"). Figure \ref{fig: pattern row example} -shows an example of a cell pre- and post-correction. - -\begin{deluxetable}{lcccc} - \tablecolumns{3} - \tablewidth{0pc} - \tablecaption{Cells which have PATTERN.ROW correction applied} - \tablehead{\colhead{OTA} & \colhead{Cell columns} & \colhead{Additional cells}} - \startdata - OTA11 & & xy02, xy03, xy04, xy07 \\ - OTA14 & & xy23 \\ - OTA15 & 0 & \\ - OTA27 & 0, 1, 2, 3, 7 & \\ - OTA31 & 7 & \\ - OTA32 & 3, 7 & \\ - OTA45 & 3, 7 & \\ - OTA47 & 0, 3, 5, 7 & \\ - OTA57 & 0, 1, 2, 6, 7 & \\ - OTA60 & & xy55 \\ - OTA74 & 2, 7 & \\ - \enddata - \label{tab:pattern_row_cells} -\end{deluxetable} - -\begin{figure} - \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/pattern_row_edit.png} - \caption{Diagram illustrating in red which cells on GPC1 require the PATTERN.ROW correction to be applied. The footprint of each OTA is outlined, and cell xy00 is marked with either a filled box or an outline. The labeling of the non-existent corner OTAs is provided to orient the focal plane.} - \label{fig: pattern row cells} -\end{figure} - -\begin{figure} - \centering - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_XY57_nopat.png} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} - \end{minipage}% - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_XY57_pat.png} -% \caption{(b)} -% \end{subfigure} - \end{minipage} - \caption{Example of the PATTERN.ROW correction on exposure o5379g0103o OTA57 cell xy01 (\ips{} filter 45s). The left panel shows the cell with all appropriate detrending except the PATTERN.ROW, and the right shows the same cell with PATTERN.ROW applied. The correction reduces the correlated noise on the right side, which is most distant from the read out amplifier. There is a slight over subtraction along the rows near the bright star. \czw{I don't think this fits the convention I stated earlier}} - \label{fig: pattern row example} -\end{figure} - -\subsubsection{Pattern Continuity} - -The background levels of cells on a single OTA do not always have the -same value. Even with dark and flat corrections applied, adjacent -cells may not match. In addition, studies of the background level -indicate that the row-by-row bias can introduce small background -gradient variations along the rows of the cells that are not stable. -This common feature across the columns of cells results in a ``saw -tooth'' pattern horizontally across an the mosaicked OTA, and as the -background model fits a smooth sky level, this induces over and under -subtraction at the cell boundaries. - -The PATTERN.CONTINUITY correction, attempts to match the edges of a -cell to those of its neighbors. For each cell, a thin box 10 pixels -wide running the full length of each edge is extracted and the median -value of unmasked values calculated for that box. These median values -are then used to construct a vector of the sum of the differences -between that cell's edges and the corresponding edge on any adjacent -cell $\Delta$. A matrix $A$ of these associations is also -constructed, with the diagonal containing the number of cells adjacent -to that cell, and the off-diagonal values being set to -1 for each -pair of adjacent cells. The offsets needed for each chip, $x$ can -then be found by solving the system $A x = \Delta$. A cell with the -maximum number of neighbors, usually cell xy11, the first cell not on -the edge of the OTA, is used to constrain the system, ensuring that -that cell has zero correction and that there is a single solution. - -For OTAs that initially show the saw tooth pattern, the effect of this -correction is to align the cells into a single ramp, at the expense of -the absolute background level. However, as we subtract off a smooth -background model prior to doing photometry, these deviations from an -absolute sky level do not affect photometry for small sources. The -fact that the final ramp is smoother than it would be otherwise also -allows for the background subtracted image to more closely match the -astronomical sky, without significant errors at cell boundaries. An -example of the effect of this correction on an image profile is shown -in Figure \ref{fig:dark switching}. - - \subsection{Fringe correction} \label{sec:fringe} @@ -1039,8 +646,8 @@ on the OTA due to defects in the semiconductor manufacturing \czw{check this fact with Peter}. To generate the mask for these -regions, a sample set of \czw{26} evenly-illuminated flat-field images -were measured to produce a map of the image variance in 20x20 pixel -bins. As the flat screen is expected to illuminate the image -uniformly, the expected variances in each bin should be Poissonian +regions, a sample set of 26 evenly-illuminated flat-field images were +measured to produce a map of the image variance in 20x20 pixel bins. +As the flat screen is expected to illuminate the image uniformly on +this scale, the expected variances in each bin should be Poissonian distributed with the flux level. However, in regions with poor CTE, adjacent pixels are not independent, as the charge in those pixels is @@ -1049,4 +656,5 @@ variance. All regions with variance less than half the average image level are added to the static mask. + The next step of mask construction is to examine the flat and dark @@ -1064,7 +672,8 @@ removing the pixels that cannot be corrected to a linear response. +% http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/StaticMasks20101215 The final step of mask construction is to examine the detector for bright columns and other static pixel issues. This is first done by -processing \czw{examining residuals in flattened flat-field images?} a set of 100 \ips{} filter science images in the same fashion as +processing a set of 100 \ips{} filter science images in the same fashion as for the DARKMASK. A median image is constructed from these inputs along with the per-pixel variance. These images are used to identify @@ -1126,5 +735,5 @@ nature, and do not completely exclude the pixel from further processing consideration. The first of these dynamic masks is the -burntool advisory mask mentioned above. These pixels are included for +burntool advisory mask described below. These pixels are included for photometry, but are rejected more readily in the stacking and difference image construction, as they are more likely to have small @@ -1195,5 +804,4 @@ \label{tab:crosstalk_rules} \end{deluxetable} - \subsubsubsection{Optical ghosts} @@ -1271,5 +879,4 @@ \label{tab:ghost_magnitudes} \end{deluxetable} - \begin{figure} @@ -1450,5 +1057,5 @@ distribution with a Gaussian. All pixels that were masked in the initial calculation are unmasked, and a histogram is again constructed -of the values, with a bin size set to $\sigma_{guess} / \left( N_{50} / +from the values, with a bin size set to $\sigma_{guess} / \left( N_{50} / 500 \right)$. With this bin size, we expect that a bin at $\pm 2 \sigma$ will have approximately 50 input points, which gives a @@ -1504,4 +1111,302 @@ scale $3\pi$ PV3 reduction. +\subsection{Burntool / Persistence effect} +\label{sec:burntool} + +Pixels that approach the saturation point on GPC1, which varies by +cell with common values around 60000 DN, introduce ``persistent +charge'' on that and subsequent images. During the read out process +of a cell with such a bright pixel, some of the charge remains in the +undepleted region of the silicon and is not shifted down the detector +column toward the amplifier. This charge remains in the starting +pixel and slowly leaks out of the undepleted region, contaminating +subsequent pixels during the read out process, resulting in a ``burn +trail'' that extends from the center of the bright source away from +the amplifier (vertically along the pixel columns toward the top of +the cell). + +This incomplete charge shifting in nearly full wells continues as each +row is read out. This results in a remnant charge being deposited in +the pixels that the full well was shifted through. In following +exposures, this remnant charge leaks out, resulting in a trail that +extends from the initial location of the bright source on the previous +image towards the amplifier (vertically down along the pixel column). +This remnant charge can remain on the detector for up to thirty +minutes. + +Both of these types of persistence trails are measured and optionally +repaired via the \IPPprog{burntool} program. This program does an +initial scan of the image, and identifies objects with pixel values +higher than a conservative threshold of 30000 DN. The trail from the +peak of that object is fit with a one-dimensional power law in each +pixel column above the threshold, based on empirical evidence that +this is the functional form of this persistence effect. This fit also +matches the expectation that a constant fraction of charge is +incompletely transferred at each shift beyond the persistence +threshold. Once the fit is done, the model can be subtracted from +the image. The location of the source is stored in a table along +with the exposure PONTIME, which denotes the number of seconds since +the detector was last powered on and provides an internally +consistent time scale. + +For subsequent exposures, the table associated with the previous image +is read in, and after correcting trails from the stars on the new +image, the positions of the bright stars from the table are used to +check for remnant trails from previous exposures on the image. These +are fit and subtracted using a one-dimensional exponential model, +again based on empirical studies. The output table retains this +remnant position for 2000 seconds after the initial PONTIME recorded. +This allows fits to be attempted well beyond the nominal lifetime of +these trails. Figure \ref{fig:burntool images} shows an example of a +cell with a persistence trail from a bright star, the post-correction +result, as well as the pre and post correction versions of the same +cell on the subsequent exposure. The profiles along the detector +columns for these two exposures are presented in Figure +\ref{fig:burntool plot}. + +Using this method of correcting the persistence trails has the +challenge that it is based on fits to the raw image data, which may +have other signal sources not determined by the persistence effect. +The presence of other stars or artifacts in the detector column can +result in a poor model to be fit, resulting in either an over- or +under-subtraction of the trail. For this reason, the image mask is +marked with a value indicating that this correction has been applied. +These pixels are not fully excluded, but they are marked as suspect, +which allows them to be excluded from consideration in subsequent +stages, such as image stacking. + +The cores of very bright stars can also be deformed by this process, +as the burntool fitting subtracts flux from only one side of the star. +As most stars that result in persistence trails already have saturated +cores, they are already ignored for the purpose of PSF determination +and are flagged as saturated by the photometry reduction. + +\begin{figure} + \centering + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_nobt.png} + \end{minipage}% + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_nobt.png} + \end{minipage} + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_bt.png} + \end{minipage}% + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_bt.png} + \end{minipage} + \caption{Example of OTA11 cell xy50 on exposures o5677g0123o (left) and o5677g0124o (right). The top panels show the image with all appropriate detrending steps, but without burntool, and the bottom show the same with burntool applied. There is some slight over subtraction in fitting the initial trail, but the impact of the trail is greatly reduced in both exposures.} + \label{fig:burntool images} +\end{figure} + + +\begin{figure} + \centering + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_bt_trail.png} + \end{minipage}% + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_bt_trail.png} + \end{minipage} + + \caption{Example of a profile cut along the y-axis through a bright star on exposure o5677g0123o OTA11 in cell xy50 (left panel) and on the subsequent exposure o5677g0124o (right panel). In both figures, the green points show the image corrected with all appropriate detrending steps, but without burntool applied, illustrating the amplitude of the persistence trails. The red points show the same data after the burntool correction, which reduces the impact of these features. Both exposures are in the \gps{} filter with exposure times of 43s} + \label{fig:burntool plot} +\end{figure} + +\subsection{Non-linearity Correction} +\label{sec:nonlinearity} + +The pixels of GPC1 are not uniformly linear at all flux levels. In +particular, at low flux levels, some pixels have a tendency to sag +relative to the expected linear value. This effect is most pronounced +along the edges of the detector cells, although some entire cells show +evidence of this effect. + +To correct this sag, we studied the behavior of a series of flat +frames for a ramp of exposure times with approximate logarithmically +equal spacing between 0.01s and 57.04s. As the exposure time +increases, the signal on each pixel also increases in what is expected +to be a linear manner. Each of the flat exposures in this ramp is +overscan corrected, and then the median is calculated for each cell, +as well as for the rows and columns within ten pixels of the edge of +the science region. From these median values at each exposure time +value, we can construct the expected trend by fitting a linear model +for the region considered. This fitting was limited to only the range +of fluxes between 12000 and 38000 counts, as these ranges were found +to match the linear model well. This range avoids the non-linearity +at low fluxes, as well as the possibility of high-flux non-linearity +effects. + +We store the average flux measurement and deviation from the linear +fit for each exposure time for each region on all detector cells in +the linearity detrend look up tables. An example of this data is +shown in figure \ref{fig: nonlinearity}. When this correction is +applied to science data, these lookup tables are loaded, and a linear +interpolation is performed to determine the correction needed for the +flux in that pixel. This look up is performed for both the row and +column of each pixel, to allow the edge correction to be applied where +applicable, and the full cell correction elsewhere. The average of +these two values is then applied to the pixel value, reducing the +effects of pixel nonlinearity. + +This non-linearity effect appears to be stable in time for the +majority of the detector pixels, with little evident change over the +survey duration. However, as the non-linearity is most pronounced at +the edges of the detector cells, those are the regions where the +correction is most likely to be incomplete. Because of this fact, +most pixels in the static mask with either the DARKMASK or FLATMASK +bit set are found along these edges. As the non-linearity correction +is unable to reliably restore these pixels, they produce inconsistent +values after the dark and flat have been applied, and are therefore +rejected. + +\begin{figure} + \centering + \includegraphics[width=0.9\hsize,angle=0,clip]{images/linearity_XY27_xy16.png} + \caption{Example plot of the linearity correction as a fraction of observed flux for OTA27, cell xy16.} + \label{fig: nonlinearity} +\end{figure} + +\subsection{Pattern correction} +\label{sec:pattern} + +\subsubsection{Pattern Row} +%% Statistics so I have them written down somewhere +%% chipProcessedImfile.bg/bg_stdev by filter for XY33 (a good chip) +%% filter bg_mean stdev median Qsig bg_stdev_mean stdev median Qsig +%% g 36.37422026669 64.64175104057 32.693 6.10284 14.696938349131 78.80460307171 8.8401 0.5417843 +%% r 200.96143304525 471.87743546238 117.105 94.55608 33.854672792146 79.01642728089 13.4564 5.3771355 +%% i 447.00504994458 938.38517801037 286.810 154.71397 57.298335510188 99.38392923935 20.0217 24.2254723 +%% z 317.54933679054 390.38930252748 241.014 114.13316 48.359069000176 94.44452756094 17.9404 9.1535209 +%% y 371.09019536218 293.57439970375 288.481 133.38769 43.724342221691 135.04286534327 19.9029 7.5396461 + +As discussed above in the dark and noisemap sections, certain +detectors have significant bias offsets between adjacent rows, caused +by drifts in the bias level due to cross talk. The magnitude of these +offsets increases as the distance from the readout amplifier and +overscan region increases, resulting in horizontal streaks that are +more pronounced along the large x pixel edge of the cell. As the +level of the offset is apparently random between exposures, the dark +correction cannot fully remove this structure from the images, and the +noisemap value only indicates the level of the average variance added +by these bias offsets. Therefore, we apply the PATTERN.ROW correction +in an attempt to mitigate the offsets and correct the image values. +To force the rows to agree, a second order clipped polynomial is fit +to each row in the cell. Four fit iterations are run, and pixels +$2.5\sigma$ deviant are excluded from subsequent fits, to minimize the +effect stars and other astronomical signals have. This final trend is +then subtracted from that row. Simply doing this subtraction will +also have the effect of removing the background sky level. To prevent +this, the constant and linear terms for each row are stored, and +linear fits are made to these parameters as a function of row, +perpendicular to the initial fits. This produces a plane that is +added back to the image to restore the background offset and any +linear ramp that exists in the sky. + +These row-by-row variations have the largest impact on data taken in +the \gps{} filter, as the read noise is the dominant noise source in +that filter. At longer wavelengths, the noise from the Poissonian +variation in the sky level increases. The PATTERN.ROW correction is +still applied to data taken in the other filters, as the increase in +sky noise does not fully obscure the row-by-row noise. + +This correction was required on all cells on all OTAs prior to +2009-12-01, at which point a modification of the camera electronics +reduced the scale of the row-by-row offsets for the majority of the +OTAs. \czw{describe modification} As a result, we only apply this +correction to the cells where it is still necessary, as shown in +Figure \ref{fig: pattern row cells}. A list of these cells is in +Table \ref{tab:pattern_row_cells}. + +Although this correction largely resolves the row-by-row offset issue +in a satisfactory way, large and bright astronomical objects can bias +the fit significantly. This results in an oversubtraction of the +offset near these objects. As the offsets are calculated on the pixel +rows, this oversubtraction is not uniform around the object, but is +preferentially along the horizontal x axis of the object. Most +astronomical objects are not significantly distorted by this, with +this only becoming on issue for only bright objects comparable to the +size of the cell (598 pixels = 150"). Figure \ref{fig: pattern row example} +shows an example of a cell pre- and post-correction. + +\begin{deluxetable}{lcccc} + \tablecolumns{3} + \tablewidth{0pc} + \tablecaption{Cells which have PATTERN.ROW correction applied} + \tablehead{\colhead{OTA} & \colhead{Cell columns} & \colhead{Additional cells}} + \startdata + OTA11 & & xy02, xy03, xy04, xy07 \\ + OTA14 & & xy23 \\ + OTA15 & 0 & \\ + OTA27 & 0, 1, 2, 3, 7 & \\ + OTA31 & 7 & \\ + OTA32 & 3, 7 & \\ + OTA45 & 3, 7 & \\ + OTA47 & 0, 3, 5, 7 & \\ + OTA57 & 0, 1, 2, 6, 7 & \\ + OTA60 & & xy55 \\ + OTA74 & 2, 7 & \\ + \enddata + \label{tab:pattern_row_cells} +\end{deluxetable} + +\begin{figure} + \centering + \includegraphics[width=0.9\hsize,angle=0,clip]{images/pattern_row_edit.png} + \caption{Diagram illustrating in red which cells on GPC1 require the PATTERN.ROW correction to be applied. The footprint of each OTA is outlined, and cell xy00 is marked with either a filled box or an outline. The labeling of the non-existent corner OTAs is provided to orient the focal plane.} + \label{fig: pattern row cells} +\end{figure} + +\begin{figure} + \centering + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_XY57_nopat.png} + \end{minipage}% + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_XY57_pat.png} + \end{minipage} + \caption{Example of the PATTERN.ROW correction on exposure o5379g0103o OTA57 cell xy01 (\ips{} filter 45s). The left panel shows the cell with all appropriate detrending except the PATTERN.ROW, and the right shows the same cell with PATTERN.ROW applied. The correction reduces the correlated noise on the right side, which is most distant from the read out amplifier. There is a slight over subtraction along the rows near the bright star.} + \label{fig: pattern row example} +\end{figure} + +\subsubsection{Pattern Continuity} + +The background levels of cells on a single OTA do not always have the +same value. Even with dark and flat corrections applied, adjacent +cells may not match. In addition, studies of the background level +indicate that the row-by-row bias can introduce small background +gradient variations along the rows of the cells that are not stable. +This common feature across the columns of cells results in a ``saw +tooth'' pattern horizontally across an the mosaicked OTA, and as the +background model fits a smooth sky level, this induces over and under +subtraction at the cell boundaries. + +The PATTERN.CONTINUITY correction, attempts to match the edges of a +cell to those of its neighbors. For each cell, a thin box 10 pixels +wide running the full length of each edge is extracted and the median +value of unmasked values calculated for that box. These median values +are then used to construct a vector of the sum of the differences +between that cell's edges and the corresponding edge on any adjacent +cell $\Delta$. A matrix $A$ of these associations is also +constructed, with the diagonal containing the number of cells adjacent +to that cell, and the off-diagonal values being set to -1 for each +pair of adjacent cells. The offsets needed for each chip, $x$ can +then be found by solving the system $A x = \Delta$. A cell with the +maximum number of neighbors, usually cell xy11, the first cell not on +the edge of the OTA, is used to constrain the system, ensuring that +that cell has zero correction and that there is a single solution. + +For OTAs that initially show the saw tooth pattern, the effect of this +correction is to align the cells into a single ramp, at the expense of +the absolute background level. However, as we subtract off a smooth +background model prior to doing photometry, these deviations from an +absolute sky level do not affect photometry for point sources and +extended sources smaller than a single cell. The fact that the +final ramp is smoother than it would be otherwise also allows for the +background subtracted image to more closely match the astronomical +sky, without significant errors at cell boundaries. An example of the +effect of this correction on an image profile is shown in Figure +\ref{fig:dark switching}. + \section{GPC1 Detrend Construction} \label{sec:detrend construction} @@ -1509,6 +1414,5 @@ The various master detrend images for GPC1 are constructed using a common approach. A series of appropriate exposures is selected from -the database, and processed with the \IPPprog{ppImage} program. This -program is used for the \IPPstage{chip} stage processing as well, and +the database, and processed with the \IPPprog{ppImage} program, which is designed to do multiple image processing operations. The processing steps applied to the images depend on the type of master @@ -1628,20 +1532,20 @@ \section{Warping} \label{sec:warping} -To provide a consistent and uniform set of coordinates for image -combination (including stacking and differences), the individual -mosaicked OTA images are projected onto a common pixel grids, called -tessellations. A tessellation can contain any number of tangent plane -projections, with those designed for single pointing surveys using -only one, while the tessellation used for the $3\pi$ survey containing -2643 tangent plane projections. These ``projection cells'' are -$4\times{}4$ degree fields spaced onto a set of centers that fully -cover the sky. They are arranged into rings of constant declination, -and allowed to overlap as $|\delta|$ increases. Each projection cell -is further subdivided into $10\times{}10$ ``skycells'' with fixed -$0.25"$ resolution pixels, and constant overlap regions between -adjacent skycells of $60"$. These skycells are the main image unit -used for processing image data beyond the initial chip stage. The +To provide a consistent and uniform set of coordinates for image +combination (including stacking and differences), the individual +mosaicked OTA images are projected onto a common pixel grids, called +tessellations. A tessellation can contain any number of tangent plane +projections, with those designed for single pointing surveys using +only one, while the tessellation used for the $3\pi$ survey contains +2643 tangent plane projection centers. These ``projection cells'' are +$4\times{}4$ degree fields spaced onto a set of centers that fully +cover the sky. They are arranged into rings of constant declination, +and allowed to overlap as $|\delta|$ increases. Each projection cell +is further subdivided into $10\times{}10$ ``skycells'' with fixed +$0.25"$ resolution pixels, and constant overlap regions between +adjacent skycells of $60"$. These skycells are the main image unit +used for processing image data beyond the initial chip stage. The coordinate system used for these images matches the parity of the sky, -with north in the positive y direction and east to the negative x +with north in the positive y direction and east to the negative x direction. @@ -1650,5 +1554,5 @@ solutions that map the detector focal plane to the sky, and map the individual OTA pixels to the detector focal plane -\citep[][see]{magnier2017.calibration}. This solution is then used to +\citep[see][]{magnier2017.calibration}. This solution is then used to determine which skycells the exposure OTAs overlap. @@ -1665,5 +1569,5 @@ With the locally linear grid defined, Lanczos interpolation -\citep{Lanczos:1950zz} with filter size parameter $a = 3$ on the input +\citep{lanczos1956applied} with filter size parameter $a = 3$ on the input image is used to determine the values to assign to the output pixel location. This process is repeated for all grid boxes, for all input @@ -1759,8 +1663,8 @@ sources. -The stacked image is comprised of all warp frames for a given skycell -in a single filter. The source catalogs and image components are -loaded into the \IPPprog{ppStack} program to prepare the inputs and -stack the frames. +For the $3\pi$ survey, the stacked image is comprised of all warp +frames for a given skycell in a single filter. The source catalogs +and image components are loaded into the \IPPprog{ppStack} program to +prepare the inputs and stack the frames. Once all files are ingested, the first step is to measure the size and @@ -1829,6 +1733,6 @@ Once the convolution kernels are defined for each image, they are used to convolve the image to match the target PSF. Any input image that -has a kernel match $chi^2$ value (defined as the sum of the RMS error -across the kernel) greater than 4.0$\sigma$ larger than the median +has a kernel match $\chi^2$ value (defined as the sum of the RMS error +across the kernel) 4.0$\sigma$ or larger than the median value is rejected from the stack. Each image also has a weight assigned, based on the image variance after convolution. A full image @@ -1968,12 +1872,13 @@ These convolved stack products are not retained, as the convolution is -only used to ensure the pixel rejection uses seeing-matched images. -Instead, we apply the normalizations and rejected pixel maps generated -from the convolved stack process to the original unconvolved input -images. This produces an unconvolved stack that has the optimum image -quality possible from the input images. Not convolving does mean that -the PSF shape changes across the image, as the different PSF widths of -the input images print through in the different regions to which they -have contributed. +used to ensure that the pixel rejection uses seeing-matched images. +This prevents any differences in the input PSF shape from skewing the +input pixel rejection. We apply the normalizations and rejected pixel +maps generated from the convolved stack process to the original +unconvolved input images. This produces an unconvolved stack that has +the optimum image quality possible from the input images. Not +convolving does mean that the PSF shape changes across the image, as +the different PSF widths of the input images print through in the +different regions to which they have contributed. %% Asinh compression @@ -1987,15 +1892,15 @@ increase in the disk space required for the stacked images. -Inspired by techniques used by SDSS \citep{2000AJ....120.1579Y} -\czw{better citation?}, we use the inverse hyperbolic sine function to -transform the data. The domain of this function allows any input -value to be converted. In addition, the quantization sampling can be -tuned by placing the zero of the inverse hyperbolic sine function at a -value where the highest sampling is desired. +Inspired by techniques used by SDSS \citep{1999AJ....118.1406L}, we +use the inverse hyperbolic sine function to transform the data. The +domain of this function allows any input value to be converted. In +addition, the quantization sampling can be tuned by placing the zero +of the inverse hyperbolic sine function at a value where the highest +sampling is desired. Formally, prior to being written to disk, the pixel values are transformed by $C = \alpha \asinh\left(\frac{L - \mathrm{BOFFSET}}{2.0 \cdot \mathrm{BSOFTEN}}\right)$, where $L$ are the linear input -pixel values, $C$ the transformed values, $\alpha = 2.5 \log_{10}(e)$. +pixel values, $C$ the transformed values, and $\alpha = 2.5 \log_{10}(e)$. BOFFSET centers the transformed values, and the mean of the linear input pixel values is used. BSOFTEN controls the stretch of the @@ -2100,7 +2005,7 @@ The image matching process used in constructing difference images is -essentially the same the stacking process. An image is chosen as a -template, another image as the input, and after matching sources to -determine the scaling and transparency, convolution kernels are +essentially the same as for the stacking process. An image is chosen +as a template, another image as the input, and after matching sources +to determine the scaling and transparency, convolution kernels are defined that are used to convolve one or both of the images to a target PSF. The images are then subtracted, and as they should now @@ -2124,9 +2029,9 @@ minus stack) and inverse (stack minus warp) to allow for the photometry of the difference image to detect sources that both rise -and fall relative to the stack. Note that the convolution process -grows the mask fraction of pixels relative to the warp (the largest -source of masked pixels in these warp stack differences). Any pixel -that after convolution has any contribution from a masked pixel is -masked as well, ensuring only fully unmasked pixels are used. +and fall relative to the stack. The convolution process grows the +mask fraction of pixels relative to the warp (the largest source of +masked pixels in these warp stack differences). Any pixel that after +convolution has any contribution from a masked pixel is masked as +well, ensuring only fully unmasked pixels are used. For warp-warp differences, such as those used for the ongoing Solar @@ -2165,5 +2070,5 @@ dependent on focal plane position. -An obvious way to make use of the PV3 catalog is to do a statistical +One obvious way to make use of the PV3 catalog is to do a statistical search for electronic crosstalk ghosts that do not match a known rule. Given that bright stars do not equally populate all fields, choosing @@ -2177,16 +2082,16 @@ There is some evidence that we have not fully identified all of these crosstalk rules, based on a study of PV3 images. For example, -extremely bright stars may be able to create crosstalk ghosts between the second -cell column of OTA01 and OTA21, with possibly fainter ghosts appearing -on OTA11. Despite the symmetry observed in the main ghost rules, -there do not appear to be clear examples of a similar ghost between -OTA47 and OTA66. Examining this further based on the PV3 catalog -should provide a clear answer to this, as well as clarify brightness -limits below which the ghost does not appear. +extremely bright stars may be able to create crosstalk ghosts between +the second cell column of OTA01 and OTA21, with possibly fainter +ghosts appearing on OTA11. Despite the symmetry observed in the main +ghost rules, there do not appear to be clear examples of a similar +ghost between OTA47 and OTA66. Examining this further based on the +PV3 catalog should provide a clear answer to this, as well as clarify +brightness limits below which the ghost does not appear. The PV3 catalog may also allow better determination of which date ranges we should use to build the dark model. The date ranges currently in use are based on limited sampling of exposures, and do -not have strong tests indicating that they are the best. By examining +not have strong tests indicating that they are optimal. By examining the scatter between the detections on a given exposure and the catalog average, we can attempt to look for increases in scatter that might @@ -2223,5 +2128,5 @@ to isolate and remove this signal in the Fourier domain. Preliminary investigations have shown that there is a small peak visible in the -power spectrum of a single cell, but determining the optimal way to +power spectrum of a single cell, but determining the best way to clip this peak to reduce the noise in the image space is not clear.