Index: trunk/doc/release.2015/ps1.detrend/Makefile =================================================================== --- trunk/doc/release.2015/ps1.detrend/Makefile (revision 40561) +++ trunk/doc/release.2015/ps1.detrend/Makefile (revision 40562) @@ -1,28 +1,14 @@ # $Id: Makefile,v 1.16 2006-01-16 01:11:40 eugene Exp $ -PDFLATEX = env TEXINPUTS=.:..:inputs:./inputs:LaTeX:$(TEXINPUTS): pdflatex -BIBTEX = env BIBINPUTS=.:..:inputs:../inputs BSTINPUTS=.:..:inputs:../inputs bibtex + +DO_PDFLATEX = 1 +DO_BIBTEX = 1 help: @echo "USAGE: make (target)" - @echo " targets: all detrend" + @echo " targets: all tgz pdf" -all: detrend.pdf - -DETREND = detrend.tex - -# pics/Metadata.ps -# pics/earthrot.ps - -detrend.pdf: $(DETREND) - rm -f detrend.aux detrend.bbl detrend.blg - $(PDFLATEX) $< - $(BIBTEX) detrend - $(PDFLATEX) $< - $(PDFLATEX) $< - $(PDFLATEX) $< - -#detrend.ps: $(DETREND) - -include ../Makefile.Common +all: pdf tgz +tgz: detrend.tgz +pdf: detrend.pdf FILES = \ @@ -63,4 +49,6 @@ images/stack_3775944_expwt.jpg -submission : - tar --transform 's%inputs/%%' -zcf waters2017.tgz $(FILES) +include ../Makefile.Common + +# submission : +# tar --transform 's%inputs/%%' -zcf waters2017.tgz $(FILES) Index: trunk/doc/release.2015/ps1.detrend/detrend.tex =================================================================== --- trunk/doc/release.2015/ps1.detrend/detrend.tex (revision 40561) +++ trunk/doc/release.2015/ps1.detrend/detrend.tex (revision 40562) @@ -136,5 +136,5 @@ (to create difference images), along with the resulting image products and their properties. -\citet[][Paper I]{chambers2017} provides an overview of the Pan-STARRS System, the +\citet[][Paper I]{chambers2017} provide an overview of the Pan-STARRS System, the design and execution of the Surveys, the resulting image and catalog data products, a discussion of the overall data quality and basic @@ -143,5 +143,5 @@ %Pan-STARRS Data Processing Stages \citet[][Paper II]{magnier2017.datasystem} -describes how the various data processing stages are organized and +describe how the various data processing stages are organized and implemented in the Imaging Processing Pipeline (IPP), including details of the @@ -154,5 +154,5 @@ %Pan-STARRS Pixel Analysis : Source Detection \citet[][Paper IV]{magnier2017.analysis} -describes the details of the source detection and photometry, including +describe the details of the source detection and photometry, including point-spread-function and extended source fitting models, and the techniques for ``forced'' photometry measurements. @@ -160,10 +160,10 @@ %Pan-STARRS Photometric and Astrometric Calibration \citet[][Paper V]{magnier2017.calibration} -describes the final calibration process, and the resulting photometric and +describe the final calibration process, and the resulting photometric and astrometric quality. %Flewelling et al. 2017 (Paper VI) %Pan-STARRS 1 Database and Data Products \citet[][Paper VI]{flewelling2017} -describes the details of the resulting catalog data and its organization +describe the details of the resulting catalog data and its organization in the Pan-STARRS database. % @@ -171,5 +171,5 @@ \citet[][Paper VII]{huber2017} %Huber et al. 2017 (Paper VII) -describes the Medium Deep Survey in detail, including the unique issues and +describe the Medium Deep Survey in detail, including the unique issues and data products specific to that survey. The Medium Deep Survey is not part of Data Release 1. (DR1) @@ -188,14 +188,20 @@ view. +\note{DS notes fonts are not consistent for keywords, etc} + +\note{DS: captions need to be clear re: illustrated effect} + \note{need to define PV3 (and PV0-2) here. see datasystem.tx} %The Processing Version 3 (PV3) reduction represents the third full DR1 contains the results of the third full reduction of the Pan-STARRS -archival data. The first two reductions were used internally for -pipeline optimization and the development of the initial photometric -and astrometric reference catalog \citep{magnier2017.calibration}. -The products from these reductions were not publicly released, but -have been used to produce a wide range of scientific papers from the -Pan-STARRS 1 Science Consortium members. +archival data, idenfied as PV3. Previous reductions \citep[PV0, PV1, + PV2; see][]{magnier2017.datasystem} +were used internally for pipeline optimization and the development of +the initial photometric and astrometric reference catalog +\citep{magnier2017.calibration}. The products from these reductions +were not publicly released, but have been used to produce a wide range +of scientific papers from the Pan-STARRS 1 Science Consortium members +\citep{chambers2017}. The Pan-STARRS image processing pipeline (IPP) is described elsewhere @@ -232,5 +238,5 @@ are provided in \citet{magnier2017.analysis}. -A limited version of same reduction procedure described above is also +A limited version of the same reduction procedure described above is also performed in real time on new exposures as they are observed by the telescope. This process is automatic, with new exposures being @@ -301,16 +307,19 @@ These corrections assume that the detector response is linear across -the full range of values. This assumption is not universally true for -GPC1, and an additional set of detrending steps are required as a +the full dynamic range and that the pixels contain only signals coming +from the imaged portion of the sky, or from linear dark current +sources within the detector. This assumption is not universally true +for GPC1, and an additional set of detrending steps are required as a result. The first of these is the \IPPprog{burntool} correction, which removes the flux trails left by the incomplete transfer of charge along the readout columns. These trails are generally only evident for the brightest stars, as only pixels that are at or beyond -the saturation point of the detector leave residual charge. More -widespread is the non-linearity at the faint end of the pixel range. -Some readout cells and some readout cell edge pixels experience a sag -relative to linear at low illumination, such that faint pixels appear -fainter than expected. The correction to this requires amplifying the -pixel values in these regions to match the expected model. +the saturation point of the detector leave residual charge. A second +confounding effect is the non-linearity at the faint end of the pixel +range. Some readout cells and some readout cell edge pixels +experience a sag relative to the linear trend at low illumination, +such that faint pixels appear fainter than expected. The correction +to this requires amplifying the pixel values in these regions to match +the linear response. Large regions of some OTA cells experience significant charge transfer @@ -326,13 +335,13 @@ field of view. -For the PV3 processing, all detrending is done by the -\IPPprog{ppImage} program. This program applies the detrend -corrections to the individual cells, and then an OTA-level mosaic is -constructed for the signal image, the mask image, and the variance map -image. The single epoch photometry is done at this stage as well. -The following subsections (\ref{sec:overscan} - \ref{sec:background}) -detail the detrending process used on GPC1 that are common to other -detectors. The GPC1 specific detrending steps are included after, -explaining these additional steps that remove the instrument signature. +Within the IPP, all detrending is done by the \IPPprog{ppImage} +program. This program applies the detrend corrections to the +individual cells, and then an OTA-level mosaic is constructed for the +signal image, the mask image, and the variance map image. The single +epoch photometry is done at this stage as well. The following +subsections (\ref{sec:overscan} - \ref{sec:background}) detail the +detrending process used on GPC1 that are common to other detectors. +The GPC1 specific detrending steps are included after, explaining +these additional steps that remove the instrument signature. \subsection{Overscan} @@ -427,8 +436,8 @@ \centering \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_M_OS_NL_XY23_b1.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_M_OS_NL_XY23.png} \end{minipage}% \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_to_DARK_XY23_b1.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_to_DARK_XY23.png} \end{minipage} \caption{An example of the dark model application to exposure o5677g0123o, OTA23 (2011-04-26, 43s \gps{} filter). The left panel shows the image data mosaicked to the OTA level, and has had the static mask applied, the overscan subtracted, and the detector non-linearity corrected. The right panel, shows the same exposure with the dark applied in addition to the processing shown on the left, removing the amplifier glows in the cell corners.} @@ -482,8 +491,8 @@ \centering \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_VIDEODARK_VDim_Rdark_XY22_b1.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_VIDEODARK_VDim_Rdark_XY22.png} \end{minipage}% \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_VIDEODARK_VDim_VDdark_XY22_b1.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_VIDEODARK_VDim_VDdark_XY22.png} \end{minipage} \caption{An example of the video dark model application to exposure o5677g0123o, OTA22 (2011-04-26, 43s \gps{} filter), which has a video cell located in cell xy16. The left panel shows the image data mosaicked to the OTA level, and has had the static mask applied, the overscan subtracted, the detector non-linearity corrected, and a regular dark applied. The right panel, shows the same exposure with a video dark applied instead of the standard dark. The main impact of this change is the improved correction of the corner glows, which are over subtracted with the standard dark.} @@ -498,5 +507,5 @@ characteristics. Instead, there is a gradient along the pixel rows, with the noise generally higher away from the read out amplifier -(higher cell x pixel positions). This is likely an effect of the +(higher cell $x$ pixel positions). This is likely an effect of the row-by-row bias issue discussed below (Section~\ref{sec:pattern.row}). As a result of this increased noise, more sources are detected in the @@ -526,5 +535,5 @@ dependent read noise. By binning the number of false positives measured on the bias frames on the noisemap inputs using 20 pixel -boxes in the cell x-axis, and comparing this to the number expected +boxes in the cell $x$-axis, and comparing this to the number expected from random Gaussian noise, we estimated the true read noise level. @@ -574,11 +583,14 @@ In addition to this flat field applied to the individual images, the -ubercal process used to calibrate the database of all detections -\citep{2012ApJ...756..158S} constructs ``in catalog'' flat field -corrections. Although a single set of image flat fields was used for -the entire PV3 survey, five separate ``seasons'' of database flat -fields were needed to ensure proper calibration. This indicates that -the flat field response is not completely fixed in time. More details -on this process are contained in \citet{magnier2017.calibration}. +``ubercal'' analysis -- in which photometric data are used define +image zero points +\citep[][]{2012ApJ...756..158S,magnier2017.calibration} and in turn +used used to calibrate the database of all detections -- constructs +``in catalog'' flat field corrections. Although a single set of image +flat fields was used for the PV3 processing of the entire $3\pi$ +survey, five separate ``seasons'' of database flat fields were needed +to ensure proper calibration. This indicates that the flat field +response is not completely fixed in time. More details on this +process are contained in \citet{magnier2017.calibration}. \subsection{Fringe correction} @@ -619,9 +631,9 @@ \begin{figure} \centering - \begin{minipage}{0.5\hsize} - \includegraphics[width=1.5\hsize,angle=0,clip]{images/o5220g0025o_XY53_nofringe.png} + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5220g0025o_nofringe_XY53.png} \end{minipage}% - \begin{minipage}{0.5\hsize} - \includegraphics[width=1.5\hsize,angle=0,clip]{images/o5220g0025o_XY53_fringe.png} + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5220g0025o_fringe_XY53.png} \end{minipage} \caption{Example of the \yps{} filter fringe pattern on exposure o5220g0025o OTA53 (\yps{} filter 30s). The left panel shows the OTA mosaic with all detrending except the fringe correction, while the right shows the same including the fringe correction. Both images have been smoothed with a Gaussian with $\sigma = 3$ pixels to highlight the faint and large scale fringe patterns. @@ -710,14 +722,15 @@ \tablewidth{0pc} \tablecaption{GPC1 Mask Values} - \tablehead{\colhead{Mask Name} & \colhead{Mask Value} & \colhead{Description}} + \tablehead{\colhead{Mask Name} & \colhead{Mask Value} & + \colhead{Description (static values listed in bold)}} \startdata - DETECTOR & 0x0001 & A detector defect is present. \\ - FLAT & 0x0002 & The flat field model does not calibrate the pixel reliably. \\ - DARK & 0x0004 & The dark model does not calibrate the pixel reliably. \\ - BLANK & 0x0008 & The pixel does not contain valid data. \\ - CTE & 0x0010 & The pixel has poor charge transfer efficiency. \\ + {\bf DETECTOR & 0x0001 & A detector defect is present.} \\ + {\bf FLAT & 0x0002 & The flat field model does not calibrate the pixel reliably.} \\ + {\bf DARK & 0x0004 & The dark model does not calibrate the pixel reliably.} \\ + {\bf BLANK & 0x0008 & The pixel does not contain valid data.} \\ + {\bf CTE & 0x0010 & The pixel has poor charge transfer efficiency.} \\ SAT & 0x0020 & The pixel is saturated. \\ LOW & 0x0040 & The pixel has a lower value than expected. \\ - SUSPECT & 0x0080 & The pixel is suspected of being bad. \\ + SUSPECT & 0x0080 & The pixel is suspected of being bad (overloaded with the BURNTOOL bit). \\ BURNTOOL & 0x0080 & The pixel contain an burntool repaired streak. \\ CR & 0x0100 & A cosmic ray is present. \\ @@ -764,23 +777,27 @@ Table \ref{tab:crosstalk_rules} summarizes the list of known crosstalk rules, with an estimate of the magnitude difference between the source -and ghost. For all of the rules, any source cell $v$ within the specified -column of cells on any of the OTAs in the specified column of OTAs $Y$ -can create a ghost in the same cell $v$ and OTA $Y$ in the target column of -cells and OTAs. In each of these cases, a source object with an -instrumental magnitude brighter than -14.47 creates a ghost object -many orders of magnitude fainter at the target location. The cell -(x,y) pixel coordinate is identical between source and ghost, as a -result of the transfer occurring as the devices are read. A circular -mask is added to the ghost location with radius $R = 3.44 \left(-14.47 -- m_{source, instrumental}\right)$ pixels. Any objects in the -photometric catalog found at the location of the ghost mask have the -GHOST mask bit set, marking the object as a likely ghost. The -majority of the crosstalk rules are bi-directional, with a source in -either position creating a ghost at the corresponding crosstalk target -position. The two faintest rules are uni-directional, due to -differences in the electronic path for the crosstalk. - -For the very brightest sources ($m_{instrumental} < -15$), there can -be crosstalk ghosts between all columns of cells during the readout. +and ghost. For all of the rules, any source cell $v$ within the +specified column of cells on any of the OTAs in the specified column +of OTAs $Y$ can create a ghost in the same cell $v$ and OTA $Y$ in the +target column of cells and OTAs. This effect depends on the number of +electrons detected for the star, thus the size of the ghost scales +with the instrumental magnitude ($m_{inst} = -2.5 \log_{10} (ADU)$) of +the star. In each of these cases, a source object with $m_{inst} < +-14.47$) (corresponding to $\rps \lesssim 14$ for the $3\pi$ survey) +creates a ghost object many orders of magnitude fainter at the target +location. The cell ($x,y$) pixel coordinate is identical between +source and ghost, as a result of the transfer occurring as the devices +are read. A circular mask is added to the ghost location with radius +$R = 3.44 \left(-14.47 - m_{inst,source}\right)$ pixels; only +positive radii are allowed. Any objects in the photometric catalog +found at the location of the ghost mask have the GHOST mask bit set, +marking the object as a likely ghost. The majority of the crosstalk +rules are bi-directional, with a source in either position creating a +ghost at the corresponding crosstalk target position. The two +faintest rules are uni-directional, due to differences in the +electronic path for the crosstalk. + +For the very brightest sources ($m_{inst} < -15$), there can be +crosstalk ghosts between all columns of cells during the readout. These ``bleed'' ghosts were originally identified as ghosts of the saturation bleeds appearing in the neighboring cells, and as such, the @@ -788,5 +805,5 @@ bottom of cells in all columns that are in the same row of cells as the bright source. The width of this box is a function of the source -magnitude, with $W = 5 * \left(-15 - m_{source, instrumental}\right)$ +magnitude, with $W = 5 \times \left(-15 - m_{inst,source}\right)$ pixels. @@ -824,10 +841,10 @@ extra travel distance, the resulting source is out of focus and elongated along the radial direction of the camera focal plane. These -optical ghosts can be modeled in the focal plane coordinates (L,M) +optical ghosts can be modeled in the focal plane coordinates ($L,M$) which has its origin at the center of the focal plane. In this -system, a bright object at location (L,M) on the focal plane creates a +system, a bright object at location ($L,M$) on the focal plane creates a reflection ghost on the opposite side of the optical axis near -(-L,-M). The exact location is fit as a third order polynomial in the -focal plane L and M directions (as listed in Table +($-L,-M$). The exact location is fit as a third order polynomial in the +focal plane $L$ and $M$ directions (as listed in Table \ref{tab:ghost_centers}). An elliptical annulus mask is constructed at the expected ghost location, with the major and minor axes defined @@ -842,5 +859,5 @@ \tablewidth{0pc} \tablecaption{Optical Ghost Center Transformations} - \tablehead{\colhead{Polynomial Term}&\colhead{L center}&\colhead{M center}} + \tablehead{\colhead{Polynomial Term}&\colhead{$L$ center}&\colhead{$M$ center}} \startdata $x^0 y^0$ & -1.215661e+02 & 2.422174e+01 \\ @@ -898,17 +915,17 @@ Prior to 2010-08-24, a reflective surface at the edge of the camera aperture was incompletely screened to light passing through the -telescope. Sources brighter than $m_{inst} = -21$ that fell on this -reflective surface resulted in light being scattered across the -detector surface in a long narrow glint. This surface was physically -masked on 2010-08-24, removing the possibility of glints in subsequent -data, but images that were taken prior to this date have an advisory -dynamic mask constructed when a reference source falls on the focal -plane within one degree of the detector edge. This mask is 150 pixels -wide, with length $L = 2500 \left(-20 - m_{inst}\right)$ pixels. -These glint masks are constructed by selecting sufficiently bright -sources in the reference catalog that fall within rectangular regions -around each edge of the GPC1 camera. These regions are separated from -the edge of the camera by 17 arcminutes, and extend outwards an -additional degree. +telescope. Sources brighter than $m_{inst} = -21$ ($\rps \lesssim +7.5$) that fell on this reflective surface resulted in light being +scattered across the detector surface in a long narrow glint. This +surface was physically masked on 2010-08-24, removing the possibility +of glints in subsequent data, but images that were taken prior to this +date have an advisory dynamic mask constructed when a reference source +falls on the focal plane within one degree of the detector edge. This +mask is 150 pixels wide, with length $L = 2500 \left(-20 - +m_{inst}\right)$ pixels. These glint masks are constructed by +selecting sufficiently bright sources in the reference catalog that +fall within rectangular regions around each edge of the GPC1 camera. +These regions are separated from the edge of the camera by 17 +arcminutes, and extend outwards an additional degree. \begin{figure} @@ -924,13 +941,13 @@ Bright sources also form diffraction spikes that are dynamically masked. These are filter independent, and are modeled as rectangles -with length $L = 10^{0.096 * (7.35 - m_{instrumental})} - 200$ and -width $W = 8 + (L - 200) * 0.01$, with negative values indicating no +with length $L = 10^{0.096 \times (7.35 - m_{inst})} - 200$ and +width $W = 8 + (L - 200) \times 0.01$, with negative values indicating no mask is constructed, as the source is likely too faint to produce the feature. These spikes are dependent on the camera rotation, and are -oriented based on the header keyword at $\theta = n * \frac{\pi}{2} - +oriented based on the header keyword at $\theta = n \times \frac{\pi}{2} - \mathrm{ROTANGLE} + 0.798$, for $n = {0,1,2,3}$. The cores of stars that are saturated are masked as well, with a -circular mask radius $r = 10.15 * (-15 - m_{instrumental})$. An +circular mask radius $r = 10.15 \times (-15 - m_{inst})$. An example of a saturated star, with the masked regions for the diffraction spikes and core saturation highlighted, is shown in Figure @@ -939,5 +956,5 @@ \begin{figure} \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o6802g0338o_XY51_b1.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o6802g0338o_SATSTAR_XY51.png} \caption{Example of saturated star, with diffraction spikes extending from the core on exposure o6802g0338o, OTA51 (2014-05-25, 45s \gps{} filter).} \label{fig:saturated star} @@ -1047,6 +1064,6 @@ median of the pixel distribution, with the standard deviation of the distribution set as the median of the $\sigma$ values calculated from -the $0.5 * (\sigma_{+1} - \sigma_{-1})$, $\sigma_{+0.5} - -\sigma_{-0.5}$, and $0.25 * (\sigma_{+2} - \sigma_{-2})$ differences. +the $0.5 \times (\sigma_{+1} - \sigma_{-1})$, $\sigma_{+0.5} - +\sigma_{-0.5}$, and $0.25 \times (\sigma_{+2} - \sigma_{-2})$ differences. If this measured standard deviation is smaller than 3 times the bin size, then all points more than 25 bins away from the calculated @@ -1057,5 +1074,5 @@ are found by interpolating the 5 bins around the expected bin as well, and the count of the number of input values within this inner -50-percentile region, $N_{50}$ is calculated. +50-percentile region, $N_{50}$, is calculated. These initial statistics are then used as the starting guesses for a @@ -1191,14 +1208,14 @@ \centering \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_nobt.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_nbt_XY11.png} \end{minipage}% \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_nobt.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_nbt_XY11.png} \end{minipage} \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_bt.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_wbt_XY11.png} \end{minipage}% \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_bt.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_wbt_XY11.png} \end{minipage} \caption{Example of OTA11 cell xy50 on exposures o5677g0123o (left) and o5677g0124o (right). The top panels show the image with all appropriate detrending steps, but without burntool, and the bottom show the same with burntool applied. There is some slight over subtraction in fitting the initial trail, but the impact of the trail is greatly reduced in both exposures.} @@ -1246,6 +1263,6 @@ We store the average flux measurement and deviation from the linear fit for each exposure time for each region on all detector cells in -the linearity detrend look up tables. An example of this data is -shown in figure \ref{fig: nonlinearity}. When this correction is +the linearity detrend look-up tables. An example of this data is +shown in Figure~\ref{fig: nonlinearity}. When this correction is applied to science data, these lookup tables are loaded, and a linear interpolation is performed to determine the correction needed for the @@ -1270,5 +1287,5 @@ \centering \includegraphics[width=0.9\hsize,angle=0,clip]{images/linearity_XY27_xy16.png} - \caption{Example plot of the linearity correction as a fraction of observed flux for OTA27, cell xy16.} + \caption{Example of the linearity correction as a fraction of observed flux for OTA27, cell xy16.} \label{fig: nonlinearity} \end{figure} @@ -1293,5 +1310,5 @@ offsets increases as the distance from the readout amplifier and overscan region increases, resulting in horizontal streaks that are -more pronounced along the large x pixel edge of the cell. As the +more pronounced along the large $x$ pixel edge of the cell. As the level of the offset is apparently random between exposures, the dark correction cannot fully remove this structure from the images, and the @@ -1299,10 +1316,11 @@ by these bias offsets. Therefore, we apply the PATTERN.ROW correction in an attempt to mitigate the offsets and correct the image values. -To force the rows to agree, a second order clipped polynomial is fit -to each row in the cell. Four fit iterations are run, and pixels -$2.5\sigma$ deviant are excluded from subsequent fits, to minimize the -effect stars and other astronomical signals have. This final trend is -then subtracted from that row. Simply doing this subtraction will -also have the effect of removing the background sky level. To prevent +To force the rows to agree, a second order clipped polynomial is +fitted to each row in the cell. Four fit iterations are run and +pixels $2.5\sigma$ deviant (chosen empirically) are excluded from +subsequent fits in order to minimize the bias from stars and other +astronomical sources in the pixels. This final trend is then +subtracted from that row. Simply doing this subtraction will also +have the effect of removing the background sky level. To prevent this, the constant and linear terms for each row are stored, and linear fits are made to these parameters as a function of row, @@ -1368,8 +1386,8 @@ \centering \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_XY57_nopat.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_npt_XY57.png} \end{minipage}% \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_XY57_pat.png} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_wpt_XY57.png} \end{minipage} \caption{Example of the PATTERN.ROW correction on exposure o5379g0103o OTA57 cell xy01 (\ips{} filter 45s). The left panel shows the cell with all appropriate detrending except the PATTERN.ROW, and the right shows the same cell with PATTERN.ROW applied. The correction reduces the correlated noise on the right side, which is most distant from the read out amplifier. There is a slight over subtraction along the rows near the bright star.} @@ -1379,18 +1397,19 @@ \subsubsection{Pattern Continuity} -The background levels of cells on a single OTA do not always have the -same value. Even with dark and flat corrections applied, adjacent -cells may not match. In addition, studies of the background level -indicate that the row-by-row bias can introduce small background -gradient variations along the rows of the cells that are not stable. -This common feature across the columns of cells results in a ``saw -tooth'' pattern horizontally across an the mosaicked OTA, and as the -background model fits a smooth sky level, this induces over and under -subtraction at the cell boundaries. +The background sky levels of cells on a single OTA do not always have +the same value. Despite having dark and flat corrections applied, +adjacent cells may not match even for images of nominally empty sky. +In addition, studies of the background level indicate that the +row-by-row bias can introduce small background gradient variations +along the rows of the cells that are not stable. This common feature +across the columns of cells results in a ``saw tooth'' pattern +horizontally across an the mosaicked OTA, and as the background model +fits a smooth sky level, this induces over- and under subtraction at +the cell boundaries. The PATTERN.CONTINUITY correction, attempts to match the edges of a cell to those of its neighbors. For each cell, a thin box 10 pixels wide running the full length of each edge is extracted and the median -value of unmasked values calculated for that box. These median values +of unmasked values is calculated for that box. These median values are then used to construct a vector of the sum of the differences between that cell's edges and the corresponding edge on any adjacent @@ -1398,6 +1417,6 @@ constructed, with the diagonal containing the number of cells adjacent to that cell, and the off-diagonal values being set to -1 for each -pair of adjacent cells. The offsets needed for each chip, $x$ can -then be found by solving the system $A x = \Delta$. A cell with the +pair of adjacent cells. The offsets needed for each chip, $\zeta$ can +then be found by solving the system $A \zeta = \Delta$. A cell with the maximum number of neighbors, usually cell xy11, the first cell not on the edge of the OTA, is used to constrain the system, ensuring that @@ -1496,5 +1515,6 @@ \tablewidth{0pc} \tablecaption{PV3 Detrends} - \tablehead{\colhead{Detrend Type} & \colhead{Detrend ID} & \colhead{Start Date} & \colhead{End Date} & \colhead{Note} } + \tablehead{\colhead{Detrend Type} & \colhead{Detrend ID} & + \colhead{Start Date (UT)} & \colhead{End Date (UT)} & \colhead{Note} } \startdata LINEARITY & 421 & 2009-01-01 00:00:00 & & \\ @@ -1541,5 +1561,5 @@ To provide a consistent and uniform set of coordinates for image combination (including stacking and differences), the individual -mosaicked OTA images are projected onto a common pixel grids, called +mosaicked OTA images are projected onto common pixel grids, called tessellations. A tessellation can contain any number of tangent plane projections, with those designed for single pointing surveys using @@ -1554,5 +1574,5 @@ used for processing image data beyond the initial chip stage. The coordinate system used for these images matches the parity of the sky, -with north in the positive y direction and east to the negative x +with north in the positive $y$ direction and east to the negative $x$ direction. @@ -1572,18 +1592,22 @@ output pixel has a unique sampling position on the input image (although it may be off the image frame and therefore not populated), -preventing gaps in the output image due to the spacing of the input -pixels. +guaranteing that all output pixels are addressed, and thus preventing +gaps in the output image due to the spacing of the input pixels. With the locally linear grid defined, Lanczos interpolation -\citep{lanczos1956applied} with filter size parameter $a = 3$ on the input -image is used to determine the values to assign to the output pixel -location. This process is repeated for all grid boxes, for all input -images, and for each output image product: the science image, the -variance, and the mask. The image values are scaled by the absolute -value of the Jacobian determinant of the transformation for each grid -box. This corrects the pixel values for the possible change in pixel -area due to the transformation. Similarly, the variance image is -scaled by the square of this value, again to correctly account for the -pixel area change. +\citep{lanczos1956applied} with filter size parameter $a = 3$ on the +input image is used to determine the values to assign to the output +pixel location. This interpolation kernel was chosen as a compromise +between simple interpolations and higher-order Lanczos kernels, with +the goal of limiting the smear in the output image while avoiding +the high-frequency ringing generated by higher order kernels. This +process is repeated for all grid boxes, for all input images, and for +each output image product: the science image, the variance, and the +mask. The image values are scaled by the absolute value of the +Jacobian determinant of the transformation for each grid box. This +corrects the pixel values for the possible change in pixel area due to +the transformation. Similarly, the variance image is scaled by the +square of this value, again to correctly account for the pixel area +change. The interpolation constructs the output pixels from more than one @@ -1599,19 +1623,20 @@ to scale the position uncertainties based on the new orientation. -The output image also contains header keywords SRC\_0000, SEC\_0000, -MPX\_0000, and MPY\_0000 that define the mappings from the warped -pixel space to the input image. The SRC keyword lists the input OTA -name, and the SEC keyword lists the image section that the mapping -covers. The MPX and MPY contain the back-transformation linearized -across the full chip. These parameters are stored in a string listing -the reference position in the chip coordinate frame, the slope of the -relation in the warp x axis, and the slope of the relation in the warp -y axis. From these keywords, any position in the warp can be mapped -back to the location in any of the input OTA images, with some -reduction in accuracy. +The output image also contains header keywords SRC\_nnnn, SEC\_nnnn, +MPX\_nnnn, and MPY\_nnnn that define the mappings from the warped +pixel space to the input images. The 'nnnn' for each keyword has the +values 0000, 0001, etc., up to the number of input images. The SRC +keyword lists the input OTA name, and the SEC keyword lists the image +section that the mapping covers. The MPX and MPY contain the +back-transformation linearized across the full chip. These parameters +are stored in a string listing the reference position in the chip +coordinate frame, the slope of the relation in the warp $x$ axis, and +the slope of the relation in the warp $y$ axis. From these keywords, +any position in the warp can be mapped back to the location in any of +the input OTA images, with some reduction in accuracy. \begin{figure} \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/warp_1046511_sci.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/warp_2046019_sci.png} \caption{Example of the warp image for skycell skycell.2047.005 centered at ($\alpha,\delta$) = (179.763, 32.1899) for exposure @@ -1629,5 +1654,5 @@ \begin{figure} \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/warp_1046511_wt.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/warp_2046019_var.png} \caption{Example of the warp variance image for skycell skycell.2047.005 of exposure o4985g0073o, the same as in Figure @@ -1644,5 +1669,5 @@ \begin{figure} \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/warp_1046511_mask.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/warp_2046019_mask.png} \caption{Example of the warp mask image for skycell skycell.2047.005 of exposure o4985g0073o, the same as in Figure \ref{fig:warp @@ -1675,9 +1700,11 @@ prepare the inputs and stack the frames. +\note{need to point out that we are convolving to a matched PSF} + Once all files are ingested, the first step is to measure the size and shapes of the input image PSFs. We exclude images that have a PSF FWHM greater than 10 pixels (2.5 arcseconds), as those images have the seeing far worse than average, and would degrade the final output -stack. For the PV3 $3\pi$ survey, this size represents a PSF larger +stack. For the PV3 processing of the $3\pi$ survey, this size represents a PSF larger than the $97$th percentile in all filters. A target PSF for the stack is constructed by finding the maximum envelope of all input PSFs, @@ -1693,13 +1720,15 @@ airmass, image exposure time, and zeropoint. All output stacks are constructed to a target zeropoint of 25.0 in all filters, and to have -an airmass of 1.0. The output exposure time is set to the sum of the -input exposure times, regardless of whether those inputs are rejected later -in the combination process. We can determine the relative +an airmass of 1.0. The target zeropoint is arbitrary; 25.0 was chosen +to be roughly consistent with the PS1 zero points, while still being a +simple number. The output exposure time is set to the sum of the +input exposure times, {\em regardless of whether those inputs are rejected +later in the combination process}. We can determine the relative transparency for each input image by comparing the magnitudes of matched sources between the different images. Each image then has a normalization factor defined, equal to $\mathrm{norm}_{input} = (ZP_\mathrm{input} - ZP_\mathrm{target}) - -\mathrm{transparency}_\mathrm{input} - 2.5 * \log_{10} -(t_\mathrm{target} / t_\mathrm{input}) - \mathrm{F}_\mathrm{airmass} * +\mathrm{transparency}_\mathrm{input} - 2.5 \times \log_{10} +(t_\mathrm{target} / t_\mathrm{input}) - \mathrm{F}_\mathrm{airmass} \times (\mathrm{airmass}_\mathrm{input} - \mathrm{airmass}_\mathrm{target})$. For the PV3 processing, the airmass factor @@ -1715,20 +1744,23 @@ the entire region of the sky imaged. This further calibration is not available at the time of stacking, and so there may be small residuals -in the transparency values as a result of this \citet{magnier2017.calibration}. +in the transparency values as a result of this \citep{magnier2017.calibration}. With the flux normalization factors and target PSF chosen, the -convolution kernels can be calculated for each image. ISIS kernels -\citep{1998ApJ...503..325A} are used with FWHM values of 1.5, 3.0, and 6.0 -pixels and polynomial orders of 6, 4, and 2. Regions around the -sources identified in the input images are extracted, convolved with -the kernel, and the residual with the target PSF used to update the -parameters of the kernel via least squares optimization. Stamps that -significantly deviate are rejected, although the squared residual -difference will increase with increasing source flux. To mitigate -this effect, a parabola is fit to the distribution of squared -residuals as a function of source flux. Stamps that deviate from this -fit by more than $2.5\sigma$ are rejected, and not used on further -kernel fit iterations. This process is repeated twice, and the final -convolution kernel is returned. +convolution kernels can be calculated for each image. To calculate +the convolution kernels, we use the algorithm described by +\cite{1998ApJ...503..325A} and \cite{2000.alard} to perform optimal +image subtraction. These `ISIS' kernels \citep[named after the + software package described by][]{1998ApJ...503..325A} are used with +FWHM values of 1.5, 3.0, and 6.0 pixels and polynomial orders of 6, 4, +and 2. Regions around the sources identified in the input images are +extracted, convolved with the kernel, and the residual with the target +PSF used to update the parameters of the kernel via least squares +optimization. Stamps that significantly deviate are rejected, +although the squared residual difference will increase with increasing +source flux. To mitigate this effect, a parabola is fit to the +distribution of squared residuals as a function of source flux. +Stamps that deviate from this fit by more than $2.5\sigma$ are +rejected, and not used on further kernel fit iterations. This process +is repeated twice, and the final convolution kernel is returned. This convolution may change the image flux scaling, so the kernel is @@ -1759,4 +1791,6 @@ identify discrepant input values that should be excluded. +\note{clarify 'should' below, e.g., with a histogram} + If only a single input is available, the initial stack contains the value from that single input. If there are only two inputs, the @@ -1770,6 +1804,6 @@ \begin{eqnarray} - \mathrm{Stack}_\mathrm{value} &=& \sum_i\left(\mathrm{value}_\mathrm{input} * W_\mathrm{input}\right) / \sum_\mathrm{inputs}\left(W_\mathrm{input}\right) \\ - \mathrm{Stack}_\mathrm{exp weight} &=& \sum_i \left(\mathrm{exptime}_\mathrm{input} * W_\mathrm{input}\right) / \sum_\mathrm{inputs}\left(W_\mathrm{input}\right) \\ + \mathrm{Stack}_\mathrm{value} &=& \sum_i\left(\mathrm{value}_\mathrm{input} \times W_\mathrm{input}\right) / \sum_\mathrm{inputs}\left(W_\mathrm{input}\right) \\ + \mathrm{Stack}_\mathrm{exp weight} &=& \sum_i \left(\mathrm{exptime}_\mathrm{input} \times W_\mathrm{input}\right) / \sum_\mathrm{inputs}\left(W_\mathrm{input}\right) \\ \end{eqnarray} @@ -1805,7 +1839,7 @@ attempt to identify outlier points. Again, if only one input is available, that input is accepted. If there are two inputs, $A$ and -$B$, then a check is made to see if $(0.5 * (\mathrm{value}_A - -\mathrm{value}_B))^2 > 16 * (\sigma^2_A + \sigma^2_B -+ (0.1 * \mathrm{value}_A)^2 + (0.1 * \mathrm{value}_B)^2)$, such that +$B$, then a check is made to see if $(0.5 \times (\mathrm{value}_A - +\mathrm{value}_B))^2 > 16 \times (\sigma^2_A + \sigma^2_B ++ (0.1 \times \mathrm{value}_A)^2 + (0.1 \times \mathrm{value}_B)^2)$, such that the deviation of the inputs from their mean position is greater than four times the sum of their measured uncertainties and a 10\% @@ -1824,7 +1858,7 @@ distribution is likely to be unimodal), or if there are insufficient inputs for this mixture model analysis, the input values are passed to -an Olympic weighted mean calculation. We reject $20\%$ of the number +an Olympic \note{define} weighted mean calculation. We reject $20\%$ of the number of inputs through this process. The number of bad inputs is set to -$N_\mathrm{bad} = 0.2 * N_\mathrm{input} + 0.5$, with the 0.5 term +$N_\mathrm{bad} = 0.2 \times N_\mathrm{input} + 0.5$, with the 0.5 term ensuring at least one input is rejected. This number is further separated into the number of low values to exclude, $N_\mathrm{low} = @@ -1843,6 +1877,6 @@ \begin{eqnarray} - \mathrm{limit}_\mathrm{mixture\ model} &=& 4^2 * (\sigma^2_\mathrm{input} + \sigma_\mathrm{mixture\ model}^2) \\ - \mathrm{limit}_\mathrm{default} &=& 4^2 * (\sigma^2_\mathrm{input} + (0.1 * \mathrm{value}_\mathrm{input})^2) + \mathrm{limit}_\mathrm{mixture\ model} &=& 4^2 \times (\sigma^2_\mathrm{input} + \sigma_\mathrm{mixture\ model}^2) \\ + \mathrm{limit}_\mathrm{default} &=& 4^2 \times (\sigma^2_\mathrm{input} + (0.1 \times \mathrm{value}_\mathrm{input})^2) \end{eqnarray} @@ -1869,5 +1903,5 @@ pixels. The ISIS kernel used in the previous step is again used to determine the largest square box that does not exceed the limit of -$0.25 * \sum_{x,y} kernel^2$. This square box is then convolved with +$0.25 \times \sum_{x,y} kernel^2$. This square box is then convolved with the rejected pixel mask to reject the neighboring pixels. This final list of rejected pixels is passed to the final combination, which @@ -1924,5 +1958,5 @@ \begin{figure} \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3775944_sci.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_sci.png} \caption{Example of the stack image for skycell skycell.2047.005 centered at ($\alpha,\delta$) = (179.763, 32.1899) in the \zps{} @@ -1940,5 +1974,5 @@ \begin{figure} \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3775944_mask.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_mask.png} \caption{Example of the stack mask image for skycell skycell.2047.005 centered at ($\alpha,\delta$) = (179.763, @@ -1954,5 +1988,5 @@ \begin{figure} \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3775944_wt.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_var.png} \caption{Example of the stack variance image for skycell skycell.2047.005 centered at ($\alpha,\delta$) = (179.763, @@ -1968,5 +2002,5 @@ \begin{figure} \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3775944_num.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_num.png} \caption{Example of the stack number image for skycell skycell.2047.005 centered at ($\alpha,\delta$) = (179.763, @@ -1982,5 +2016,5 @@ \begin{figure} \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3775944_exp.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_exp.png} \caption{Example of the stack exposure time image for skycell skycell.2047.005 centered at ($\alpha,\delta$) = (179.763, @@ -1995,5 +2029,5 @@ \begin{figure} \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3775944_expwt.jpg} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/stack_3956997_expwt.png} \caption{Example of the stack weighted exposure image for skycell skycell.2047.005 centered at ($\alpha,\delta$) = (179.763, @@ -2071,9 +2105,9 @@ Although the detrending and image combination algorithms work well to -produce a consistent and calibrated images, having the full PV3 data -set allows issues to be identified and solutions created for future -improvements to the IPP pipeline. In addition, the existence of the -final calibrated catalog can be used to look for issues that appear -dependent on focal plane position. +produce consistent and calibrated images, having the PV3 processing of +the full $3\pi$ data set allows issues to be identified and solutions +created for future improvements to the IPP pipeline. In addition, the +existence of the final calibrated catalog can be used to look for +issues that appear dependent on focal plane position. One obvious way to make use of the PV3 catalog is to do a statistical @@ -2171,4 +2205,6 @@ University (ELTE), and the Los Alamos National Laboratory. +\note{ApJ, etc latex macros have an extra comma} + \bibliography{lib}{} \bibliographystyle{apj} @@ -2176,4 +2212,2 @@ \end{document} - -