Index: /branches/czw_branch/20170908/Changes.txt =================================================================== --- /branches/czw_branch/20170908/Changes.txt (revision 40482) +++ /branches/czw_branch/20170908/Changes.txt (revision 40483) @@ -1,2 +1,44 @@ + +2017.01.09 : EAM + + * ppMerge : comment out a free (this introduces a leak and should be conditional) + * stacktool : sort the skycells used for a summary skycell by skycell_id to ensure lists match. + * psLib : + psMetadata.c : F64 data needs to be printed with %g not %f + psFitsScale.c : allow quartile stats to return a sigma based on +/- 1-sigma range + * psastro : + fit a mosaic-wide distortion model, if it exists (e.g., RA---ZPN), otherwise maintain the mosaic fit model + allow missing model for a given chip (e.g., model not defined for bad chips) + save the calibrated magnitudes in MATCHED_REFS (using measured zero point) + add ROT_PARITY concept to the astrometry model (defined sign of ROTANGLE) + tpa->fpa transformation needs to have more polynomial terms than fpa->tpa (where the fit is performed) + * ippScripts : add HSC to cameras not corrected for background (in camera_exp.pl, should be a recipe-thing) + + * Ohana: + various changes to relastro, relphot, fakeastro, addstar, delstar, uniphot, photdbc, dvomerge which do not affect nightly processing + libdvo/coorops.c : upgrade ZPN to full terms (remove UKIRT exemption) + libdvo : add test galaxy models; add object / measure bits for GAIA, 2MASS, TYCHO, SDSS, HSC, CFH, DEC + libfits : do not allow gfits_modify, etc to match FOO to FOOBAR, etc + libkapa & kapa2 : change some hardwired numbers to enum values, allow names for e.g., -pt and -lt + opihi : + unify random number seeding; + allow user-defined names for header NAXISi keywords for imbox; + fix csystem conversion of motions + allow sphere-based 2d matches (matches2d) + define some physical constants as variables at startup (e.g., M_PI, M_c, M_c_cgs) + + * psModules: + allow photometry-based match in astrometry : downweight photometrically discrepant values + add ZPN model + do not attempt to force CRPIX1,2 to be 0.0 + support for ROT_PARITY + add some tracing for detrending + + * Nebulous & Nebulous-Server : changes to support proximity + + * ippconfig : + GAIA and HSC photcodes + support for ROT_PARITY and psastro photometry window + accurate HSC zero points 2010.10.25 : EAM Index: /branches/czw_branch/20170908/Nebulous/bin/whichnode =================================================================== --- /branches/czw_branch/20170908/Nebulous/bin/whichnode (revision 40482) +++ /branches/czw_branch/20170908/Nebulous/bin/whichnode (revision 40483) @@ -31,5 +31,5 @@ my $dbh = getDBHandle(); -my $query = "SELECT volume.name AS volname, uri, mountedvol.*" +my $query = "SELECT volume.name AS volname, uri, mountedvol.*, volume.xattr AS volxattr" . " FROM storage_object JOIN instance USING(so_id)" . " JOIN volume using(vol_id)" @@ -53,8 +53,12 @@ my $volname = $instance->{volname}; my $uri = $instance->{uri}; + my $volxattr = $instance->{volxattr}; + $num_instances++; if ($instance->{available}) { $num_available++; print "$volname available\n"; + } elsif ($volxattr == 5) { + print "$volname permanently offline\n"; } else { print "$volname not available\n"; Index: /branches/czw_branch/20170908/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /branches/czw_branch/20170908/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 40482) +++ /branches/czw_branch/20170908/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 40483) @@ -1664,7 +1664,45 @@ my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps); my $npoints = 0; + ## MEH hack -- + my $pi=3.14159265359; + my $tra=0.0; + my $tdec=0.0; + ## foreach my $row (@$rowList) { print $pointsList "$npoints $row->{CENTER_X} $row->{CENTER_Y}\n"; - $npoints++; + #$npoints++; + ## MEH hack -- add width, height corners -- chip gap problem and need to add more points to try + ## -- need to adjust RA for DEC and check 0/360, 90 boundary (dont assume coord to image lookup will do it) + if ( ($row->{OPTION_MASK} & $PSTAMP_MULTI_OVERLAP_IMAGE) && !($row->{COORD_MASK} & $PSTAMP_RANGE_IN_PIXELS) ){ + foreach my $f (-1.0, 1.0, -1.5, 1.5, -2.0, 2.0, -2.5, 2.5) { + ## +/-ra +/-dec corner + #printf $pointsList "$npoints %f %f\n", $row->{CENTER_X}+$row->{WIDTH}/2.0/3600.0, $row->{CENTER_Y}+$row->{HEIGHT}/2.0/3600.0; + $tra = $row->{CENTER_X}+$row->{WIDTH}/$f/2.0/3600.0/cos($row->{CENTER_Y}/180.0*$pi); + $tdec = $row->{CENTER_Y}+$row->{HEIGHT}/$f/2.0/3600.0; + $tra = ($tra>360.0) ? $tra-360.0 : $tra; + $tdec = ($tdec>90.0) ? 90.0-($tdec-90.0) : $tdec; + printf $pointsList "$npoints %f %f\n",$tra,$tdec; + ## +/-ra -/+dec corner + $tra = $row->{CENTER_X}+$row->{WIDTH}/$f/2.0/3600.0/cos($row->{CENTER_Y}/180.0*$pi); + $tdec = $row->{CENTER_Y}-$row->{HEIGHT}/$f/2.0/3600.0; + $tra = ($tra>360.0) ? $tra-360.0 : $tra; + $tdec = ($tdec>90.0) ? 90.0-($tdec-90.0) : $tdec; + printf $pointsList "$npoints %f %f\n",$tra,$tdec; + ## -/+ra side + $tra = $row->{CENTER_X}-$row->{WIDTH}/$f/2.0/3600.0/cos($row->{CENTER_Y}/180.0*$pi); + $tdec = $row->{CENTER_Y}; + $tra = ($tra>360.0) ? $tra-360.0 : $tra; + $tdec = ($tdec>90.0) ? 90.0-($tdec-90.0) : $tdec; + printf $pointsList "$npoints %f %f\n",$tra,$tdec; + ## -/+dec side + $tra = $row->{CENTER_X}; + $tdec = $row->{CENTER_Y}-$row->{HEIGHT}/$f/2.0/3600.0; + $tra = ($tra>360.0) ? $tra-360.0 : $tra; + $tdec = ($tdec>90.0) ? 90.0-($tdec-90.0) : $tdec; + printf $pointsList "$npoints %f %f\n",$tra,$tdec; + } + } + #### + $npoints++; # this gets overwitten if an overlapping image is found $row->{error_code} = $PSTAMP_NO_OVERLAP; @@ -1725,5 +1763,8 @@ $ref = $components{$component} = []; } - push @$ref, $ptnum; + #push @$ref, $ptnum; + ## MEH hack -- only push ptnum if not already, so check if ptnum in list + push(@$ref, $ptnum) unless grep{$_ == $ptnum} @$ref; + ## my $row = $rowList->[$ptnum]; # this row found a match Index: /branches/czw_branch/20170908/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- /branches/czw_branch/20170908/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 40482) +++ /branches/czw_branch/20170908/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 40483) @@ -35,4 +35,5 @@ $PSTAMP_SELECT_UNCONV $PSTAMP_RESTORE_BACKGROUND + $PSTAMP_MULTI_OVERLAP_IMAGE $PSTAMP_USE_IMFILE_ID $PSTAMP_NO_WAIT_FOR_UPDATE @@ -81,6 +82,6 @@ our $PSTAMP_SELECT_UNCONV = 2048; our $PSTAMP_RESTORE_BACKGROUND = 4096; -# unused 8192 - +# MEH -- previously unused 8192 +our $PSTAMP_MULTI_OVERLAP_IMAGE = 8192; our $PSTAMP_USE_IMFILE_ID = 16384; our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768; Index: /branches/czw_branch/20170908/doc/release.2015/Makefile.Common =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/Makefile.Common (revision 40482) +++ /branches/czw_branch/20170908/doc/release.2015/Makefile.Common (revision 40483) @@ -28,4 +28,6 @@ %.tgz: tar --transform 's%inputs/%%' -zcf $@ $(FILES) $*.bbl +%.zip: + zip -j $@ $? clean : $(RM) *.bib *.log *.dvi *.aux *.toc *.tbd *.tbr *.tpm *.lof *.out *~ core body.tmp head.tmp Index: /branches/czw_branch/20170908/doc/release.2015/inputs/lib.bib =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/inputs/lib.bib (revision 40482) +++ /branches/czw_branch/20170908/doc/release.2015/inputs/lib.bib (revision 40483) @@ -16291,2 +16291,13 @@ adsnote = {Provided by the SAO/NASA Astrophysics Data System} } + +@book{lanczos1956applied, + title={Applied analysis}, + author={Lanczos, C.}, + lccn={lc56012218}, + series={Prentice-Hall mathematics series}, + url={https://books.google.com/books?id=JmNKAAAAMAAJ}, + year={1956}, + publisher={Prentice-Hall} +} + Index: /branches/czw_branch/20170908/doc/release.2015/ps1.datasystem/datasystem.tex =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/ps1.datasystem/datasystem.tex (revision 40482) +++ /branches/czw_branch/20170908/doc/release.2015/ps1.datasystem/datasystem.tex (revision 40483) @@ -1,6 +1,6 @@ % \documentclass[iop,floatfix]{emulateapj} % \documentclass[iop,floatfix,onecolumn]{emulateapj} -% \documentclass[12pt,preprint]{aastex} -\documentclass[10pt,preprint]{aastex} +\documentclass[12pt,preprint]{aastex} +% \documentclass[10pt,preprint]{aastex} % \pdfoutput=1 @@ -93,4 +93,6 @@ \label{sec:intro} +\note{missing figures: analysis elements, DVO schema} + The 1.8m Pan-STARRS\,1 telescope is located on the summit of Haleakala on the Hawaiian island of Maui. The wide-field optical design of the @@ -104,10 +106,10 @@ The \PSONE\ camera \citep{2009amos.confE..40T}, known as GPC1, consists of a mosaic of 60 back-illuminated CCDs manufactured by Lincoln Laboratory. -The CCDs each consist of an $8\times8$ grid of $\sim 600\times 600$ -pixel readout regions, yielding an effective $4800\times4800$ +The CCDs each consist of an $8\times8$ grid of $590 \times 598$ +pixel readout regions, yielding an effective $4846 \times 4868$ detector. Initial performance assessments are presented in \cite{2008SPIE.7014E..0DO}. Routine observations are conducted remotely from the Advanced Technology Research Center in Kula, the main facility of the -University of Hawaii's Institute for Astronomy operations on Maui. +University of Hawaii's Institute for Astronomy (IfA) operations on Maui. The Pan-STARRS1 filters and photometric system have already been described in detail in \cite{2012ApJ...750...99T}. @@ -167,5 +169,5 @@ %Pan-STARRS Pixel Analysis : Source Detection \citet[][Paper IV]{magnier2017.analysis} -describes the details of the source detection and photometry, including point-spread-function and extended source fitting models, and the techniques for ``forced" photometry measurements. +describes the details of the source detection and photometry, including point-spread-function and extended source fitting models, and the techniques for ``forced'' photometry measurements. %Magnier et al. 2017 (Paper V) @@ -202,35 +204,4 @@ reducing data from other cameras and telescopes. -\note{overview discussion of Pan-STARRS: the telescope, survey time - period, surveys. 2 paragraphs.} - -The Pan-STARRS Image Processing Pipeline consists of a suite of -software programs and data systems that are designed to reduce -astronomical images, with a focus on parallelization necessary to -speed the processing of the large images produced by the GPC1 camera. -Part of this parallelization is derived from the fact that this camera -consists of 60 independent orthogonal transfer array (OTA) devices, -and can therefore be processed simultaneously. Although there are -multiple stages that operate on an entire exposure at once, the -majority of stages operate only on smaller segments of a full exposure -to allow the processing tasks to be spread over the machines in the -processing cluster. - - -\note{fix this summary once outline is solidified} - -This paper presents a description of the IPP data handling system. -Section \ref{sec:subsystems} describes the major IPP subsystems that -underlie the main pipeline, providing a set of common interfaces and -tools used at multiple stages. The main processing stages of the -pipeline are described in Section \ref{sec:stages}, although all -exposures may not necessarily pass through each of these stages. The -hardware systems that have done the processing for the PV3 data -release are listed in Section \ref{sec:hardware}, with some details -on the scale of computing needed to reduce this large number of -exposures. Finally, Section \ref{sec:discussion} presents a -discussion of some of the lessons learned in the creation of the IPP, -and its utility in reducing data from other cameras and telescopes. - {\color{red} {\em Note: These papers are being placed on arXiv.org to provide crucial support information at the time of the public @@ -244,15 +215,17 @@ \label{sec:overview} -The Pan-STARRS Data Analysis system consists of many elements to -support the wide range of activities: archiving and management of the +\subsection{Elements of the Pan-STARRS Data Processing System} + +The Pan-STARRS data analysis system consists of many elements to +support a wide range of activities: archiving and management of the raw and processed image files; real-time nightly processing of images for transient and moving object science; large-scale re-processing and calibration to produce measurements for the science collaboration and -the wider public; specialized image processing tasks to facilitate -research and development of the analysis system itself; distribution -of the resulting data products to various consumers in a variety of -formats and modes. - -The Pan-STARRS Data Analysis system is divided internally into several major +the wider public; specialized image processing to facilitate research +and development of the analysis system itself; and distribution of the +resulting data products to various consumers in a variety of formats +and modes. + +The Pan-STARRS data analysis system is divided internally into several major components: \begin{itemize} @@ -260,5 +233,5 @@ data analysis tasks needed to support the on-going observations. In this article, we focus only on those aspects used by the off-summit - analysis stages. \note{is summit processing discussed anywhere?} + analysis stages. \item Image Processing Pipeline (IPP) : this portion of the data analysis system takes the data from raw pixels on the summit @@ -295,4 +268,18 @@ the summit systems are described by \note{REF?}. +\begin{figure*}[htbp] + \begin{center} + \includegraphics[width=\hsize,clip]{PS1_Data_Analysis_System_Overview.pdf} + \caption{\label{fig:analysis.elements} Elements of the Pan-STARRS\,1 + Data Analysis System. Rectangles represent data analysis steps; + ellipses represent databases; rounded rectangles represent + external groups (``customers''). The arrows show a simplified representation + of the major flow of data between the analysis stages and data + processing elements.} + \end{center} +\end{figure*} + +\subsection{Nightly Processing Analysis Stages} + Data analysis to support nightly science operations is driven by two main goals: 1) rapid detection of the moving and transient sources to @@ -309,33 +296,36 @@ (\IPPstage{warp}). Warped images may either be added together (\IPPstage{stack}) or used in an image subtraction (\IPPstage{diff}). -For nightly science operations, images for certain fields such as the -Medium Deep survey fields \citep[see][]{MDref}, are stacked together -in nightly chunks, providing deeper detection capability on 1-day -timescales. Depending on the survey mode, difference images are -generated for the nightly stack images (vs a deep stack template) or -for individual warp images. In the later case, the warp images may be -difference against another warp from the same night or against a +As part of nightly science processing, images for certain fields such +as the Medium Deep survey fields \citep[see][]{MDref}, are stacked +together in nightly chunks, providing deeper detection capability on +1-day timescales. Depending on the survey mode, difference images are +generated for the nightly stack images (using a deep stack template) +or for individual warp images. In the later case, the warp images may +be differenced against another warp from the same night or against a reference stack from the appropriate part of the sky. +\subsection{Re-processing Analysis Stages} + Pan-STARRS has performed several large-scale reprocessings of both the -Medium Deep and 3pi Survey data for internal consumption. For the 3pi -Survey data, we identify these large-scale reprocessings as PV1, PV2, -and PV3, with PV3 the analysis used for the first public data release, -DR1. We also refer to the nightly science analysis of the data as -PV0. For these reprocessing stages, the standard steps of chip -through warp, plus stack and diff are performed, starting from raw -data, usually using a single homogenous version of the data analysis -procedures. PV2 was a special case in which we started from the -camera level products of PV1 to speed up the turn-around to the -community. In addition to the analysis stages listed above which are -shared with the nightly processing, these large-scale reprocessing -analyses include additional processing. A more detailed photometric -analysis is performed on the stacks, including morphological analysis -appropriate to galaxies. The results of the stack photometry analysis -are used to drive a forced-photometry analysis of the warp images. -The data products from the camera, stack photometry, and forced-warp -photometry analysis stages are ingested into the internal calibration -database (DVO, the Desktop Virtual Observatory) and used for -photometric and astrometric calibrations. +Medium Deep and $3\pi$ Survey data for internal consumption. For the +$3\pi$ Survey data, we identify these large-scale reprocessings as +PV1, PV2, and PV3, with PV3 the analysis used for the first public +data release, DR1. We also refer to the nightly science analysis of +the data as PV0. For these reprocessing stages, the standard steps of +\ippstage{chip} through \ippstage{warp}, plus \ippstage{stack} and +\ippstage{diff} are performed, starting from raw data, usually using a +single homogenous version of the data analysis procedures. PV2 was a +special case in which we started from the camera level products of PV1 +to speed up the turn-around to the community. In addition to the +analysis stages listed above which are shared with the nightly +processing, these large-scale reprocessing analyses include additional +processing. A more detailed photometric analysis is performed on the +stacks, including morphological analysis appropriate to galaxies. The +results of the stack photometry analysis are used to drive a +forced-photometry analysis of the warp images. The data products from +the camera, stack photometry, and forced-warp photometry analysis +stages are ingested into the internal calibration database (DVO, the +Desktop Virtual Observatory) and used for photometric and astrometric +calibrations (see Section~\ref{sec:DVO}). \subsection{Data Access and Distribution} @@ -371,7 +361,26 @@ {\bf Stage} & {\bf Primary Table} & {\bf Secondary Table(s)} & {\bf Key} & {\bf Notes} \\ \hline - \ippstage{addstar} & \ippdbtable{addRun} & \ippdbtable{addProcessedExp} & \ippdbcolumn{add_id} & \\ + \ippstage{summitcopy} & \ippdbtable{pzDataStore} & & & Lists locations to check for new exposures.\\ + & \ippdbtable{summitExp} & \ippdbtable{summitImfile} & \ippdbcolumn{summit_id} & Exposures available at the telescope.\\ + & \ippdbtable{pzDownloadExp}& \ippdbtable{pzDownloadImfile} & & Exposures that are being downloaded.\\ + & \ippdbtable{newExp} & \ippdbtable{newImfile} & \ippdbcolumn{exp_id} & Exposures that have been saved to IPP cluster.\\ + + \ippstage{registration} & \ippdbtable{rawExp} & \ippdbtable{rawImfile} & \ippdbcolumn{exp_id} & \\ + \ippstage{chip} & \ippdbtable{chipRun} & \ippdbtable{chipProcessedImfile} & \ippdbcolumn{chip_id} & \\ \ippstage{camera} & \ippdbtable{camRun} & \ippdbtable{camProcessedExp} & \ippdbcolumn{cam_id} & \\ - \ippstage{chip} & \ippdbtable{chipRun} & \ippdbtable{chipProcessedImfile} & \ippdbcolumn{chip_id} & \\ + \ippstage{fake} & \ippdbtable{fakeRun} & \ippdbtable{fakeProcessedImfile} & \ippdbcolumn{fake_id} & \\ + \ippstage{warp} & \ippdbtable{warpRun} & \ippdbtable{warpImfile} & \ippdbcolumn{warp_id} & \\ + & & \ippdbtable{warpSkyCellMap} & & Mapping of input chips to projection skycells.\\ + & & \ippdbtable{warpSkyfile} & & \\ + \ippstage{stack} & \ippdbtable{stackRun} & \ippdbtable{stackInputSkyfile} & \ippdbcolumn{stack_id} & \\ + & & \ippdbtable{stackSumSkyfile} & & \\ + \ippstage{staticsky} & \ippdbtable{staticskyRun} & \ippdbtable{staticskyInput} & \ippdbcolumn{sky_id} & \\ + & & \ippdbtable{staticskyResult} & & \\ + \ippstage{skycal} & \ippdbtable{skycalRun} & \ippdbtable{skycalResult} & \ippdbcolumn{skycal_id} & \\ + \ippstage{fullforce} & \ippdbtable{fullForceRun} & \ippdbtable{fullForceInput} & \ippdbcolumn{ff_id} & \\ + & & \ippdbtable{fullForceResult} & & \\ + & & \ippdbtable{fullForceSummary} & & Properties about average parameters from all results.\\ + \ippstage{diff} & \ippdbtable{diffRun} & \ippdbtable{diffSkyfile} & \ippdbcolumn{diff_id} & \\ + & & \ippdbtable{diffInputSkyfile} & & \\ \ippstage{detrend} & \ippdbtable{detRun} & \ippdbtable{detRunSummary} & \ippdbcolumn{det_id} & \\ & & \ippdbtable{detInputExp} & & \\ @@ -381,31 +390,12 @@ & \ippdbtable{detResidExp} & \ippdbtable{detResidImfile} & & \\ & \ippdbtable{detNormalizedExp} & \ippdbtable{detNormalizedImfile} & & \\ - \ippstage{diff} & \ippdbtable{diffRun} & \ippdbtable{diffSkyfile} & \ippdbcolumn{diff_id} & \\ - & & \ippdbtable{diffInputSkyfile} & & \\ + \ippstage{addstar} & \ippdbtable{addRun} & \ippdbtable{addProcessedExp} & \ippdbcolumn{add_id} & \\ \ippstage{distribution} & \ippdbtable{distRun} & \ippdbtable{distComponent} & \ippdbcolumn{dist_id} & \\ & & \ippdbtable{distTarget} & & \\ - \ippstage{fake} & \ippdbtable{fakeRun} & \ippdbtable{fakeProcessedImfile} & \ippdbcolumn{fake_id} & \\ - \ippstage{fullforce} & \ippdbtable{fullForceRun} & \ippdbtable{fullForceInput} & \ippdbcolumn{ff_id} & \\ - & & \ippdbtable{fullForceResult} & & \\ - & & \ippdbtable{fullForceSummary} & & Properties about average parameters from all results.\\ + \ippstage{publish} & \ippdbtable{publishRun} & \ippdbtable{publishDone} & \ippdbcolumn{pub_id} & \\ + & & \ippdbtable{publishClient} & & \\ \ippstage{lap} & \ippdbtable{lapSequence} & \ippdbtable{lapRun} & \ippdbcolumn{seq_id} & Sequence of full reprocessing\\ & \ippdbtable{lapRun} & \ippdbtable{lapExp} & \ippdbcolumn{lap_id} & \\ - \ippstage{publish} & \ippdbtable{publishRun} & \ippdbtable{publishDone} & \ippdbcolumn{pub_id} & \\ - & & \ippdbtable{publishClient} & & \\ - \ippstage{summitcopy} & \ippdbtable{pzDataStore} & & & Lists locations to check for new exposures.\\ - & \ippdbtable{summitExp} & \ippdbtable{summitImfile} & \ippdbcolumn{summit_id} & Exposures available at the telescope.\\ - & \ippdbtable{pzDownloadExp}& \ippdbtable{pzDownloadImfile} & & Exposures that are being downloaded.\\ - & \ippdbtable{newExp} & \ippdbtable{newImfile} & \ippdbcolumn{exp_id} & Exposures that have been saved to IPP cluster.\\ - - \ippstage{registration} & \ippdbtable{rawExp} & \ippdbtable{rawImfile} & \ippdbcolumn{exp_id} & \\ \ippstage{remote} & \ippdbtable{remoteRun} & \ippdbtable{remoteComponent} & \ippdbcolumn{remote_id} & \\ - \ippstage{skycal} & \ippdbtable{skycalRun} & \ippdbtable{skycalResult} & \ippdbcolumn{skycal_id} & \\ - \ippstage{stack} & \ippdbtable{stackRun} & \ippdbtable{stackInputSkyfile} & \ippdbcolumn{stack_id} & \\ - & & \ippdbtable{stackSumSkyfile} & & \\ - \ippstage{staticsky} & \ippdbtable{staticskyRun} & \ippdbtable{staticskyInput} & \ippdbcolumn{sky_id} & \\ - & & \ippdbtable{staticskyResult} & & \\ - \ippstage{warp} & \ippdbtable{warpRun} & \ippdbtable{warpImfile} & \ippdbcolumn{warp_id} & \\ - & & \ippdbtable{warpSkyCellMap} & & Mapping of input chips to projection skycells.\\ - & & \ippdbtable{warpSkyfile} & & \\ \hline \end{tabular} @@ -424,10 +414,12 @@ successive processing stages to begin their own tasks. -The processing database is colloquially referred to as the `gpc1' +The processing database is colloquially referred to as the ``gpc1'' database, since a single instance of the database is used to track the processing of images and data products related to the PS1 GPC1 camera. This same database engine also has instances (same schema, different data) for other cameras processed by the IPP, e.g., GPC2, the test -cameras TC1, TC3, and the Imaging Sky Probe (ISP). +cameras TC1, TC3, and the Imaging Sky Probe (ISP). In general, +processing information for different cameras is separate in different +processing database; merging of output products takes place in DVO. Within the processing database, the various processing stages are @@ -435,24 +427,25 @@ primary table which defines the conceptual list of processing items either to be done, in progress, or completed. An associated secondary -table (or set of tables) lists the details of elements which have been -processed. Table \ref{tab: database schema} contains an outline of -the database schema, showing the relations between tables organized by -processing stage. As an example, one critical stage is the -\ippstage{chip} processing stage (see \S\ref{sec:chip}) in which the -individual chips from an exposure are detrended and sources are -detected. Within the gpc1 database, the primary table is called -\ippdbtable{chipRun} in which each exposure has a single entry. -Associated with this table is the \ippdbtable{chipProcessedImfile} -table, which contains one row for each of the chips -associated with the exposure (up to 60 for gpc1). The primary tables, such as -\ippdbtable{chipRun}, are populated once the system has decided that a -specific item (e.g., an exposure) should be processed at that stage. -Initially, the entry is given a state of ``run'', denoting that the -exposure is ready to be processed. The low-level table entries, such -as the \ippdbtable{chipProcessedImfile} entries, are only populated -once the element (e.g., the chip) has been processed by the analysis -system. Once all elements for a given stage, e.g., chips in this -case, are completed, then the status of the top-level table entry -(\ippdbtable{chipRun}) are switched from ``run'' to ``full''. +table (or set of tables) lists the details of component elements which +have been processed for each top-level item. Table \ref{tab: database + schema} contains an outline of the database schema, showing the +relations between tables organized by processing stage. As an +example, one critical stage is the \ippstage{chip} processing stage +(see \S\ref{sec:chip}) in which the individual chips from an exposure +are detrended and sources are detected. Within the gpc1 database, the +primary table is called \ippdbtable{chipRun} in which each exposure +has a single entry. Associated with this table is the +\ippdbtable{chipProcessedImfile} table, which contains one row for +each of the chips associated with the exposure (up to 60 for gpc1). +The primary tables, such as \ippdbtable{chipRun}, are populated once +the system has decided that a specific item (e.g., an exposure) should +be processed at that stage. Initially, the entry is given a state of +``run'', denoting that the exposure is ready to be processed. The +low-level table entries, such as the \ippdbtable{chipProcessedImfile} +entries, are only populated once the element (e.g., the chip) has been +processed by the analysis system. Once all elements for a given +stage, e.g., chips in this case, are completed, then the status of the +top-level table entry (\ippdbtable{chipRun}) are switched from ``run'' +to ``full''. If the analysis of an element (e.g., the individual OTA chip) @@ -467,5 +460,5 @@ other hand, if the analysis failed because of a problem with the input data, this is noted by setting a non-zero value in a different table -field, \ippdbcolumn{quality}. For example, if the chip analysis +field, \ippdbcolumn{quality}. For example, if the \ippstage{chip} analysis failed to discover any stars because the image was completely saturated, the analysis can complete successfully (\ippdbcolumn{fault} @@ -483,9 +476,9 @@ of the \ippdbcolumn{fault}s which occur are ephemeral due to current conditions of the processing cluster, the processing stages are set up -to occasionally clear and re-try the faulted entries. Some faults +to occasionally clear and re-try the faulted entries. Some \ippdbcolumn{fault}s represent software bugs and in the early stages of processing were accumulated until the corresponding software issue could be addressed; since the start of the PS1 Science Consortium Surveys, these types of -faults have largely been eliminated. Thus, automatic processing is +\ippdbcolumn{fault}s have largely been eliminated. Thus, automatic processing is able to keep the data flowing even in the face of occasional network glitches or hardware crashes. @@ -496,8 +489,10 @@ As exposures are taken by the PS1 telescope \& GPC1 camera system, the data from the 60 OTA devices are read out by the camera software -wsystem and written to disk on a collection of computers at the summit +system and written to disk on a collection of computers at the summit in the PS1 facility called ``pixel servers.'' After the images are written to disk, a summary listing of the information about the -exposure and the chip images are added to the summit datastore. +exposure and the chip images are added to the summit datastore (an +internal http-based data sharing tool, see +Section~\ref{sec:datastore}). During night-time operations, while the summit datastore is being @@ -531,12 +526,12 @@ Once the chips for an exposure have all been downloaded, the exposure -is ready to be registered. In this context, `registration' refers to +is ready to be registered. In this context, ``registration'' refers to the process of adding them to the database listing of known, raw -exposures (not to be confused with `registration' in the sense of -pixel re-alignment). The result of the registration analysis is an +exposures (not to be confused with ``registration'' in the sense of +pixel re-alignment). The result of the \ippstage{registration} analysis is an entry for each exposure in the \ippdbtable{rawExp} table, and one for each chip in the \ippdbtable{rawImfile} table. These tables are critical for downstream processing to identify what exposures are -available for processing in any other stage. At the registration +available for processing in any other stage. At the \ippstage{registration} stage, a large amount of descriptive metadata for each chip is added to the \ippdbtable{rawImfile} table, the majority of which is @@ -552,8 +547,8 @@ Unlike much of the rest of the IPP stage, the raw exposures may only -have a single entry in the registration tables of the processing +have a single entry in the \ippstage{registration} tables of the processing database tables (\ippdbtable{rawExp} and \ippdbtable{rawImfile}). -For GPC1, the image registration stage is also the stage at which the +For GPC1, the \ippstage{registration} stage is also the stage at which the \ippprog{burntool} analysis is run. This analysis is more completely described in \citet{waters2017}. In brief, the \ippprog{burntool} @@ -564,16 +559,16 @@ observation date and time listed in the headers, with the results stored in an text table. As a result of the sequential nature of this -analysis, the registration of exposures is blocked until the +analysis, the \ippstage{registration} of exposures is blocked until the \ippprog{burntool} has been run on the previous exposures. -Once the registration process has finished, new science exposures that -have an \ippdbcolumn{obs_mode} value that indicates they are part of -a particular science survey are automatically launched into the -science analysis by defining entries for the \ippstage{chip} -processing stage, as described above. This analysis can be relaunched -multiple times, such as for the large scale PV3 reprocessing. -However, this automatic process ensures the shortest time between -observation and analysis, which is particularly important in the -search for transient sources. +Once the \ippstage{registration} process has finished, new science +exposures that have an \ippdbcolumn{obs_mode} value that indicates +they are part of a particular science survey are automatically +launched into the science analysis by defining entries for the +\ippstage{chip} processing stage, as described above. The science +analysis of a given exposure can be relaunched multiple times, such as +for the large scale PV3 reprocessing. The automatically-launched +analysis process ensures the shortest time between observation and +analysis, particularly important in the search for transient sources. \subsection{Chip Processing} @@ -619,5 +614,5 @@ %% attempts to target the processing for each OTA to the machine on which %% the data for that detector is stored. The output products are then -%% primarily saved back to the same machine. This `targetted' processing +%% primarily saved back to the same machine. This ``targetted'' processing %% was an early design choice to minimize the system wide network load %% during processing. In practice, as computer disks filled up at @@ -647,26 +642,26 @@ The results of the image processing are then written to disk, -including the science, mask, and variance images, the background model -subtracted, the PSF model used in the photometry process, and a FITS -catalog of detected sources. Additional binned images of the full OTA -are also saved, providing $16\times{}16$ and $256\times{}256$ pixel -binning scales for quick visualization. The processing log and a -selection of summary metadata describing the processing results are -also written to disk. This metadata is used to populate a row in the -\ippdbtable{chipProcessedImfile} table (linked to the -\ippdbtable{chipRun} entry by a shared \ippdbcolumn{chip_id} value) -to indicate that the processing of this OTA is complete. +including the science, mask, and variance images, the binned +background model subtracted, the PSF model used in the photometry +process, and a FITS catalog of detected sources. Additional binned +images of the full OTA are also saved, using $16\times{}16$ and +$256\times{}256$ pixel binning scales for quick visualization. The +processing log and a selection of summary metadata describing the +processing results are also written to disk. This metadata is used to +populate a row in the \ippdbtable{chipProcessedImfile} table to +indicate that the processing of this OTA is complete. As each OTA is processed independently of the others across a number -of computers, the \ippprog{pantasks} managing the jobs periodically -runs an \ippmisc{advance} task that checks that the number of rows in -\ippdbtable{chipProcessedImfile} with \ippdbcolumn{fault} equal to -zero matches the associated number of rows in \ippdbtable{rawImfile}. -If this condition is met, than all processing for that exposure is -finished, and the \ippdbcolumn{state} field is set to ``full''. If -the \ippdbtable{chipRun}.\ippdbcolumn{end_stage} field is set to +of computers, the \ippprog{pantasks} server managing the jobs +periodically runs an \ippmisc{advance} task that checks that the +number of rows in \ippdbtable{chipProcessedImfile} with +\ippdbcolumn{fault} equal to zero matches the associated number of +rows in \ippdbtable{rawImfile}. If this condition is met, than all +processing for that exposure is finished, and the \ippdbcolumn{state} +field is set to ``full''. If the +\ippdbtable{chipRun}.\ippdbcolumn{end_stage} field is set to \ippstage{chip}, then no further action is taken. However, this field is usually set to a subsequent stage (most often \ippstage{warp}), -then an entry for this exposure is added to the \ippdbtable{camRun} +in which case an entry for this exposure is added to the \ippdbtable{camRun} table, and processing continues. @@ -710,6 +705,8 @@ to help guarantee a solution in the case of a modest pointing error. The guess astrometry is used to match the reference catalog to the -observed stellar positions in the focal plane coordinate system. Once -an acceptable match is found, the astrometric calibration of the +observed stellar positions in the focal plane coordinate system +\citep[see][]{magnier2017.calibration}). + +Once an acceptable match is found, the astrometric calibration of the individual chips is performed, including a fit to a single model for the distortion introduced by the camera optics. After the astrometic @@ -720,7 +717,7 @@ used to generate synthetic w-band photometry for areas where no PS1-based calibrated w-band photometry is available. For more -details, see \cite{magnier2017.calibration}. The result of these calibrations is -stored as a single multi-extension FITS table containing the results -from each OTA as a separate extension. +details, see \cite{magnier2017.calibration}. The result of these +calibrations is stored as a single multi-extension FITS table +containing the results from each OTA as a separate extension. In addition to the astrometric and photometric calibrations, the @@ -740,25 +737,25 @@ processed all at once, this update also updates the associated \ippdbtable{camRun} entry, linked by the \ippdbcolumn{cam_id}. As -with the \ippstage{chip} stage, the +with the \ippstage{chip} stage, if the \ippdbtable{camRun}.\ippdbcolumn{end_stage} is for a subsequent stage, an appropriate entry is added to the \ippdbtable{fakeRun} -table. - -%% \subsection{Fake Analysis} -%% \label{sec:fake} -%% -%% The \ippstage{fake} stage was originally designed to do false source -%% injection and recovery, in order to determine the detection efficiency -%% of sources on the exposure. However, early in the design of the IPP, -%% this task was moved to the rest of the photometry analysis done at the -%% \ippstage{chip} stage. Removing the stage would require significant -%% changes to the database schema. As a result, this conveniently named -%% stage generally does no actual data processing, and consists mainly of -%% database operations to move the exposure on to the \ippstage{warp} -%% stage. The operations mimic the \ippstage{chip} stage, with -%% individual jobs run for each OTA that update rows in the -%% \ippdbtable{fakeProcessedImfile}, and an \ippmisc{advance} task that -%% updates the \ippdbtable{fakeRun} table and promotes the exposure to -%% the next stage by adding a row to the \ippdbtable{warpRun} table. +table. + +\subsection{Fake Analysis} +\label{sec:fake} + +The \ippstage{fake} stage was originally designed to do false source +injection and recovery, in order to determine the detection efficiency +of sources on the exposure. However, early in the design of the IPP, +this task was moved to the rest of the photometry analysis done at the +\ippstage{chip} stage. Removing the stage would require significant +changes to the database schema. As a result, this conveniently named +stage generally does no actual data processing, and consists mainly of +database operations to move the exposure on to the \ippstage{warp} +stage. The operations mimic the \ippstage{chip} stage, with +individual jobs run for each OTA that update rows in the +\ippdbtable{fakeProcessedImfile}, and an \ippmisc{advance} task that +updates the \ippdbtable{fakeRun} table and promotes the exposure to +the next stage by adding a row to the \ippdbtable{warpRun} table. \subsection{Image Warping} @@ -776,5 +773,5 @@ described by a single tangent plane projection, or for larger regions which have multiple projection centers. For the $3\pi$ survey, the -\ippmisc{RINGS.V3} tessellation was used that used projection centers +\ippmisc{RINGS.V3} tessellation was used that arrange projection centers spaced every four degrees in both RA and DEC, with $0\farcs{}25$ pixels. These projections are further broken down into ``skycells'' @@ -822,6 +819,6 @@ \label{sec:stack} -The skycell images generated by the \ippstage{warp} process are added -together to make deeper, higher signal-to-noise images in the +The skycell images generated by the \ippstage{warp} process can be +added together to make deeper, higher signal-to-noise images in the \ippstage{stack} stage. These stacked images also fill in coverage gaps between different exposures, resulting in an image of the sky @@ -831,5 +828,5 @@ input images. During nightly science processing, the 8 exposures per filter for each Medium Deep field are combined into a set of stacks -for that field. These so-called `nightly stacks' are used by the +for that field. These so-called ``nightly stacks'' are used by the transient survey projects to detect faint supernovae, among other transient events. For the PV3 $3\pi$ analysis, all images in each @@ -840,8 +837,8 @@ For the PV3 processing of the Medium Deep fields, stacks have been generated for the nightly groups and for the full depth using all -exposures, producing ``deep stacks''. In addition, a `best seeing' +exposures, producing ``deep stacks''. In addition, a ``best seeing'' set of stacks have been produced \note{using image quality cuts to be described: need input from MEH}. We have also generated -out-of-season stacks for the Medium Deep fields, in which all image +out-of-season stacks for the Medium Deep fields, in which all images not from a particular observing season for a field are combined into a stack. These later stacks are useful as deep templates when studying @@ -850,16 +847,14 @@ season. -When a given set of \ippstage{stack} stage are defined, exposures with -existing \ippstage{warp} entries that match the filter, position, and -other criteria such as seeing are grouped by their skycell. An entry +When a given set of \ippstage{stack} stage processing is defined, +exposures with existing \ippstage{warp} entries that match the filter, +position, and other criteria such as seeing are identified. An entry is then added for each skycell in the \ippdbtable{stackRun} table, with the \ippdbcolumn{warp_id} entries for the exposures added to the \ippdbtable{stackInputSkyfile} table, linked to the -\ippdbtable{stackRun} entry by the \ippdbcolumn{stack_id} field. -This defines the mapping for which exposures contribute to the -\ippstage{stack}. This breaks exposures into single skycells, but as -adjacent \ippstage{stack} skycells may contain inputs from different -exposures, there is no simple way to group the processing at the -\ippstage{stack} stage into exposures. +\ippdbtable{stackRun} entry by the \ippdbcolumn{stack_id} field. This +defines the mapping for which exposures contribute to the +\ippstage{stack}. The \ippstage{stack} stage processing is performed +at the skycell level. The \ippstage{stack} jobs pass the information about the input images @@ -867,5 +862,5 @@ image combinations. See~\cite{waters2017} for details on the stack combination algorithm. In addition to the standard image, mask, and -variance produced at other stage, additional images are constructed +variance produced at other stages, additional images are constructed with information about the contributions to each pixel. A number image contains the number of input exposures used for each pixel, @@ -887,13 +882,14 @@ deferred to the \ippstage{staticsky} stage. This separation is maintained because the photometry analysis of the \ippstage{stack} -images is performed on all 5 filters simultaneously. By deferring -this analysis, the processing system may also decouple the generation -of the pixels from the source detection. This makes the sequencing of -analysis somewhat easier and less subject to blocks due to a failure -in the stacking analysis. Similar to the \ippstage{stack} stage, an -entry is created in the \ippdbtable{staticskyRun} table, linked to a -series of rows in the \ippdbtable{staticskyInput} table by a common -\ippdbcolumn{sky_id}, each of which also contains the appropriate -\ippdbcolumn{stack_id} entries for the skycell under consideration. +images, including convolved galaxy model fitting, is performed on all +5 filters simultaneously. By deferring this analysis, the processing +system may also decouple the generation of the pixels from the source +detection. This makes the sequencing of analysis somewhat easier and +less subject to blocks due to a failure in the stacking analysis. +Similar to the \ippstage{stack} stage, an entry is created in the +\ippdbtable{staticskyRun} table, linked to a series of rows in the +\ippdbtable{staticskyInput} table by a common \ippdbcolumn{sky_id}, +each of which also contains the appropriate \ippdbcolumn{stack_id} +entries for the skycell under consideration. The input images are passed to the \ippprog{psphotStack} program, @@ -927,16 +923,24 @@ The stack photometry output catalogs are re-calibrated for both photometry and astrometry in a process very similar to the -\ippstage{camera} calibration stage. In the case of this -\ippstage{skycal} stage, each skycell is processed independently. -Because of this independence, when queued for processing, the entries -in the \ippdbtable{skycalRun} table contain the \IPPdbcolumn{sky_id} -and \ippdbcolumn{stack_id} entries of the parent data directly. As -in the \ippstage{camera} stage, the \ippprog{psastro} program reads in -the stack photometry catalog, and produces a calibrated output, with -format matching the input. A different processing recipe is supplied -to \ippprog{psastro}, which controls for the different data. The same -reference catalog is used for the \ippstage{camera} and -\ippstage{stack} calibration stages. Upon completion, the analysis -statistics are written to the \ippdbtable{skycalResult} table. +\ippstage{camera} calibration stage. Although the individual warps +which go into the stack are calibrated based on the \ippstage{camera} +stage analysis, there was some concern that these calibrations might +not be sufficiently well-defined for some of the input warps, biasing +the photometry of the stack. By re-calibrating the stacks, we can be +sure that the stack photometry as measured is tied to the photometric +reference system. + +In the case of this \ippstage{skycal} stage, each skycell is processed +independently. Because of this independence, when queued for +processing, the entries in the \ippdbtable{skycalRun} table contain +the \ippdbcolumn{sky_id} and \ippdbcolumn{stack_id} entries of the +parent data directly. As in the \ippstage{camera} stage, the +\ippprog{psastro} program reads in the stack photometry catalog, and +produces a calibrated output, with format matching the input. A +different processing recipe is supplied to \ippprog{psastro}, which +controls for the different data. The same reference catalog is used +for the \ippstage{camera} and \ippstage{stack} calibration stages. +Upon completion, the analysis statistics are written to the +\ippdbtable{skycalResult} table. \subsection{Forced Warp Photometry} @@ -995,12 +999,14 @@ individual warp images used to generate the stack. This \ippstage{fullforce} analysis is performed on all warps for a single -skycell and filter as a single unit, as this matches the arrangement -of the input source catalog from the \ippstage{skycal} stage. When -processing is queued for this stage, an entry is added to the -\ippdbtable{fullForceRun} primary database table linking to the -specific \ippdbcolumn{skycal_id} entry that will be used as the -catalog for the photometry. The \ippdbcolumn{warp_id} values for the -input \ippstage{warp} stage images that contributed to the -\ippstage{stack} associated with that \ippdbcolumn{skycal_id} are +skycell and filter as a single unit within the processing database, +while individual warps are processed individually in parallel as +separate processing jobs. + +When processing is queued for this stage, an entry is added to the +\ippdbtable{fullForceRun} primary database table with a reference to +the corresponding stack and \ippdbcolumn{skycal_id} entry that is the +input source of detections to be measured. The \ippdbcolumn{warp_id} +values for the input \ippstage{warp} stage images that contributed to +the \ippstage{stack} associated with that \ippdbcolumn{skycal_id} are then added to the \ippdbtable{fullForceInput} table, linked to the primary table by the \ippdbcolumn{ff_id} identifier. The individual @@ -1008,4 +1014,22 @@ stage image products along with the \ippstage{skycal} catalog to the \ippprog{psphotFullForce} program. + +%% In this program, the positions of sources are loaded from the input +%% catalog. PSF stars are pre-identified from the stack image and a PSF +%% model generated for each \ippstage{warp} image based on those stars, +%% using the same stars for all warps to the extent possible (PSF stars +%% which are excessively masked on a particular image are not used to +%% model the PSF). The PSF model is fitted to all of the known source +%% positions in the warp images. Aperture magnitudes, Kron magnitudes, +%% and moments are also measured at this stage for each warp. Note that +%% the flux measurement for a faint, but significant, source from the +%% stack image may be at a low significance (less than the $5\sigma$ +%% criterion used when the photometry is not run in this forced mode) in +%% any individual warp image; the flux may even be negative for specific +%% warps. When combined together, these low-significance measurements +%% will result in a signficant measurement as the signal-to-noise +%% increases by the square root of the number of measurements. The +%% individual warp measurements are combined together to generate +%% averages values within DVO. The convolved galaxy models are also re-measured on the @@ -1053,10 +1077,10 @@ images are matched. \note{discuss Alard-Lupton}. -In the \ippstage{diff} stage, the IPP generates diffferece images for +In the \ippstage{diff} stage, the IPP generates difference images for appropriately specified pairs of images. It is possible for the difference image to be generated from a pair of \ippstage{warp} stage images, from a \ippstage{warp} and a \ippstage{stack} of some variety, or from a pair of \ippstage{stack} stage images. During the PS1 -survey, pairs of exposures, call TTI pairs (see~\note{Survey +survey, pairs of exposures, called TTI pairs (see~\note{Survey Strategy in Chambers et al}), were obtained for each pointing within a $\approx$ 1 hour period in the same filter, and to the extent possible with the @@ -1074,5 +1098,5 @@ \ippdbtable{diffRun} table, and the appropriate input images are added to the \ippdbtable{diffInputSkyfile} table, with one entry for each -skycell that are covered by the images. For a \ippstage{diff} +skycell that is covered by the images. For a \ippstage{diff} generated from two \ippstage{warp} stage products, the input images have their \ippdbcolumn{warp_id} values recorded in the @@ -1095,7 +1119,9 @@ catalogs passed to the \ippprog{ppSub} program. This does the subtraction, as well as the photometry of any sources detected in the -\ippstage{diff} image. The algorithm used for PSF matching is -described in \citet{waters2017}. Upon completion of these jobs, -statistics about the processing are written to an entry in the +\ippstage{diff} image. Sources may be detected as a positive source +(flux in the minuend is higher than the subtrahend) or as a negative +source (flux in the subtrahend is higher). The algorithm used for PSF +matching is described in \citet{waters2017}. Upon completion of these +jobs, statistics about the processing are written to an entry in the \ippdbtable{diffSkyfile} table. An \ippmisc{advance} checks for the completion of all of the components listed in @@ -1111,5 +1137,6 @@ \begin{table}[hb] \begin{center} -\caption{DVO Database Tables\label{tab:DVO_schema}} +\caption{DVO Database Tables\label{tab:DVO_schema} \note{fix order, + drop invalid tables}} \begin{tabular}{ll} \hline @@ -1155,5 +1182,5 @@ DVO tracks three main classes of information: 1) average properties of astronomical objects; 2) measurements of those objects (from which the -average properties are derived); 3) properties of image which provided +average properties are derived); 3) properties of the images which provided some or all of the measuements. Figure~\ref{fig:DVO_schema} illustrates the schematic relationship between these types of @@ -1182,14 +1209,4 @@ measurements; those which store information about the images; those which store supporting information (metadata). - -\subsubsubsection{Photcodes} - -% photcodes -DVO has a special metadata table called \ippdbcolumn{photcode} which -identifies the photometry filter systems. Entries in this table are -used to identify the source of measurements and images. Each row in -the \ippdbcolumn{photcode} table includes a \ippdbcolumn{photcode} -name, a unique numerical ID, and information about that photometry -system. DVO includes two major classes of database tables: those containing @@ -1208,4 +1225,25 @@ levels each containing a finer mesh of regions covering the sky. +\subsubsubsection{Photcodes} + +% photcodes +DVO has a special metadata table called \ippdbtable{photcode} which +identifies the photometry filter systems. Entries in this table are +used to identify the source of measurements and images. Each row in +the \ippdbtable{photcode} table includes a \ippdbtable{photcode} +name, a unique numerical ID, and information about that photometry +system. + +There are 3 classes of photcodes defined within the DVO system. One +class of photcodes define the filter systems for the average +photometry measurements; these are called \ippmisc{SEC} photcodes. A +second class of photcode is associated with measurements from a +specific camera for which image metadata is available are called +\ippmisc{DEP} photcodes. There are also those measurements which come +from external data sources for which DVO does not have any information +to determine a calibration (e.g., instrumental magnitudes and detector +coordinates). These are measurements are reference values and are +assigned \ippmisc{REF} photcodes. + The names for \ippmisc{SEC} photcodes are the names of filter systems, such as $g,r,i$ or $J,H,K$. For \ippmisc{DEP} and \ippmisc{REF} @@ -1229,5 +1267,5 @@ properties derived from multiple measurements, and for which the measurement-to-image relationship is not provided. Ingests methods -have been defined for example for 2MASS, WISE, Gaia, USNO-B. In each +have been defined, for example, for 2MASS, WISE, Gaia, USNO-B. In each of these cases, the astrometric and photometric measurements are stored in the \ippdbtable{Measure} table, with the data source @@ -1258,6 +1296,6 @@ discussed below) and the astrometrically calibrated position. Astrometric offsets for several systematic corrections discussed below -are also defined for each measurement. Photometry from chip, warp, -and stack are all placed in the same table with photcodes +are also defined for each measurement. Photometry from \ippstage{chip}, \ippstage{warp}, +and \ippstage{stack} are all placed in the same table with photcodes distinguishing the source \note{show example of stack and warp photcodes}. Since stacks and forced warp fluxes may have @@ -1269,5 +1307,5 @@ For the warp images, we also measure the weak lensing KSB parameters related to the shear and smear tensors \citep{1995ApJ...449..460K}. -These measurements are stored in the \ippdbcolumn{Lensing} table, +These measurements are stored in the \ippdbtable{Lensing} table, along with the radial aperture fluxes for radii numbers 5, 6, \& 7 (respectively 3.0, 4.63, and 7.43 arcsec). This table contains one @@ -1281,4 +1319,6 @@ sorted \ippdbtable{Lensing} table entries. \note{discuss failure of the Lensing to Measure indexing} + +\note{Average used above but defined below} \subsubsubsection{Object Tables} @@ -1359,5 +1399,5 @@ these photometric distance modulus measurements are not extremely precise (see below), they provide a constraint on the distance is used -in our analysis of the astrometry \citep[][see]{magnier2017.calibration}. +in our analysis of the astrometry \citep[see][]{magnier2017.calibration}. In the \ippdbtable{Measure} table, there are three fields which @@ -1416,9 +1456,10 @@ determined by the photometry calibration analysis and the astrometric flat-field corrections determined by the astrometry calibration -analysis \citep[][see]{magnier2017.calibration}. +analysis \citep[see][]{magnier2017.calibration}. +\note{use names and match DVO schema table} \subsubsection{Sky Partition} -DVO includes two major classes of database tables: those containing +\note{re-word this sentence} DVO includes two major classes of database tables: those containing information about astronomical objects in the sky and those containing other supporting information. The object-related tables are @@ -1438,6 +1479,6 @@ on the one used by the Hubble Space Telescope Guide Star Catalog files. \note{add figure} Level 0 is a single region covering the full -sky. Level 1 divides the sky in Declination into bands -7.5\degree\ high. Level 2 subdivides these Declination bands in the +sky. Level 1 divides the sky in declination into bands +7.5\degree\ high. Level 2 subdivides these declination bands in the RA direction, with spacing related to the stellar density. Level 3 divides these RA chunks into 4 - 8 smaller partitions. This level @@ -1459,5 +1500,5 @@ astronomical objects in the database files, with an associated maximum of \approx 30 million measurements in these files. With the compression -scheme described above, the largest database files are \approx +scheme described below, the largest database files are \approx 3GB, which can be loaded into memory in 30 seconds on the processing machines that contain partition data. @@ -1499,5 +1540,5 @@ tables are compressed using the (to date) experimental FITS binary table compression strategy outlined by \note{REF}. Table compression -is in general an option in DVO; for the PV3 database, the large data +is an option in DVO; for the PV3 database, the large data volume (70TB compressed) drove the decision to compress the tables. @@ -1505,5 +1546,5 @@ The FITS binary table compression scheme uses a strategy similar to that used for FITS image compression (\note{REF}). The binary tabular -data is compressed and stored in the `HEAP' section of the FITS table +data is compressed and stored in the ``HEAP'' section of the FITS table extension, with pointers to the compressed data stored in the regular data section. Each column in the FITS table is compressed as one (or @@ -1511,5 +1552,5 @@ column format (e.g., TFORM1) are replaced with keywords which describe the location and size of the compressed data in the HEAP section; the -information about the uncompressed data is moved to a keyword with `Z' +information about the uncompressed data is moved to a keyword with ``Z'' prepended (e.g., ZFORM1) and an additional field is added to define the compression algorithm (e.g., ZCTYP1). The column names (e.g., @@ -1533,5 +1574,5 @@ in the tables. In practice, we have chosen a default in which floating point numbers use \code{GZIP_2}, character strings use -\code{GZIP_1}, integers use \code{RICE}. +\code{GZIP_1}, and integers use \code{RICE}. \subsubsection{Addstar : DVO Ingest} @@ -1540,18 +1581,18 @@ Upon completion of the processing of each stage, the results of the photometry analysis are stored in a large number of individual catalog -files as described in~\ref{XXX}. The data from these files are loaded -into a DVO database to define the astronomical objects and to allow -for calibration analysis. The program which loads the data into the -DVO database is called \ippprog{addstar}, and is associated with the -the \ippstage{addstar} processing stage. The measurement catalogs -generated by the \ippstage{camera}, \ippstage{staticsky}, -\ippstage{skycal}, \ippstage{fullforce}, and \ippstage{diff} stages -are processed loaded into DVOs in this fashion, although not every -measurement in each catalog are included in the master DVO that is -constructed. For a particular re-processing version, a single master -DVO is constructed for the positive image stages (\ippstage{camera}, -\ippstage{staticsky}, \ippstage{skycal}, \ippstage{fullforce}) and a -separate one is constructed for the difference image analysis stage -results. +files as described in \cite{magnier2017.analysis}. The data from +these files are loaded into a DVO database to define the astronomical +objects and to allow for calibration analysis. The program which +loads the data into the DVO database is called \ippprog{addstar}, and +is associated with the the \ippstage{addstar} processing stage. The +measurement catalogs generated by the \ippstage{camera}, +\ippstage{staticsky}, \ippstage{skycal}, \ippstage{fullforce}, and +\ippstage{diff} stages are processed loaded into DVOs in this fashion, +although not every measurement in each catalog are included in the +master DVO that is constructed. For a particular re-processing +version, a single master DVO is constructed for the positive image +stages (\ippstage{camera}, \ippstage{staticsky}, \ippstage{skycal}, +\ippstage{fullforce}) and a separate one is constructed for the +difference image analysis stage results. The construction of the master DVO is performed in a hierarchical @@ -1564,5 +1605,5 @@ databases together. In the merge, astronomical objects are joined together using essentially the same rules as those used to associated -detections into objects. One exception: the match radius may be +detections into objects with one exception: the match radius may be chosen to be a different size depending on the data source. For example, when WISE data is merged with PS1 data, as discussed below, a @@ -1612,6 +1653,6 @@ a function of position in the camera (essentially an astrometric flat-field correction), as a function of the brightness of the star -(the so-called Koppenh\"offer effect, see~\ref{magnier2017.calibration}), and as -a function of airmass and color (Differential chromatic refraction). +(the so-called Koppenh\"offer effect, see~\citealt{magnier2017.calibration}), and as +a function of airmass and color (differential chromatic refraction). Once the systematic errors have been measured, they are applied back to the measurements in the database. Within the DVO @@ -1624,11 +1665,13 @@ astrometry is again performed this time using the corrected positions. +\note{have eddie suggest wording here?} + Photometric calibration consists of determination of zero points for each exposure along with corrections for systematic effects. In this case, we rely on efforts of our external collaborators for the initial zero point determination. The team at CfA downloaded the per-exposure -catalog files (`smf files') and determined the zero points of those +catalog files (``smf files'') and determined the zero points of those exposures which were believed to be obtained in photometric -conditions. This process, called `\"ubercal', is described in detail +conditions. This process, called ``\"ubercal'', is described in detail by \cite{2012ApJ...756..158S} for the first (PV1) version. In brief, photometric periods, with time-scales of at least \note{half of a night}, are @@ -1638,6 +1681,6 @@ parameters in this solution consist of a single zero point and airmass slope for each photometric period along with a collection of -flat-field offsets for several large time range (`flat-field -seasons'). For the PV3 \"ubercal analysis, the flat-field offsets +flat-field offsets for several large time range (``flat-field +seasons''). For the PV3 \"ubercal analysis, the flat-field offsets were determined on a $2\times2$ grid for each chip and 5 flat-field seasons were chosen (listed in Table~\ref{tab:flat-field-seasons}). @@ -1673,5 +1716,5 @@ Telescope Sciences Institute through their Mikulski Archive for Space Telescopes (MAST). The underying database at MAST is a copy of a -database generated at the Institute for Astronomy by the subsystem +database generated at the IfA by the subsystem called PSPS : the \note{define PSPS}. The construction of the PSPS version of the PS1 database starts once the PS1 photometry and @@ -1681,5 +1724,5 @@ The first stage of constructing the PSPS database consists of the -generation of small files called `batches' which contain a complete +generation of small files called ``batches'' which contain a complete set of measurements for a small chunk of the database tables. The program which is responsible for the construction of these batches is @@ -1690,5 +1733,5 @@ One type of batch consists of measurements from the individual exposures. These batches are generated based on the output catalog -files generated at the \ippstage{camera} stage (`smf files'). The +files generated at the \ippstage{camera} stage (``smf files''). The \ippprog{ipptopsps} program loads the complete set of measurements and metadata from the smf catalog file, then queries the DVO database for @@ -1757,9 +1800,9 @@ might be run and to regularly generate new commands based on that concept. The ``tasks'' are defined using the opihi scripting language -(also shared by DVO and other user-interative programs within the +(also shared by DVO and other user-interactive programs within the IPP). -Pantasks repeatedly checks each task in an attempt to generate a new -command: we say pantasks attempts to `execute' the task in each of +\ippprog{Pantasks} repeatedly checks each task in an attempt to generate a new +command: we say \ippprog{pantasks} attempts to ``execute'' the task in each of these attempts. Tasks may specify the time between execution attempts, with a 1 second default. @@ -1773,5 +1816,5 @@ opihi language) which is run each time the task is executed. The \code{task.exec} code may refer to variables or other data structures -defined by the opihi language within the pantasks environment. Within +defined by the opihi language within the \ippprog{pantasks} environment. Within a single \ippprog{pantasks} instance, all opihi variables and data structures have global context (\ie, all are visible to all tasks). @@ -1782,14 +1825,14 @@ Within the \ippprog{task.exec} macro, the command to be run must be -defined with the function `command'. Once the \ippprog{task.exec} -macro exits successfully, the defined command is the added to the list of jobs +defined with the function ``command''. Once the \ippprog{task.exec} +macro exits successfully, the defined command is then added to the list of jobs to be run within the UNIX environment. Jobs may be run in one of two ways: locally or via the parallel processing system. The task, or the -\ippprog{task.exec} macro, uses the `host' command to define how to -run the job. If the host is set to `local', then the job is run in -the background by pantasks itself (using the C \code{execvp} +\ippprog{task.exec} macro, uses the ``host'; command to define how to +run the job. If the host is set to ``local'', then the job is run in +the background by \ippprog{pantasks} itself (using the C \code{execvp} function). Otherwise, the job is sent to the parallel processing system to be run on another machine within the cluster. If the host -is set to the special value `anyhost', then the parallel processing +is set to the special value ``anyhost'', then the parallel processing system is allowed to choose the processing computer arbitrarily. Any other value is taken to be the DNS name of the computer on which this @@ -1798,10 +1841,10 @@ that the job only runs on the specifically named computer. Otherwise, the parallel processing system may choose to redirect the command to -another computer (based on whatever rules are defined for the parallel -processing system). +another computer using its own rules, e.g. to balance processing load +across the cluster. When the \ippprog{task.exec} macro is run, the code may choose (e.g., based on tests of some global variables) to exit the macro with an -error condition, e.g., with the `break' command. In this +error condition, e.g., with the ``break'' command. In this circumstance, no job is produced by the task. The task will be tried again the next time it is executed. This feature allows for the user @@ -1818,18 +1861,18 @@ online user guide?} -The option `npending' may be used to limit the number of jobs which +The option ``npending'' may be used to limit the number of jobs which are simultaneously executed for a specific task. For example, some classes of jobs should only be run one-at-a-time because they are not protected against collisions or they may overload a resource. The use -of `npending' allows these situations to be handled cleanly within -pantasks (avoiding cumbersome coding within with program or supporting +of ``npending'' allows these situations to be handled cleanly within +\ippprog{pantasks} (avoiding cumbersome coding within with program or supporting script). -The option `nmax' limits the total number of jobs which a task +The option ``nmax'' limits the total number of jobs which a task generates. This option may be useful in cases where \ippprog{pantasks} is used to perform a limited set of operations. \note{do we actually use this in IPP?} -The option `trange' allows the user to restrict the time period during +The option ``trange'' allows the user to restrict the time period during which the specific tasks is executed. This option is given with a start and an end time for the limiting time range. These times may be @@ -1846,10 +1889,10 @@ ranges may be specified \note{how are they evaluated?} -The option \code{nice} specifies the `nice' level at which the job is +The option \code{nice} specifies the ``nice'' level at which the job is run when it is executed. The parallel processing system must respect this concept. The option \code{active} can be used to turn on and off a task for -periods. Since a user command or a macro run by pantasks can +periods. Since a user command or a macro run by \ippprog{pantasks} can re-define task options, the \code{active} state may be changed independently of the task execute. This is useful for keeping tasks @@ -1857,5 +1900,5 @@ prevent them from running for some reason. -\subsubsection{pantasks passes jobs to pcontrol} +\subsubsection{pcontrol} Jobs which are generated by \ippprog{pantasks} may be run locally on @@ -1883,21 +1926,21 @@ Similarly, the hosts may also have one of several states: off, down, busy, idle, etc. A single host can accept a single job at a time. -Multiple hosts instances corresponding to the same machine may be +Multiple host instances corresponding to the same machine may be specified allowing a single computer to run more than one simultaneous job. -During operation, pcontrol accepts new jobs from pantasks and adds -them to the list of jobs to execute. It also accepts from pantasks +During operation, \ippprog{pcontrol} accepts new jobs from \ippprog{pantasks} and adds +them to the list of jobs to execute. It also accepts from \ippprog{pantasks} the names of computers on which it is allowed to run those jobs. -\subsubsection{pcontrol passes jobs to pclient} - -When pcontrol is provided with the name of a computer, it will attempt +\subsubsection{pclient} + +When \ippprog{pcontrol} is provided with the name of a computer, it will attempt to make an connection to that machine via ssh (or rsh?). When a connection is made, the remote shell is used to run a special interface program call \ippprog{pclient}. This program accepts -command lines from pcontrol and is responsible for executing the +command lines from \ippprog{pcontrol} and is responsible for executing the individual commands in the local shell environment. A single ssh -connection to a remote host keeps a single pclient shell running for a +connection to a remote host keeps a single \ippprog{pclient} shell running for a somewhat arbirarly long time, excuting many shell commands as needed. This architecture avoids wasting overhead making the ssh connection to @@ -1906,25 +1949,25 @@ architecture is allowed to be very light and short running if needed. -After pcontrol sends a job (commands) to a specific pclient, it checks +After \ippprog{pcontrol} sends a job (commands) to a specific \ippprog{pclient}, it checks back occasionally to see if the command has been run and executed. If -it has finished, then pcontrol will query for the exit status, the +it has finished, then \ippprog{pcontrol} will query for the exit status, the standard output and standard error streams from the command. (where -do these go, back to pantasks?), with the results associated with the -job statistics. At that point, the pclient on the remote machine is -ready to accept a new job from pcontrol. If any jobs are pending in -the list of jobs known to pcontrol, it will send those jobs to any +do these go, back to \ippprog{pantasks}?), with the results associated with the +job statistics. At that point, the \ippprog{pclient} on the remote machine is +ready to accept a new job from \ippprog{pcontrol}. If any jobs are pending in +the list of jobs known to \ippprog{pcontrol}, it will send those jobs to any machines which are idle. -While pcontrol interacts with the many remote machines, it -occasionally interacts with pantasks to report the results from the -jobs it has been monitoring. Pantasks occasionally requests a list of +While \ippprog{pcontrol} interacts with the many remote machines, it +occasionally interacts with \ippprog{pantasks} to report the results from the +jobs it has been monitoring. \ippprog{Pantasks} occasionally requests a list of the completed jobs. It then requests the status information for each completed job, including the standard error and standard output. As -pantasks receives this completion information, the jobs are removed -from the list managed by pcontrol. Thus pcontrol maintains at most a -modest list of jobs which are `in flight', leaving all interpretation -work to pantasks. - -At the pantasks level, the tasks define how pantasks should use the +\ippprog{pantasks} receives this completion information, the jobs are removed +from the list managed by \ippprog{pcontrol}. Thus \ippprog{pcontrol} maintains at most a +modest list of jobs which are ``in flight'' , leaving all interpretation +work to \ippprog{pantasks}. + +At the \ippprog{pantasks} level, the tasks define how \ippprog{pantasks} should use the exit status and output products from each job. For example, the stderr and stdout may be specified to go to a file (with static name @@ -1936,10 +1979,10 @@ started. This mode is useful for testing as all errors are reported back to the opihi shell. However, when the user exits the shell, the -pantasks instance exits, shutting down pcontrol and all remote client -connections. In standard operations, pantasks is run in a client +\ippprog{pantasks} instance exits, shutting down \ippprog{pcontrol} and all remote client +connections. In standard operations, \ippprog{pantasks} is run in a client server mode. The server runs continuously in the background and multiple users may connect via the \ippprog{pantasks_client} program. Users can the send commands to the server to load scripts, add -parallel hosts, check status, and start or stop the pantasks +parallel hosts, check status, and start or stop the \ippprog{pantasks} operations. @@ -1956,9 +1999,9 @@ end \end{verbatim} - \caption{\label{fig:task_example} Example of a simple static - task in the opihi-based scripting language used by pantasks. In - this example, pantasks would run a single instance of the command - ({\tt ls /tmp}) every 5 seconds, sending the stdout and stderr to - the listed files. } +\caption{\label{fig:task_example} Example of a simple static + task in the opihi-based scripting language used by ippprog{pantasks}. In + this example, ippprog{pantasks} would run a single instance of the command + ({\tt ls /tmp}) every 5 seconds, sending the stdout and stderr to + the listed files. } \end{center} \end{figure} @@ -1968,8 +2011,8 @@ \subsubsection{Pantasks scripts: ippTasks} -Pantasks provides an environment in which commands can be generated +\ippprog{Pantasks} provides an environment in which commands can be generated and extensive parallel processing managed. The details of how to implement the different stages of IPP processing are captured in a -collection of scripts written for pantasks in the \code{opihi} +collection of scripts written for \ippprog{pantasks} in the \code{opihi} language. In general, each stage is defined by an associated script collected together under the \ippmisc{ippTasks} collection. While @@ -2001,7 +2044,7 @@ row in the result set, each column in the row is stored as a separate line on the \ippmisc{page}, identified by the database column name. An -additional line, the \ippdbcolumn{pantasksState}, is added so pantasks +additional line, the \ippdbcolumn{pantasksState}, is added so \ippprog{pantasks} can manage the processing of the job which will be generated by this -page. When the page is first generate, the +page. When the page is first generated, the \ippdbcolumn{pantasksState} is set to \ippmisc{INIT}, indicating that this \ippmisc{page} is a new addition to the \ippmisc{book}. Once all @@ -2018,7 +2061,7 @@ construct the appropriate command-line (e.g., lines in the page may include input file names and output file names for the specific item -in the database). The resulting command becomes a job in the pantasks +in the database). The resulting command becomes a job in the \ippprog{pantasks} collection of jobs. Most IPP analysis stages specify that the jobs -are then sent to pcontrol for parallel process. Before task generates +are then sent to \ippprog{pcontrol} for parallel process. Before task generates the job, the \ippdbcolumn{pantasksState} is set to \ippmisc{RUN} so a future execution of the task will not attempt to re-run this specific job. @@ -2029,9 +2072,9 @@ this responsibility is left to the program which ran the analysis. IPP analysis steps normally consist of two main elements: a C-language -program to do the data analysis work and a supporting perl script +program to do the data analysis work and a supporting Perl script which performs the database update upon completion. Upon completion, -the pantasks \ippmisc{RUN} tasks is responsible for updating the +the \ippprog{pantasks} \ippmisc{RUN} tasks is responsible for updating the status within the book, but not within the processing database. This -split keeps the interactions at the pantasks level relatively light, +split keeps the interactions at the \ippprog{pantasks} level relatively light, leaving the overhead of the database interaction within the job running on one of the computing machines in the cluster. @@ -2042,6 +2085,6 @@ clear jobs which have failed with one of the ephemeral failure modes (see the discussion in Section~\ref{sec:processing.database}). This -step allows these failures to be cleared from the system, and schedule -those jobs again for a retry. +step allows these failures to be cleared from the system, allowing +those jobs to be scheduled again. Similarly, some stages have \ippmisc{advance} tasks that update the @@ -2066,8 +2109,8 @@ discussed above, the query to the processing database for new items is restricted to a set of user-defined labels. A given instance of -pantasks will be supplied a set of labels which are then applied to -all tasks managed by that pantasks. For example, the pantasks which +\ippprog{pantasks} will be supplied a set of labels which are then applied to +all tasks managed by that \ippprog{pantasks}. For example, the \ippprog{pantasks} which manages the nightly processing of the basic science analysis stages -(chip - warp, stack, diff) is supplied with several labels which +(\ippstage{chip} - \ippstage{warp}, \ippstage{stack}, \ippstage{diff}) is supplied with several labels which correspond to the different kinds of observations being performed. In this way, the analysis of the nightly observations is kept separate @@ -2083,5 +2126,5 @@ \note{then discuss the addstar sequences with manual triggering} -Outside of the basic sequence of chip to warp, there is no single +Outside of the basic sequence of \ippstage{chip} to \ippstage{warp}, there is no single natural next step. For example: a stack can be generated with any number of input warps; a difference image can be generated between a @@ -2103,5 +2146,5 @@ significantly reduced from the arbitrary case. -{\em Queuing the diffs} is done by first examining the set of all +Queuing the diffs is done by first examining the set of all exposures that have been taken at the summit on the current night of observing, and querying information from each stage up through @@ -2111,5 +2154,5 @@ group are then sorted by increasing observation date (\ippdbcolumn{dateobs}). The database results for each stage -(chip-warp) are checked to ensure that the selected exposures have +(\ippstage{chip}-\ippstage{warp}) are checked to ensure that the selected exposures have been successfully processed for all stages through \ippstage{warp}. Exposure groups are ignored until all exposures have either been @@ -2129,5 +2172,5 @@ that were excluded due to an odd number of exposures to be paired with the exposure closest in time (with the exposure that was previously -first ignored). Exposure pairs in which at least one exposures does +first ignored). Exposure pairs in which at least one exposure does not have a pre-existing difference image are queued for difference image analysis. @@ -2138,9 +2181,9 @@ exposures, as this is the number of exposures taken for each field. Once this number was reached, no more exposures are expected, so -\ippstage{stack} database entries can be queued with the +\ippstage{stack} database entries can be queued from the \ippstage{warp} entries. Again, failures and weather can reduce the number of usable exposures. If no stack could be made for a given MD field with the minimum number of inputs by the time of the -end-of-night darks, stacks are generated using using whatever +end-of-night darks, stacks are generated using whatever exposures are available. @@ -2161,21 +2204,23 @@ \ippdbtable{lapRun} entries can be queued that define a \ippdbcolumn{filter} and a \ippdbcolumn{projection_cell} to be -considered. A \ippdbcolumn{projection_cell} is a unit of sky defined -to be a square four degrees on each side which has a single tangent -plane projection \citep[][see]{waters2017}. \note{does waters2017 - discuss RINGS.V3? if not, where?} Once this entry is defined, is is -populated with exposures (stored in the \ippdbtable{lapExp} table in -the database), with any exposure located within 5 degrees of the -center of the projection cell included. This radius ensures that any -exposure that overlaps the projection cell will be included. Once the -exposures have been added, the other exposures within the same -sequence are checked to see if a \ippstage{chip} stage entry has been -generated, and if so, the \ippdbcolumn{chip_id} for that entry is -saved into the \ippdbtable{lapExp} as well. This linkage ensures that -each exposure is only processed once. If no entry is found, a new -\ippstage{chip} entry is queued for processing. The task periodically -checks the status of the exposures in each \ippdbtable{lapRun} entry, -and if they have all completed the \ippstage{warp} stage, then a -\ippstage{stack} is queued for each skycell contained within the +considered. These projection cells match the tangent plane centers +used for the warp tessellation. A \ippdbcolumn{projection_cell} is a +unit of sky defined to be a square four degrees on each side which has +a single tangent plane projection \citep[][see]{waters2017}. +\note{does waters2017 discuss RINGS.V3? if not, where?} Once this +entry is defined, it is populated with all exposures (stored in the +\ippdbtable{lapExp} table in the database) that are located +within 5 degrees of the center of the projection cell included. This +radius ensures that any exposure that overlaps the projection cell +will be included. Once the exposures have been added, the other +exposures within the same sequence are checked to see if a +\ippstage{chip} stage entry has been generated, and if so, the +\ippdbcolumn{chip_id} for that entry is saved into the +\ippdbtable{lapExp} as well. This linkage ensures that each exposure +is only processed once. If no entry is found, a new \ippstage{chip} +entry is queued for processing. The task periodically checks the +status of the exposures in each \ippdbtable{lapRun} entry, and if they +have all completed the \ippstage{warp} stage, then a \ippstage{stack} +is queued for each skycell contained within the \ippdbcolumn{projection_cell}. @@ -2192,6 +2237,6 @@ system per-se, but only method of tracking the locations of files within the file system, and of tracking duplicate copies of the same -file. The core of \ippprog{Nebulous} is a dedicated database engine -which tracks ``storage objects'', the concept of a file exists in the +file. The core of \ippprog{Nebulous} is a mysql database which tracks +``storage objects'', the equivalent concept of a file within the system. Each storage object may be associated with a number of copies of the actual files on the disks in the storage system (called @@ -2213,6 +2258,6 @@ stored on a specific computer (for at least one of the instances). All of the analysis stages which interact with that chip could then be -preferentially targetted to be run on that computer. The localization -in \ippprog{Nebulous} and the host targetted processing in pantasks +preferentially targeted to be run on that computer. The localization +in \ippprog{Nebulous} and the host targeted processing in \ippprog{pantasks} can therefore work together to encourage processing to require only local disk access, reducing the I/O local on the network @@ -2221,6 +2266,6 @@ practice, the as-built IPP has had sufficient network bandwidth that this targetting was not required. In practice, due to the timing of -hardware aquisition, occasional hardware failures, and other -organizational details, targetted processing has only been used to a +hardware acquisition, occasional hardware failures, and other +organizational details, targeted processing has only been used to a moderate degree within the Pan-STARRS cluster. \note{can we get a number here?} @@ -2229,14 +2274,26 @@ The user interfaces to Nebulous consist of command-line programs as -well as APIs in both C and Perl. The basic user commands to interact -with Nebulous are to 1) create a new storage object and associated -instance; 2) add a new instance to an existing storage object; 3) -remove (cull) an instance; 4) delete a storage object; and 5) find a -file associated with a given storage objects. Note that these user -commands do not affect the files on disk \note{true for cull?} -(exception: the create function will create an empty file if one does -not exist). They only change the state of the Nebulous database; it -is the responsibility of the user program to read and write data to a -file and to create the copies, etc. +well as APIs in both C and Perl. + +"The basic user commands to interact with Nebulous are to 1) query the +database for an existing storage object, and find a valid file +instance associated with that object; 2) create a new storage object, +which instantiates an empty file that can be opened for writing; 3) +replicate an existing storage object to create more file instances; 4) +cull a single file instance of storage object from the cluster; and 5) +remove a storage object, and ensure that all file instances are +removed. The filehandles returned for newly created instances can +then be opened for reading and writing data to that instance. + +% The basic user commands to interact +% with Nebulous are to 1) create a new storage object and associated +% instance; 2) add a new instance to an existing storage object; 3) +% remove (cull) an instance; 4) delete a storage object; and 5) find a +% file associated with a given storage objects. Note that these user +% commands do not affect the files on disk \note{true for cull?} +% (exception: the create function will create an empty file if one does +% not exist). They only change the state of the Nebulous database; it +% is the responsibility of the user program to read and write data to a +% file and to create the copies, etc. For the Nebulous users, the identifier of a storage object is a unique @@ -2247,8 +2304,8 @@ computer (HOST) and disk (VOL). The path and filename portions become the identifier and are recorded in the \ippmisc{storage_object} table -in the \ippmisc{extern_id} field. A storage object entry is then -created in the database for this id, and an instance of the file -created on the specified node (or at random from available nodes if -left empty). +in the \ippmisc{ext_id} field. A storage object entry is then created +in the database for this id, and an instance of the file created on +the specified node. If the host is unspecified, or if the specified +volume is full, then a host is chosen at random from available nodes. Files are stored on specific computers in a \ippprog{Nebulous} @@ -2258,9 +2315,9 @@ \code{nebulous}. Beneath the top-level directory are 256 subdirectories with names of the form 00 - ff (i.e., 2 digit -hexadecimate number). Each subdirectory again as 256 subdirectories -with the same naming scheme. +hexadecimal number). Each subdirectory has 256 subdirectories with +the same naming scheme. The filename of an instance in Nebulous is deterministic and derived -from the \ippmisc{extern_id}: the \ippmisc{extern_id} is hashed using +from the \ippmisc{ext_id}: the \ippmisc{ext_id} is hashed using the SHA-1 function, and the first four hexadecimal digits of this hash are separated into two two-digit strings and used as the top and @@ -2269,5 +2326,5 @@ provide a unique SQL ID for each instance. Under the subdirectory identified above, the disk file name is by appending the database -instance id with a string derived from the \code{extern_id}: forward +instance id with a string derived from the \code{ext_id}: forward slash characters are replaced in the name with colons so the string can represent a file in the UNIX filesystem. For the example URI @@ -2333,6 +2390,16 @@ using only the low-latency SOAP communications. -\note{need a paragraph or two on stats: how many objects, how many - instances?} +The Nebulous database currently (2017 July) contains information about +5,560,533,654 file instances for 3,543,240,981 storage objects. All +raw data, along with permanent products such as catalogs and the +current versions of full-sky stacks, are replicated to ensure at least +two copies exist in case of hardware failure. Based on the most +recent database ID values (which are unique and never reused), this +corresponds to roughly half of all the storage objects and file +instances ever created, due to the transient nature of many pipeline +products. + +% those numbers are so_id 6758205602 ins_id 9971666505, with ratios +% 0.5242, 0.5576) \subsection{Datastore repositories} @@ -2343,5 +2410,5 @@ that exposes data in a common form. \note{add Isani / Hoblitt reference?} One of the main datastores used by the IPP is the one -located at the summit. This datastore exposes, a list of the +located at the summit. This datastore exposes a list of the exposures obtained since the start of the PS1 operations. Requests to this server may restrict to the latest by time. Each row in the @@ -2353,5 +2420,5 @@ associated with that exposure. This listing includes a link to the individual chip FITS files as well as an md5 checksum. Systems which -are allowed access may download chip FITS files via http requests to +are allowed access may download the raw chip FITS files via http requests to the provided links. @@ -2509,10 +2576,10 @@ These storage nodes are not fully capable of completing all processing on the short timescale necessary for each night's worth of data. To -increase the processing capability, we have a large number -\note{actual number?} of ``compute'' nodes, that have small amounts of -local storage, but are able to add processing power. In addition to -the direct processing of image data, these nodes are also used to -manage the \ippprog{Nebulous} file interface, as well as controlling -the job scheduling for the processing. +increase the processing capability, we have 212 ``compute'' nodes that +have small amounts of local storage, but are able to provide +additional processing power. In addition to the direct processing of +image data, these nodes are also used to manage the \ippprog{Nebulous} +file interface, as well as controlling the job scheduling for the +processing. The final type of computer in the cluster are the database servers. @@ -2631,18 +2698,18 @@ products are present. -Approximately half of the chip through warp processing for the PV3 -reduction was performed on Mustang, with 201,040 / 375,573 of the -\ippstage{camera} stage products reduced there. Only processing -through the \ippstage{stack} stage was attempted, although with a -smaller fraction of the total compared to the \ippstage{camera} stage, -with 290,257 / 998,886 being produced at Los Alamos. One reason for -this decrease is that due to the memory constraints on the Mustang -processing nodes, we were unable to run stacks with more than 25 -inputs there. Stacks with this larger number of inputs overflow the -memory of the processing node, and as they do not have disk space -available for use as virtual memory, cause the machine to hang until -the job time limit is reached. These stacks were instead processed on -the regular IPP cluster, where hosts with sufficent memory were -available. +Approximately half of the \ippstage{chip} through \ippstage{warp} +processing for the PV3 reduction was performed on Mustang, with +201,040 / 375,573 of the \ippstage{camera} stage products reduced +there. Only processing through the \ippstage{stack} stage was +attempted, although with a smaller fraction of the total compared to +the \ippstage{camera} stage, with 290,257 / 998,886 being produced at +Los Alamos. One reason for this decrease is that due to the memory +constraints on the Mustang processing nodes, we were unable to run +stacks with more than 25 inputs there. Stacks with larger numbers of +inputs overflow the memory of the processing node, and as they do not +have disk space available for use as virtual memory, cause the machine +to hang until the job time limit is reached. These stacks were +instead processed on the regular IPP cluster, where hosts with +sufficent memory were available. \subsection{UH Cray Cluster} Index: /branches/czw_branch/20170908/doc/release.2015/ps1.detrend/detrend.bbl =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/ps1.detrend/detrend.bbl (revision 40482) +++ /branches/czw_branch/20170908/doc/release.2015/ps1.detrend/detrend.bbl (revision 40483) @@ -1,3 +1,3 @@ -\begin{thebibliography}{10} +\begin{thebibliography}{15} \expandafter\ifx\csname natexlab\endcsname\relax\def\natexlab#1{#1}\fi @@ -47,4 +47,41 @@ {Huber}, M., {TBD}, A., {TBD}, B., \& et~al. 2017, ArXiv e-prints +\bibitem[{Lanczos(1956)}]{lanczos1956applied} +Lanczos, C. 1956, Applied analysis, Prentice-Hall mathematics series + (Prentice-Hall) + +\bibitem[{{Lupton} {et~al.}(1999){Lupton}, {Gunn}, \& + {Szalay}}]{1999AJ....118.1406L} +{Lupton}, R.~H., {Gunn}, J.~E., \& {Szalay}, A.~S. 1999, \aj, 118, 1406 + +\bibitem[{{Magnier} \& {Cuillandre}(2004)}]{2004PASP..116..449M} +{Magnier}, E.~A. \& {Cuillandre}, J.-C. 2004, \pasp, 116, 449 + +\bibitem[{{Magnier} {et~al.}(2017){Magnier}, {Schlafly}, {Finkbeiner}, \& + et~al.}]{magnier2017.datasystem} +{Magnier}, E.~A., {Schlafly}, E.~F., {Finkbeiner}, D.~P., \& et~al. 2017, ArXiv + e-prints + +\bibitem[{{Magnier} {et~al.}(2016{\natexlab{a}}){Magnier}, {Schlafly}, + {Finkbeiner}, {Tonry}, {Goldman}, {R{\"o}ser}, {Schilbach}, {Chambers}, + {Flewelling}, {Huber}, {Price}, {Sweeney}, {Waters}, {Denneau}, {Draper}, + {Hodapp}, {Jedicke}, {Kudritzki}, {Metcalfe}, {Stubbs}, \& + {Wainscoast}}]{magnier2017.calibration} +{Magnier}, E.~A., {Schlafly}, E.~F., {Finkbeiner}, D.~P., {Tonry}, J.~L., + {Goldman}, B., {R{\"o}ser}, S., {Schilbach}, E., {Chambers}, K.~C., + {Flewelling}, H.~A., {Huber}, M.~E., {Price}, P.~A., {Sweeney}, W.~E., + {Waters}, C.~Z., {Denneau}, L., {Draper}, P., {Hodapp}, K.~W., {Jedicke}, R., + {Kudritzki}, R.-P., {Metcalfe}, N., {Stubbs}, C.~W., \& {Wainscoast}, R.~J. + 2016{\natexlab{a}}, ArXiv e-prints + +\bibitem[{{Magnier} {et~al.}(2016{\natexlab{b}}){Magnier}, {Sweeney}, + {Chambers}, {Flewelling}, {Huber}, {Price}, {Waters}, {Denneau}, {Draper}, + {Jedicke}, {Hodapp}, {Kudritzki}, {Metcalfe}, {Stubbs}, \& + {Wainscoast}}]{magnier2017.analysis} +{Magnier}, E.~A., {Sweeney}, W.~E., {Chambers}, K.~C., {Flewelling}, H.~A., + {Huber}, M.~E., {Price}, P.~A., {Waters}, C.~Z., {Denneau}, L., {Draper}, P., + {Jedicke}, R., {Hodapp}, K.~W., {Kudritzki}, R.-P., {Metcalfe}, N., {Stubbs}, + C.~W., \& {Wainscoast}, R.~J. 2016{\natexlab{b}}, ArXiv e-prints + \bibitem[{{Price} {et~al.}(2017){Price}, {TBD}, {TBD}, \& et~al.}]{price2017} {Price}, P.~A., {TBD}, A., {TBD}, B., \& et~al. 2017, ArXiv e-prints @@ -73,58 +110,3 @@ IAU General Assembly, 22, 2251124 -\bibitem[{{York} {et~al.}(2000){York}, {Adelman}, {Anderson}, {Anderson}, - {Annis}, {Bahcall}, {Bakken}, {Barkhouser}, {Bastian}, {Berman}, {Boroski}, - {Bracker}, {Briegel}, {Briggs}, {Brinkmann}, {Brunner}, {Burles}, {Carey}, - {Carr}, {Castander}, {Chen}, {Colestock}, {Connolly}, {Crocker}, {Csabai}, - {Czarapata}, {Davis}, {Doi}, {Dombeck}, {Eisenstein}, {Ellman}, {Elms}, - {Evans}, {Fan}, {Federwitz}, {Fiscelli}, {Friedman}, {Frieman}, {Fukugita}, - {Gillespie}, {Gunn}, {Gurbani}, {de Haas}, {Haldeman}, {Harris}, {Hayes}, - {Heckman}, {Hennessy}, {Hindsley}, {Holm}, {Holmgren}, {Huang}, {Hull}, - {Husby}, {Ichikawa}, {Ichikawa}, {Ivezi{\'c}}, {Kent}, {Kim}, {Kinney}, - {Klaene}, {Kleinman}, {Kleinman}, {Knapp}, {Korienek}, {Kron}, {Kunszt}, - {Lamb}, {Lee}, {Leger}, {Limmongkol}, {Lindenmeyer}, {Long}, {Loomis}, - {Loveday}, {Lucinio}, {Lupton}, {MacKinnon}, {Mannery}, {Mantsch}, {Margon}, - {McGehee}, {McKay}, {Meiksin}, {Merelli}, {Monet}, {Munn}, {Narayanan}, - {Nash}, {Neilsen}, {Neswold}, {Newberg}, {Nichol}, {Nicinski}, {Nonino}, - {Okada}, {Okamura}, {Ostriker}, {Owen}, {Pauls}, {Peoples}, {Peterson}, - {Petravick}, {Pier}, {Pope}, {Pordes}, {Prosapio}, {Rechenmacher}, {Quinn}, - {Richards}, {Richmond}, {Rivetta}, {Rockosi}, {Ruthmansdorfer}, {Sandford}, - {Schlegel}, {Schneider}, {Sekiguchi}, {Sergey}, {Shimasaku}, {Siegmund}, - {Smee}, {Smith}, {Snedden}, {Stone}, {Stoughton}, {Strauss}, {Stubbs}, - {SubbaRao}, {Szalay}, {Szapudi}, {Szokoly}, {Thakar}, {Tremonti}, {Tucker}, - {Uomoto}, {Vanden Berk}, {Vogeley}, {Waddell}, {Wang}, {Watanabe}, - {Weinberg}, {Yanny}, {Yasuda}, \& {SDSS Collaboration}}]{2000AJ....120.1579Y} -{York}, D.~G., {Adelman}, J., {Anderson}, Jr., J.~E., {Anderson}, S.~F., - {Annis}, J., {Bahcall}, N.~A., {Bakken}, J.~A., {Barkhouser}, R., {Bastian}, - S., {Berman}, E., {Boroski}, W.~N., {Bracker}, S., {Briegel}, C., {Briggs}, - J.~W., {Brinkmann}, J., {Brunner}, R., {Burles}, S., {Carey}, L., {Carr}, - M.~A., {Castander}, F.~J., {Chen}, B., {Colestock}, P.~L., {Connolly}, A.~J., - {Crocker}, J.~H., {Csabai}, I., {Czarapata}, P.~C., {Davis}, J.~E., {Doi}, - M., {Dombeck}, T., {Eisenstein}, D., {Ellman}, N., {Elms}, B.~R., {Evans}, - M.~L., {Fan}, X., {Federwitz}, G.~R., {Fiscelli}, L., {Friedman}, S., - {Frieman}, J.~A., {Fukugita}, M., {Gillespie}, B., {Gunn}, J.~E., {Gurbani}, - V.~K., {de Haas}, E., {Haldeman}, M., {Harris}, F.~H., {Hayes}, J., - {Heckman}, T.~M., {Hennessy}, G.~S., {Hindsley}, R.~B., {Holm}, S., - {Holmgren}, D.~J., {Huang}, C.-h., {Hull}, C., {Husby}, D., {Ichikawa}, - S.-I., {Ichikawa}, T., {Ivezi{\'c}}, {\v Z}., {Kent}, S., {Kim}, R.~S.~J., - {Kinney}, E., {Klaene}, M., {Kleinman}, A.~N., {Kleinman}, S., {Knapp}, - G.~R., {Korienek}, J., {Kron}, R.~G., {Kunszt}, P.~Z., {Lamb}, D.~Q., {Lee}, - B., {Leger}, R.~F., {Limmongkol}, S., {Lindenmeyer}, C., {Long}, D.~C., - {Loomis}, C., {Loveday}, J., {Lucinio}, R., {Lupton}, R.~H., {MacKinnon}, B., - {Mannery}, E.~J., {Mantsch}, P.~M., {Margon}, B., {McGehee}, P., {McKay}, - T.~A., {Meiksin}, A., {Merelli}, A., {Monet}, D.~G., {Munn}, J.~A., - {Narayanan}, V.~K., {Nash}, T., {Neilsen}, E., {Neswold}, R., {Newberg}, - H.~J., {Nichol}, R.~C., {Nicinski}, T., {Nonino}, M., {Okada}, N., {Okamura}, - S., {Ostriker}, J.~P., {Owen}, R., {Pauls}, A.~G., {Peoples}, J., {Peterson}, - R.~L., {Petravick}, D., {Pier}, J.~R., {Pope}, A., {Pordes}, R., {Prosapio}, - A., {Rechenmacher}, R., {Quinn}, T.~R., {Richards}, G.~T., {Richmond}, M.~W., - {Rivetta}, C.~H., {Rockosi}, C.~M., {Ruthmansdorfer}, K., {Sandford}, D., - {Schlegel}, D.~J., {Schneider}, D.~P., {Sekiguchi}, M., {Sergey}, G., - {Shimasaku}, K., {Siegmund}, W.~A., {Smee}, S., {Smith}, J.~A., {Snedden}, - S., {Stone}, R., {Stoughton}, C., {Strauss}, M.~A., {Stubbs}, C., {SubbaRao}, - M., {Szalay}, A.~S., {Szapudi}, I., {Szokoly}, G.~P., {Thakar}, A.~R., - {Tremonti}, C., {Tucker}, D.~L., {Uomoto}, A., {Vanden Berk}, D., {Vogeley}, - M.~S., {Waddell}, P., {Wang}, S.-i., {Watanabe}, M., {Weinberg}, D.~H., - {Yanny}, B., {Yasuda}, N., \& {SDSS Collaboration}. 2000, \aj, 120, 1579 - \end{thebibliography} Index: /branches/czw_branch/20170908/doc/release.2015/ps1.detrend/detrend.tex =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/ps1.detrend/detrend.tex (revision 40482) +++ /branches/czw_branch/20170908/doc/release.2015/ps1.detrend/detrend.tex (revision 40483) @@ -109,6 +109,6 @@ \begin{abstract} -The Pan-STARRS1 Science Consortium have carried out a set of imaging -surveys using the 1.4 giga-pixel GPC1 camera on the PS1 telescope. As +The Pan-STARRS1 Science Consortium has carried out a set of imaging +surveys using the 1.4 gigapixel GPC1 camera on the PS1 telescope. As this camera is composed of many individual electronic readouts, and covers a very large field of view, great care was taken to ensure that @@ -133,6 +133,6 @@ the data reduction techniques and the resulting data products. This paper (Paper III) describes the details of the pixel processing algorithms, including -detrending, warping, and adding (to create stacked images) and subtracting -(to create difference images) and resulting image products and their +detrending, warping, adding (to create stacked images), and subtracting +(to create difference images), along with the resulting image products and their properties. \citet[][Paper I]{chambers2017} provides an overview of the Pan-STARRS System, the @@ -142,5 +142,5 @@ %Magnier et al. 2017 (Paper II) %Pan-STARRS Data Processing Stages -\citet[][Paper II]{magnier2017a} +\citet[][Paper II]{magnier2017.datasystem} describes how the various data processing stages are organized and implemented @@ -153,5 +153,5 @@ %Magnier et al. 2017 (Paper IV) %Pan-STARRS Pixel Analysis : Source Detection -\citet[][Paper IV]{magnier2017b} +\citet[][Paper IV]{magnier2017.analysis} describes the details of the source detection and photometry, including point-spread-function and extended source fitting models, and the @@ -159,5 +159,5 @@ %Magnier et al. 2017 (Paper V) %Pan-STARRS Photometric and Astrometric Calibration -\citet[][Paper V]{magnier2017c} +\citet[][Paper V]{magnier2017.calibration} describes the final calibration process, and the resulting photometric and astrometric quality. @@ -178,33 +178,34 @@ -The Pan-STARRS 1 Science Survey uses the 1.4 giga-pixel GPC1 camera +The Pan-STARRS 1 Science Survey uses the 1.4 gigapixel GPC1 camera with the PS1 telescope on Haleakala Maui to image the sky north of $-30^\circ$ declination. The GPC1 camera is composed of 60 orthogonal transfer array (OTA) devices, each of which is an $8\times{}8$ grid of -readout cells. This parallelizes the readout process, reducing the -overhead in each exposure. However, as a consequence of this large -number of individual detector readouts, many calibrations are needed -to ensure the response is consistent across the entire field of view. - -The Processing Version 3 (PV3) reduction represents the third full -reduction of the Pan-STARRS archival data. The first two reductions -were used internally for pipeline optimization and the development of -the initial photometric and astrometric reference catalog \citep{magnier2017c}. The -products from these reductions were not publicly released, but have -been used to produce a wide range of scientific papers from the +readout cells. The large number of cells parallelizes the readout +process, reducing the overhead in each exposure. However, as a +consequence, many calibration operations are needed to ensure the +response is consistent across the entire seven square degree field of +view. + +%The Processing Version 3 (PV3) reduction represents the third full +DR1 contains the results of the third full reduction of the Pan-STARRS +archival data. The first two reductions were used internally for +pipeline optimization and the development of the initial photometric +and astrometric reference catalog \citep{magnier2017.calibration}. +The products from these reductions were not publicly released, but +have been used to produce a wide range of scientific papers from the Pan-STARRS 1 Science Consortium members. The Pan-STARRS image processing pipeline (IPP) is described elsewhere -\citep{magnier2017a}, but a short summary follows. The -archive of raw exposures is stored on disk, with a database storing -the metadata of exposure parameters. For the PV3 processing, large -contiguous regions were defined, and the images for all exposures -within that region launched for the \IPPstage{chip} stage processing. -This stage performs the image detrending (described below in section +\citep{magnier2017.datasystem}, but a short summary follows. The raw +image data is stored on the processing cluster, with a database +containing the metadata of exposure parameters. These raw images can +be launched for the initial \IPPstage{chip} stage processing. This +stage performs the image detrending (described below in section \ref{sec:detrending}), as well as the single epoch photometry -\citep{magnier2017b}, in parallel on the individual OTA device data. -Following the \IPPstage{chip} stage is the \IPPstage{camera} stage, in -which the astrometry and photometry for the entire exposure is -calibrated by matching the detections against the reference catalog. +\citep{magnier2017.analysis}, in parallel on the individual OTA device +data. Following the \IPPstage{chip} stage is the \IPPstage{camera} +stage, in which the astrometry and photometry for the entire exposure +is calibrated by matching the detections against a reference catalog. This stage also performs masking updates based on the now-known positions and brightnesses of stars that create dynamic features (see @@ -213,62 +214,62 @@ \IPPstage{chip} stage images onto common sky oriented images that have fixed sky projections (Section \ref{sec:warping}). When all -\IPPstage{warp} stage processing is done for the region of the sky, +\IPPstage{warp} stage processing is done for a region of the sky, \IPPstage{stack} processing is performed (Section \ref{sec:stacking}) to construct deeper, fully populated images from the set of -\IPPstage{warp} images that cover that region of the sky. Beyond the -\IPPstage{stack} stage, a series of additional stages are done that -are more fully described in other papers. Transient features are -identified in the \IPPstage{diff} stage, which takes input -\IPPstage{warp} and/or \IPPstage{stack} data and performs image +\IPPstage{warp} images that cover that region of the sky. Transient +features are identified in the \IPPstage{diff} stage, which takes +input \IPPstage{warp} and/or \IPPstage{stack} data and performs image differencing (Section \ref{sec:diffs}). Further photometry is performed in the \IPPstage{staticsky} and \IPPstage{skycal} stages, which add extended source fitting to the point source photometry of -objects detected in the \IPPstage{stack} images, and calibrate the -results against the reference catalog. The \IPPstage{fullforce} stage -takes the catalog output of the \IPPstage{skycal} stage, and uses the -objects detected in that to perform forced photometry on the +objects detected in the \IPPstage{stack} images, and again calibrate +the results against a reference catalog. The \IPPstage{fullforce} +stage takes the catalog output of the \IPPstage{skycal} stage, and +uses the objects detected in that to perform forced photometry on the individual \IPPstage{warp} stage images. The details of these stages -are provided in \citet{magnier2017b}. - -The same reduction procedure described above is also performed in real -time on new exposures as they are observed by the telescope. This -process is largely automatic, with new exposures being downloaded from -the summit to the main IPP processing cluster at the Maui Research and -Technology Center in Kihei, and registered into the processing -database. This triggers a new \IPPstage{chip} stage reduction for -science exposures, advancing processing upon completion through to the -\IPPstage{diff} stage. This allows the ongoing solar system moving -object search to identify candidates for follow up observations within -24 hours of the initial set of observations \citep{2015IAUGA..2251124W}. +are provided in \citet{magnier2017.analysis}. + +A limited version of same reduction procedure described above is also +performed in real time on new exposures as they are observed by the +telescope. This process is automatic, with new exposures being +downloaded from the summit to the main IPP processing cluster at the +Maui Research and Technology Center in Kihei, and registered into the +processing database. New \IPPstage{chip} stage reductions are +launched for science exposures, advancing processing upon completion +through to the \IPPstage{diff} stage, skipping the additional stack +and forced warp photometry stages. This automatic processing allows +the ongoing solar system moving object search to identify candidates +for follow up observations within 24 hours of the initial set of +observations \citep{2015IAUGA..2251124W}. Section \ref{sec:detrending} provides an overview of the detrending process that corrects the instrumental signatures of GPC1, with -details of the construction of those detrends in Section -\ref{sec:detrend construction}. An analysis of the algorithms used to -complete the \IPPstage{warp} (section \ref{sec:warping}), +details of the construction of the reference detrend templates in +Section \ref{sec:detrend construction}. An analysis of the algorithms +used to perform the \IPPstage{warp} (section \ref{sec:warping}), \IPPstage{stack} (section \ref{sec:stacking}), and \IPPstage{diff} -(section \ref{sec:diffs}) stage transformations of the image data to -from the detector frame to a common sky frame, and the co-adding of -those common sky frame images continues after the list of detrend -steps. Finally, a discussion of the remaining issues and possible -future improvements is presented in section \ref{sec:discussion}. - -Image products presented in figures have been -mosaicked to arrange pixels as follows. Single cell images are -arranged such that pixel $(1,1)$ is at the lower left corner. Images -mosaicked to the OTA level have cell xy00 in the lower left corner, -with cells xy10, xy20, etc. sequentially to the right, and cells xy01, -xy02, etc. sequentially to the top of this cell. Again, pixel $(1,1)$ -of cell xy00 is located in the lower left corner of the image. For -mosaicks of the full field of view, the OTAs are arranged as they see -the sky. The lower left corner is the empty location where OTA70 -would exist. Toward the right, the OTA labels decrease in $X$ label, -with the empty OTA00 located in the lower right. The OTA $Y$ labels -increase upward in the mosaic. The OTAs to the left of the midplane -(OTA4Y-OTA7Y) are oriented with cell xy00 and pixel $(1,1)$ to the -lower left of their position. Due to the electronic connections of +(section \ref{sec:diffs}) stage transformations of the image data +follows after the list of detrend steps. Finally, a discussion of the +remaining issues and possible future improvements is presented in +section \ref{sec:discussion}. + +Image products presented in figures have been mosaicked to arrange +pixels as follows. Single cell images are arranged such that pixel +$(1,1)$ is at the lower right corner (for example Figure +\ref{fig:burntool images}). This corrects the parity difference +between the raw data and the sky. Images mosaicked to show a full OTA +detector are arranged as they are on the focal plane (as in Figure +\ref{fig:dark image}. The OTAs to the left of the midplane +(OTA4Y-OTA7Y) are oriented with cell xy00 and pixel $(590,1)$ to the +lower right of their position. Due to the electronic connections of the OTAs in the focal plane, the OTAs to the right of the midplane (OTA0Y-OTA3Y) are rotated 180 degrees, and are oriented with cell xy00 -and pixel $(1,1)$ to the top right of their position. +and pixel $(590,1)$ to the top left of their position. For mosaics of +the full field of view, the OTAs are arranged as they see the sky, +with the cells arranged as in the single OTA images (Figure \ref{fig:optical ghosts}). The lower left +corner is the empty location where OTA70 would exist. Toward the +right, the OTA labels decrease in $X$ label, with the empty OTA00 +located in the lower right. The OTA $Y$ labels increase upward in the +mosaic. \textit{Note: These papers are being placed on the arXiv.org to @@ -290,155 +291,45 @@ level, dark frame subtraction to remove temperature and exposure time dependent detector glows, and flat field correction to remove pixel to -pixel response functions. We also construct fringe correction for the -reddest data in the \yps{} filter, to remove the interference patterns that -arise in that filter due to the variations in the thickness of the -detector surface. - -These corrections, however, assume that the detector response is -linear across the full range of values. This is not universally the -case with GPC1, and this requires an additional set of detrending -steps to remove these non-linear responses. The first of these is the -\IPPprog{burntool} correction, which removes the persistence trails -caused by the incomplete transfer of charge along the readout columns. -This bright-end nonlinearity is generally only evident for the -brightest stars, as only pixels that are at or beyond the saturation -point of the detector have this issue. More widespread is the -non-linearity at the faint end of the pixel range. Some readout cells -and some readout cell edge pixels experience a sag relative to linear -at low illumination, such that faint pixels appear fainter than -expected. The correction to this requires amplifying the pixel values -in these regions to match the expected model. - -The final non-linear response issue has no good option for correction. +pixel response functions. We also perform fringe correction for the +reddest data in the \yps{} filter to remove the interference patterns +that arise in that filter due to the variations in the thickness of +the detector surface. + +These corrections assume that the detector response is linear across +the full range of values. This assumption is not universally true for +GPC1, and an additional set of detrending steps are required to remove +these artifacts. The first of these is the \IPPprog{burntool} +correction, which removes the flux trails left by the incomplete +transfer of charge along the readout columns. These trails are +generally only evident for the brightest stars, as only pixels that +are at or beyond the saturation point of the detector leave residual +charge. More widespread is the non-linearity at the faint end of the +pixel range. Some readout cells and some readout cell edge pixels +experience a sag relative to linear at low illumination, such that +faint pixels appear fainter than expected. The correction to this +requires amplifying the pixel values in these regions to match the +expected model. + Large regions of some OTA cells experience significant charge transfer issues, making them unusable for science observations. These regions are therefore masked in processing, with these CTE regions making up the largest fraction of masked pixels on the detector. Other regions -are masked for other regions, such as static bad pixel features or -temporary readout masking caused by issues in the camera electronics -that make these regions unreliable. These all contribute to the -detector mask, which is augmented in each exposure for dynamic +are masked for reasons such as static bad pixel features or temporary +readout masking caused by issues in the camera electronics that make +these regions unreliable. These all contribute to the detector mask, +a 16 bit value which records the reason a pixel is masked based on the +value added. This mask is augmented in each exposure for dynamic features that are masked based on the astronomical features within the field of view. For the PV3 processing, all detrending is done by the -\IPPprog{ppImage} program. This program applies the detrends to the -individual cells, and then an OTA level mosaic is constructed for the -science image, the mask image, and the variance map image. The single -epoch photometry is done at this stage as well. The following -subsections (\ref{sec:burntool} - \ref{sec:background}) detail these -detrending steps, presented in the order in which they are applied to -the individual OTA image data. - -\subsection{Burntool / Persistence effect} -\label{sec:burntool} - -Pixels that approach the saturation point on GPC1, which varies by -readout with common values around 60000 DN, cause persistence problems -on that and subsequent images. During the read out process of an -image with such a bright pixel, some of the charge associated with it -is not fully shifted down the detector column toward the amplifier. -As a result, this charge remains in the starting cell, and is -partially collected in subsequent shifts, resulting in a ``burn -trail'' that extends from the center of the bright source away from -the amplifier (vertically along the pixel columns toward the top of -the cell). - -This incomplete charge shifting in nearly full wells continues as each -row is read out. This results in a remnant charge being deposited in -the pixels that the full well was shifted through. In following -exposures, this remnant charge leaks out, resulting in a trail that -extends from the initial location of the bright source on the previous -image towards the amplifier (vertically down along the pixel column). -This remnant charge can remain on the detector for up to thirty -minutes, requiring the locations of these ``burns'' be retained -between exposures. - -Both of these types of persistence trails are measured and optionally -repaired via the \IPPprog{burntool} program. This program does an -initial scan of the images, and identifies objects with pixel values -brighter than a conservative threshold of 30000 DN. The trail from -the peak of that object is fit with a one-dimensional power law in -each pixel column above the threshold, based on empirical evidence -that this is the functional form of this persistence effect. This -also matches the expectation that a constant fraction of charge is -incompletely transferred at each shift beyond the persistence -threshold. Once this fit is done, the model can be subtracted from -the image, and the location of the star is stored in a table along -with the exposure PONTIME, which denotes the number of seconds since -the detector was last powered on, and provides an internally consistent -time scale. - -For subsequent exposures, the table associated with the previous image -is read in, and after correcting trails from the stars on the new -image, the positions of the bright stars from the table are used to -check for remnant trails on the image. These are fit and subtracted -using a one-dimensional exponential model, again based on empirical -studies. If a significant model is found, then this location is -retained in the image output table. If not, the old burn is allowed -to expire. - -The main concern with this method of correcting the persistence trails -is that it is based on fits to the raw image data, which may have -other signal sources not determined by the persistence effect. The -presence of other stars or artifacts along the path of the burn can -result in a poor model to be fit, resulting in either an over- or -under-subtraction of the persistence burn. For this reason, the image -mask is marked with a value indicating that this correction has been -applied. These pixels are not fully excluded, but they are marked as -suspect, which allows them to be excluded from consideration in -subsequent stages, such as image stacking. - -Another concern is that the cores of very bright stars are deformed by -this process, as the burntool fitting subtracts flux -from only one side of the star. As most stars that result in burns already -have saturated cores, they are already ignored for the purpose of -PSF determination and are flagged as saturated by the photometry -reduction. - -\begin{figure} - \centering - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_bt_trail.png} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} - \end{minipage}% - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_bt_trail.png} -% \caption{(b)} -% \end{subfigure} - \end{minipage} - - \caption{Example of a profile cut along the y-axis through a bright star on exposure o5677g0123o OTA11 in cell xy60 (left panel) and on the subsequent exposure o5677g0124o (right panel). In both figures, the green points show the image corrected with all appropriate detrending steps, but without burntool applied, illustrating the amplitude of the persistence trails. The red points show the same data after the burntool correction, which reduces the impact of these features. Both exposures are in the \gps{} filter with exposure times of 43s} -\end{figure} - -\begin{figure} - \centering - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_nobt.png} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} - \end{minipage}% - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_nobt.png} -% \caption{(b)} -% \end{subfigure} - \end{minipage} - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_bt.png} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} - \end{minipage}% - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_bt.png} -% \caption{(b)} -% \end{subfigure} - \end{minipage} - \caption{Example of OTA11 cell xy60 on exposures o5677g0123o (left) and o5677g0124o (right). The top panels show the image with all appropriate detrending steps, but without burntool, and the bottom show the same with burntool applied. There is some slight over subtraction in fitting the initial trail, but the impact of the trail is greatly reduced in both exposures.} -\end{figure} - +\IPPprog{ppImage} program. This program applies the detrend +corrections to the individual cells, and then an OTA-level mosaic is +constructed for the signal image, the mask image, and the variance map +image. The single epoch photometry is done at this stage as well. +The following subsections (\ref{sec:overscan} - \ref{sec:background}) +detail the detrending process used on GPC1 that are common to other +detectors. The GPC1 specific detrending steps are included after, +explaining these additional steps that remove the instrument signature. \subsection{Overscan} @@ -447,82 +338,32 @@ Each cell on GPC1 has an overscan region that covers the first 34 columns of each row, and the last 10 rows of each column. No light -lands on these pixels, so the image region is trimmed to exclude them. -Each row has an overscan value subtracted, calculated by finding the -median value of that row's overscan pixels and then smoothing between -rows with a three-row boxcar median. - -\subsection{Non-linearity Correction} -\label{sec:nonlinearity} - -The pixels of GPC1 are not uniformly linear at all flux levels. In -particular, at low flux levels, some pixels have a tendency to sag -relative to the expected linear value. This effect is most pronounced -along the edges of the detector cells, although some entire cells show -evidence of this effect. - -To correct this sag, we studied the flux behavior of a series of flat -frames for a ramp of exposure times with approximate logarithmically -equal spacing between 0.01s and 57.04s. As the exposure time -increases, the flux on each pixel also increases in what is expected -to be a linear manner. Each of these flat exposures in this ramp is -overscan corrected, and then the median is calculated for each cell, -as well as for the rows and columns within ten pixels of the edge of -the science region. From these median values at each exposure time -value, we can construct the expected trend by fitting a linear model, -$f_{region} = G * t_{exp} + B$, to determine the gain, $G$, and the -bias, $B$, for the region considered. This fitting was limited to only -the range of fluxes between 12000 and 38000 counts, as these ranges -were found to match the linear model well. This range avoids the -non-linearity at low fluxes, as well as the possibility of high-flux -non-linearity effects. - -We store the average flux measurement and deviation from the linear -fit for each exposure time for all regions on all detector cells in -the linearity detrend look up tables. When this is applied to science -data, these lookup tables are loaded, and a linear interpolation is -performed to determine the correction needed for the flux in that -pixel. This look up is performed for both the row and column of each -pixel, to allow the edge correction to be applied where applicable, -and the full cell correction elsewhere. The average of these two -values is then applied to the pixel value, reducing the effects of -pixel nonlinearity. - -This non-linearity effect appears to be stable in time for the -majority of the detector pixels, with little evident change over the -survey duration. However, as the non-linearity is most pronounced at -the edges of the detector cells, those are the regions where the -correction is most likely to be incomplete. Because of this fact, -most pixels in the static mask with either the DARKMASK or FLATMASK -bit set are found along these edges. As the non-linearity correction -is unable to reliably restore these pixels, they produce inconsistent -values after the dark and flat have been applied, and are therefore -rejected. - -\begin{figure} - \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/linearity_XY27_xy16.png} - \caption{Example plot of the linearity correction as a fraction of observed flux for OTA27, cell xy16.} -\end{figure} +lands on these pixels, so the science region is trimmed to exclude +them. Each row has an overscan value subtracted, calculated by +finding the median value of that row's overscan pixels and then +smoothing between rows with a three-row boxcar median. \subsection{Dark/Bias Subtraction} \label{sec:dark} -The dark model we make for GPC1 considers each pixel individually, -independent of any neighbors. To construct this model, we fit a -multi-dimensional model to the array of input pixels from a randomly -selected set of 100-150 overscan and non-linearity corrected dark -frames chosen from a given date range. The model fits each pixel as a -function of the exposure time $t_{exp}$ and the detector temperature -$T_{chip}$ of the input images such that $\mathrm{dark} = a_0 + a_1 -t_{exp} + a_2 T_{chip} t_{exp} + a_3 T_{chip}^2 t_{exp}$. This -fitting uses two iterations to produce a clipped fit, rejecting at the -$3\sigma$ level. The final coefficients $a_i$ for the dark model are -stored in the detrend image. The constant $a_0$ term includes the -residual bias signal after overscan subtraction, and as such, a -separate bias subtraction is not necessary. +The dark current in the GPC1 detectors has significant variations +across each cell. The model we make to remove this signal considers +each pixel individually, independent of any neighbors. To construct +this model, we fit a multi-dimensional model to the array of input +pixels from a randomly selected set of 100-150 overscan and +non-linearity corrected dark frames chosen from a given date range. +The model fits each pixel as a function of the exposure time $t_{exp}$ +and the detector temperature $T_{chip}$ of the input images such that +$\mathrm{dark} = a_0 + a_1 t_{exp} + a_2 T_{chip} t_{exp} + a_3 +T_{chip}^2 t_{exp}$. This fitting uses two iterations to produce a +clipped fit, rejecting at the $3\sigma$ level. The final coefficients +$a_i$ for the dark model are stored in the detrend image. The +constant $a_0$ term includes the residual bias signal after overscan +subtraction, and as such, a separate bias subtraction is not +necessary. Applying the dark model is simply a matter of calculating the response -to the exposure time and detector temperature for the image to be +for the exposure time and detector temperature of the image to be corrected, and subtracting the resulting dark signal from the image. +Figure \ref{fig:dark image} shows the results of the dark subtraction. \subsubsection{Time evolution} @@ -531,14 +372,16 @@ significant drift over the course of multiple months. Some of the changes in the dark can be attributed to changes in the voltage -settings of the GPC1 controller electronics, but the majority seem to -be the result of some unknown parameter. We can separate the dark -model history of GPC1 into three epochs. The first epoch covers all -data taken prior to 2010-01-23. This epoch used a different header -keyword for the detector temperature, making data from this epoch -incompatible with later dark models. +settings of the GPC1 controller electronics, but the causes of others +are unknown. We can separate the dark model history of GPC1 into +three epochs. The first epoch covers all data taken prior to +2010-01-23. This epoch used a different header keyword for the +detector temperature, making data from this epoch incompatible with +later dark models. In addition, the temperatures recorded in this +value were not fully calibrated, making the dark model generated less +reliable. The second epoch covers data between 2010-01-23 and 2011-05-01, and is characterized by a largely stable but oscillatory dark solution. -There are two modes that the dark model switches between apparently at +The dark model switches between two modes apparently at random. No clear cause has been established for the switching, but there are clear differences between the two modes that require the @@ -570,28 +413,22 @@ After 2011-05-01, the two-mode behavior of the dark disappears, and is -replaced with a slow observation date dependent drift in the magnitude +replaced with a slow observation-date-dependent drift in the magnitude of the gradient. This drift is sufficiently slow that we have modeled -it using three observation date independent dark model for different -date ranges. These darks cover the range from 2011-05-01 to -2011-08-01, 2011-08-01 to 2011-11-01, and 2011-11-01 and on. The -reason for this time evolution is unknown, but as it is correctable -with a small number of dark models, this does not significantly impact -detrending. +it by generating models for different date ranges. These darks cover +the range from 2011-05-01 to 2011-08-01, 2011-08-01 to 2011-11-01, and +2011-11-01 and on. The reason for this time evolution is unknown, but +as it is correctable with a small number of dark models, this does not +significantly impact detrending. \begin{figure} \centering -% \begin{subfigure}[]{.45\hsize} \begin{minipage}{0.45\hsize} \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_M_OS_NL_XY23_b1.jpg} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} \end{minipage}% \begin{minipage}{0.45\hsize} \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_to_DARK_XY23_b1.jpg} -% \caption{(b)} -% \end{subfigure} \end{minipage} - \caption{An example of the dark model application to exposure o5677g0123o, OTA23 (2011-04-26, 43s \gps{} filter). The left panel shows the image data mosaicked to the OTA level, and has had the static mask applied, the overscan subtracted, and the detector non-linearity corrected. The right panel, shows the same exposure with the dark applied in addition to the processing shown on the left.} + \caption{An example of the dark model application to exposure o5677g0123o, OTA23 (2011-04-26, 43s \gps{} filter). The left panel shows the image data mosaicked to the OTA level, and has had the static mask applied, the overscan subtracted, and the detector non-linearity corrected. The right panel, shows the same exposure with the dark applied in addition to the processing shown on the left, removing the amplifier glows in the cell corners.} + \label{fig:dark image} \end{figure} @@ -599,5 +436,5 @@ \centering \includegraphics[width=0.9\hsize,angle=0,clip]{images/B_profile_ex.png} - \caption{Example showing a profile cut across exposure o5676g0195, OTA67 (2011-04-25, 43s \gps{} filter). The entire first row of cells (xy00-xy07) have had a median calculated along each pixel column on the OTA mosaicked image. Arbitrary offsets have been applied to shift the curves to not overlap. The top curve (in purple) shows the initial raw profile, with no dark model applied. The next curve (in green) shows the smoother profile after applying the correct B-mode dark model. Applying the incorrect A-mode dark results in the blue curve, which shows a significant increase in gradients across the cells. The orange curve shows the result of the PATTERN.CONTINUITY correction. Although this creates a larger gradient across the mosaicked images, it decreases the cell-to-cell level changes. The final yellow curve shows the final image profile after all detrending and background subtraction, and has not had an offset applied. The bright source at the cell xy00 to xy01 transition is a result of a large optical ghost, which due to the area covered, increases the median level more than the field stars.} + \caption{Example showing a profile cut across exposure o5676g0195, OTA67 (2011-04-25, 43s \gps{} filter). The entire first row of cells (xy00-xy07) have had a median calculated along each pixel column on the OTA mosaicked image. Arbitrary offsets have been applied to shift the curves to not overlap. The top curve (in purple) shows the initial raw profile, with no dark model applied. The next curve (in green) shows the smoother profile after applying the appropriate B-mode dark model. Applying the A-mode dark instead results in the blue curve, which shows a significant increase in gradients across the cells. The orange curve shows the result of the PATTERN.CONTINUITY correction. Although this creates a larger gradient across the mosaicked images, it decreases the cell-to-cell level changes. The final yellow curve shows the final image profile after all detrending and background subtraction, and has not had an offset applied. The bright source at the cell xy00 to xy01 transition is a result of a large optical ghost, which due to the area covered, increases the median level more than the field stars.} \label{fig:dark switching} \end{figure} @@ -606,53 +443,44 @@ \label{sec:video_darks} -The dark signal is stronger in cell corners due to glow from the -read-out amplifiers. The standard dark model corrects this for most -observations. However, as mentioned above, when a cell is repeatedly -read in video mode, the dark model for the OTA containing it changes. -Surprisingly, added reads for the video cell do not amplify the -amplifier glow, but rather decrease the dark signal in these regions. -As a result, using the standard dark model on the data for these OTAs -results in oversubtraction of the corner glow. - -Video darks have been constructed to eliminate the effect this -observational change has on the final image quality. This was done by -running the standard dark construction process on a series of dark -frames that have had the video signal enabled for some cells. GPC1 -can only run video signals on a subset of the OTAs at a given time. -This requires two passes to enable the video signal across the full -set of OTAs that support video cells. This is convenient for the -process of creating darks, as those OTAs that do not have video -signals enabled create standard dark models, while the video dark is -created for those that do. +Individual cells on GPC1 can be repeatedly read to create a video +signal used for telescope guiding. However, when a cell is used for +this purpose, the dark signal for the entire OTA is changed. The most +noticeable feature of this change is that the glows in cell corners +caused by the read-out amplifiers are suppressed. As a result, using +the standard dark model on the data for these OTAs results in +oversubtraction of the corner glow. + +To generate a correction for this change, a set of video dark models +were created by running the standard dark construction process on a +series of dark frames that had the video signal enabled for some +cells. GPC1 can only run video signals on a subset of the OTAs at a +given time. This requires two passes to enable the video signal +across the full set of OTAs that support video cells. This is +convenient for the process of creating darks, as those OTAs that do +not have video signals enabled create standard dark models, while the +video dark is created for those that do. This simultaneous construction of video and standard dark models is -useful, as it provides the ability to isolate the response on the -standard dark from the video signals. Isolating this response is -essential for attempting to create archival video darks. We only have -raw video dark frame data after 2012-05-16, when this problem was -initially identified, so any data prior to that can not be directly -corrected for the video dark signal. Isolating the video signal -response allows linear corrections to the pre-existing standard dark -models for archival data. Testing this shows that constructing a -video dark for older data simply as $VD_{2009} = D_{2009} - D_{Modern} -+ VD_{Modern}$ produces a satisfactory result that does not -over subtract the amplifier glow. This is shown in figure -\ref{fig:video_darks}, which shows video cells from before 2012-05-16, -corrected with both the standard and video darks, with the early video -dark constructed in such a manner. +useful, as it provides a way to isolate the response on the standard +dark from the video signals. If the standard and video dark signals +are separable, then archival video darks can be constructed by +applying the video dark response to the previously constructed dark +models. Raw video dark frame data only exists after 2012-05-16, when +this problem was initially identified, so any data prior to that can +not be directly corrected for the video dark signal. Testing the +separability shows that constructing a video dark for older data +simply as $VD_{Old} = D_{Old} - D_{Modern} + VD_{Modern}$ produces a +satisfactory result that does not over subtract the amplifier glow. +This is shown in figure \ref{fig:video_darks}, which shows video cells +from before 2012-05-16, corrected with both the standard and video +darks, with the early video dark constructed in such a manner. \begin{figure} \centering -% \begin{subfigure}[]{.45\hsize} \begin{minipage}{0.45\hsize} \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_VIDEODARK_VDim_Rdark_XY22_b1.jpg} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} \end{minipage}% \begin{minipage}{0.45\hsize} \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_VIDEODARK_VDim_VDdark_XY22_b1.jpg} -% \caption{(b)} -% \end{subfigure} \end{minipage} \caption{An example of the video dark model application to exposure o5677g0123o, OTA22 (2011-04-26, 43s \gps{} filter), which has a video cell located in cell xy16. The left panel shows the image data mosaicked to the OTA level, and has had the static mask applied, the overscan subtracted, the detector non-linearity corrected, and a regular dark applied. The right panel, shows the same exposure with a video dark applied instead of the standard dark. The main impact of this change is the improved correction of the corner glows, which are over subtracted with the standard dark.} @@ -664,18 +492,16 @@ Based on a study of the positional dependence of all detected sources, -we have discovered that the cells in GPC1 do not have uniform noise +we discovered that the cells in GPC1 do not have uniform noise characteristics. Instead, there is a gradient along the pixel rows, with the noise generally higher away from the read out amplifier (higher cell x pixel positions). This is likely an effect of the -row-by-row bias issue discussed below. This gradient causes the read -noise to increase as the row is read out. As a result of this +row-by-row bias issue discussed below. As a result of this increased noise, more sources are detected in the higher noise regions -when the read noise is assumed constant across the readout. Read noise is the - -To -mitigate this noise gradient, we constructed an initial set of -noisemap images by measuring the median variance on bias frames. The -variance is calculated in boxes of 20x20 pixels, and then linearly -interpolated to cover the full image. +when the read noise is assumed constant across the readout. + +To mitigate this noise gradient, we constructed an initial set of +noisemap images by measuring the median variance on bias frames +processed as science images. The variance is calculated in boxes of +20x20 pixels, and then linearly interpolated to cover the full image. Unfortunately, due to correlations within this noise, the variance @@ -689,35 +515,31 @@ contaminating the final object catalogs. -In the detection process, we expect false positives at a rate equal to -the one-tailed probability beyond the detection threshold. For these -tests, only detections measured at the $\sigma_{thresh} = 5\sigma$ -level are used, to match that used in the photometry on science data. -This probability can be converted into a number of false number by -considering a given area. As the detections must be isolated to not -be detected as an extended object, this area must be reduced by the -area a given PSF occupies. Combining this, we find that we expect a -probability $P = 1 - \Phi_{normal}(5) = \frac{1}{2} -\erfcinv\left(\frac{5}{\sqrt{2}}\right)$, and an area given $N$ -exposures of area $X\times Y$, $A = \frac{X \times Y \times - N}{A_{PSF}}$. For a typical $1"$ seeing, $A_{PSF}$ is approximately -16 pixels. Using this model for the false positives, we found that -the added read noise was insufficient to account for the observed -false positive rate. Inverting this relation, we can measure -$\sigma_{obs}$, the true threshold level based on the number of false -positives observed. This $\sigma_{obs}$ is the combined to form a -boost factor $B = \sigma_{thresh} / \sigma_{obs}$ that amplifies the - noisemap to match the observed false detection rate. - -The row-to-row variations that contribute to the extra noise are -related to the dark model, and because of this, as the dark model -changes, the effective noise also changes. To ensure that the -noisemap accurately matches the true noise level, we have created -different noisemap models for the three major time ranges of the dark -model. We do not see any strong evidence that the noisemaps have the -A/B modes visible in the dark, and so we do not generate different -models for each individual dark model. The additional pixel-to-pixel -variance from this noisemap is added to the Poissonian variance to -form the science variance image generated by the \IPPstage{chip} -processing. +In the detection process, we expect false positives at a low rate, +given that all sources are required to be significant at the $5\sigma$ +level. Since the observed false positive rate was significantly +higher than expected, we implemented an empirical ``boost'' to +increase the noisemap to more accurately account for the position +dependent read noise. By binning the number of false positives +measured on the bias frames on the noisemap inputs using 20 pixel +boxes in the cell x-axis, and comparing this to the number expected +from random Gaussian noise, we estimated the true read noise level. + +As the noisemap uses bias frames that have had a dark model +subtracted, we constructed noisemaps for each dark model used for +science processing. There is some evidence that the noise has changed +over time as measured on full cells, so matching the noisemap to the +dark model allows for these changes to be tracked. There is no +evidence that the noisemap has the A/B modes found in the dark, so we +do not generate separate models for that time period. + +The noisemap detrend is not directly applied to the science image. +Instead, it is used to construct the weight image that contains the +pixel-by-pixel variance for the \IPPstage{chip} stage image. The +initial weight image is constructed by dividing the science image by +the cell gain (approximately 1.0 e$^{-} /$ DN). This weight image +contains the expected Poissonian variance in electrons measured. The +square of the noisemap is then added to this initial weight, adding +the additional empirical variance term in place of a single read noise +value. \subsection{Flat} @@ -736,5 +558,6 @@ correction to remove the effect of the illumination differences over the detector surface. This is done by dithering a series of science -exposures with a given pointing. By fully calibrating these exposures +exposures with a given pointing, as described in +\citet{2004PASP..116..449M}. By fully calibrating these exposures with the initial flat model, and then comparing the measured fluxes for the same star as a function of position on the detector, we can @@ -748,163 +571,10 @@ In addition to this flat field applied to the individual images, the ubercal process used to calibrate the database of all detections -\citep{2012ApJ...756..158S} constructs internal ``flat field'' corrections. -Although a single set of image flat fields was used for the entire PV3 -survey, five separate ``seasons'' of database flat fields were needed -to ensure proper calibration. This indicates that the flat field -response is not completely fixed in time. More details on this -process are contained in \citet{magnier2017c}. - -\subsection{Pattern correction} -\label{sec:pattern} - -Due to detector specific issues that are not cleanly removed by the -dark model, we have a set of ``pattern'' corrections that are applied -to some selection of the OTAs in the camera. This is done to reduce -the effect that detector differences have on the measured astronomical -signal that are not stable enough to be corrected with a static model. -Because of this, the pattern corrections attempt to identify and -correct the detector issues based on appropriate filtering the -individual science exposures. - -The PATTERN.ROW correction is used to remove any remaining row-by-row -bias variation, and the PATTERN.CONTINUITY correction attempts to -ensure that the cells of a given OTA are consistent with the other -cells on that OTA. - -\subsubsection{Pattern Row} -%% Statistics so I have them written down somewhere -%% chipProcessedImfile.bg/bg_stdev by filter for XY33 (a good chip) -%% filter bg_mean stdev median Qsig bg_stdev_mean stdev median Qsig -%% g 36.37422026669 64.64175104057 32.693 6.10284 14.696938349131 78.80460307171 8.8401 0.5417843 -%% r 200.96143304525 471.87743546238 117.105 94.55608 33.854672792146 79.01642728089 13.4564 5.3771355 -%% i 447.00504994458 938.38517801037 286.810 154.71397 57.298335510188 99.38392923935 20.0217 24.2254723 -%% z 317.54933679054 390.38930252748 241.014 114.13316 48.359069000176 94.44452756094 17.9404 9.1535209 -%% y 371.09019536218 293.57439970375 288.481 133.38769 43.724342221691 135.04286534327 19.9029 7.5396461 - -As discussed above in the dark and noisemap sections, certain -detectors have significant bias offsets between adjacent rows, caused -by noise in the camera control electronics. The magnitude of these -offsets increases as the distance from the readout amplifier -increases, resulting in horizontal streaks that are more pronounced -along the large x pixel edge of the cell. As the level of the offset -is apparently random between exposures, the dark correction cannot -fully remove this structure from the images, and the noisemap value -only indicates the level of the average variance added by these bias -offsets. Therefore, we apply the PATTERN.ROW correction in an attempt -to mitigate the offsets and correct the image values. To force the -rows to agree, a second order clipped polynomial is fit to each row in -the cell. Four fit iterations are run, and pixels $2.5\sigma$ deviant -are excluded from subsequent fits, to minimize the effect stars and -other astronomical signals have. This final trend is then subtracted -from that row. Simply doing this subtraction will also have the -effect of removing the background sky level. To prevent this, the -constant and linear terms for each row are stored, and linear fits are -made to these parameters as a function of row, perpendicular to the -initial fits. This produces a plane that is added back to the image -to restore the background offset and any linear ramp that exists in -the sky. - -These row-by-row variations have the largest impact on data taken in -the \gps{} filter, as the read noise is the dominant noise source in -that filter. At longer wavelengths, the noise from the Poissonian -variation in the sky level increases. Although the PATTERN.ROW correction is still applied to data taken in the other filters, - -This correction was required on all cells on all OTAs prior to -2009-12-01, at which point a modification of the camera electronics -reduced the scale of the row-by-row offsets for the majority of the -OTAs. As a result, we only apply this correction to the cells where -it is still necessary, as shown in Figure \ref{fig: pattern row - cells}. A list of these cells is listed in Table -\ref{tab:pattern_row_cells}. - -Although this correction does largely resolve the row-by-row offset -issue in a satisfactory way, large and bright astronomical objects can -bias the fit significantly. This results in an oversubtraction of the -offset near these objects. As the offsets are calculated on the pixel -rows, this oversubtraction is not uniform around the object, but is -preferentially along the horizontal x axis of the object. Most -astronomical objects are not significantly distorted by this, with -this only becoming on issue for only bright objects comparable to the -size of the cell (598 pixels = 150"). - -\begin{deluxetable}{lcccc} - \tablecolumns{3} - \tablewidth{0pc} - \tablecaption{Cells which have PATTERN.ROW correction applied} - \tablehead{\colhead{OTA} & \colhead{Cell columns} & \colhead{Additional cells}} - \startdata - OTA11 & & xy02, xy03, xy04, xy07 \\ - OTA14 & & xy23 \\ - OTA15 & 0 & \\ - OTA27 & 0, 1, 2, 3, 7 & \\ - OTA31 & 7 & \\ - OTA32 & 3, 7 & \\ - OTA45 & 3, 7 & \\ - OTA47 & 0, 3, 5, 7 & \\ - OTA57 & 0, 1, 2, 6, 7 & \\ - OTA60 & & xy55 \\ - OTA74 & 2, 7 & \\ - \enddata - \label{tab:pattern_row_cells} -\end{deluxetable} - -\begin{figure} - \centering - \includegraphics[width=0.9\hsize,angle=0,clip]{images/pattern_row_edit.png} - \caption{Diagram illustrating in red which cells on GPC1 require the PATTERN.ROW correction to be applied. The footprint of each OTA is outlined, and cell xy00 is marked with either a filled box or an outline. The labeling of the non-existent corner OTAs is provided to orient the focal plane.} - \label{fig: pattern row cells} -\end{figure} - -\begin{figure} - \centering - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_XY57_nopat.png} -% \caption{(a)} -% \end{subfigure}% -% \begin{subfigure}[]{.45\hsize} - \end{minipage}% - \begin{minipage}{0.45\hsize} - \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_XY57_pat.png} -% \caption{(b)} -% \end{subfigure} - \end{minipage} - \caption{Example of the PATTERN.ROW correction on exposure o5379g0103o OTA57 cell xy00 (\ips{} filter 45s). The left panel shows the cell with all appropriate detrending except the PATTERN.ROW, and the right shows the same cell with PATTERN.ROW applied. The correction reduces the correlated noise on the right side, which is most distant from the read out amplifier. There is a slight over subtraction along the rows near the bright star.} -\end{figure} - -\subsubsection{Pattern Continuity} - -After previous attempts to ensure that adjacent cells on an OTA -matched background levels were insufficient in many situations, we -designed a replacement correction that would reduce the background -distortion for large objects. In addition, studies of the background -level illustrated that the row-by-row bias can introduce small -background gradient variations along the rows of the cells that is not -stable enough to be completely fit by the dark model. This common -feature across the columns of cells results in a ``saw tooth'' pattern -horizontally across an OTA, and as the background model fits a smooth -sky level, this induces over and under subtraction at the cell -boundaries. - -The PATTERN.CONTINUITY correction, attempts to match the edges of a -cell to those of its neighbors. For each cell, a thin box 10 pixels -wide on each edge is extracted and the median value of unmasked values -calculated for that box. These median values are then used to -construct a vector of differences $\Delta_i = \sum_{j} \mathrm{Edge}_{i} - -\mathrm{Edge}_{j}$, along with a matrix of associations $A_{i,i'} = \sum_{j} -\delta(i,j) \delta(j,i')$ denoting which cell boundaries are adjacent. -By solving the system $A x = \Delta$, we find the set of offsets $x_i$ -to be applied to each cell to ensure the minimum differences between -all cell edges and their neighbors. - -For OTAs that initially show the saw tooth pattern, the effect of this -correction is to align the cells into a single ramp, at the expense of -the absolute background level. However, as we subtract off a smooth -background model prior to doing photometry, these deviations from an -absolute sky level are unimportant. The fact that the final ramp is -smoother than it would be otherwise also allows for the background -subtracted image to more closely match the astronomical sky, without -significant errors at cell boundaries. An example of the effect of -this correction on an image profile is shown in Figure \ref{fig:dark switching}. - +\citep{2012ApJ...756..158S} constructs ``in catalog'' flat field +corrections. Although a single set of image flat fields was used for +the entire PV3 survey, five separate ``seasons'' of database flat +fields were needed to ensure proper calibration. This indicates that +the flat field response is not completely fixed in time. More details +on this process are contained in \citet{magnier2017.calibration}. \subsection{Fringe correction} @@ -928,11 +598,11 @@ input images with two iteration of clipping at the $3\sigma$ level. -A course background model for each cell is constructed by calculating +A coarse background model for each cell is constructed by calculating the median on a 3x3 grid (approximately 200x200 pixels each). A set -of 1000 randomly selected points are then selected on the fringe image -for each cell, and a median calculated for this position in a 10x10 -pixel box, with the background level subtracted. These sample -locations provide scale points to allow the amplitude of the measured -fringe to be compared to that found on science images. +of 1000 points are randomly selected from the fringe image for each +cell, and a median calculated for this position in a 10x10 pixel box, +with the background level subtracted. These sample locations provide +scale points to allow the amplitude of the measured fringe to be +compared to that found on science images. To apply the fringe, the same sample locations are measured on the @@ -962,25 +632,31 @@ \label{sec:static_masks} -Due to the large size of the detector, it is expected that there -are a number of pixel defects that do not have the detection -sensitivity on par with their neighbors. To remove these pixels, we -have constructed a static mask that identifies the known defects. -This mask is constructed in three phases. - -First, a CTEMASK is constructed to mask out regions in which the -charge transfer efficiency is low compared to the rest of the -detector. Twenty-five of the sixty OTAs in GPC1 show some evidence of -CTE issues, with this pattern appearing (to varying degrees) in -roughly triangular patches on the OTA due to defects in the -semiconductor manufacturing. To generate the mask for these regions, -a sample set of 26 evenly illuminated flat field images were measured -to produce a map of the image variance in 20x20 pixel bins. As the -flat image is expected to illuminate the image uniformly, the expected -variances in each bin should be Poissonian distributed with the flux -level. However, in regions with CTE issues, adjacent pixels are not -independent, as the charge in those pixels is more free to spread. -This reduces the pixel-to-pixel differences, resulting in a lower than -expected variance. All regions with variance less than half the -average image level are added to the static CTEMASK. +Due to the large size of the detector, it is expected that there are a +number of pixels that respond poorly. To remove these pixels, we have +constructed a mask that identifies the known defects. This mask is +referred to as the ``static'' mask, as it is applied to all images +processed. The ``dynamic'' mask (Section \ref{sec:dynamic_masks}) is +calculated based on objects in the field, and so changes between +images. Construction of the static mask consists of three phases. + +First, regions in which the charge transfer efficiency (CTE) is low +compared to the rest of the detector are identified. Twenty-five of +the sixty OTAs in GPC1 show some evidence of poor CTE, with this +pattern appearing (to varying degrees) in roughly triangular patches. +During the manufacture of the devices, an improperly tuned +semiconductor process step resulted in a radial pattern of poor +performance on some silicon wafers. When the OTAs were cut from these +wafers, the outer corners exhibited the issue. To generate the mask +for these regions, a sample set of 26 evenly-illuminated flat-field +images were measured to produce a map of the image variance in 20x20 +pixel bins. As the flat screen is expected to illuminate the image +uniformly on this scale, the expected variances in each bin should be +Poissonian distributed with the flux level. However, in regions with +poor CTE, adjacent pixels are not independent, as the charge in those +pixels is more free to spread along the image columns. This reduces +the pixel-to-pixel differences, resulting in a lower than expected +variance. All regions with variance less than half the average image +level are added to the static mask. + The next step of mask construction is to examine the flat and dark @@ -998,4 +674,5 @@ removing the pixels that cannot be corrected to a linear response. +% http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/StaticMasks20101215 The final step of mask construction is to examine the detector for bright columns and other static pixel issues. This is first done by @@ -1020,7 +697,7 @@ \centering \includegraphics[width=0.9\hsize,angle=0,clip]{images/gpc1_mask_indexed.png} - \label{fig:static mask} \caption{Image map of the GPC1 static mask. The CTE regions are clearly visible as roughly triangular patches covering the corners of some OTAs. Some entire cells are masked, including an entire column of cells on OTA14. Calcite cells remove large areas from OTA17 AND OTA76.} + \label{fig:static mask} \end{figure} @@ -1060,14 +737,14 @@ nature, and do not completely exclude the pixel from further processing consideration. The first of these dynamic masks is the -burntool advisory mask mentioned above. These pixels are included for +burntool advisory mask described below. These pixels are included for photometry, but are rejected more readily in the stacking and difference image construction, as they are more likely to have small deviations due to imperfections in the burntool correction. -The remaining dynamic masks are not generated until the IPP -\IPPstage{camera} stage, at which point all object photometry is -complete, and an astrometric solution is known for the exposure. This -added information provides the positions of bright sources based on -the reference catalog, including those that fall slightly out of the +The remaining dynamic masks are generated in the IPP \IPPstage{camera} +stage, at which point all object photometry is complete, and an +astrometric solution is known for the exposure. This added +information provides the positions of bright sources based on the +reference catalog, including those that fall slightly out of the detector field of view or within the inter chip gaps, where internal photometry may not identify them. These bright sources are the origin @@ -1083,7 +760,7 @@ Table \ref{tab:crosstalk_rules} summarizes the list of known crosstalk rules, with an estimate of the magnitude difference between the source -and ghost. For all of the rules, any cell $v$ within the specified +and ghost. For all of the rules, any source cell $v$ within the specified column of cells on any of the OTAs in the specified column of OTAs $Y$ -creates the ghost in the same $v$ and $Y$ in the target column of +can create a ghost in the same cell $v$ and OTA $Y$ in the target column of cells and OTAs. In each of these cases, a source object with an instrumental magnitude brighter than -14.47 creates a ghost object @@ -1129,30 +806,31 @@ \label{tab:crosstalk_rules} \end{deluxetable} - \subsubsubsection{Optical ghosts} \label{sec:optical_ghosts} -Due to imperfections in the anti-reflective coating on the optical -surfaces of GPC1, bright sources can also result in large out of focus -objects, particularly in the \gps{} filter data. These objects are the -result of light reflecting back off the surface of the detector, -reflecting again off the lower surfaces of the optics (particularly -the L1 corrector lens), and then back down onto the focal plane. Due -to the extra travel distance, the resulting source is out of focus and +The anti-reflective coating on the optical surfaces of GPC1 is less +effective at shorter wavelengths, which can allow bright sources to +reflect back onto the focal plane and generate large out-of-focus +objects. Due to the wavelength dependence, these objects are most +prominent in the \gps{} filter data. These objects are the result of +light reflecting back off the surface of the detector, reflecting +again off the lower surfaces of the optics (particularly the L1 +corrector lens), and then back down onto the focal plane. Due to the +extra travel distance, the resulting source is out of focus and elongated along the radial direction of the camera focal plane. These optical ghosts can be modeled in the focal plane coordinates (L,M) which has its origin at the center of the focal plane. In this system, a bright object at location (L,M) on the focal plane creates a -reflection ghost on the opposite side of the optical axis at (-L,-M). -The exact location is fit as a third order polynomial in the focal -plane L and M directions (as listed in Table \ref{tab:ghost_centers}). -An elliptical annulus mask is constructed at the expected ghost -location, with the major and minor axes defined by linear functions of -the ghost distance from the optical axis, and oriented with the -ellipse major axis is along the radial direction (Table -\ref{tab:ghost_radii}). All stars brighter than a filter-dependent -threshold (listed in Table \ref{tab:ghost_magnitudes}) have such masks -constructed. +reflection ghost on the opposite side of the optical axis near +(-L,-M). The exact location is fit as a third order polynomial in the +focal plane L and M directions (as listed in Table +\ref{tab:ghost_centers}). An elliptical annulus mask is constructed +at the expected ghost location, with the major and minor axes defined +by linear functions of the ghost distance from the optical axis, and +oriented with the ellipse major axis is along the radial direction +(Table \ref{tab:ghost_radii}). All stars brighter than a +filter-dependent threshold (listed in Table +\ref{tab:ghost_magnitudes}) have such masks constructed. \begin{deluxetable}{lcc} @@ -1204,9 +882,9 @@ \end{deluxetable} - \begin{figure} \centering \includegraphics[width=0.9\hsize,angle=0,clip]{images/full_fpa_ghosts.jpg} \caption{Example of the full GPC1 field of view illustrating the sources and destinations of optical ghosts on exposure o5677g0123o (2011-04-26, 43s \gps{} filter). The bright stars on OTA33 and OTA44 result in nearly circular ghosts on the opposite OTA. In contrast, the trio of stars on OTA11 result in very elongated ghosts on OTA66.} + \label{fig:optical ghosts} \end{figure} @@ -1220,12 +898,13 @@ detector surface in a long narrow glint. This surface was physically masked on 2010-08-24, removing the possibility of glints in subsequent -data, but that taken prior have an advisory dynamic mask constructed -when a reference source falls on the focal plane within one degree of -the detector edge. This mask is 150 pixels wide, with length $L = -2500 \left(-20 - m_{inst}\right)$ pixels. These glint masks are -constructed by selecting sufficiently bright sources in the reference -catalog that fall within rectangular regions around each edge of the -GPC1 camera. These regions are separated from the edge of the camera -by 17 arcminutes, and extend outwards an additional degree. +data, but images that were taken prior to this date have an advisory +dynamic mask constructed when a reference source falls on the focal +plane within one degree of the detector edge. This mask is 150 pixels +wide, with length $L = 2500 \left(-20 - m_{inst}\right)$ pixels. +These glint masks are constructed by selecting sufficiently bright +sources in the reference catalog that fall within rectangular regions +around each edge of the GPC1 camera. These regions are separated from +the edge of the camera by 17 arcminutes, and extend outwards an +additional degree. \begin{figure} @@ -1233,4 +912,5 @@ \includegraphics[width=0.9\hsize,angle=0,clip]{images/glint_example_o5379g0103o.jpg} \caption{Example of a glint on exposure o5379g0103o (2010-07-02, 45s \ips{} filter). The source star out of the field of view creates a long reflection that extends through OTA73 and OTA63.} + \label{fig:optical glints} \end{figure} @@ -1244,6 +924,6 @@ mask is constructed, as the source is likely too faint to produce the feature. These spikes are dependent on the camera rotation, and are -oriented at $\theta = n * \frac{\pi}{2} - \mathrm{ROTANGLE} + 0.798$, -based on the header keyword. +oriented based on the header keyword at $\theta = n * \frac{\pi}{2} - +\mathrm{ROTANGLE} + 0.798$, for $n = {0,1,2,3}$. The cores of stars that are saturated are masked as well, with a @@ -1263,25 +943,24 @@ \label{sec:masking_fraction} -Although there are a large number of masked pixels within the sixty -OTAs of GPC1, the camera was designed to move chips with problematic -areas (most notably CTE issues) to the edges of the detector. Because -of this, the main analysis of the mask fraction is based not on the -total footprint of the detector, but upon a circular reference field -of view with a radius of 1.5 degrees. This field of view corresponds -approximately to half the width and height of the detector. This -field of view underestimates the unvignetted region of GPC1. A second -``maximum'' field of view is also used to estimate the mask fraction -within a larger 1.628 degree radius. This larger radius includes far -larger missing fractions due to the circular regions outside region -populated with OTAs, but does include the contribution from -well-illuminated pixels that are ignored by the reference radius. - -The results of simulating simulating the footprint of the detector as -a grid of uniformly sized pixels of $0\farcs{}258$ size are provided -in Table \ref{tab:mask fraction}. Both fields of view contain -circular segments outside of the footprint of the detector, which -increase the area estimate that is unpopulated. This category also -accounts for the inter-OTA and inter-cell gaps. The regions with poor -CTE also account for a significant fraction of the masked pixels. The +The GPC1 camera was designed such that where possible, OTAs with CTE +issues were placed towards the edge of the detector. Because of this, +the main analysis of the mask fraction is based not on the total +footprint of the detector, but upon a circular reference field of view +with a radius of 1.5 degrees. This radius corresponds approximately +to half the width and height of the detector. This field of view +underestimates the unvignetted region of GPC1. A second ``maximum'' +field of view is also used to estimate the mask fraction within a +larger 1.628 degree radius. This larger radius includes far larger +missing fractions due to the circular regions outside region populated +with OTAs, but does include the contribution from well-illuminated +pixels that are ignored by the reference radius. + +The results of simulating the footprint of the detector as a grid of +uniformly sized pixels of $0\farcs{}258$ size are provided in Table +\ref{tab:mask fraction}. Both fields of view contain circular +segments outside of the footprint of the detector, which increase the +area estimate that is unpopulated. This category also accounts for +the inter-OTA and inter-cell gaps. The regions with poor CTE also +contribute to a significant fraction of the masked pixels. The remaining mask category accounts for known bad columns, cells that do not calibrate well, and vignetting. There are also a small fraction @@ -1297,10 +976,10 @@ input masks already account for the inter-cell gaps). This estimate does not include the circular segments outside of the detector -footprint. This is minor for the reference field of view (1\% -difference), but does underestimate the static mask fraction for the -maximum radius by 7.3\%. This analysis does provide a the observed -dynamic and advisory mask fractions, which are 0.03\% and 3\% -respectively. The significant advisory value is a result of applying -such masks to all burntool corrected pixels. +footprint. This difference is minor for the reference field of view +(1\% difference), but underestimates the static mask fraction for the +maximum radius by 7.3\%. This analysis provides the observed dynamic +and advisory mask fractions, which are 0.03\% and 3\% respectively. +The significant advisory value is a result of applying such masks to +all burntool corrected pixels. \begin{deluxetable}{lcc} @@ -1326,33 +1005,33 @@ background model for the full OTA is then determined prior to the photometric analysis. The mosaicked image is subdivided into -$800\times{}800$ pixel segments that define each pixel of the -background model, with the segments centered on the image center, and -overlapping adjacent subdivisions by 400 pixels. These overlaps help -smooth the background model, as adjacent model pixels share input +$800\times{}800$ pixel segments that define each superpixel of the +background model, with the superpixels centered on the image center +and overlapping adjacent superpixels by 400 pixels. These overlaps +help smooth the background model, as adjacent model pixels share input pixels. -From each subdivision, 10000 random unmasked pixels are drawn. In the +From each segment, 10000 random unmasked pixels are drawn. In the case where the mask fraction is large (such as on OTAs near the edge of the field of view), and there are insufficient unmasked pixels to meet this criterion, all possible unmasked pixels are used instead. If this number is still small (less than 100 good pixels), the -subdivision does not have a background model calculated, and instead, +superpixel does not have a background model calculated. Instead, the value assigned to that model pixel is set as the average of the adjacent model pixels. This allows up to eight neighboring background values to be used to patch these bad pixels. -For the remaining subdivisions that have sufficient unmasked pixels -for the background to be measured, the pixel values are used to -calculate a set of robust statistics for the initial background guess. -The minimum and maximum of the values are found, and checked to ensure +For the subdivisions that have sufficient unmasked pixels for the +background to be measured, the pixel values are used to calculate a +set of robust statistics for the initial background guess. The +minimum and maximum of the values are found, and checked to ensure that these are not the same value, which would indicate some problem with the input values. The values are then inserted into a histogram -with 1000 bins between the minimum and maximum values, and again -checked for issues with the inputs by ensuring that the bin with the -most input pixels does not contain more than half of the input values. -In this case, the minimum and maximum do not constrain the true -distribution of the input values well, and any values outside of the -20 bins closest to the bin with the peak are masked for future -consideration. A cumulative distribution is then constructed from the +with 1000 bins between the minimum and maximum values. If the bin +with the most input pixels contains more than half of the input +values, the bin size is too coarse for the population of interest. In +this case, a new histogram is constructed using a range corresponding +to the 20 bins closes to the peak, again dividing the range into 1000 +bins. This process is iterated up to 20 times until a binsize is +determined. A cumulative distribution is then constructed from the histogram, which saves the computational cost of sorting all the input values. The bins containing the 50-percentile point, as well as the @@ -1368,11 +1047,11 @@ If this measured standard deviation is smaller than 3 times the bin size, then all points more than 25 bins away from the calculated -median are masked, and the process is repeated until the bin size is -sufficiently small to ensure that the distribution width is well -sampled. Once this iterative process converges, or 20 iterations are -run, the 25- and 75-percentile values are found by interpolating the 5 -bins around the expected bin as well, and the count of the number of -input values within this inner 50-percentile region, $N_{50}$ is -calculated. +median are masked, and the process is repeated with a new 1000 bin +histogram until the bin size is sufficiently small to ensure that the +distribution width is well sampled. Once this iterative process +converges, or 20 iterations are run, the 25- and 75-percentile values +are found by interpolating the 5 bins around the expected bin as well, +and the count of the number of input values within this inner +50-percentile region, $N_{50}$ is calculated. These initial statistics are then used as the starting guesses for a @@ -1380,8 +1059,8 @@ distribution with a Gaussian. All pixels that were masked in the initial calculation are unmasked, and a histogram is again constructed -of the values, with a bin size set to $\sigma_{guess} / \left( N_{50} / +from the values, with a bin size set to $\sigma_{guess} / \left( N_{50} / 500 \right)$. With this bin size, we expect that a bin at $\pm 2 \sigma$ will have approximately 50 input points, which gives a -Poissonian signal to noise estimate around 7. In the case where +Poissonian signal-to-noise estimate around 7. In the case where $N_{50}$ is small (due to a poorly populated input image), this bin size is fixed to be no larger than the guess of the standard @@ -1393,5 +1072,5 @@ Two second order polynomial fits are then performed to the logarithm of the histogram counts set at the midpoint of each bin. The first -fit considers the ``lower half'' of the distribution, under the +fit considers the lower portion of the distribution, under the assumption that deviations from a normal distribution are caused by real astrophysical sources that will be brighter than the true @@ -1407,18 +1086,19 @@ fit. The Gaussian mean and standard deviation are calculated from the polynomial coefficients, and the symmetric fit results are accepted -unless the lower-half fit results in a smaller mean. This process is -repeated again if the calculated standard deviation is not larger than -75\% of the initial guess (suggesting an issue with the initial bin -size). +unless the lower-half fit results in a smaller mean. This histogram +and polynomial fit process is repeated again, with updated bin size +based on the previous iteration standard deviation, if the calculated +standard deviation is not larger than 75\% of the initial guess +(suggesting an issue with the initial bin size). With this two-stage calculation performed across all subdivisions of the mosaicked OTA image, and missing model pixels filled with the average of their neighbors, the final background model is stored on -disk as a $13\times{}13$ image with header entries listing the binning -used. The full scale background image is then constructed by -bilinearly interpolating this binned model, and this is subtracted -from the science image. Each object in the photometric catalog has a -SKY and SKY\_SIGMA value that is the evaluation of this model at the -location of that object. +disk as a $13\times{}13$ image for the GPC1 chips with header entries +listing the binning used. The full scale background image is then +constructed by bilinearly interpolating this binned model, and this is +subtracted from the science image. Each object in the photometric +catalog has a SKY and SKY\_SIGMA value determined from the background +model mean and standard deviation. Although this background modeling process works well for most of the @@ -1431,40 +1111,339 @@ of GPC1, the measured background was added back to the \IPPstage{chip} stage images, but this special processing was not used for the large -scale $3\Pi$ PV3 reduction. +scale $3\pi$ PV3 reduction. + +\subsection{Burntool / Persistence effect} +\label{sec:burntool} + +Pixels that approach the saturation point on GPC1, which varies by +cell with common values around 60000 DN, introduce ``persistent +charge'' on that and subsequent images. During the read out process +of a cell with such a bright pixel, some of the charge remains in the +undepleted region of the silicon and is not shifted down the detector +column toward the amplifier. This charge remains in the starting +pixel and slowly leaks out of the undepleted region, contaminating +subsequent pixels during the read out process, resulting in a ``burn +trail'' that extends from the center of the bright source away from +the amplifier (vertically along the pixel columns toward the top of +the cell). + +This incomplete charge shifting in nearly full wells continues as each +row is read out. This results in a remnant charge being deposited in +the pixels that the full well was shifted through. In following +exposures, this remnant charge leaks out, resulting in a trail that +extends from the initial location of the bright source on the previous +image towards the amplifier (vertically down along the pixel column). +This remnant charge can remain on the detector for up to thirty +minutes. + +Both of these types of persistence trails are measured and optionally +repaired via the \IPPprog{burntool} program. This program does an +initial scan of the image, and identifies objects with pixel values +higher than a conservative threshold of 30000 DN. The trail from the +peak of that object is fit with a one-dimensional power law in each +pixel column above the threshold, based on empirical evidence that +this is the functional form of this persistence effect. This fit also +matches the expectation that a constant fraction of charge is +incompletely transferred at each shift beyond the persistence +threshold. Once the fit is done, the model can be subtracted from +the image. The location of the source is stored in a table along +with the exposure PONTIME, which denotes the number of seconds since +the detector was last powered on and provides an internally +consistent time scale. + +For subsequent exposures, the table associated with the previous image +is read in, and after correcting trails from the stars on the new +image, the positions of the bright stars from the table are used to +check for remnant trails from previous exposures on the image. These +are fit and subtracted using a one-dimensional exponential model, +again based on empirical studies. The output table retains this +remnant position for 2000 seconds after the initial PONTIME recorded. +This allows fits to be attempted well beyond the nominal lifetime of +these trails. Figure \ref{fig:burntool images} shows an example of a +cell with a persistence trail from a bright star, the post-correction +result, as well as the pre and post correction versions of the same +cell on the subsequent exposure. The profiles along the detector +columns for these two exposures are presented in Figure +\ref{fig:burntool plot}. + +Using this method of correcting the persistence trails has the +challenge that it is based on fits to the raw image data, which may +have other signal sources not determined by the persistence effect. +The presence of other stars or artifacts in the detector column can +result in a poor model to be fit, resulting in either an over- or +under-subtraction of the trail. For this reason, the image mask is +marked with a value indicating that this correction has been applied. +These pixels are not fully excluded, but they are marked as suspect, +which allows them to be excluded from consideration in subsequent +stages, such as image stacking. + +The cores of very bright stars can also be deformed by this process, +as the burntool fitting subtracts flux from only one side of the star. +As most stars that result in persistence trails already have saturated +cores, they are already ignored for the purpose of PSF determination +and are flagged as saturated by the photometry reduction. + +\begin{figure} + \centering + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_nobt.png} + \end{minipage}% + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_nobt.png} + \end{minipage} + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_bt.png} + \end{minipage}% + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_bt.png} + \end{minipage} + \caption{Example of OTA11 cell xy50 on exposures o5677g0123o (left) and o5677g0124o (right). The top panels show the image with all appropriate detrending steps, but without burntool, and the bottom show the same with burntool applied. There is some slight over subtraction in fitting the initial trail, but the impact of the trail is greatly reduced in both exposures.} + \label{fig:burntool images} +\end{figure} + + +\begin{figure} + \centering + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0123o_XY11_bt_trail.png} + \end{minipage}% + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5677g0124o_XY11_bt_trail.png} + \end{minipage} + + \caption{Example of a profile cut along the y-axis through a bright star on exposure o5677g0123o OTA11 in cell xy50 (left panel) and on the subsequent exposure o5677g0124o (right panel). In both figures, the green points show the image corrected with all appropriate detrending steps, but without burntool applied, illustrating the amplitude of the persistence trails. The red points show the same data after the burntool correction, which reduces the impact of these features. Both exposures are in the \gps{} filter with exposure times of 43s} + \label{fig:burntool plot} +\end{figure} + +\subsection{Non-linearity Correction} +\label{sec:nonlinearity} + +The pixels of GPC1 are not uniformly linear at all flux levels. In +particular, at low flux levels, some pixels have a tendency to sag +relative to the expected linear value. This effect is most pronounced +along the edges of the detector cells, although some entire cells show +evidence of this effect. + +To correct this sag, we studied the behavior of a series of flat +frames for a ramp of exposure times with approximate logarithmically +equal spacing between 0.01s and 57.04s. As the exposure time +increases, the signal on each pixel also increases in what is expected +to be a linear manner. Each of the flat exposures in this ramp is +overscan corrected, and then the median is calculated for each cell, +as well as for the rows and columns within ten pixels of the edge of +the science region. From these median values at each exposure time +value, we can construct the expected trend by fitting a linear model +for the region considered. This fitting was limited to only the range +of fluxes between 12000 and 38000 counts, as these ranges were found +to match the linear model well. This range avoids the non-linearity +at low fluxes, as well as the possibility of high-flux non-linearity +effects. + +We store the average flux measurement and deviation from the linear +fit for each exposure time for each region on all detector cells in +the linearity detrend look up tables. An example of this data is +shown in figure \ref{fig: nonlinearity}. When this correction is +applied to science data, these lookup tables are loaded, and a linear +interpolation is performed to determine the correction needed for the +flux in that pixel. This look up is performed for both the row and +column of each pixel, to allow the edge correction to be applied where +applicable, and the full cell correction elsewhere. The average of +these two values is then applied to the pixel value, reducing the +effects of pixel nonlinearity. + +This non-linearity effect appears to be stable in time for the +majority of the detector pixels, with little evident change over the +survey duration. However, as the non-linearity is most pronounced at +the edges of the detector cells, those are the regions where the +correction is most likely to be incomplete. Because of this fact, +most pixels in the static mask with either the DARKMASK or FLATMASK +bit set are found along these edges. As the non-linearity correction +is unable to reliably restore these pixels, they produce inconsistent +values after the dark and flat have been applied, and are therefore +rejected. + +\begin{figure} + \centering + \includegraphics[width=0.9\hsize,angle=0,clip]{images/linearity_XY27_xy16.png} + \caption{Example plot of the linearity correction as a fraction of observed flux for OTA27, cell xy16.} + \label{fig: nonlinearity} +\end{figure} + +\subsection{Pattern correction} +\label{sec:pattern} + +\subsubsection{Pattern Row} +%% Statistics so I have them written down somewhere +%% chipProcessedImfile.bg/bg_stdev by filter for XY33 (a good chip) +%% filter bg_mean stdev median Qsig bg_stdev_mean stdev median Qsig +%% g 36.37422026669 64.64175104057 32.693 6.10284 14.696938349131 78.80460307171 8.8401 0.5417843 +%% r 200.96143304525 471.87743546238 117.105 94.55608 33.854672792146 79.01642728089 13.4564 5.3771355 +%% i 447.00504994458 938.38517801037 286.810 154.71397 57.298335510188 99.38392923935 20.0217 24.2254723 +%% z 317.54933679054 390.38930252748 241.014 114.13316 48.359069000176 94.44452756094 17.9404 9.1535209 +%% y 371.09019536218 293.57439970375 288.481 133.38769 43.724342221691 135.04286534327 19.9029 7.5396461 + +As discussed above in the dark and noisemap sections, certain +detectors have significant bias offsets between adjacent rows, caused +by drifts in the bias level due to cross talk. The magnitude of these +offsets increases as the distance from the readout amplifier and +overscan region increases, resulting in horizontal streaks that are +more pronounced along the large x pixel edge of the cell. As the +level of the offset is apparently random between exposures, the dark +correction cannot fully remove this structure from the images, and the +noisemap value only indicates the level of the average variance added +by these bias offsets. Therefore, we apply the PATTERN.ROW correction +in an attempt to mitigate the offsets and correct the image values. +To force the rows to agree, a second order clipped polynomial is fit +to each row in the cell. Four fit iterations are run, and pixels +$2.5\sigma$ deviant are excluded from subsequent fits, to minimize the +effect stars and other astronomical signals have. This final trend is +then subtracted from that row. Simply doing this subtraction will +also have the effect of removing the background sky level. To prevent +this, the constant and linear terms for each row are stored, and +linear fits are made to these parameters as a function of row, +perpendicular to the initial fits. This produces a plane that is +added back to the image to restore the background offset and any +linear ramp that exists in the sky. + +These row-by-row variations have the largest impact on data taken in +the \gps{} filter, as the read noise is the dominant noise source in +that filter. At longer wavelengths, the noise from the Poissonian +variation in the sky level increases. The PATTERN.ROW correction is +still applied to data taken in the other filters, as the increase in +sky noise does not fully obscure the row-by-row noise. + +This correction was required on all cells on all OTAs prior to +2009-12-01, at which point a modification of the camera electronics +reduced the scale of the row-by-row offsets for the majority of the +OTAs. \czw{describe modification} As a result, we only apply this +correction to the cells where it is still necessary, as shown in +Figure \ref{fig: pattern row cells}. A list of these cells is in +Table \ref{tab:pattern_row_cells}. + +Although this correction largely resolves the row-by-row offset issue +in a satisfactory way, large and bright astronomical objects can bias +the fit significantly. This results in an oversubtraction of the +offset near these objects. As the offsets are calculated on the pixel +rows, this oversubtraction is not uniform around the object, but is +preferentially along the horizontal x axis of the object. Most +astronomical objects are not significantly distorted by this, with +this only becoming on issue for only bright objects comparable to the +size of the cell (598 pixels = 150"). Figure \ref{fig: pattern row example} +shows an example of a cell pre- and post-correction. + +\begin{deluxetable}{lcccc} + \tablecolumns{3} + \tablewidth{0pc} + \tablecaption{Cells which have PATTERN.ROW correction applied} + \tablehead{\colhead{OTA} & \colhead{Cell columns} & \colhead{Additional cells}} + \startdata + OTA11 & & xy02, xy03, xy04, xy07 \\ + OTA14 & & xy23 \\ + OTA15 & 0 & \\ + OTA27 & 0, 1, 2, 3, 7 & \\ + OTA31 & 7 & \\ + OTA32 & 3, 7 & \\ + OTA45 & 3, 7 & \\ + OTA47 & 0, 3, 5, 7 & \\ + OTA57 & 0, 1, 2, 6, 7 & \\ + OTA60 & & xy55 \\ + OTA74 & 2, 7 & \\ + \enddata + \label{tab:pattern_row_cells} +\end{deluxetable} + +\begin{figure} + \centering + \includegraphics[width=0.9\hsize,angle=0,clip]{images/pattern_row_edit.png} + \caption{Diagram illustrating in red which cells on GPC1 require the PATTERN.ROW correction to be applied. The footprint of each OTA is outlined, and cell xy00 is marked with either a filled box or an outline. The labeling of the non-existent corner OTAs is provided to orient the focal plane.} + \label{fig: pattern row cells} +\end{figure} + +\begin{figure} + \centering + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_XY57_nopat.png} + \end{minipage}% + \begin{minipage}{0.45\hsize} + \includegraphics[width=0.9\hsize,angle=0,clip]{images/o5379g0103o_XY57_pat.png} + \end{minipage} + \caption{Example of the PATTERN.ROW correction on exposure o5379g0103o OTA57 cell xy01 (\ips{} filter 45s). The left panel shows the cell with all appropriate detrending except the PATTERN.ROW, and the right shows the same cell with PATTERN.ROW applied. The correction reduces the correlated noise on the right side, which is most distant from the read out amplifier. There is a slight over subtraction along the rows near the bright star.} + \label{fig: pattern row example} +\end{figure} + +\subsubsection{Pattern Continuity} + +The background levels of cells on a single OTA do not always have the +same value. Even with dark and flat corrections applied, adjacent +cells may not match. In addition, studies of the background level +indicate that the row-by-row bias can introduce small background +gradient variations along the rows of the cells that are not stable. +This common feature across the columns of cells results in a ``saw +tooth'' pattern horizontally across an the mosaicked OTA, and as the +background model fits a smooth sky level, this induces over and under +subtraction at the cell boundaries. + +The PATTERN.CONTINUITY correction, attempts to match the edges of a +cell to those of its neighbors. For each cell, a thin box 10 pixels +wide running the full length of each edge is extracted and the median +value of unmasked values calculated for that box. These median values +are then used to construct a vector of the sum of the differences +between that cell's edges and the corresponding edge on any adjacent +cell $\Delta$. A matrix $A$ of these associations is also +constructed, with the diagonal containing the number of cells adjacent +to that cell, and the off-diagonal values being set to -1 for each +pair of adjacent cells. The offsets needed for each chip, $x$ can +then be found by solving the system $A x = \Delta$. A cell with the +maximum number of neighbors, usually cell xy11, the first cell not on +the edge of the OTA, is used to constrain the system, ensuring that +that cell has zero correction and that there is a single solution. + +For OTAs that initially show the saw tooth pattern, the effect of this +correction is to align the cells into a single ramp, at the expense of +the absolute background level. However, as we subtract off a smooth +background model prior to doing photometry, these deviations from an +absolute sky level do not affect photometry for point sources and +extended sources smaller than a single cell. The fact that the +final ramp is smoother than it would be otherwise also allows for the +background subtracted image to more closely match the astronomical +sky, without significant errors at cell boundaries. An example of the +effect of this correction on an image profile is shown in Figure +\ref{fig:dark switching}. \section{GPC1 Detrend Construction} \label{sec:detrend construction} -The various detrends for GPC1 are constructed in similar ways. A -series of appropriate exposures is selected from the database, and -processed with the \IPPprog{ppImage} program. This program is used -for the \IPPstage{chip} stage processing as well, and is designed to -do multiple image processing operations. The extent of this -processing is dependent on the order in which the detrend to be -constructed is applied to science data. In general, the input -exposures to the detrend have all prior stages of detrend processing -applied. Table \ref{tab:detrend ppImage} summarizes stages applied -for the detrends we construct. +The various master detrend images for GPC1 are constructed using a +common approach. A series of appropriate exposures is selected from +the database, and processed with the \IPPprog{ppImage} program, which +is designed to do multiple image processing operations. The +processing steps applied to the images depend on the type of master +detrend to be constructed. In general, the input exposures to the +detrend have all prior stages of detrend processing applied. Table +\ref{tab:detrend ppImage} summarizes stages applied for the detrends +we construct. Once the input data has been prepared, the \IPPprog{ppMerge} program -is used to construct some sort of ``average'' of the inputs. This -step need not be a mathematical average, but is used to combine the -signal from the individual exposures into a single output product. -Table \ref{tab:detrend ppMerge} lists some of the properties of the -process for the detrends, including how discrepant values are removed -and the combination method used. The outputs from this step have the -format of the detrend under construction, and after construction, are -applied to the processed input data. This creates a set of residual -files that are checked to determine if the newly created detrend -correctly removes the detector dependent signal. +is used to combine the inputs. In some cases, this is the +mathematical average, but in other cases it is a fit across the +inputs. Table \ref{tab:detrend ppMerge} lists some of the properties +of the process for the detrends, including how discrepant values are +removed and the combination method used. The outputs from this step +have the format of the detrend under construction. After +construction, these combined outputs are applied to the processed +input data. This creates a set of residual files that are checked to +determine if the newly created detrend correctly removes the detector +dependent signal. This process of detrend construction and testing can be iterated, with individual exposures excluded if they are found to be contaminating -the output. If the final detrend has sufficiently small residuals, -then the iterations are stopped and the detrend is finalized by -selecting the date range to which it applies. This allows subsequent -science processing to select the detrends needed based on the -observation date. Table \ref{tab:detrend list} lists the set of -detrends used in the PV3 processing. +the output. The construction of detrends is largely automatic, but +manual intervention is needed to accept the detrend for use on science +data. If the final detrend has sufficiently small residuals, then the +iterations are stopped and the detrend is finalized by selecting the +date range to which it applies. This allows subsequent science +processing to select the detrends needed based on the observation +date. Table \ref{tab:detrend list} lists the set of detrends used in +the PV3 processing. \begin{deluxetable}{lcccc} @@ -1475,11 +1454,15 @@ \startdata LINEARITY & Y & & & \\ +%% DARKMASK & Y & Y & Y & \\ +%% FLATMASK & Y & Y & Y & Y \\ +%% CTEMASK & Y & Y & Y & Y \\ + DARK & Y & Y & & \\ +%% NOISEMAP & Y & Y & & \\ + FLAT & Y & Y & Y & \\ + FRINGE & Y & Y & Y & Y \\ DARKMASK & Y & Y & Y & \\ FLATMASK & Y & Y & Y & Y \\ CTEMASK & Y & Y & Y & Y \\ - DARK & Y & Y & & \\ NOISEMAP & Y & Y & & \\ - FLAT & Y & Y & Y & \\ - FRINGE & Y & Y & Y & Y \\ \enddata \label{tab:detrend ppImage} @@ -1551,56 +1534,57 @@ \section{Warping} \label{sec:warping} -To provide a consistent and uniform set of images for co-added image -stacking and differences, the individual mosaicked OTA images are -projected onto a common set of tangent plane projected regions called -projection cells. These projection cells are $4\times{}4$ degree -fields spaced onto a set of centers that fully cover the sky. They are -arranged into rings of constant declination, and allowed to overlap as -$|\delta|$ increases. Each projection cell is further subdivided into -$10\times{}10$ sky cells with fixed $0.25"$ resolution pixels, and -constant overlap regions between adjacent skycells of $60"$. These -skycells are the main image unit used for processing image data beyond -the initial chip stage. The coordinate system used for these images -matches the parity of the sky, with north in the positive y direction -and east to the negative x direction. +To provide a consistent and uniform set of coordinates for image +combination (including stacking and differences), the individual +mosaicked OTA images are projected onto a common pixel grids, called +tessellations. A tessellation can contain any number of tangent plane +projections, with those designed for single pointing surveys using +only one, while the tessellation used for the $3\pi$ survey contains +2643 tangent plane projection centers. These ``projection cells'' are +$4\times{}4$ degree fields spaced onto a set of centers that fully +cover the sky. They are arranged into rings of constant declination, +and allowed to overlap as $|\delta|$ increases. Each projection cell +is further subdivided into $10\times{}10$ ``skycells'' with fixed +$0.25"$ resolution pixels, and constant overlap regions between +adjacent skycells of $60"$. These skycells are the main image unit +used for processing image data beyond the initial chip stage. The +coordinate system used for these images matches the parity of the sky, +with north in the positive y direction and east to the negative x +direction. After the detrending and photometry, the detection catalog for the -full camera is fit to the reference catalog, producing third-order -astrometric solutions that map the detector focal plane to the sky, -and map the individual OTA pixels to the detector focal plane. This -solution is then used to determine which skycells the exposure OTAs -overlap. +full camera is fit to the reference catalog, producing astrometric +solutions that map the detector focal plane to the sky, and map the +individual OTA pixels to the detector focal plane +\citep[see][]{magnier2017.calibration}. This solution is then used to +determine which skycells the exposure OTAs overlap. For each output skycell, all overlapping OTAs and the calibrated -catalog are read into the \IPPprog{pswarp} program. Each input image -is examined in order, and the same transformation performed. This -transformation breaks the output warp image into $128\times{}128$ -pixel grid boxes. Each grid box has a locally linear map calculated -that converts the output warp image coordinates to the input chip -image coordinates. By doing the transformation in this direction, -each output pixel has a unique sampling position on the input image +catalog are read into the \IPPprog{pswarp} program. The output warp +image is broken into $128\times{}128$ pixel grid boxes. For purposes +of speed, each grid box has a locally linear map calculated that +converts the output warp image coordinates to the input chip image +coordinates. By doing the transformation in this direction, each +output pixel has a unique sampling position on the input image (although it may be off the image frame and therefore not populated), preventing gaps in the output image due to the spacing of the input pixels. -With the locally linear grid defined, Lanczos interpolation with -filter size parameter $a = 3$ on the input image is used to determine -the values to assign to the output pixel location. The output -locations are shifted by 0.5 pixels to let the interpolation select -the value that would be assigned to the center of the output -pixel. This process is repeated for all grid boxes, for all input +With the locally linear grid defined, Lanczos interpolation +\citep{lanczos1956applied} with filter size parameter $a = 3$ on the input +image is used to determine the values to assign to the output pixel +location. This process is repeated for all grid boxes, for all input images, and for each output image product: the science image, the variance, and the mask. The image values are scaled by the absolute -value of the Jacobian determinant of the transformation. This -corrects the pixel values for the possible change in pixel area due to -the transformation. Similarly, the variance image is scaled by the -square of this value, again to correctly account for the pixel area -change. - -As the interpolation constructs the output pixels from more than one -input pixel, there is a covariance term that is must be included. For -each locally linear grid box, the covariance is calculated from the +value of the Jacobian determinant of the transformation for each grid +box. This corrects the pixel values for the possible change in pixel +area due to the transformation. Similarly, the variance image is +scaled by the square of this value, again to correctly account for the +pixel area change. + +The interpolation constructs the output pixels from more than one +input pixel, which introduces covariance between pixels. For each +locally-linear grid box, the covariance matrix is calculated from the kernel in the center of the 128 pixel range. Once the image has been -fully populated, this set of individual covariance matrices is +fully populated, this set of individual covariance matrices are averaged to create the final covariance for the full image. @@ -1608,17 +1592,17 @@ catalog, including only those objects that fall on the new warped image. These detections are transformed to match the new image location, and -to scale the position errors based on the new orientation. +to scale the position uncertainties based on the new orientation. The output image also contains header keywords SRC\_0000, SEC\_0000, -MPX\_0000, and MPY\_0000 that contain the mappings from the warped +MPX\_0000, and MPY\_0000 that define the mappings from the warped pixel space to the input image. The SRC keyword lists the input OTA -name, and the SEC keyword lists the image section corresponding to the -locally linear grid box. The MPX and MPY contain the transformation -parameters for the locally linear grid. These parameters are stored -in a string listing the reference position in the chip coordinate -frame, the slope of the relation in the warp x axis, and the slope of -the relation in the warp y axis. From these keywords, any position in -the warp can be mapped back to the location in any of the input OTA -images. +name, and the SEC keyword lists the image section that the mapping +covers. The MPX and MPY contain the back-transformation linearized +across the full chip. These parameters are stored in a string listing +the reference position in the chip coordinate frame, the slope of the +relation in the warp x axis, and the slope of the relation in the warp +y axis. From these keywords, any position in the warp can be mapped +back to the location in any of the input OTA images, with some +reduction in accuracy. \begin{figure} @@ -1671,17 +1655,18 @@ Once individual exposures have been warped onto a common projection -system, they can then be combined pixel-by-pixel regardless of their +system, they can be combined pixel-by-pixel regardless of their original orientation. Creating a stacked image by co-adding the individual warps increases the signal to noise, allowing for the -detection of objects that would not be sufficiently significant to be measured from a single image. -Creating this stack also allows a complete image to be -constructed that does not have regions masked due to the gaps between -cells and OTAs. This fully populated static sky image can also be -used as a template for subtraction to find transient sources. - -The stacked image is comprised of all warp frames for a given skycell -in a single filter. The source catalogs and image components are -loaded into the \IPPprog{ppStack} program to prepare the inputs and -stack the frames. +detection of objects that would not be sufficiently significant to be +measured from a single image. Creating this stack also allows a more +complete image to be constructed that has fewer regions masked due to +the gaps between cells and OTAs. This deeper and more complete image +can also be used as a template for subtraction to find transient +sources. + +For the $3\pi$ survey, the stacked image is comprised of all warp +frames for a given skycell in a single filter. The source catalogs +and image components are loaded into the \IPPprog{ppStack} program to +prepare the inputs and stack the frames. Once all files are ingested, the first step is to measure the size and @@ -1689,5 +1674,5 @@ FWHM greater than 10 pixels (2.5 arcseconds), as those images have the seeing far worse than average, and would degrade the final output -stack. For the PV3 $3\Pi$ survey, this size represents a PSF larger +stack. For the PV3 $3\pi$ survey, this size represents a PSF larger than the $97$th percentile in all filters. A target PSF for the stack is constructed by finding the maximum envelope of all input PSFs, @@ -1697,23 +1682,24 @@ input images when matched to the target. -The input images also need to have their fluxes normalized to prevent -differences in seeing and sky transparency from causing discrepancies -during pixel rejection. From the reference catalog calibrated input -catalogs, we have the instrumental magnitudes of all sources, along -with the airmass, image exposure time, and zeropoint. All output -stacks are calibrated to a zeropoint of 25.0 in all filters, and to -have an airmass of 1.0. The output exposure time is set to the sum of -the input exposure times, regardless of if those inputs are rejected -later in the combination process. We can determine the relative +The input image fluxes are normalized to prevent differences in seeing +and sky transparency from causing discrepancies during pixel +rejection. From the reference catalog calibrated input catalogs, we +have the instrumental magnitudes of all sources, along with the +airmass, image exposure time, and zeropoint. All output stacks are +constructed to a target zeropoint of 25.0 in all filters, and to have +an airmass of 1.0. The output exposure time is set to the sum of the +input exposure times, regardless of whether those inputs are rejected later +in the combination process. We can determine the relative transparency for each input image by comparing the magnitudes of matched sources between the different images. Each image then has a -normalization factor defined, equal to $\mathrm{norm}_{input} = (ZP_\mathrm{input} -- ZP_\mathrm{target}) - \mathrm{transparency}_\mathrm{input} - 2.5 * -\log_{10} (t_\mathrm{target} / t_\mathrm{input}) - -\mathrm{F}_\mathrm{airmass} * (\mathrm{airmass}_\mathrm{input} - -\mathrm{airmass}_\mathrm{target})$. For the PV3 processing, the -airmass factor $\mathrm{F}_\mathrm{airmass}$ was set to zero, such -that all flux differences from differing exposure airmasses are -assumed to be included in the zeropoint and transparency values. +normalization factor defined, equal to $\mathrm{norm}_{input} = +(ZP_\mathrm{input} - ZP_\mathrm{target}) - +\mathrm{transparency}_\mathrm{input} - 2.5 * \log_{10} +(t_\mathrm{target} / t_\mathrm{input}) - \mathrm{F}_\mathrm{airmass} * +(\mathrm{airmass}_\mathrm{input} - \mathrm{airmass}_\mathrm{target})$. +For the PV3 processing, the airmass factor +$\mathrm{F}_\mathrm{airmass}$ was set to zero, such that all flux +differences from differing exposure airmasses are assumed to be +included in the zeropoint and transparency values. The zeropoint calibration performed here uses the calibration of the @@ -1724,5 +1710,5 @@ the entire region of the sky imaged. This further calibration is not available at the time of stacking, and so there may be small residuals -in the transparency values as a result of this \citet{magnier2017c}. +in the transparency values as a result of this \citet{magnier2017.calibration}. With the flux normalization factors and target PSF chosen, the @@ -1733,5 +1719,5 @@ the kernel, and the residual with the target PSF used to update the parameters of the kernel via least squares optimization. Stamps that -significantly deviate are rejected, but as the squared residual +significantly deviate are rejected, although the squared residual difference will increase with increasing source flux. To mitigate this effect, a parabola is fit to the distribution of squared @@ -1741,6 +1727,6 @@ convolution kernel is returned. -This convolution may change the image flux scaling, so a normalization -factor is used to correct this. This normalization factor is equal to +This convolution may change the image flux scaling, so the kernel is +normalized to account for this. The normalization factor is equal to the ratio of $10^{-0.4 \mathrm{norm}_{input}}$ to the sum of the kernel. The image is multiplied by this factor, and the variance by @@ -1749,24 +1735,27 @@ Once the convolution kernels are defined for each image, they are used to convolve the image to match the target PSF. Any input image that -has a kernel match $\chi^2$ value greater than 4.0$\sigma$ larger than -the median value is rejected from the stack. Each image also has a -weight assigned, based on the image variance after convolution. A -full image weight is then calculated for each input, with the weight, -$W_\mathrm{input}$ is equal to the inverse of the median of the image -variance multiplied by the peak of the image covariance (due to the -warping process). +has a kernel match $\chi^2$ value (defined as the sum of the RMS error +across the kernel) 4.0$\sigma$ or larger than the median +value is rejected from the stack. Each image also has a weight +assigned, based on the image variance after convolution. A full image +weight is then calculated for each input, with the weight, +$W_\mathrm{input}$ equal to the inverse of the median of the image +variance multiplied by the peak of the image covariance (from the +warping process). This ensures that low signal-to-noise images are +down-weighted in the final combination. Following the convolution, an initial stack is constructed. For a given pixel coordinate, the values at that coordinate are extracted -from all input images. Images that have a suspect mask bit (including -the SUSPECT, BURNTOOL, SPIKE, STREAK, STARCORE, and CONV.POOR bit -values) are appended to a suspect pixel list for preferential -exclusion. Following this, the pixel values are combined and tested -to attempt to identify discrepant input values that should be excluded. +from all input images, with pixels masked excluded from consideration. +Images that only have a suspect mask bit (including the SUSPECT, +BURNTOOL, SPIKE, STREAK, STARCORE, and CONV.POOR bit values) are +appended to a suspect pixel list for preferential exclusion. +Following this, the pixel values are combined and tested to attempt to +identify discrepant input values that should be excluded. If only a single input is available, the initial stack contains the value from that single input. If there are only two inputs, the average of the two is used. These cases should occur only rarely in -the $3\Pi$ survey, as there are many input exposures that overlap each +the $3\pi$ survey, as there are many input exposures that overlap each point on the sky. For the more common case of three or more inputs, a weighted average from the inputs is used, with the weight for each @@ -1789,21 +1778,22 @@ there were no valid inputs, in which case the BLANK mask bit is set. -Due to the various non-astronomical ghosts that can occur on GPC1, and -the fact that they may not be fully masked to ensure all bad pixels -are removed, it is expected that some of the inputs for a given stack -pixel are not in agreement with the others. In general, there is the -population of input pixel values around the correct astronomical -level, as well as possible populations at lower pixel value (such as -due to an over-subtracted burntool trail) and at higher pixel values -(such as that caused by an incompletely masked optical ghost). Due to -the observation strategy to image a given field twice to allow for +Due to uncorrected artifacts that can occur on GPC1, and the fact that +they may not be fully masked to ensure all bad pixels are removed, it +is expected that some of the inputs for a given stack pixel are not in +agreement with the others. In general, there is the population of +input pixel values around the correct astronomical level, as well as +possible populations at lower pixel value (such as due to an +over-subtracted burntool trail) and at higher pixel values (such as +that caused by an incompletely masked optical ghost). Due to the +observation strategy to observe a given field twice to allow for warp-warp difference images to be constructed to identify transient detections, higher pixel values that come from sources like optical -ghosts that depend on the telescope pointing will come in pairs as well. -The higher pixel value contaminants are also potentially problematic -as they may appear to be real sources, prompting photometry to be -performed on false objects. Because of the expectation that there are -more bright contaminants than faint ones, there is a slight preference -to reject higher pixel values than lower pixel values. +ghosts that depend on the telescope pointing will come in pairs. +Detector artifacts will appear in pairs as well. The higher +pixel value contaminants are also potentially problematic as they may +appear to be real sources, prompting photometry to be performed on +false objects. Because of the expectation that there are more positive +deviations than negative ones, there is a slight preference to reject +higher pixel value outliers than lower pixel values, as described below. Following the initial combination, a ``testing'' loop iterates in an @@ -1818,7 +1808,7 @@ and both are flagged for rejection -If the number of inputs is larger than 6, then a Gaussian mixture -model analysis is run on the inputs to fit two sub populations, and -determine an the likelihood that the distribution is best described by +If the number of input pixels is larger than 6, then a Gaussian mixture +model analysis is run on the inputs fitting two sub populations, to +determine the likelihood that the distribution is best described by an uni-modal model. If this probability is less than $5\%$, then the mean is taken from the bimodal sub population with the largest @@ -1826,24 +1816,24 @@ comprised of high pixel value outliers. -If this is not the case, and the distribution is likely unimodal, or -if there are insufficient inputs for this mixture model analysis, the -input values are passed to an Olympic weighted mean calculation. We -reject $20\%$ of the number of inputs through this process. The -number of bad inputs is set to $N_\mathrm{bad} = 0.2 * -N_\mathrm{input} + 0.5$, with the 0.5 term ensuring at least one input -is rejected. This number is further separated into the number of low -values to exclude $N_\mathrm{low} = N_\mathrm{bad} / 2$, which will -default to zero if there are few inputs, and $N_\mathrm{high} = -N_\mathrm{input} + N_\mathrm{low} - N_\mathrm{bad}$. After sorting -the input values to determine which values fall into the low and high -groups, the remaining input values are used in a weighted mean using -the image weights above. +If the unimodal probability is greater than $5\%$ (indicating the +distribution is likely to be unimodal), or if there are insufficient +inputs for this mixture model analysis, the input values are passed to +an Olympic weighted mean calculation. We reject $20\%$ of the number +of inputs through this process. The number of bad inputs is set to +$N_\mathrm{bad} = 0.2 * N_\mathrm{input} + 0.5$, with the 0.5 term +ensuring at least one input is rejected. This number is further +separated into the number of low values to exclude, $N_\mathrm{low} = +N_\mathrm{bad} / 2$, which will default to zero if there are few +inputs, and $N_\mathrm{high} = N_\mathrm{low} - N_\mathrm{bad}$. +After sorting the input values to determine which values fall into the +low and high groups, the remaining input values are used in a weighted +mean using the image weights above. A systematic variance term is necessary to correctly scale how discrepant points can be from the ensemble mean. If the mixture model -analysis was run, the Gaussian sigma from the largest sub population -is squared and used. If this is not available, a $10\%$ systematic -error on the input values is used. Each point then has a limit -calculated using a $4\sigma$ rejection +analysis has been run, the Gaussian sigma from the largest sub +population is squared and used. Otherwise, a $10\%$ systematic error +on the input values is used. Each point then has a limit calculated +using a $4\sigma$ rejection \begin{eqnarray} @@ -1856,6 +1846,6 @@ \mathrm{mean})^2$ exceeding this limit is identified. If there are suspect pixels in the set, those pixels are marked for rejection, -otherwise this worst pixel is marked for rejection. Following this, -the combine and test loop is repeated for until no more pixels are +otherwise this worst pixel is marked for rejection. Following this step, +the combine and test loop is repeated until no more pixels are rejected, up to a maximum number of iterations equal to $50\%$ of the number of inputs. @@ -1870,8 +1860,8 @@ the entire image is rejected as it likely has some systematic issue. -Finally, a second pass at rejecting pixels is conducted, by growing the -current list to include pixels that are neighbors to many rejected +Finally, a second pass at rejecting pixels is conducted, by extending +the current list to include pixels that are neighbors to many rejected pixels. The ISIS kernel used in the previous step is again used to -determine the largest square box that contains under the limit of +determine the largest square box that does not exceed the limit of $0.25 * \sum_{x,y} kernel^2$. This square box is then convolved with the rejected pixel mask to reject the neighboring pixels. This final @@ -1883,40 +1873,36 @@ a map of the number of inputs per pixel. -These convolved stack products are not retained, as the convolution -reduces the resolution of the final image. Instead, we apply the -normalizations and rejected pixel maps generated from the convolved -stack process to the original unconvolved input images. This produces -an unconvolved stack that has the optimum image quality possible from -the input images. Not convolving does mean that the PSF shape changes -across the image, as the different PSF widths of the input images -print through in the different regions to which they have contributed. +These convolved stack products are not retained, as the convolution is +used to ensure that the pixel rejection uses seeing-matched images. +This prevents any differences in the input PSF shape from skewing the +input pixel rejection. We apply the normalizations and rejected pixel +maps generated from the convolved stack process to the original +unconvolved input images. This produces an unconvolved stack that has +the optimum image quality possible from the input images. Not +convolving does mean that the PSF shape changes across the image, as +the different PSF widths of the input images print through in the +different regions to which they have contributed. %% Asinh compression -Due to the expected large range of data values in the final stacked -image, saving them as compressed 16-bit integer images with linear -BSCALE and BZERO scaling values is likely to offer poor -reconstructions of the stacked image. This will lead either to -truncation of the extrema of the image, or quantized values that are -poorly spaced for the image histogram. Saving the images as 32-bit -floating point values would alleviate this quantization issue, at the -cost of a large increase in the disk space required for the stacked -images. - -Transforming the data prior to writing to disk by taking the logarithm -of the pixel values can resolve this, with the complication that all -data values must first be made positive, which then sets the highest -quantization sampling near the lowest values in the image. Following -techniques used by SDSS \citep{2000AJ....120.1579Y}, we have instead opted to use the -inverse hyperbolic sine function to transform the data. The domain of -this function allows any input value to be converted. In addition, -the quantization sampling can be tuned by placing the zero of the -inverse hyperbolic sine function at a value where the highest sampling -is desired. +While IPP image products from single exposures use compressed 16-bit +integer images, this dynamic range is insufficient for the expected +scale of the stacked images. This will lead either to truncation of +the extrema of the image, or quantized values that poorly sample the +image noise distribution. Saving the images as 32-bit floating point +values would alleviate this quantization issue, at the cost of a large +increase in the disk space required for the stacked images. + +Inspired by techniques used by SDSS \citep{1999AJ....118.1406L}, we +use the inverse hyperbolic sine function to transform the data. The +domain of this function allows any input value to be converted. In +addition, the quantization sampling can be tuned by placing the zero +of the inverse hyperbolic sine function at a value where the highest +sampling is desired. Formally, prior to being written to disk, the pixel values are transformed by $C = \alpha \asinh\left(\frac{L - \mathrm{BOFFSET}}{2.0 \cdot \mathrm{BSOFTEN}}\right)$, where $L$ are the linear input -pixel values, $C$ the transformed values, $\alpha = 2.5 \log_{10}(e)$. +pixel values, $C$ the transformed values, and $\alpha = 2.5 \log_{10}(e)$. BOFFSET centers the transformed values, and the mean of the linear input pixel values is used. BSOFTEN controls the stretch of the @@ -2020,16 +2006,16 @@ \label{sec:diffs} -Constructing difference images is essentially the same as that used in -the stacking process. An image is chosen as a template, another image -as the input, and after matching sources to determine the scaling and -transparency, convolution kernels are defined that are used to -convolve one or both of the images to a target PSF. The images are -then subtracted, and as they should now share a common PSF, static -sources are largely subtracted (completely in an ideal case), whereas -sources that are not static between the two images leave a significant -remnant. More information on the difference image construction is -contained in \citet{price2017}. The follow section contains a -overview of the difference image construction used for the data in -DR2. +The image matching process used in constructing difference images is +essentially the same as for the stacking process. An image is chosen +as a template, another image as the input, and after matching sources +to determine the scaling and transparency, convolution kernels are +defined that are used to convolve one or both of the images to a +target PSF. The images are then subtracted, and as they should now +share a common PSF, static sources are largely subtracted (completely +in an ideal case), whereas sources that are not static between the two +images leave a significant remnant. More information on the +difference image construction is contained in \citet{price2017}. The +following section contains an overview of the difference image +construction used for the data in DR2. The images used to construct difference images can be either @@ -2045,35 +2031,36 @@ minus stack) and inverse (stack minus warp) to allow for the photometry of the difference image to detect sources that both rise -and fall relative to the stack. Note that the convolution process -grows the mask fraction of pixels relative to the warp (the largest -source of masked pixels in these warp stack differences). Any pixel -that after convolution has any contribution from a masked pixel is -masked as well, ensuring only fully unmasked pixels are used. +and fall relative to the stack. The convolution process grows the +mask fraction of pixels relative to the warp (the largest source of +masked pixels in these warp stack differences). Any pixel that after +convolution has any contribution from a masked pixel is masked as +well, ensuring only fully unmasked pixels are used. For warp-warp differences, such as those used for the ongoing Solar -System moving object search in nightly observations \citep{2013PASP..125..357D}, the -warp that was taken first is used as the template. As there is less -certainty in which of the two input images will have better seeing, a -``dual'' convolution method is used. Both inputs are convolved to a -target PSF that is not identical to either input. This intermediate -target is essential for the case in which the PSFs of the two inputs -have been distorted in orthogonal directions. Simply convolving one -to match the other would require some degree of deconvolution along -one axis. As this convolution method by necessity uses more free -parameters, the ISIS kernels used are chosen to be simpler than those -used in the warp-stack differences. The ISIS widths are kept the same -(1.5, 3.0, 6.0 pixel FWHMs), but each Gaussian kernel is constrained -to only use a second-order polynomial. As with the warp-stack -differences, the mask fraction grows between the input warp and the -final difference image due to the convolution. For the warp-warp -differences, each image mask grows based on the appropriate -convolution kernel, so the final usable image area is highly dependent -on ensuring that the telescope pointings are as close to identical as -possible. The observing strategy to enable this is discussed in more -detail in \citet{chambers2017}. - - - -\section{Discussion} +System moving object search in nightly observations +\citep{2013PASP..125..357D}, the warp that was taken first is used as +the template. As there is less certainty in which of the two input +images will have better seeing, a ``dual'' convolution method is used. +Both inputs are convolved to a target PSF that is not identical to +either input. This intermediate target is essential for the case in +which the PSFs of the two inputs have been distorted in orthogonal +directions. Simply convolving one to match the other would require +some degree of deconvolution along one axis. As this convolution +method by necessity uses more free parameters, the ISIS kernels used +are chosen to be simpler than those used in the warp-stack +differences. The ISIS widths are kept the same (1.5, 3.0, 6.0 pixel +FWHMs), but each Gaussian kernel is constrained to only use a +second-order polynomial. As with the warp-stack differences, the mask +fraction grows between the input warp and the final difference image +due to the convolution. For the warp-warp differences, each image +mask grows based on the appropriate convolution kernel, so the final +usable image area is highly dependent on ensuring that the telescope +pointings are as close to identical as possible. The observing +strategy to enable this is discussed in more detail in +\citet{chambers2017}. + + + +\section{Future Plans} \label{sec:discussion} @@ -2085,5 +2072,5 @@ dependent on focal plane position. -An obvious way to make use of the PV3 catalog is to do a statistical +One obvious way to make use of the PV3 catalog is to do a statistical search for electronic crosstalk ghosts that do not match a known rule. Given that bright stars do not equally populate all fields, choosing @@ -2097,16 +2084,16 @@ There is some evidence that we have not fully identified all of these crosstalk rules, based on a study of PV3 images. For example, -extremely bright stars may be able to create crosstalk ghosts between the second -cell column of OTA01 and OTA21, with possibly fainter ghosts appearing -on OTA11. Despite the symmetry observed in the main ghost rules, -there do not appear to be clear examples of a similar ghost between -OTA47 and OTA66. Examining this further based on the PV3 catalog -should provide a clear answer to this, as well as clarify brightness -limits below which the ghost does not appear. +extremely bright stars may be able to create crosstalk ghosts between +the second cell column of OTA01 and OTA21, with possibly fainter +ghosts appearing on OTA11. Despite the symmetry observed in the main +ghost rules, there do not appear to be clear examples of a similar +ghost between OTA47 and OTA66. Examining this further based on the +PV3 catalog should provide a clear answer to this, as well as clarify +brightness limits below which the ghost does not appear. The PV3 catalog may also allow better determination of which date ranges we should use to build the dark model. The date ranges currently in use are based on limited sampling of exposures, and do -not have strong tests indicating that they are the best. By examining +not have strong tests indicating that they are optimal. By examining the scatter between the detections on a given exposure and the catalog average, we can attempt to look for increases in scatter that might @@ -2143,5 +2130,5 @@ to isolate and remove this signal in the Fourier domain. Preliminary investigations have shown that there is a small peak visible in the -power spectrum of a single cell, but determining the optimal way to +power spectrum of a single cell, but determining the best way to clip this peak to reduce the noise in the image space is not clear. @@ -2150,5 +2137,5 @@ The Pan-STARRS1 PV3 processing has reduced an unprecedented volume of -image data, and has produced a catalog for the $3\Pi$ Survey +image data, and has produced a catalog for the $3\pi$ Survey containing hundreds of billions of individual measurements of three billion astronomical objects. Accurately calibrating Index: /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/Makefile =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/Makefile (revision 40482) +++ /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/Makefile (revision 40483) @@ -3,6 +3,6 @@ # WARNING : pdflatex does not do .ps # -DO_PDFLATEX = 0 -DO_BIBTEX = 1 +DO_PDFLATEX = 1 +DO_BIBTEX = 0 help: @@ -11,20 +11,62 @@ all: pdf tgz -pdf: systematics.pdf -tgz: systematics.tgz +pdf: diffusion.pdf +tgz: diffusion.tgz # if you have PDF pics, they may need to be converted to EPS -PDFPICS = -# pics/peaks.pdf \ -# pics/FWHM.smooth.trend.ps1.pdf \ -# pics/moment.class.pdf \ -# pics/radial.profiles.pdf +PDFPICS = \ +pics/dmag.pdf \ +pics/dapmag.pdf \ +pics/all_effects_r.pdf \ +pics/radial_p1_r.pdf \ +pics/radial_p2_r.pdf \ +pics/radial_p3_r.pdf \ +pics/filter_trends.pdf + +PSPICS = \ +pics/dmag.ps \ +pics/dapmag.ps \ +pics/all_effects_r.ps \ +pics/radial_p1_r.ps \ +pics/radial_p2_r.ps \ +pics/radial_p3_r.ps \ +pics/filter_trends.ps + +EPSPICS = \ +pics/dmag.eps \ +pics/dapmag.eps \ +pics/all_effects_r.eps \ +pics/radial_p1_r.eps \ +pics/radial_p2_r.eps \ +pics/radial_p3_r.eps \ +pics/filter_trends.eps + +OLD_PDFPICS = \ +pics/all_effects_r.pdf \ +pics/astrom_trends.pdf \ +pics/astrom_vs_radius.pdf \ +pics/dapmag.pdf \ +pics/dastrom_vs_flat.pdf \ +pics/dflat.pdf \ +pics/dmag.pdf \ +pics/drad.pdf \ +pics/dsmear_vs_astrom.pdf \ +pics/flat_trends.pdf \ +pics/flat_vs_radius.pdf \ +pics/psfmag_trends.pdf \ +pics/psfmag_vs_radius.pdf \ +pics/shear.pdf \ +pics/smear.pdf \ +pics/smear_trends.pdf \ +pics/smear_vs_astrom.pdf \ +pics/smear_vs_dastrom.pdf \ +pics/smear_vs_psfmag.pdf \ +pics/smear_vs_radius.pdf FILES = \ -../inputs/astro.sty \ ../inputs/code.sty \ ../inputs/apj.bst \ -../inputs/lib.bib \ -systematics.tex +$(PDFPICS) \ +diffusion.tex pics/%.pdf : pics/%.ps @@ -36,6 +78,7 @@ pdfpics: $(PDFPICS) -systematics.pdf: $(FILES) -systematics.tgz: $(FILES) +diffusion.pdf: $(FILES) ../inputs/lib.bib +diffusion.tgz: $(FILES) ../inputs/astro.sty +diffusion.zip: diffusion.tex $(EPSPICS) magnier.tex magnier_bib.tex include ../Makefile.Common Index: /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/antilogus.etal.bib =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/antilogus.etal.bib (revision 40483) +++ /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/antilogus.etal.bib (revision 40483) @@ -0,0 +1,14 @@ + + +@article{1748-0221-9-03-C03048, + author={P Antilogus and P Astier and P Doherty and A Guyonnet and N Regnault}, + title={The brighter-fatter effect and pixel correlations in CCD sensors}, + journal={Journal of Instrumentation}, + volume={9}, + number={03}, + pages={C03048}, + url={http://stacks.iop.org/1748-0221/9/i=03/a=C03048}, + year={2014}, + abstract={We present evidence that spots imaged using astronomical CCDs do not exactly scale with flux: bright spots tend to be broader than faint ones, using the same illumination pattern. We measure that the linear size of spots or stars, of typical size 3 to 4 pixels FWHM, increase linearly with their flux by up to 2 % over the full CCD dynamic range. This brighter-fatter effect affects both deep-depleted and thinned CCD sensors. We propose that this effect is a direct consequence of the distortions of the drift electric field sourced by charges accumulated within the CCD during the exposure and experienced by forthcoming light-induced charges in the same exposure. The pixel boundaries then become slightly dynamical: overfilled pixels become increasingly smaller than their neighbors, so that bright star sizes, measured in number of pixels, appear larger than those of faint stars. This interpretation of the brighter-fatter effect implies that pixels in flat-fields should exhibit statistical correlations, sourced by Poisson fluctuations, that we indeed directly detect. We propose to use the measured correlations in flat-fields to derive how pixel boundaries shift under the influence of a given charge pattern, which allows us in turn to predict how star shapes evolve with flux. We show that, within the precision of our tests, we are able to quantitatively relate the correlations of flat-field pixels and the broadening of stars with flux. This physical model of the brighter-fatter effect also explains the commonly observed phenomenon that the spatial variance of CCD flat-fields increases less rapidly than their average.} +} + Index: /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/arxiv.metadata =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/arxiv.metadata (revision 40483) +++ /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/arxiv.metadata (revision 40483) @@ -0,0 +1,18 @@ +Authors: Eugene A. Magnier (1), +J.~L. Tonry (1), +D. Finkbeiner (2), +E. Schlafly (4,5), +W.~S. Burgett (1), +K.~C. Chambers (1), +H.~A. Flewelling (1), +K. W. Hodapp (1), +N. Kaiser (1), +R.-P. Kudritzki (1), +N. Metcalfe (3), +R.~J. Wainscoat (1), +C.~Z. Waters (1) +((1) UH Institute for Astronomy, +(2) Harvard-Smithsonian Center for Astrophysics, +(3) Durham University Department of Physics, +(4) Lawrence Berkeley National Laboratory, +(5) Hubble Fellow) Index: /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/diffusion.tex =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/diffusion.tex (revision 40483) +++ /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/diffusion.tex (revision 40483) @@ -0,0 +1,1332 @@ +% \documentclass[iop,floatfix]{emulateapj} +% \documentclass[10pt,preprint]{aastex} +\documentclass[10pt,preprint]{aastex} +% \pdfoutput=1 + +% see latex.readme.txt for notes on using the PS1 template +%\documentclass[12pt,preprint]{aastex} +%\documentclass[manuscript]{aastex} +%\documentclass[preprint2]{aastex} +%\documentclass[preprint2,longabstract]{aastex} + +\RequirePackage{graphicx} +\RequirePackage{color} +% \RequirePackage{code} +% \RequirePackage{pbox} +% \input{magnier.tex} +\input{astro.sty} + +%\newcommand\oldtext[1]{\color{red}#1} +%\newcommand\newtext[1]{\textbf{\color{blue}#1}} + +\definecolor{light-gray}{gray}{0.50} +% \newcommand\oldtext[1]{\textbf{\color{light-gray}#1}} +\newcommand\oldtext[1]{\ignorespaces} +% \newcommand\newtext[1]{\textbf{\color{blue}#1}} +\newcommand\newtext[1]{#1} +\newcommand\fixtext[1]{\textbf{\color{red}#1}} + +\usepackage[T1]{fontenc}% (2) specify encoding + +% online version may use color, but print version needs b/w +\def\plotmode{col} +%\def\plotmode{bw} + +\def\plotext{pdf} +%\def\plotext{eps} + +%\def\picdir{/home/eugene/chipresid.20140404} +%\def\picdir{/data/kukui.2/eugene/chipresid.20140404} +%\def\picdir{pics} %%% need to set this for local processing +\def\picdir{.} %%% need to set this for the zip archive + +% Pick a terse version of the title here; +\shorttitle{Charge Diffusion Variations in PS1} +\shortauthors{E.A. Magnier et al} +\begin{document} +\title{Charge Diffusion Variations in Pan-STARRS\,1 CCDs} + +% this is a crude trick to get the order of affiliations right. These +% names are used in the affiliations below. The user needs to (1) set +% the order and numbers to have the correct sequence in the author +% list and (2) re-order the list at the bottom (and comment-out as needed) +\def\IfA{1} +\def\CfA{2} +\def\LBL{3} +\def\Hubble{4} +\def\DUH{5} + +% This example has a first author from UH: +\author{ +Eugene A. Magnier,\altaffilmark{\IfA} +J.~L. Tonry, \altaffilmark{\IfA} +D. Finkbeiner,\altaffilmark{\CfA} +E. Schlafly,\altaffilmark{\LBL,\Hubble} +%PS Builder List +W.~S. Burgett,\altaffilmark{\IfA} +K.~C. Chambers,\altaffilmark{\IfA} +% L. Denneau,\altaffilmark{\IfA} +% P. Draper,\altaffilmark{\DUR} +H.~A. Flewelling,\altaffilmark{\IfA} +% T. Grav,\altaffilmark{\IfA} +% J. N. Heasley,\altaffilmark{\IfA} +K. W. Hodapp,\altaffilmark{\IfA} +% M. E. Huber,\altaffilmark{\IfA} +% R. Jedicke,\altaffilmark{\IfA} +N. Kaiser,\altaffilmark{\IfA} +R.-P. Kudritzki,\altaffilmark{\IfA} +% G. A. Luppino,\altaffilmark{\IfA} +% R. H. Lupton,\altaffilmark{\Princeton} +% E. A. Magnier,\altaffilmark{\IfA} +N. Metcalfe,\altaffilmark{\DUH} +% D. G. Monet,\altaffilmark{\USNO} +% J.~S. Morgan,\altaffilmark{\IfA} +% P. M. Onaka,\altaffilmark{\IfA} +% P.~A. Price,\altaffilmark{\Princeton} +% C.~W. Stubbs,\altaffilmark{\CfA} +% W.~E. Sweeney,\altaffilmark{\IfA} +% J.~L. Tonry, \altaffilmark{\IfA} +R. J. Wainscoat,\altaffilmark{\IfA} and +C. Z. Waters,\altaffilmark{\IfA} +} % this bracket terminates author list + +% The ordering here should be sequential, matching the sequence in the list of authors: +\altaffiltext{\IfA}{Institute for Astronomy, University of Hawaii, 2680 Woodlawn Drive, Honolulu HI 96822} +\altaffiltext{\CfA}{Harvard-Smithsonian Center for Astrophysics, 60 Garden Street, Cambridge, MA 02138} +% \altaffiltext{\Princeton}{Department of Astrophysical Sciences, Princeton University, Princeton, NJ 08544, USA} +% \altaffiltext{\USNO}{US Naval Observatory, Flagstaff Station, Flagstaff, AZ 86001, USA} +% \altaffiltext{\JHU}{Department of Physics and Astronomy, Johns Hopkins University, 3400 North Charles Street, Baltimore, MD 21218, USA} +% \altaffiltext{\MPIA}{Max Planck Institute for Astronomy, K\"onigstuhl 17, D-69117 Heidelberg, Germany} +\altaffiltext{\DUH}{Department of Physics, Durham University, South Road, Durham DH1 3LE, UK} +\altaffiltext{\LBL}{Lawrence Berkeley National Laboratory, One Cyclotron Road, Berkeley, CA 94720, USA} +\altaffiltext{\Hubble}{Hubble Fellow} +\begin{abstract} + +Thick back-illuminated deep-depletion CCDs have superior quantum +efficiency over previous generations of thinned and traditional thick +CCDs. As a result, they are being used for wide-field imaging cameras +in several major projects. We use observations from the Pan-STARRS +$3\pi$ survey to characterize the behavior of the deep-depletion +devices used in the Pan-STARRS\,1 Gigapixel Camera. We have +identified systematic spatial variations in the photometric +measurements and stellar profiles which are similar in pattern to the +so-called ``tree rings'' identified in devices used by other +wide-field cameras (e.g., DECam and Hypersuprime Camera). The +tree-ring features identified in these other cameras result from +lateral electric fields which displace the electrons as they are +transported in the silicon to the pixel location. In contrast, we +show that the photometric and morphological modifications observed in +the GPC1 detectors are caused by variations in the vertical charge +transportation rate and resulting charge diffusion variations. +\end{abstract} + +% insert additional keywords as appropriate: +\keywords{Surveys:\PSONE } + +\section{INTRODUCTION}\label{sec:intro} + +CCD detectors have evolved greatly since they were first introduced +for astronomical imaging in the mid 1970s. In addition to the +well-known increases in the size of CCDs over the past 4 decades, CCD +architecture has gone through three major evolutionary stages. + +The first generation of CCDs used a silicon substrate a few hundred +microns thick on top of which gate structures were deposited to define +the pixels. A positive voltage applied to the gate layers would +create a shallow region (\approx 10 microns thick) in which the holes +were depleted. This ``depletion region'' acted as a potential well to +trap electrons, specifically those generated by absorbed photons. The +thick silicon substrate required illumination from the ``front'' side +containing the thin gate structures to allow the photons to reach the +depletion region and be detected. These early CCDs had modest quantum +efficiency as photons were easily absorbed by the several-micron-thick +gate structures. For an excellent review of the history of CCD +development, see \cite{1992ASPC...23....1J}. + +Thinned, backside-illuminated CCDs such as the TI 3PCCD +\citep{1981SPIE..290....6B} were developed to address the quantum +efficiency limitations of the first generation thick CCDs. The +silicon substrate was removed using a chemical process, leaving a +delicate device only \approx 10 - 20\micron\ thick, exposing the +depletion region on the backside. Photons entering the backside of +the device are not blocked by the gate structures and are thus more +easily absorbed and detected. Thinned backside-illuminated CCDs have +high quantum efficiency to blue photons. However, as the wavelength +increases beyond \approx 800 nm, the silicon becomes more transparent +to the photons with a corresponding drop in quantum efficiency for +red photons. In addition, thin-film interference between the entering +photons and those reflecting off the front side of the CCD result in +``fringe'' patterns for redder photons. + +Early generations of CCDs were made of low-resistivity (\approx 10 - +50 $\Omega$-cm) silicon. Following experiments beginning in the early +1990s \citep{Holland.1996}, CCDs made from thick, high-resistivity ($ +> 10 k\Omega$-cm) silicon were developed for astronomical instruments +in the early 2000s \citep{Holland.2003}. The high-resistivity of the +silicon allows for depletion regions of hundreds of microns in depth, +compared to \approx 10\micron\ for the low-resistivity silicon. This +modification allows for a back-illuminated CCD with a relatively thick +silicon subtrate of 75 - 300\micron. Blue photons impinging on the +back of the device are absorbed near the back surface of the device +and are caried through the depletion region to the gates on the front +side. The thick silicon allows red photons to have a greater chance +to be absorbed, increasing quantum efficiency in the red. Because +these thick, deep-depletion devices have near-unity quantum efficiency +across a very wide spectral range, they have become the design of +choice for many modern, large-scale CCD cameras (e.g., Pan-STARRS +GPC1, \citealt{2009amos.confE..40T}; Subaru Hypersuprime Camera, +\citealt{2010SPIE.7735E..3FK}; Dark Energy Survey Camera, +\citealt{2015AJ....150..150F}). + +While these deep-depletion CCDs seem to be ideal, they do have +features which can cause challenges for precise measurements. For +example, as a result of the ``Brighter-Fatter Effect'' +\citep{2014JInst...9C3048A,2015JInst..10C5032G}, the profile of bright +stars are measured to be wider than the profiles of faint stars. The +accepted interpretation is that the electric fields produced by the +electrons accumulated from a star repel successive incoming electrons, +with the repulsion increasing the more electrons have accumulated. + +The effects of lateral electric fields are likewise identified as the +cause of the so-called ``tree rings'' observed in the flat-field, +astrometry, and photometry response of thick deep-depletion detectors +\citep{2014PASP..126..750P}. These tree-ring patterns have been noted +in the flat-field response of deep depletion devices since their early +testing \citep[see, e.g., Figure 2 in][]{2010SPIE.7735E..1RE} and were +initially considered to be a sensitivity response which could be +removed with a flat-field. As discussed in detail by +\cite{2014PASP..126..750P}, these tree rings are more correctly +interpretted as variations in the effective pixel area due to +migration of the electrons pushed by lateral electric fields induced +by small changes in the doping used to set the resistivity of the +silicon. The changes in the effective area result in changes to the +apparent flat-field response as well as the astrometric response of +the detector. More subtly, the changes in the flat-field response, +since they do not reflect actual variations in sensitivity, can lead +to systematic photometry errors for astronomical sources if flat-field +images are used in the standard fashion. + +In this paper, we examine the behavior of an apparently-similar kind +of tree-ring pattern observed in the Pan-STARRS\,1 Gigapixel Camera 1 +CCDs. Although we also observe the changes in effective pixel area +caused by lateral electric fields as described by +\cite{2014PASP..126..750P}, we show below a second effect which is +more important in these devices in driving systematic photometry +errors. We find that variations in charge diffusion, also resulting +from changes in the silicon doping structures, affect both the +observed stellar profiles as well as the photometry measured with +profile fitting techniques. In Section~\ref{sec:PS1}, we discuss the +Pan-STARRS\,1 telescope, camera, and survey data used in this analysis. +In Section~\ref{sec:tree.rings}, we present the tree-ring patterns as +observed in several different types of measurements: flat-field +response, systematic photometric residuals, systematic astrometric +residuals, and stellar profile shape variations. In +Section~\ref{sec:discussion}, we discuss the interpretation of +patterns we observe and present a simple model to explain the observed +behavior. We conclude with a discussion of the implications of this +effect on astronomical measurements from deep depletion instruments + +\section{Pan-STARRS1} +\label{sec:PS1} + +The 1.8m Pan-STARRS\,1 telescope (PS1), located on the summit of +Haleakala on the Hawaiian island of Maui, has been surveying the sky +regularly since May 2010 \citep{chambers2017}. From May 2010 through +March 2014, PS1 was run under the aegis of the Pan-STARRS\,1 Science +Consortium (PS1SC) to perform a set of wide-field science surveys; +since March 2014, operations have been supported primarily by NASA's +Near Earth Object Observation program +\citep[see][]{2015IAUGA..2251124W}. Under the PS1SC, the largest +survey, both in terms of area of the sky covered ($3\pi$ steradians) +and fraction of observing time (56\%), was the \TPS\ in which the +entire sky north of Declination $-30$\degrees\ was imaged \approx 80 +times over 4 years. These observations were distributed over five +filters, \grizy, and have been astrometrically and photometrically +calibrated to good precision \citep{magnier2017.calibration}. + +% 2004SPIE.5489..667H == PS1.optics +% 2008SPIE.7014E..0DO == PS1.GPCB +% 2009amos.confE..40T == PS1.GPCA +% 2012ApJ...756..158S == ubercal +The wide-field PS1 telescope optics \citep{2004SPIE.5489..667H} image +a 3.3 degree field of view on a 1.4 gigapixel camera +\citep[GPC1;][]{2009amos.confE..40T}, with low distortion and generally +good image quality. The median seeing for the \TPS\ data vary +somewhat by filter: (\grizy) = (1.31, 1.19, 1.11, 1.07, 1.02) +arcseconds. Routine observations are conducted remotely from the +Advanced Technology Research Center in Kula, the main facility of the +University of Hawaii's Institute for Astronomy operations on Maui. + +GPC1, currently the largest astronomical camera in terms of number of +pixels, consists of a mosaic of 60 edge-abutted $4800\times4800$ pixel +detectors, with 10~$\mu$m pixels subtending 0.258~arcsec. These CCID58 +detectors, manufactured by Lincoln Laboratory, are 75\micron-thick +back-illuminated CCDs \citep{2006amos.confE..47T,2008SPIE.7021E..05T}. +Initial performance assessments are presented in +\cite{2008SPIE.7014E..0DO}. The active, usable pixels cover \approx +80\% of the FOV. + +\subsection{Data Processing and Calibration} + +% PS1_IPP = \bibitem[Magnier(2006)]{PS1.IPP} Magnier, E.\ 2006, +% Proceedings of The Advanced Maui Optical and Space Surveillance +% Technologies Conference, Ed.: S. Ryan, The Maui Economic Development +% Board, p.E5 + +Images obtained by PS1 are processed by the Pan-STARRS Image +Processing Pipeline (IPP; +\citealp{2006amos.confE..50M,magnier2017.datasystem}). All +observations are processed nightly, with results sent to groups within +the science consortium (i.e., PS1SC during the \TPS) performing +short-term science projects (e.g., searching for transient and moving +objects). In addition, the \TPS\ dataset has been re-processed +several times with improved calibration and analysis techniques. To +date (2017 September), 3 re-processings starting from raw pixel data +have been performed. The labels PV0, PV1, PV2, PV3 are used identify +the nightly processing and successive re-processing versions. PV3 has +been used for the public release of the Pan-STARRS \TPS\ data via the +{\it Barbara A. Mikulski Archive for Space Telescopes} (MAST) at the +Space Telescope Science Institute.\footnote{http//panstarrs.stsci.edu} +The process of the construction of this database and the schema +details are discussed in detail by \cite{flewelling2017}. + +The data processing and calibration operations are discussed in detail +in elsewhere +\citep{magnier2017.analysis,magnier2017.calibration,waters2017}. +We re-visit here a number of points that are of significance to this +study. Images are processed following a fairly standard sequence of +image detrending, source detection, and initial calibration +(astrometric and photometric) of those detected sources. Additional +standard processing critical to PS1 science operations includes +geometric transformation (`warping') and image combinations (summed +stacks and differences). For the purposes of this analysis, we are +only considering the sources detected in the individual exposures from +the initial analysis steps. + +% Magnier.belgium: +% \bibitem[Magnier(2007)]{PS1.photometry} Magnier, E.\ 2007, The Future +% of Photometric, Spectrophotometric and Polarimetric Standardization, ASP Conference Series {\bf 364}, 153 + +%IPP astrometry (NOT USED) +% \bibitem[Magnier {\it et al.}(2008)]{PS1.astrometry} Magnier, E.~A., Liu, +% M., Monet, D.~G., \& Chambers, K.~C.\ 2008, IAU Symposium, {\bf 248}, 553 + +As discussed in \cite{waters2017}, image detrending includes +flat-field processing with a single epoch flat-field image for each +filter. The flat-field image used for this analysis has been +generated by median-combining dome flat-field images (after +pre-processing and pixel outlier rejections) and then multiplying by a +photometric flat-field correction image generated by the analysis of a +grid of images of a dense stellar field. The purpose of this second +step is to correct the basic flat-field image for errors arising from +the non-uniformity of the illumination, from \newtext{variations in + the effective pixel size} \oldtext{non-pixel uniformity} due to the +varying optical \newtext{distortion} \oldtext{distorition} across the +field, and any other factors which may make the flat-field image +inconsistent with stellar photometry, e.g., SED, filter band-pass +variations, etc +\citep[see][]{waters2017,2004PASP..116..449M,2007ASPC..364..153M}. +This correction was made on a relatively coarse \newtext{(\approx 1200 + CCD pixels per sample)} grid across the focal plane in order to +accumulate sufficient statistics from the stars in the relatively +small number of images available at the time. We have found that a +single flat-field set can be used for all PS1 observations to yield +photometric systematic errors at the level of \approx 2\%. PS1 +benefits in this regard from the stability of having a single +instrument which is rarely removed. + +Photometry of the PS1 images is performed using a +point-spread-function (PSF) model as well as multiple kinds of +apertures \citep{magnier2017.analysis}. In this analysis, we refer to +aperture photometry performed using an aperture defined based on the +image quality observed for a given chip. The aperture diameter is set +to be 3.75 times the FWHM for the image. + +To improve the photometric systematic errors beyond the level achieved +with a single (photometrically corrected) flat-field set, the PS1 +photometry is re-calibrated within the databasing system based on the +properties of the measured photometry. The calibration process is +discussed by \cite{2012ApJ...756..158S} and +\cite{2013ApJS..205...20M,magnier2017.calibration}. As part of this +process, several flat-field corrections have been determined. For the +PV0 analysis discussed here, a flat-field correction determined during +the ubercal analysis \citep[see][]{2012ApJ...756..158S} consisted of +an $8\times 8$ grid of corrections for each GPC1 chip, corresponding +to a correction for each OTA ``cell'' and filter for each of 4 +seasons. The boundaries of those seasons are tentatively identified +with modifications to the baffle structures or the system optics. The +critical point here is that the final effective flat-field image for +the PV0 dataset is based on a dome-flat at the highest resolution, +with very low resolution (hundreds of pixels) corrections based on +photometry, resulting in photometric systematic uncertainties in the +range 7 - 12 millimagnitudes, depending on the filter +\citep{2013ApJS..205...20M}. \newtext{We note that the PV3 analysis + used for the public release includes a flat-field correction + measured with a much finer spatial sampling than the PV0 analysis, + with 40 CCD pixels per superpixel. As a result, some of the + fine-grained structure discussed below are corrected in the public + release (see however the caveats in the discussion section below).} + +For all objects, positions are measured from the PSF model for the +brighter sources (using a non-linear fitting process) and from a +simple centroid (1st moment) for the fainter source +\citep{magnier2017.analysis}. These position measurements are +used in the astrometric analysis. The astrometric calibration is +discussed by \cite{magnier2017.calibration}; for the PV0 +dataset, the typical systematic floor is \approx 15 - 20 +milliarcsecond for individual measurements of brighter stars. + +\section{Tree-Ring Patterns} +\label{sec:tree.rings} + +%% \begin{table} +%% % \tiny +%% \begin{center} +%% \caption{Systematic Trends : Standard deviation by filter\label{table:sigmas.by.filter}} +%% \begin{tabular}{|l|rrrrr|} +%% \hline +%% {\bf Filter} & {\bf psf mags} & {\bf ap mags} & {\bf astrom} & {\bf smear} & {\bf flat} \\ +%% & mmags & mmags & mas & pixels$^2$ & mmags \\ +%% \hline +%% \gps & 11.8 & 13 & 8.0 & 0.169 & 3.0 \\ +%% \rps & 10.9 & 12 & 6.7 & 0.133 & 2.2 \\ +%% \ips & 8.5 & 10 & 6.0 & 0.069 & 1.7 \\ +%% \zps & 8.7 & 12 & 5.5 & 0.052 & 3.2 \\ +%% \yps & 16.5 & 26 & 6.8 & 0.059 & 15.3 \\ +%% \hline +%% \end{tabular} +%% \end{center} +%% \end{table} + +For many of the GPC1 OTA CCDs, we observe a spatial pattern in the +photometric residuals for each device which is similar in appearence +to the tree rings described in the Dark Energy Camera (DECam) by +\cite{2014PASP..126..750P}. This pattern consists of systematic +deviations which are consistent in a set of circular arcs centered on +the corner of the CCD, as shown in Figure~\ref{fig:psfmags.by.filter}. +The details of the analysis used to generate +Figure~\ref{fig:psfmags.by.filter} are given below. For now, we note +that the GPC1 CCDs are constructed by dividing the circular silicon +wafer into 4 inscribed squares. Thus the corners of the CCDs lie in +the center of the silicon boule, just as the center of the circular +tree rings described by \cite{2014PASP..126..750P} match the center of +the boule from which they came. This gives the impression that a +similar mechanism is responsible for the pattern observed in the PS1 +photometry and the DECam photometry, namely the \oldtext{diffusive + effects of} \newtext{migration of charge by} lateral electric +\newtext{fields} \oldtext{field variations} in the detectors. In the +next section, we will make the case that the patterns observed in the +PS1 photometry residuals are {\em not} caused by this mechanism, but +are instead caused by variations in the {\em vertical} electric field +(the field direction perpendicular to the CCD surface). + +First, in this section, we will describe how we have measured the +presence or absence of these tree-ring patterns in 5 types of data. +For all of these examples, we use a single GPC1 CCD (XY40) to +illustrate the effects in detail, but a similar set of effects are +seen in many, if not all, of the GPC1 detectors with varying +strengths. First, we show the residual PSF photometry. Second, we +show the residual aperture photometry. Third, we show the astrometric +residual patterns. Fourth, we show the patterns observed in the +flat-field images. Finally, we show measurements derived from the +second-moments of the stars. + +For all effects discussed below, we are measuring the mean value of +the effect in 10x10 pixel superpixels across the detector. The +resulting images are all constructed so that a given superpixel +represents the same range of true GPC1 XY40 pixels regardless of the +type of measurement. To generate the photometry, astrometry, or +second-moment plots, measurements were extracted from the PV0 DVO +database \citep{magnier2017.calibration} for observations covering +the region ($\alpha$,$\delta$) = (90\degree\ -- 150\degree, +-25\degree\ -- 10\degree). This region of the sky provides a fairly +high density of stars, but avoids the Galactic Plane where confusion +may potentially contaminate the measurement. We limit the analysis to +good measurements (\textsf{PSF\_QF} $>$ 0.85, see +\citealt{magnier2017.analysis}) of likely stars ($|m_{psf} - +m_{aper}| < 0.2$). Only measurements with instrumental magnitude $< +-8.0$ ($-2.5\log \mbox{cts sec}^{-1} < -8.0$) are included to ensure +reasonable signal-to-noise per measurement. We require at least 2 +measurements in a given filter and at least 5 measurements total for +any star included in the analysis. + +\subsection{Photometric Residuals} + +% PSF Magnitudes +\def\figwidth{5.2in} +\def\jumpleft{-2.6in} +\def\capwidth{2.4in} +\begin{figure*}[htbp] +\begin{center} +\parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/dmag.\plotext}} +\hspace{\jumpleft} +\parbox[b]{\capwidth}{ +\caption{PSF Magnitude residuals by filter (\grizy) for a single + example GPC1 device (XY40). White boxes are + GPC1 cells which have been masked due to poor response. Superpixels + representing regions of $10\times10$ pixels are used to determine + the median deviation for measurements at the given chip pixel + location compared with the average photometry for the given + object. Fringing dominates the \yps-band signal.} \label{fig:psfmags.by.filter}} +\end{center} +\end{figure*} + +% Aperture Magnitudes +\begin{figure*}[htbp] +\begin{center} +\parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/dapmag.\plotext}} +\hspace{\jumpleft} +\parbox[b]{\capwidth}{ +\caption{Aperture Magnitude residuals by filter (\grizy) for a single + example GPC1 device (XY40). White boxes + are GPC1 cells which have been masked due to poor response. + Superpixels representing regions of $10\times10$ pixels are used to + determine the median deviation for measurements at the given chip + pixel location compared with the average photometry for the given + object. Fringing dominates the \yps-band signal, saturating the + color scale to black or white in areas.} \label{fig:apmags.by.filter}} +\end{center} +\end{figure*} + +Figure~\ref{fig:psfmags.by.filter} shows the 2D patterns of PSF +photometry residuals. In this case, we select PSF magnitude +measurements for detections of stars which fall in the given +superpixel. We subtract each measurement from the average magnitude +for the object in the selected filter ($\delta m_{psf} = +\overline{m}_{psf} - m_{psf}$) to determine the residual magnitude, +excluding as an outlier any measurement with $|\delta m_{psf}| > 0.5$. +For a given superpixel, we measure the median of the $\delta m_{psf}$ +distribution. The figure shows $\delta m_{psf}$ for each filter +(\grizy). The dynamic range of the color scale is from -20 to +20 +millimagnitudes for all 5 plots. + +The tree-ring pattern is clearly visible for the four blue filters, +but fringing dominates the pattern for \yps. Small offsets of +individual cells are also apparent for \zps. While the patterns are +clear across the image, the signal-to-noise of the structures per +pixel is not very strong in these images. \oldtext{The per-pixel standard +deviations of these plots are listed in +Table~1.} The \oldtext{per-pixel} standard deviation \newtext{of the pixel values in the images (a measure of the noise in the absence of any systematic signal)} +is comparable to the amplitude of the correlated structures, so we +need to integrate along the radial structures to make stronger +statements about these patterns. + +Figure~\ref{fig:apmags.by.filter} shows the equivalent measurement for +aperture photometry instead of PSF photometry. The fringing pattern +again dominates the plot for \yps, but the tree rings are not seen in +any of the filters. A diagonal pattern is visible in \gps\ which is +not observed in the PSF magnitudes. While the \newtext{standard + deviation of the pixel values} \oldtext{per-pixel scatter} is +somewhat (10\% to 20\%) higher for these aperture magnitudes than for +the PSF magnitudes\oldtext{ (Table~1)}, a structure with the amplitude +of the PSF magnitude tree-rings would certainly have been obvious. + +\newtext{Figure~\ref{fig:all.effects.rband} shows the complete set of + measured effects for the \rps\ filter. In addition to the PSF and + aperture photometry, this figure shows the astrometric residuals, + the high-frequency flat-field structures, along with two + measurements derived from the second moments: the ``smear'' and the + ``shear'', discussed below.} + +\subsection{Astrometric Residuals} + +%% % astrometry radial term +%% \begin{figure*}[htbp] +%% \begin{center} +%% \parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/drad.\plotext}} +%% \hspace{\jumpleft} +%% \parbox[b]{\capwidth}{ +%% \caption{Astrometric residuals of the displacement in the radial +%% direction, relative to the chip coordinate -5,4960 (upper left +%% corner), by filter (\grizy). White boxes are GPC1 cells which have +%% been masked due to poor response. Superpixels representing regions +%% of $10\times10$ pixels are used to determine the median deviation +%% for measurements at the given chip pixel location compared with the +%% average astrometry for the given +%% object. } \label{fig:astrom.by.filter}} +%% \end{center} +%% \end{figure*} + +\oldtext{Figure~3} \newtext{Figure~\ref{fig:all.effects.rband} + (middle-left)} shows a similar type of measurement for astrometric +residuals \newtext{in \rps-band}. To generate this plot, we use the +same selection of measurements for astrometry as for photometry. In +this case, we extract the residual in both the RA and DEC directions +($\delta RA = \overline{RA} - RA_i$, $\delta DEC = \overline{DEC} - +DEC_i$) and rotate these values to the chip coordinate system ($\delta +X,\delta Y$) using our knowledge of the chip orientation on the sky. +We again exclude as bad any measurement with $|\delta X|$ or $|\delta +Y| > 0.5$ arcsec before measuring the median values for each +superpixel. We have determined the approximate center of the circular +tree-ring pattern as (-5,4960) for this particular chip based on the +pattern of the X astrometry displacements. Using this coordinate as +the center of the pattern, we have converted the $\delta X,\delta Y$ +offsets into $\delta R,\delta \theta$ measurements ($\delta R$ : +radial component away from the center of the pattern, $\delta \theta$ +: tangential component). \newtext{The dynamic range of the color scale + is from -20 to +20 milliarcseconds for this plot.} + +\oldtext{Figure~\ref{fig:astrom.by.filter} shows the 2D patterns of + $\delta R$ for each filter (\grizy). The dynamic range of the color + scale is from -20 to +20 milliarcseconds for all 5 plots.} A +tree-ring pattern is visible for all five filters, with systematic +structures following a circular pattern centered on the chip corner; +the fringing pattern is not apparent in the \yps\ astrometry. +\oldtext{The per-pixel standard deviations of these plots are listed + in Table~1.} The signal-to-noise of these structures is again +somewhat weak, but the pattern is clearly visible in \oldtext{these + figures} \newtext{Figure~\ref{fig:all.effects.rband} (middle-left)}. + +\subsection{Flat-field Structures} +\label{sec:flat-fields} + +% All Effects in r-band +\begin{figure*}[htbp] +\begin{center} +\parbox[b]{\figwidth}{\includegraphics[width=5.0in]{\picdir/all_effects_r.\plotext}} +\caption{All 6 measured effects for \rps for a single + example GPC1 device (XY40). This figure illustrates the + different spatial structure observed for each of the 6 patterns + measured in this work. The PSF magnitude (upper-left) and smear + residuals (lower-left) have a very clear common tree-ring structure, + while the astrometric residual (middle-left) and flat-field + residuals (middle-right) have their own common tree-ring pattern with + much higher frequencies than the previous two effects. Aperture + magnitude (upper-right) and shear residuals (lower-right) do not + show a strong signal consistent with either of the two patterns.} +\label{fig:all.effects.rband} +\end{center} +\end{figure*} + +% flat-field residual +%% \begin{figure*}[htbp] +%% \begin{center} +%% \parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/dflat.\plotext}} +%% \hspace{\jumpleft} +%% \parbox[b]{\capwidth}{ +%% \caption{Flat-field high-frequency structues, by filter (\grizy). +%% White boxes are GPC1 cells which have been masked due to poor +%% response. Flat-field images generated using a tunable laser have +%% been combined (see text); a smoothed version has been subtracted to +%% high-pass the response. Flat-field pixels are averaged for +%% $10\times10$ superpixels. } \label{fig:flats.by.filter}} +%% \end{center} +%% \end{figure*} + +% 2012ApJ...750...99T = Tonry et al PS1 phot system +\oldtext{Figure~4} \newtext{Figure~\ref{fig:all.effects.rband} + (middle-right)} shows the high-spatial-frequency structures in the +\newtext{\rps-band} flat-field\oldtext{ images}. For this +measurement, we have used a set of monochromatic flat-field images +obtained with a tunable laser. The laser is used to illuminate our +flat-field screen which is then observed by the PS1 telescope. These +flat-field images were obtained 2011 Feb 09 as part of a campaign to +study the PS1 system response \citep{2012ApJ...750...99T}. Flats were +obtain in a set of 4nm steps sampling the spectral response curve of +each filter. To enhance the signal-to-noise, we have median-combined +a set of 6 flats at the wavelength center of the corresponding filter. +\newtext{Note that the flat-field images used for the science analysis + are made from broad-band dome flat, not these monochromatic flats. + The monochromatic flats were used here to avoid smearing out any + effects which changed as a function of wavelength.} + +In order to mask pixels which do not flatten well, we generate a copy +of the image smoothed with a Gaussian kernel with $\sigma = 1.5$ +pixels. Any pixels in the smoothed image which deviate from the +median value in the image by more than 4 standard deviations are +masked. We generate the superpixel image by averaging the unmasked +pixels associated with each superpixel. + +\oldtext{Figure~\ref{fig:flats.by.filter} shows the superpixel images + for the flat-fields in the five filters. These flat-field images + are} \newtext{The flat-field image is} displayed as fractional +deviations relative to the median of the flat-field image and can thus +be compared to the magnitude residuals. When flattening an image, +\oldtext{these flat-fields} \newtext{the flat-field image} would be +divided into the flux of the raw image. The residuals are thus +defined in the sense that a positive feature in \oldtext{these flats} +\newtext{the flat} which does {\em not} represent a real quantum +efficiency deviation would induce a {\em reduction} in the measured +flux in those pixels, and thus a {\em negative} deviation in $\delta +m_{psf}$ as defined above. The dynamic range of the color scale in +\oldtext{these plots} \newtext{this plot} is -0.01 to +0.01. The +tree-ring pattern is strong in the (\gps,\rps,\ips) images, but nearly +swamped by fringing in \zps, and completely lost to fringing in \yps. +\newtext{For the broad-band dome flats used for the science analysis, + the tree-ring patterns are apparent for all filters: the fringe + patterns seen in the \zps\ and \yps\ monochromatic flats are + apparently washed out by the range of wavelengths in the broad-band + flats.} + +A diagonal banding pattern is also apparent in \gps\ and \rps, though +it is largely removed in Figure~\ref{fig:all.effects.rband} by the +high-pass filtering mentioned above. This feature is thought to be due +to the lithography process used to generate the CCD. A blob can also +been seen covering 4 cells near the center of this chip; this is +apparently a deposit of some kind on the detector. Both of the latter +two effects behave like quantum efficiency variations and are removed +well by standard flat-field techniques. Note that a small amount of +the diagonal banding pattern remains in the aperture magnitude +residuals for \gps. For the rest of this article, we ignore these +features and concentrate on the tree-ring features. + +In order to suppress the large-scale structures for a quantitative +analysis of the tree rings, we high-pass filter the superpixel image +by subtracting a copy smoothed with a Gaussian of $\sigma = 3.0$ +superpixels. \newtext{This smoothing kernel is large enough compared + to the tree ring structures that they are not suppressed + significantly. Without this smoothing, features from the diagonal + banding pattern remain in the \rps-band image and contaminate the + tree-ring signal.} + +\subsection{Second Moments} + +% Smear Images +%% \begin{figure*}[htbp] +%% \begin{center} +%% \parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/smear.\plotext}} +%% \hspace{\jumpleft} +%% \parbox[b]{\capwidth}{ +%% \caption{Average residual smear variations, by filter (\grizy). White +%% boxes are GPC1 cells which have been masked due to poor response. +%% The residual smear ($\sigma^2_{\mbox{major}} + \sigma^2_{\mbox{minor}}$) has been +%% determined after the after PSF second moments have been subtracted +%% for each image; these values are averaged for each $10\times10$ +%% superpixels. } \label{fig:smear.by.filter}} +%% \end{center} +%% \end{figure*} + +% Shear Images +%% \begin{figure*}[htbp] +%% \begin{center} +%% \parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/shear.\plotext}} +%% \hspace{\jumpleft} +%% \parbox[b]{\capwidth}{ +%% \caption{Average residual shear variations, by filter (\grizy). White +%% boxes are GPC1 cells which have been masked due to poor response. +%% The residual shear ($\sigma^2_{\mbox{major}} - \sigma^2_{\mbox{minor}}$) has been +%% determined after the after PSF second moments have been subtracted +%% for each image; these values are averaged for each $10\times10$ +%% superpixels. } \label{fig:shear.by.filter}} +%% \end{center} +%% \end{figure*} + +During the image analysis, the second moments are measured for all +stars. The values can be used to assess changes in the shape of stars +on the image. To measure changes in the shapes, we have extracted the +second moments for all stellar detections, subject to the same +selections as for the photometry and astrometry residuals (good stars, +multiple detections). The second moments are measured with a Gaussian +weighting function, with the $\sigma_{w}$ scaled by the PSF size so +that the $\sigma$ measured for PSF stars is \approx 65\% of +$\sigma_{w}$. (Note that, since the measured $\sigma$ of stellar +objects is biased down by the weighting function, this is not quite +the same as having $\sigma_{w} = 1.6$ times the true PSF $\sigma$; see +discussion in \citealt{magnier2017.analysis}). For each stellar +detection, we extract the values $M_{xx,xy,yy} = \sum F_i w_i (x^2, x +y, y^2) / \sum F_i w_i$. For each exposure, we find the median second +moments for PSF objects on this chip (XY40) and subtract those median +values from the instantaneous measurements of $M_{xx,xy,yy}$. We then +determine the median of the residual second moments for each +superpixel, resulting in 3 images ($\delta M_{xx,xy,yy}$) for each +filter. + +Using the second moment images, we can construct certain interesting +combinations, inspired by discussions of lensing measurements \citep{1995ApJ...449..460K}: +\begin{eqnarray} +e_0 & = & \delta M_{xx} + \delta M_{yy} \\ +e_1 & = & \delta M_{xx} - \delta M_{yy} \\ +e_2 & = & \sqrt{e_1^2 + 4 \delta M_{xy}} +\end{eqnarray} +For a 2D Gaussian profile with an elliptical contour, these values are +related to the shape of the elliptical contour as follows: +\begin{eqnarray} +e_0 & = & \sigma^2_{\mbox{major}} + \sigma^2_{\mbox{minor}} \\ +e_1 & = & (\sigma^2_{\mbox{major}} - \sigma^2_{\mbox{minor}}) \cos (2 \theta) \\ +e_2 & = & \sigma^2_{\mbox{major}} - \sigma^2_{\mbox{minor}} +\end{eqnarray} +Where $\sigma_{\mbox{major}}$ and $\sigma_{\mbox{minor}}$ are the +major and minor axis dimensions of the ellipse and $\theta$ is the +position angle. Thus, $e_0$ is a measurement of the change in the +size of the stellar PSFs as a function of position in the detector +(``smear''), $e_2$ is a measurement of the change in ellipticity of +the stellar PSFs (``shear''), and we can determine the angle of the +PSF ellipticity from the $e_1$ term. + +\oldtext{Figure~5} \newtext{Figure~\ref{fig:all.effects.rband} + (lower-left)} shows the spatial trend of the smear, $e_0$. The +dynamic range of \oldtext{these images} \newtext{this image} is -0.3 +to +0.3 pixel$^2$. A tree-ring pattern is visible for all 5 filters, +though \yps\ is dominated by the fringing pattern. Structures with +relatively low spatial frequencies can also be seen. + +% All Effects in r-band +\begin{figure*}[htbp] +\begin{center} +\parbox[b]{\figwidth}{\includegraphics[width=5.0in]{\picdir/filter_trends.\plotext}} +\caption{Amplitude of the 4 effects which follow the tree-rings as a + function of filter, relative to the amplitude in the \gps-band.} +\label{fig:filter.trend} +\end{center} +\end{figure*} + +\oldtext{Figure~6} \newtext{Figure~\ref{fig:all.effects.rband} (lower-right)} +shows the spatial trend of the shear, +$e_2$. This value is positive definite and is plotted with a color +scale ranging from -0.02 to 0.22 pixel$^2$. Overlayed on +\oldtext{Figure~6} \newtext{Figure~\ref{fig:all.effects.rband} (lower-right)} +is a set of vectors representing the +ellipse orientation as a function of postion. The length of the +vectors corresponds to the value of $e_2$. The tree-ring structure is +{\em not} apparent in \oldtext{this figure} \newtext{the shear} for any filter. The spatial +variations are low-frequency and unrelated to the radial trend from +the upper-left corner. + +\subsection{Correlations Between Tree-Ring Patterns} + +%% \begin{table} +%% % \tiny +%% \begin{center} +%% \caption{\newtext{Amplitude of the four systematic trends in each filter +%% relative to \gps.} \oldtext{Systematic Trends : Correlations by filter}\label{table:correlation.by.filter}} +%% \begin{tabular}{|l|rrrr|} +%% \hline +%% {\bf Filter} & {\bf smear} & {\bf psf mags} & {\bf astrom} & {\bf flat} \\ +%% \hline +%% \gps & 1.00 & 1.00 & 1.00 & 1.00 \\ +%% \rps & 0.78 & 0.84 & 0.84 & 0.76 \\ +%% \ips & 0.40 & 0.50 & 0.66 & 0.64 \\ +%% \zps & 0.16 & 0.26 & 0.37 & 0.33 \\ +%% \yps & 0.10 & 0.10 & 0.25 & 0.30 \\ +%% \hline +%% \end{tabular} +%% \end{center} +%% \end{table} + +Tree-ring patterns are clearly seen in 4 of the measurement types +above: the PSF photometry, the astrometry, the flat-field, and the +smear terms. As discussed above, the signal-to-noise per pixel in the +plots of the systematic trends is relatively low (\approx 1.0). While +the tree-ring patterns are apparent in many of these figures, +there are also some other systematic structures which may degrade the +signal further. + +To quantitatively compare the tree-ring trends between filters and +between the types of measurements, we need to measure the tree-ring +structure explicitly and filter out the other effects if possible. To +do this, we have applied a high-pass filter to all of the relevant +images (PSF photometry residuals, astrometric residuals in the radial +direction, flat-field residuals, and second moment smear terms) to +remove unrelated spatial structures. We have then measured the median +of the signal in radial bins centered on (-5,4960) across an arc from +$\phi$ = -20\degrees\ to -50\degrees (as measured relative to the top +row of the images). We have selected a small fraction of the arc to +minimize the error associated with the choice of the pattern center +and to avoid several bad cells near the bottom of the chip. + +% \note{include the arc on one of the figures?} + +% \note{do plots of all filter pairs in a triangle? is that interesting?} + +For a given type of measurement, the systematic effect is strongly +correlated between filters. The strongest correlation is the smear +term\oldtext{: Figure~8 shows the correlation of the smear +pattern between \gps\ and the other four filters}. Even \yps\ is +strongly correlated with \gps\ despite the presence of the fringe +pattern. PSF photometric residuals are also correlated between +filters\oldtext{, as shown in Figure~9}. Here, the +\yps\ correlation with \gps\ is quite weak: the fringing pattern +dominates the tree rings for PSF photometry. The radial component of +the astrometric residual is also well correlated between filters, with +no loss of correlation due to fringing in \yps. Finally, the +flat-field residuals are generally correlated between filters, but +both \zps\ and \yps\ are affected by fringing. For \yps, the +correlation is completely washed out by the very strong fringing +pattern. + +For all four types of measurements, the \oldtext{slope of the fitted + lines} \newtext{amplitudes relative to \gps} are \oldtext{listed in + Table~2} \newtext{plotted in Figure~\ref{fig:filter.trend}}. There +is a consistency in the trend from \gps, with the strongest systematic +tree-ring effects, to \yps, with the weakest effects. Note that the +relative strength of the second moment smear in the reddest bands +compared to \gps\ is quite different from the relative strength of the +astrometry and flat-field terms in the reddest bands. + +% smear trends by filter +%% \def\figwidth{6.5in} +%% \begin{figure*}[htbp] +%% \begin{center} +%% \includegraphics[width=\figwidth]{\picdir/smear_trends.\plotext} +%% \caption{Correlation of the smear ($\sigma^2_{\mbox{major}} + +%% \sigma^2_{\mbox{minor}}$) signal in \gps\ with the other 4 bands: +%% \rps\ (upper-left), \ips\ (upper-right), \zps\ (lower-left), \yps\ (lower-right). +%% } \label{fig:smear.trends} +%% \end{center} +%% \end{figure*} + +% psfmag trends by filter +%% \def\figwidth{6.5in} +%% \begin{figure*}[htbp] +%% \begin{center} +%% \includegraphics[width=\figwidth]{\picdir/psfmag_trends.\plotext} +%% \caption{Correlation of the PSF magnitude residuals ($\delta m_{psf}$) +%% in \gps\ with the other 4 bands: \rps\ (upper-left), \ips\ +%% (upper-right), \zps\ (lower-left), \yps\ (lower-right). +%% } \label{fig:psfmag.trends} +%% \end{center} +%% \end{figure*} + +% astrom trends by filter +%% \def\figwidth{6.5in} +%% \begin{figure*}[htbp] +%% \begin{center} +%% \includegraphics[width=\figwidth]{\picdir/astrom_trends.\plotext} +%% \caption{Correlation of the radial astrometric residual displacement ($\delta R$) +%% in \gps\ with the other 4 bands: \rps\ (upper-left), \ips\ +%% (upper-right), \zps\ (lower-left), \yps\ (lower-right). +%% } \label{fig:astrom.trends} +%% \end{center} +%% \end{figure*} + +% flat trends by filter +%% \def\figwidth{6.5in} +%% \begin{figure*}[htbp] +%% \begin{center} +%% \includegraphics[width=\figwidth]{\picdir/flat_trends.\plotext} +%% \caption{Correlation of the flat-field tree-ring structures in \gps\ +%% with the other 4 bands: \rps\ (upper-left), \ips\ (upper-right), \zps\ +%% (lower-left), \yps\ (lower-right). } \label{fig:flat.trends} +%% \end{center} +%% \end{figure*} + +An important question is the relationship of the tree-ring +pattern between the different types of measurements. Different models +for the tree-ring structures make different predictions about the +correlations between different effects. +% +\newtext{Figure~\ref{fig:effects.vs.radius} shows the radial run of the + four effects which show clear tree rings (in \rps). Since the tree + rings are relatively narrow, this figure shows only the radial range + of 150 - 300 pixels to allow the reader to see the relationship + between structures in the different effects. } +% +Note the very different +spatial structure between the different measurements in a given +filter: the radial variations do not all follow the same patterns. +Instead, we find the following relationships hold: + +% tangentially-averaged patterns vs radial position +\def\figwidth{6.5in} +\begin{figure*}[htbp] +\begin{center} +\includegraphics[width=\figwidth]{\picdir/radial_p1_r.\plotext} +\caption{Radial run of the four tree-ring trends for \rps: smear + ($\sigma^2_{\mbox{major}} + \sigma^2_{\mbox{minor}}$, pixel$^2$), PSF magnitude + residuals ($\delta m_{PSF}$, magnitudes), flat-field (fractional + deviation), and astrometric residuals + ($\delta R$, arcseconds). } \label{fig:effects.vs.radius} +\end{center} +\end{figure*} + +% tangentially-averaged patterns vs radial position +\def\figwidth{6.5in} +\begin{figure*}[htbp] +\begin{center} +\includegraphics[width=\figwidth]{\picdir/radial_p2_r.\plotext} +\caption{Radial run of the derivative of the smear + ($\frac{\partial (\sigma^2_{major} + \sigma^2_{minor})}{\partial + radius}$, pixels) + and astrometric residuals ($\delta R$, arcseconds) for \rps. +} \label{fig:dsmear.and.astrom} +\end{center} +\end{figure*} + +% tangentially-averaged patterns vs radial position +\def\figwidth{6.5in} +\begin{figure*}[htbp] +\begin{center} +\includegraphics[width=\figwidth]{\picdir/radial_p3_r.\plotext} +\caption{Radial run of + the derivative of the astrometric residuals ($\frac{\partial \delta + R}{\partial radius}$, pixels pixel$^{-1}$) and the flat-field + (fractional deviation) for \rps.} \label{fig:dastrom.and.flat} +\end{center} +\end{figure*} + +First, the PSF magnitude residuals and the second-moment smear trends +are strongly anti-correlated: regions which have larger PSFs than the +mean tend to have smaller measured PSF fluxes than the mean (note that +$\delta m_{psf}$ is defined so that positive values correspond to +larger fluxes). \oldtext{These trends are shown in Figure 12.} + +Second, the radial derivative of the smear is anti-correlated with the +radial component of the astrometric residuals. +\newtext{Figure~\ref{fig:dsmear.and.astrom} shows the radial run of + $\frac{\partial (\sigma^2_{major} + \sigma^2_{minor})}{\partial radius}$ + and $\delta R$ together to illustrate this relationship.} +\oldtext{: $\frac{\partial(\sigma^2_{major} + \sigma^2_{minor})}{\partial radius} \sim \delta R$. (see Figure~13).} + +Finally, the radial derivative of the radial component of the +astrometric residual is correlated with the flat-field residual +errors. \newtext{Figure~\ref{fig:dastrom.and.flat} shows the radial + run of $\frac{\partial \delta R}{\partial radius}$ and the + flat-field together to illustrate this relationship.} \oldtext{: + $\frac{\partial \delta R}{\partial radius} \sim \delta flat$ (see + Figure~14).} + +This last relationship is somewhat weakly measured. Because of the +periodic nature of the tree rings, it is also difficult to be +completely certain that the flat-field is proportional to the +derivative of the astrometry residual, rather than the astrometry +residual being proportional to the derivative of the flat-field. +\newtext{Careful examination of Figures~\ref{fig:effects.vs.radius} + and \ref{fig:dastrom.and.flat} convince us that we have the sense of + the derivative correct.} +% +\oldtext{The correlation is somewhat weaker for derivative of the + flat-field vs astrometry residual. The correlation is very weak + between the flat-field and the astrometry residual values without a + derivative. We are convinced that we have the sense of the + derivative correct by examination of specific features in each + image.} + +%% \begin{table} +%% % \tiny +%% \begin{center} +%% \caption{Systematic Trends : Correlations between trends\label{table:correlation.by.trend}} +%% \begin{tabular}{|l|rrr|} +%% \hline +%% {\bf Filter} & {\bf psf mags} & {\bf $\grad$ smear} & {\bf $\grad$ astrom} \\ +%% & {\bf vs smear} & {\bf vs astrom} & {\bf vs flat} \\ +%% \hline +%% \gps & -0.056 & -0.060 & -0.47 \\ +%% \rps & -0.071 & -0.073 & -0.45 \\ +%% \ips & -0.077 & -0.095 & -0.45 \\ +%% \zps & -0.082 & -0.078 & -0.17 \\ +%% \hline +%% \end{tabular} +%% \end{center} +%% \end{table} + +% smear vs psfmag +%% \def\figwidth{6.5in} +%% \begin{figure*}[htbp] +%% \begin{center} +%% \includegraphics[width=\figwidth]{\picdir/smear_vs_psfmag.\plotext} +%% \caption{Correlation of the PSF magnitude residuals ($\delta m_{PSF}$) +%% with the smear ($\sigma^2_{\mbox{major}} + \sigma^2_{\mbox{minor}}$) +%% signal for \gps\ (upper-left), \rps\ (upper-right), \ips\ (lower-left), +%% \zps\ (lower-right). +%% } \label{fig:smear.vs.psfmag} +%% \end{center} +%% \end{figure*} + +% dsmear vs astrom +%% \def\figwidth{6.5in} +%% \begin{figure*}[htbp] +%% \begin{center} +%% \includegraphics[width=\figwidth]{\picdir/dsmear_vs_astrom.\plotext} +%% \caption{ +%% Correlation of the radial astrometric residual displacement ($\delta +%% R$) with the derivative of the smear ($\partial +%% \sigma^2_{\mbox{major}} + \sigma^2_{\mbox{minor}}$) signal with +%% respect to the radial postion for \gps\ (upper-left), \rps\ +%% (upper-right), \ips\ (lower-left), \zps\ (lower-right). +%% } \label{fig:dsmear.vs.astrom} +%% \end{center} +%% \end{figure*} + +% dastrom vs flat +%% \def\figwidth{6.5in} +%% \begin{figure*}[htbp] +%% \begin{center} +%% \includegraphics[width=\figwidth]{\picdir/dastrom_vs_flat.\plotext} +%% \caption{ +%% Correlation of the derivative of the radial astrometric residual +%% displacement ($\delta R$) with respect to the radial position with the +%% flat-field tree-ring signal for \gps\ (upper-left), \rps\ (upper-right), +%% \ips\ (lower-left), \zps\ (lower-right). +%% } \label{fig:dastrom.vs.flat} +%% \end{center} +%% \end{figure*} + +\section{Discussion} +\label{sec:discussion} + +The trends measured above (Section~\ref{sec:tree.rings}) help to +illuminate the underlying causes of these different effects. + +First, if we consider the smear pattern +\oldtext{(Figure~5)}\newtext{(Figure~\ref{fig:all.effects.rband}, lower-left)}, +the measurement shows that the intrinsic sizes of the stellar images +are varying in a radial sense between the different tree-ring regions. +Although images experience an average image quality \oldtext{(due to + seeing and focus)} across the chip which may vary substantially from +exposure to exposure \newtext{(due to seeing and focus)}, stars +landing in the different tree-ring regions are consistently somewhat +larger or somewhat smaller than that average. + +Next, we can explain the correlation between the PSF photometry +residuals and the observed smear +\newtext{(Figure~\ref{fig:effects.vs.radius})}\oldtext{(Figure~12)}. +In the photometry analysis, we model the PSF allowing for some spatial +variation in the shape. However, we have a limited number of stars to +measure any spatial variation. Thus the 2D variations are sampled on +a very coarse (e.g., $3 \times 3$) grid for each chip: the PSF +parameters may vary smoothly across the chip following the bilinear +interpolation between the $3 \times 3$ grid points. Thus, the spatial +scale on which we model PSF variations is much larger than the spatial +scale on which PSF variations are actually occuring, as illustrated by +the changes in the smear plot +\oldtext{(Figure~5)}\newtext{(Figure~\ref{fig:all.effects.rband}, lower-left)}. +When the true PSF is larger than the model PSF, our model fits +systematically underestimate the amount of flux in a given object. +Conversely, when the true PSF is smaller, we overestimate the flux -- +this type of offset is a typical effect when mis-estimating the PSF +size. The slope of the trend depends on the mean typical seeing for +the given filter. For example, the \gps\ seeing is typically +1.3\arcsec, corresponding to a Gaussian $\sigma$ of 2.15 pixels. A +smearing of $\sigma^2_{major} + \sigma^2_{minor} = 0.1$ pixels$^2$ +would increase the size by about 0.02 pixels, or 1\%, roughly +consistent with the observed photometric deviation of about 5 to 10 +millimags for this amount of smearing. + +The correlation between the flat-field structures and the radial +derivative of the astrometric residual displacements in the radial +direction +\oldtext{(Figure~14)}\newtext{(Figure~\ref{fig:dastrom.and.flat})} +is consistent with radial variations in the plate-scale. The +tree-rings observed in DECam are completely attributed to effective +plate scale changes. Effective plate scale changes result in +flat-field deviations because the flat-field illumination is a source +of constant surface brightness. Pixels see a varying amount of flux +depending on their effective area. This changing plate scale also +affects the astrometry since these variations occur on spatial scales +much smaller than the astrometric model. In this description of the +tree rings, the flat-field deviations are \newtext{proportional to $\frac{\partial +\delta R}{\partial r}$, as observed in Figure~\ref{fig:dastrom.and.flat}.} +\oldtext{$-1 \times \frac{\partial + \delta R}{\partial r}$. The best-fit slopes of our correlations are +\approx 0.5, but the signal-to-noise is rather low. A slope of -1 +appears to be consistent with our measurements.} + +The fact that the PSF ellipticity changes are {\em not} correlated +with the tree-ring structure +\oldtext{(Figure~6)}\newtext{(Figure~\ref{fig:all.effects.rband})} +tells us that, unlike the case for DECam, the effective plate-scale +changes seen in the flat-field and astrometry signals are not the +dominant cause of the PSF photometry errors. Also, the fact that we +do not measure significant aperture photometry errors correlated with +the tree rings confirms this point. The amplitude of the flat-field +errors are 1-2 millimagnitudes, much smaller than the PSF photometry +errors, and far below the pixel-to-pixel noise in the aperture +magnitude residuals. It is likely in our opinion that the plate-scale +changes causing the flat-field and astrometry effects are affecting +both the ellipticity and the aperture magnitudes, but the level of the +effect is too small to see given the other systematic structures (in +the shear plot) and the noise level (in the aperture magnitudes). + +Finally, the correlation between the smear structures and the +astrometry residuals shows that these two effects are connected. +Although the correlation is weak in +\oldtext{Figure~13} \newtext{Figure~\ref{fig:effects.vs.radius}}, +careful inspection of the location of these two tree ring patterns +shows that the locations of the rings in the radial astrometric +residual images occurs at the boundaries between regions with +substantially different values of the smear signal. + +We suggest that the underlying connection between all of these +tree-ring effects is the pattern of the doping variations in the +silicon. As discussed by \cite{2014PASP..126..750P}, the tree-ring +patterns seen by the DECam team are caused by lateral electic fields in +the detector silicon (in the plane of the CCD wafer) generated by +variations in the space charges embedded in the silicon, in turn +coming from low-level changes in the doping as the silicon boule is +grown. We conclude that the astrometric and flat-field variations +seen in our detectors are caused by these same types of doping +variations. The changes in the smear (and thus the PSF magnitudes) +are apparently also related to the doping variations. The lateral +electric fields which introduce the astrometry and flat-field +variations occur at the boundary between regions with higher and lower +space charges from the dopant. Regions with high (or low) space +charge density thus correspond to regions with relatively high (or +low) amounts of smear; the astrometric deviations follow the gradient +between these regions. + +We interpret the changes in the smear term as changes in the amount of +charge diffusion as the photoelectrons travel to the bottom of the +pixel well. The blue filters exhibit the strongest changes in the +amount of smear. These are also the filters for which the detected +electrons have travelled the longest distance in the silicon, and are +thus most affected by diffusion effects. Charge diffusion (as opposed +to the charge drift caused by the lateral electric fields) results in +a Gaussian smearing of the stellar profile: as the photoelectrons +migrate from the site where they were generated by the incoming photon +to the bottom of the pixel well, they follow a random walk in the +plane of the detector. The longer the electrons take to make the +journey down to the bottom of the pixel, the further they are able to +wander from their creation coordinate in the detector. Following the +discussion in \cite{Holland.2003}, the amount of charge diffusion is +thus related to the velocity of the electrons in the direction of the +optical axis: $\sigma \sim \sqrt{2Dt}$ where $\sigma$ is the size of +the smearing kernel, $t$ is the time required for the electrons to +traverse the thickness of the silicon wafer, and $D$ is the diffusion +coefficient. The velocity of the photoelectron, and thus the time to +traverse the silicon, is related to the vertical electric fields in +the silicon, which are caused by a combination of the applied voltages +and the distribution of the space charges from the dopant. As shown +by \cite{Holland.2003}, the charge diffusion is related to the space +charge density by $\sigma \sim \rho^{-\frac{1}{2}}$ (their equation +6). Regions with high space charge densities increase the electric +field in the depletion region for a fixed voltage, and thus increase +the migration speed of the photoelectrons, reducing the amount of +charge diffusion smearing; and vice versa for regions of low +space-charge densities. + +In summary, the variations in the space-charge density caused by +variations in the dopant result in regions of higher and lower charge +diffusion, and in turn regions with PSF photometry systematic +residuals. The lateral gradients in the space-charge density induce +lateral electric fields which in turn cause lateral motions of the +photoelectrons, resulting in astrometric and flat-field deviations. + +The DECam team did not detect these charge diffusion variations. In +that case, the amplitude of the photometric effects due to the lateral +field are dominant; these include both the modification of the +flat-field as well as PSF fitting errors due to the changing PSF sizes +introduced by the varying effective pixels sizes. If the smearing +effect reported here were as large for DES compared with the lateral +PSF size changes as they are for GPC1, then the reported PSF +photometry residuals for would have had very different +characteristics. We conclude that, for DES, the lateral effects are +much larger than the diffusion variations, compared with GPC1. The +relative amplitude of these two effects depends on the details of the +applied voltages, the amplitude of the space-charge density variations +compared with the typical space-charge density, and the detector +thicknesses. It is beyond the scope of this article to model these +effects in detail. + +% http://adsabs.harvard.edu/abs/2006NIMPA.568...41K + +The origin of the fringing patterns observed in the \yps\ PSF and +aperture photometry is uncertain. The photometry fringe patterns are +similar to the fringe patterns seen in the monochromatic flat-fields. +However, since the broad-band flat-field images actually used for the +science do not exhibit the fringes, the photometry fringes are not +simply the result of having an inappropriate fringe term in the +flat-field images. One possible cause could be the interaction +between spectral features in the (largely M and K) stars used for the +photometry analysis interacting with the fringe effect -- in other +words, a flat-field image generated with a uniform spectral density +source may not be exactly right for sources with strong spectral +features. However, this explanation is clearly incomplete since it +does not explain the difference in the amplitude of the fringes seen +in the PSF vs the aperture photometry. In any case, the presence of +the fringe pattern does not affect our conclusions regarding the +charge diffusion effect. + +\section{Conclusion} + +The tree rings observed in the Pan-STARRS GPC1 data show two different +effects, though they are related. First, the images are experiencing +circularly-symmetric changes in the PSF size correlated with the +tree-ring pattern. These PSF size changes drive errors in the PSF +photometry on the scale of a few millimagnitudes, and are also +correlated with the tree-ring pattern. These PSF size changes are +consistent with changes in the charge diffusion, which also introduces +a circularly symmetric smearing. + +In addition, there are radial plate-scale changes correlated with the +tree rings. These plate-scale changes introduce flat-field errors on +the scale of \approx 1 millimagnitude and astrometric errors on the +scale of 5-10 milliarcseconds. The observed relationship between the +flat-field deviations and the radial derivative of the astrometric +deviations confirms this interpretation \citep[see also discussion + in][]{2014PASP..126..750P}. + +The spatial correlation of the gradient in the smear variations and +the astrometric variations imply that both of these two types of tree +ring effects are related, even though they manifest through different +mechanisms. We conclude that the variations in both the vertical charge +diffusion and the lateral charge migration are driven by changes +in the electric field structures in the silicon due to the same +variations in the doping structures in the silicon. + +% The small-scale variations in the charge diffusion observed in these +% devices has not been reported for DECam, Hypersuprime Cam, or +% prototype LSST sensors. + +The small-scale variations in the charge diffusion observed in the +Pan-STARRS detectors represents a new type of systematic effect in +deep depletion devices. This feature, if present in other detectors, +could manifest in systematic errors in several ways. Like in the +Pan-STARRS analysis example, the charge diffusion variations result in +fine-structure in the observed stellar point-spread functions. For +very precise photometry or morphological analysis, it will be +necessary for the PSF models to account for the extra charge +diffusion. Unlike the non-uniform pixel-size effects, correction of +the PSF photometry cannot simply be performed as an average flat-field +correction on the measurements after they have been processed. The +additional smearing acts as a convolution with a Gaussian kernel of +fixed size for a given filter. The photometry bias is a function of +the fractional change of the PSF size. Thus, the introduced error +depends on the average PSF for the image in question: an image with +good image quality will suffer larger PSF model errors than an image +with poor image quality. To account for this effect in a rigorous +way, the analysis should use the measured diffusion variations to +modify the model PSFs as a function of position before they are used +for the image analysis. + +The PV3 analysis of the Pan-STARRS $3\pi$ dataset applied an average +correction to the photometry and astrometry for each exposure as a +function of camera position with fine-enough resolution to follow +these tree-ring effects. However, since the photometry was only +corrected with an average flat-field-like correction, the full impact +of the smearing on the PSF photometry is not corrected. The remaining +systematic structure will tend to average out with many observations +in which the stars land on different portions of the detector. A +future re-processing will be required to completely correct for this +effect. + +The charge diffusion variations may also have an impact on +spectroscopic measurements. Modern, precise spectroscopic +measurements rely on precise measurements of the stellar line +profiles. If such an analysis ignores variations in the charge +diffusion, the measured line widths may be systematically biased. + +This analysis points to the importance of careful instrumental +characterization, especially for those instruments which are used for +large-scale surveys with largely automatic data analysis systems and +stringent precision goals. + +\acknowledgments + +The Pan-STARRS1 Surveys (PS1) have been made possible through +contributions of the Institute for Astronomy, the University of +Hawaii, the Pan-STARRS Project Office, the Max-Planck Society and its +participating institutes, the Max Planck Institute for Astronomy, +Heidelberg and the Max Planck Institute for Extraterrestrial Physics, +Garching, The Johns Hopkins University, Durham University, the +University of Edinburgh, Queen's University Belfast, the +Harvard-Smithsonian Center for Astrophysics, the Las Cumbres +Observatory Global Telescope Network Incorporated, the National +Central University of Taiwan, the Space Telescope Science Institute, +the National Aeronautics and Space Administration under Grant +No. NNX08AR22G issued through the Planetary Science Division of the +NASA Science Mission Directorate, the National Science Foundation +under Grant No. AST-1238877, the University of Maryland, and Eotvos +Lorand University (ELTE) and the Los Alamos National Laboratory. + +% \note{Ken: please add NASA ops grants} + +\bibliographystyle{apj} +%\bibliography{lib}{} +\input{diffusion.bbl} +%\input{magnier_bib.tex} + +\end{document} + +%% Some refs to be added as appropriate: +% Bernstein DEC astrometry : arxiv 1703.01679 +% Baumer et al arxiv 1706.07400 (Flat-fielding) Index: /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/gruen.etal.bib =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/gruen.etal.bib (revision 40483) +++ /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/gruen.etal.bib (revision 40483) @@ -0,0 +1,14 @@ + + +@article{1748-0221-10-05-C05032, + author={D. Gruen and G.M. Bernstein and M. Jarvis and B. Rowe and V. Vikram and A.A. Plazas and S. Seitz}, + title={Characterization and correction of charge-induced pixel shifts in DECam}, + journal={Journal of Instrumentation}, + volume={10}, + number={05}, + pages={C05032}, + url={http://stacks.iop.org/1748-0221/10/i=05/a=C05032}, + year={2015}, + abstract={Interaction of charges in CCDs with the already accumulated charge distribution causes both a flux dependence of the point-spread function (an increase of observed size with flux, also known as the brighter/fatter effect) and pixel-to-pixel correlations of the {Poissonian} noise in flat fields. We describe these effects in the Dark Energy Camera (DECam) with charge dependent shifts of effective pixel borders, i.e. the Antilogus et al. (2014) model, which we fit to measurements of flat-field {Poissonian} noise correlations. The latter fall off approximately as a power-law r −2.5 with pixel separation r , are isotropic except for an asymmetry in the direct neighbors along rows and columns, are stable in time, and are weakly dependent on wavelength. They show variations from chip to chip at the 20% level that correlate with the silicon resistivity. The charge shifts predicted by the model cause biased shape measurements, primarily due to their effect on bright stars, at levels exceeding weak lensing science requirements. We measure the flux dependence of star images and show that the effect can be mitigated by applying the reverse charge shifts at the pixel level during image processing. Differences in stellar size, however, remain significant due to residuals at larger distance from the centroid.} +} + Index: /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/notes.proof.v0.txt =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/notes.proof.v0.txt (revision 40483) +++ /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/notes.proof.v0.txt (revision 40483) @@ -0,0 +1,13 @@ + + Magnier, E. 2006 The Advanced Maui Optical and Space Surveillance Technologies Conf., 2, 455. + Publisher: Maui Economic Development Board + http://toc.proceedings.com/01913webtoc.pdf + + Tonry, J. 2006 The Advanced Maui Optical and Space Surveillance Technologies Conf., 1, 439. + Publisher: Maui Economic Development Board + http://toc.proceedings.com/01913webtoc.pdf + + Tonry, J. 2009 The Advanced Maui Optical and Space Surveillance Technologies Conf., 1, 364. + Publisher: Maui Economic Development Board + http://toc.proceedings.com/06773webtoc.pdf + Index: /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/proof.txt =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/proof.txt (revision 40483) +++ /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/proof.txt (revision 40483) @@ -0,0 +1,49 @@ + +Please find herein my corrections to the proof: + +Answers to the questions: + +* Q1: country name for affiliations 1 & 2 : country is USA for both. + +* Q2 : check references without a link: + + * Chambers et al 2016 : http://adsabs.harvard.edu/abs/2016arXiv161205560C + * Flewelling et al 2016 : http://adsabs.harvard.edu/abs/2016arXiv161205243F + * Hodapp et al 2004 : http://adsabs.harvard.edu/abs/2004SPIE.5489..667H, https://doi.org/10.1117/12.550179 + * Holland et al 1996 + * Magnier 2006 : + * Magnier, Chambers, Flewelling et al 2016 X : http://adsabs.harvard.edu/abs/2016arXiv161205240M [data system] + * Magnier, Sweeney, Chambers et al 2016 Y : http://adsabs.harvard.edu/abs/2016arXiv161205244M [analysis (psphot)] + * Magnier, Schlafly, Finkbeiner et al 2016 Z : http://adsabs.harvard.edu/abs/2016arXiv161205242M [calibration] + * Tonry & Onaka 2009 + * Tonry et al 2006 + * Waters et al 2016 : http://adsabs.harvard.edu/abs/2016arXiv161205245W + +* Q3 : publisher details for 3 references: + + Hodapp et al 2004 : publisher is SPIE + Janesick \& Elliott 1992 : publisher is Astronomical Society of the Pacific + Magnier 2007 : publisher is Astronomical Society of the Pacific + +* Q4 : volume & page number for 3 refs + + Magnier 2006, The Advanced Maui Optical and Space Surveillance Technologies Conf., Vol 2, Page 455. + Publisher: Maui Economic Development Board + + Tonry et al 2006, The Advanced Maui Optical and Space Surveillance Technologies Conf., Vol 1, Page 439. + Publisher: Maui Economic Development Board + + Tonry & Onaka 2009, The Advanced Maui Optical and Space Surveillance Technologies Conf., Vol 1, Page 364. + Publisher: Maui Economic Development Board + +* Q5 : update refs to arxiv if possible + +------ + +Other Notes: + +P. 13 References: + +Antilogus et al 2014 has a floating comma after the journal (JInst) + +Gruen et al 2015 has an extra space after the journal (JInst) Index: /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/response.v1.tex =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/response.v1.tex (revision 40483) +++ /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/response.v1.tex (revision 40483) @@ -0,0 +1,228 @@ +\documentstyle[gletter,epsfig]{article} +\begin{document} +\letterhead + +We thank the referee for the careful and thoughtful examination of our +article. We particularly appreciate the suggestion to reduce the +number of figures and remove those that are superfluous. We have +removed the various scatter plots and replaced them with plots of the +radial run of the important effects, along with two plots showing the +radial run of the derivative of the smear compared with the +astrometric residuals and the derivative of the astrometric residuals +compared with the flat-field features. We have also removed the +5-filter panels for the astrometric, flat-field, smear, and shear +effects. We instead adjust the discussion to focus on the single +figure with all 6 effects shown for r-band. We have adjusted the +wording to account for the change in the figures. + +Below are our point-by-point responses to the {\tt referee's suggestions}. + +\begin{verbatim} +Figures: there are substantially more than are needed; I don't think you need +to show every band of every effect in Figs 3,4,5, and 6 - Fig 7 is by far the +most informative, allowing comparison of your diagnostics. Similarly, showing +all the scatter plots for all bands in Figs 8,9,10,11,12 is overkill, in my +view. I'm not sure any of these x vs y scatter plots are really needed; for me +it suffices to give the amplitudes of each effect relative to g band in +tabular or graphical form. The most important figure that I would want to see +is, in fact, missing: a plot of the radial run of each effect (a 1d plot of +Fig 7, basically), so that I can e.g. visually see whether the flat field is +the derivative of the astrometric displacements, as the lateral-field model +predicts. +\end{verbatim} + +This is a good point. We have greatly reduced the number of plots, +removing the 5-filter images in Figs 3,4,5,6 (astrometry, flat-field, +smear, shear) and the scatter plots for figures 8-14. We have +replaced these with 3 new figures: one showing the radial run of the 4 +effects with significant tree-ring patters (smear, psf mags, +flat-field, and astrometry) as suggested; a second plot showing the derivative +of the smear and the astrometry; a third plot showing the derivative +of the astrometry with the flat-field. We feel these new plots make +the relationships between the different trends much clearer than the +scatter plots. + +\begin{verbatim} +Also the spacing of the arrows in the ellipticity figures is too coarse to see +whether there are radial shear variations that are coherent with any of the +ring patterns. Here a 1d radial plot would be essential. +\end{verbatim} + +We disagree : the arrows are only showing the direction; the image in +the background shows the amplitude of the ellipticity variations, +which are completely dominated by the large-scale (optical) effects. +The resolution of the image is sufficient to follow the tree rings, +but there is no signal matching that pattern. + +\begin{verbatim} +page 4, Line 18: "non-pixel uniformity" ?? +\end{verbatim} + +Changed the wording to be clearer. + +\begin{verbatim} +4/22: "relatively coarse grid" - can you give the grid size? +\end{verbatim} + +Added the grid size. + +\begin{verbatim} +4/36: Can you say whether there are any differences between the "PV2 analysis +discussed here" and the public data releases, i.e. corrections that this work +implies need to be made to the public data? +\end{verbatim} + +Added a sentence about this. + +\begin{verbatim} +5/12: "diffusive effects of the lateral fields" - the lateral fields effects I +would not consider "diffusive" since they simply are conducting the charge +packets in a tranverse direction. The effect you find dominating the PS1 +devices differs in being primarily diffusive. +\end{verbatim} + +True, we mean charge motions or migrations caused by the lateral +fields. We've reworded this to be more accurate. + +\begin{verbatim} +Table 1, 8/13, other places: these values are referred to as the "per-pixel +standard deviations" which is unclear to me. Is this the SD of different +stellar measurements or flat-field pixels contributing to the mean of a given +super-pixel in the plots? Or is it the SD of the super-pixel averages, i.e. +the SD of the plotted values? Or something else? The most relevant statistic +is the SD after the shot noise of the individual measurements has been +suppressed, since we don't care about measurement noise in this paper. +\end{verbatim} + +This referred to the standard deviation of the pixel values in the +superpixel images. In retrospect, this table did not add much useful, +so we've removed it and changed the wording in the text to be clearer. + +\begin{verbatim} +Fig 1: In the caption it would be good to state that this is for one device in +the camera (even though this is stated in the text). +\end{verbatim} + +Done. + +\begin{verbatim} +Figs 1,2, 4: Why does Y band have so many missing (white) pixels? +impossible to look for patterns. +\end{verbatim} + +The effect from y-band fringing is much larger than the relevant +effects, saturating the dynamic range of the color scale. Added +wording to the captions to explain. + +\begin{verbatim} +Fig 3: what are the units of the plot? +\end{verbatim} + +This figure has now been removed; the text gives the dynamic range for +all 6 sub-figures in the r-band figure showing all effects. + +\begin{verbatim} +Table 2 (and in text in many places): you talk about "correlations" between +filters or different effects, but my guess is that what you are really +plotting is the amplitude of variations relative to those in g band. The +correlation coefficient would be something different - and less interesting, +since the correlation coefficient will depend heavily on size of irrelevant +sources of variation like measurement noise and the g-band striping. A graph +of these values (grizY along the x axis, different symbols for each effect) +might be more effective than this table. +\end{verbatim} + +This is correct. The table shows the relative amplitudes of the +effects. We have replaced this table with a plot of the relative +amplitudes of the 4 effects which show the tree-ring patterns. + +\begin{verbatim} +Table 3: same comment, what does "correlation" mean here? +\end{verbatim} + +Again, the is the relative amplitude of the effects compared with +g-band. Along with Table 1, we have decided to remove this table. +The conclusions of the paper are not dependent on the relative +amplitude of the effects, except to the extent that the blue filters +show stronger effects than the red filters. + +\begin{verbatim} +11/9: "deviations relative to the median flat-field image" - do you mean that +you are plotting the deviations relative to the median *of* the flat-field +image? Otherwise I don't understand. +\end{verbatim} + +Correct. The wording is fixed. + +\begin{verbatim} +11/25: A Gaussian smoothing of 3 superpixels - is this well above the max +"wavelength" of the rings? If not it can suppress a lot of them. Seems small +compared to the features visible in some of the figures. +\end{verbatim} + +This smoothing scale is large enough that it does not suppress +significantly the tree-ring features; without the smoothing, some of +the brick-wall pattern remains. The wording has been updated to clarify. + +\begin{verbatim} +Fig 14: what are the red and blue lines? +\end{verbatim} + +This figure has been removed and the information replaced with the new +Figure 6. + +\begin{verbatim} +24/19-21: These statements seem contradictory: the slope should be -1; the +data give a slope of 0.5; they are consistent. +\end{verbatim} + +The best-fit slope was actually $\sim -0.5$, with errors large enough +that it was consistent with -1.0: the missing minus sign was a typo. +However, we had a sign error in the calculation of the +proportionality; with the correct definition, the proprotionality is +positive. However, the noisy measurement of the slope was not very +informative. Referring to the new Figure 6, it is clear that that the +derivative of the astrometric variations is roughly proportional to +the flat-field variations. The point of this portion of this +discussion is simply that the astrometry and flat-field variations +behave in the way we would expect if they were caused by the lateral +charge migration effect, and thus this is a plausible explanation for +those observed effects. + +We have reworded the text to be clearer about this. + +\begin{verbatim} +24/23, 25/28, 25/30: Here you should probably be referring to DECam (the +camera) rather than DES (the survey). +\end{verbatim} + +Fixed. + +\begin{verbatim} +25/17: Is there any concise way to explain why adding dopants (space charge) +will *decrease* the transit time and charge diffusion at fixed voltage and +distance across the device? +\end{verbatim} + +The increased space charge increase the electric field in the device +for a fixed applied voltage, thus increasing the migration speed. +The description has been somewhat clarified. + +\begin{verbatim} +Conclusions: you should probably say something about which published +quantities of the PS1 catalogs are affected by this effect, by how much, and +whether any corrections have been applied, whether the effects are reduced by +averaging over multiple exposures. +\end{verbatim} + +Done. + +\epsfig{figure=signature1.ps,width=2.5in,angle=0} + +Eugene Magnier \\ +Specialist / Astronomer \\ +Pan-STARRS IPP Lead \\ +IfA / Pan-STARRS \\ +University of Hawaii + +\end{document} Index: /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/response.v2.tex =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/response.v2.tex (revision 40483) +++ /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/response.v2.tex (revision 40483) @@ -0,0 +1,76 @@ +\documentstyle[gletter,epsfig]{article} +\begin{document} +\letterhead + +We thank the referee for the updated comments on our article. Below +are our point-by-point responses to the {\tt referee's suggestions}. + +\begin{verbatim} +Page 5, line 52: the presence of fringing is odd here since normally neither +photometry nor dome flats exhibit fringes. Is this because the dome flats are +being made with a monochromatic source (as suggested later in the paper at +8/19)? If so, it might be worth changing this sentence to say "fringing from +the monochromatic dome-flat illumination..." +\end{verbatim} + +The flat-fields shown in the article use the monochromatic flats, but +the flats we use for science analysis are broad-band dome flats. We +used the monochromatic flats to avoid washing out any effects across +the wide band-pass. And as the referee notes, the broad-band z- and +y-band flats do not show the fringing effect. We have changed the +wording to explain the difference between the monochromatic flats used +for this study and the broad-band flats used for science. We have +also expanded the discussion of the fringing effect somewhat in the +Discussion, noting in addition that the fringing is not relevant to +the our interpretation of the charge diffusion effect. + +\begin{verbatim} +10/14: here is says a diagonal pattern "is seen in g and r" - since these +panels are no longer being shown, and also I think the banding is absent from +Fig 3 because it's been high-passed, the reader can't actually "see" this. +You might clarify the wording to keep people from expecting looking for it in +the figure. +\end{verbatim} + +We fixed the wording and noted that the high-pass filtering removes +the diagonal banding from the r-band flats. + +\begin{verbatim} +12/27: I think you want a comma in "with the strongest systematic tree-ring +effects, to yP1,..." +\end{verbatim} + +Fixed. + +\begin{verbatim} +12/28: do you mean that smear & astrometry are different from each other, or +that these two are different from the other two? +\end{verbatim} + +Fixed the wording to clarify that the trend of the smear is different +from the trend of the astrometry. + +\begin{verbatim} +16/8: someone who knows device physics better than me would not ask this +question, but: if the voltage is the integral of the electric field, how can +the higher space charge change the electric field while keeping the same +voltage? +\end{verbatim} + +It is true that the integral of the electric field between the two +conductors which define the voltage across the device should be the +same regardless of the dopant density. However, the detailed +structure of the electric field will be different. If the dopant adds +space charges in the depleted region, the electric field will be +increased slightly within, and correspondingly reduced slightly +before. + +\epsfig{figure=signature1.ps,width=2.5in,angle=0} + +Eugene Magnier \\ +Specialist / Astronomer \\ +Pan-STARRS IPP Lead \\ +IfA / Pan-STARRS \\ +University of Hawaii + +\end{document} Index: anches/czw_branch/20170908/doc/release.2015/systematics.20140411/systematics.tex =================================================================== --- /branches/czw_branch/20170908/doc/release.2015/systematics.20140411/systematics.tex (revision 40482) +++ (revision ) @@ -1,1165 +1,0 @@ -% \documentclass[iop,floatfix]{emulateapj} -\documentclass[10pt,preprint]{aastex} -% \pdfoutput=1 - -% see latex.readme.txt for notes on using the PS1 template -%\documentclass[12pt,preprint]{aastex} -%\documentclass[manuscript]{aastex} -%\documentclass[preprint2]{aastex} -%\documentclass[preprint2,longabstract]{aastex} - -\RequirePackage{graphicx} -\RequirePackage{color} -\RequirePackage{code} -\RequirePackage{pbox} -\input{astro.sty} - -\usepackage[T1]{fontenc}% (2) specify encoding - -% online version may use color, but print version needs b/w -\def\plotmode{col} -%\def\plotmode{bw} - -%\def\plotext{pdf} -\def\plotext{ps} - -%\def\picdir{/home/eugene/chipresid.20140404} -%\def\picdir{/data/kukui.2/eugene/chipresid.20140404} -\def\picdir{pics} - -% Pick a terse version of the title here; -\shorttitle{Charge Diffusion Variations in PS1} -\shortauthors{E.A. Magnier et al} -\begin{document} -\title{Charge Diffusion Variations in Pan-STARRS\,1 CCDs} - -% this is a crude trick to get the order of affiliations right. These -% names are used in the affiliations below. The user needs to (1) set -% the order and numbers to have the correct sequence in the author -% list and (2) re-order the list at the bottom (and comment-out as needed) -\def\IfA{1} -\def\CfA{2} -\def\MPIA{3} -\def\Princeton{3} -\def\USNO{4} -\def\JHU{1} - -% This example has a first author from UH: -\author{ -Eugene A. Magnier,\altaffilmark{\IfA} -J.~L. Tonry, \altaffilmark{\IfA} -D. Finkbeiner,\altaffilmark{\CfA} -E. Schlafly,\altaffilmark{\MPIA} -%PS Builder List -% W.~S. Burgett,\altaffilmark{\IfA} -K.~C. Chambers,\altaffilmark{\IfA} -% L. Denneau,\altaffilmark{\IfA} -% P. Draper,\altaffilmark{\DUR} -% H.~A. Flewelling,\altaffilmark{\IfA} -% T. Grav,\altaffilmark{\IfA} -% J. N. Heasley,\altaffilmark{\IfA} -% K. W. Hodapp,\altaffilmark{\IfA} -% M. E. Huber,\altaffilmark{\IfA} -% R. Jedicke,\altaffilmark{\IfA} -% N. Kaiser,\altaffilmark{\IfA} -% R.-P. Kudritzki,\altaffilmark{\IfA} -% G. A. Luppino,\altaffilmark{\IfA} -% R. H. Lupton,\altaffilmark{\Princeton} -% E. A. Magnier,\altaffilmark{\IfA} -% N. Metcalfe,\altaffilmark{\DUH} -% D. G. Monet,\altaffilmark{\USNO} -% J.~S. Morgan,\altaffilmark{\IfA} -% P. M. Onaka,\altaffilmark{\IfA} -% P.~A. Price,\altaffilmark{\Princeton} -% C.~W. Stubbs,\altaffilmark{\CfA} -% W.~E. Sweeney,\altaffilmark{\IfA} -% J.~L. Tonry, \altaffilmark{\IfA} -% R. J. Wainscoat,\altaffilmark{\IfA} and -% C. Z. Waters,\altaffilmark{\IfA} -PS Builders TBA -} % this bracket terminates author list - -% The ordering here should be sequential, matching the sequence in the list of authors: -\altaffiltext{\IfA}{Institute for Astronomy, University of Hawaii, 2680 Woodlawn Drive, Honolulu HI 96822} -\altaffiltext{\CfA}{Harvard-Smithsonian Center for Astrophysics, 60 Garden Street, Cambridge, MA 02138} -% \altaffiltext{\Princeton}{Department of Astrophysical Sciences, Princeton University, Princeton, NJ 08544, USA} -% \altaffiltext{\USNO}{US Naval Observatory, Flagstaff Station, Flagstaff, AZ 86001, USA} -% \altaffiltext{\JHU}{Department of Physics and Astronomy, Johns Hopkins University, 3400 North Charles Street, Baltimore, MD 21218, USA} -\altaffiltext{\MPIA}{Max Planck Institute for Astronomy, K\"onigstuhl 17, D-69117 Heidelberg, Germany} -\begin{abstract} - -Thick back-illuminated deep-depletion CCDs have superior quantum -efficiency over previous generations of thinned and traditional thick -CCDs. As a result, they are being used for major wide-field imaging -cameras in several projects. We use observations from the Pan-STARRS -$3\pi$ survey to characterize the behavior of the deep-depletion -devices used in the Pan-STARRS\,1 Gigapixel Camera. We have -identified systematic spatial variations in the photometric behavior and -stellar profiles which are similar to the so-called ``tree rings'' -identified in devices used by other wide-field cameras (DECam and -Hypersuprime Camera). The tree-ring features identified in these -other cameras result from lateral electric fields which displace the -electrons as they are transported in the silicon to the pixel -location. In contrast, we show that the photometric and morphological -modifications observed in the GPC1 detectors are caused by variations -in the vertical charge transportation rate and resulting charge -diffusion variations. -\end{abstract} - -% insert additional keywords as appropriate: -\keywords{Surveys:\PSONE } - -\section{INTRODUCTION}\label{sec:intro} - -CCD detectors have evolved greatly since they were first introduced -for astronomical imaging in the mid 1970s. In addition to the -well-known increases in the size of CCDs over the past 4 decades, CCD -architecture has gone through three major evolutionary stages. - -The first generation of CCDs used a silicon substrate a few hundred -microns thick on top of which gate structures were deposited to define -the pixels. A positive voltage applied to the gate layers would -create a shallow region (\approx 10 microns thick) in which the holes -were depleted. This ``depletion region'' acted as a potential well to -trap electrons, specifically those generated by absorbed photons. The -thick silicon substrate required illumination from the ``front'' side -with the thin gate structures to allow the photons to reach the -depletion region and be detected. These early CCDs had modest quantum -efficiency as photons were easily absorbed by the several micron thick -gate structures. For an excellent review of the history of CCD -development, see \cite{1992ASPC...23....1J}. - -Thinned, backside-illuminated CCDs such as the TI 3PCCD -\citep{1981SPIE..290....6B} were developed to address the quantum -efficiency limitations of the first generation thick CCDs. The -silicon substrate was removed using a chemical process, leaving a -delicate device only \approx 10 - 20\micron\ thick, exposing the -depletion region on the backside. Photons entering the backside of -the device are not blocked by the gate structures and thus more easily -absorbed and detected. Thinned backside-illuminated CCDs have high -quantum efficiency to blue photons. However, as the wavelength -increases beyond \approx 800 nm, the silicon becomes more transparent -to the photons, with a corresponding drop in quantum efficiency for -red photons. In addition, thin film interference between the entering -photons and those reflecting off the front side of the CCD result in -``fringe'' patterns for redder photons. - -Early generations of CCDs were made of low-resistivity (\approx 10 - -50 $\Omega$-cm) silicon. Following experiments beginning in the early -1990s \citep{Holland.1996}, CCDs made from thick, high-resistivity ($ -> 10 k\Omega$-cm) silicon were developed for astronomical instruments -in the early 2000s \citep{Holland.2003}. The high-resistivity of the -silicon allows for depletion regions of hundreds of microns in depth, -compared to \approx 10\micron\ for the low-resistivity silicon. This -modification allows for a back-illuminated CCD with a relatively thick -silicon subtrate of 75 - 300\micron. Blue photons impinging on the -back of the device are absorbed near the back surface of the device -and are caried through the depletion region to the gates on the front -side. The thick silicon allows red photons to have a greater chance -to be absorbed, increasing quantum efficiency in the red. Because -these thick, deep-depletion devices have near-unity quantum efficiency -across a very wide spectral range, they have become the design of -choice for many modern, large-scale CCD cameras (e.g., Pan-STARRS -GPC1, \citealt{2009amos.confE..40T}; Subaru Hypersuprime Camera, -\citealt{2010SPIE.7735E..3FK}; Dark Energy Survey Camera, -\citealt{2015AJ....150..150F}). - -While these deep-depletion CCDs seem to be ideal, they do have -features which can cause challenges for precise measurements. As a -result of the ``Brighter-Fatter Effect'' -\citep{2014JInst...9C3048A,2015JInst..10C5032G}, the profile of bright -stars are measured to be wider than the profiles of faint stars. The -accepted interpretation is that the electric fields produced by the -electrons accumulated from a star repel successive incoming electrons, -with the repulsion increasing the more electrons have accumulated. - -The effects of lateral electric fields are likewise identified as the -cause of the so-called ``tree rings'' observed in the flat-field, -astrometry, and photometry response of thick deep depletion detectors -\citep{2014PASP..126..750P}. These tree-ring patterns have been noted -in the flat-field response of deep depletion devices since their early -testing \citep[see, e.g., Figure 2 in][]{2010SPIE.7735E..1RE} and were -initially considered to be a sensitivity response which could be -removed with a flat-field. As discussed in detail by -\cite{2014PASP..126..750P}, these tree rings are more correctly -interpretted as variations in the effective pixel area due to -migration of the electrons pushed by lateral electric fields induced -by small changes in the doping used to set the resistivity of the -silicon. The changes in the effective area result in changes to the -apparent flat-field response as well as the astrometric response of -the detector. More subtly, the flat-field response changes, since -they do not reflect actual variations in sensitivity, can lead to -systematic photometry errors for astronomical sources if the -flat-field images are used in the standard fashion. - -In this paper, we examine the behavior of an apparently-similar kind -of tree ring observed in the Pan-STARRS GPC1 CCDs. Although we also -observe the pixel effective area changes caused by lateral electric -fields as described by \cite{2014PASP..126..750P}, we show below a -second effect which is more important in driving systematic photometry -errors. We find that variations in charge diffusion, also resulting -from changes in the silicon doping structures, affect both the -observed stellar profiles as well as the photometry measured with -profile fitting techniques. In Section~\ref{sec:PS1}, we discuss the -Pan-STARRS telescope, camera, and survey data used in this analysis. -In Section~\ref{sec:tree.rings}, we present the tree-ring -patterns as observed in several different types of measurements: -flat-field response, systematic photometry residuals, systematic -astrometric residuals, and stellar profile shape variations. In -Section~\ref{sec:discussion}, we discuss the interpretation of -patterns we observe and present a simple model to explain the observed -behavior. We conclude with a discussion of the implications of this -effect on astronomical measurements from deep depletion instruments - -\section{Pan-STARRS1} -\label{sec:PS1} - -The 1.8m Pan-STARRS\,1 telescope (PS1), located on the summit of -Haleakala on the Hawaiian island of Maui, has been surveying the sky -regularly since May 2010 \citep{chambers2017}. From May 2010 through -March 2014, PS1 was run under the aegis of the Pan-STARRS Science -Consortium to perform a set of wide-field science surveys; since March -2014, operations have been supported primarily by NASA's Near Earth -Object Observation program, see \cite{2015IAUGA..2251124W}. Under the -PS1SC, the largest survey, both in terms of area of the sky covered -($3\pi$ steradians) and fraction of observing time (56\%), was the -\TPS\ in which the entire sky north of Declination $-30$\degrees\ was -imaged up \approx 80 times over 4 years. These observations were -distributed over five filters, \grizy, and have been astrometrically -and photometrically calibrated to good precision -\citep{magnier2017.calibration}. - -% 2004SPIE.5489..667H == PS1.optics -% 2008SPIE.7014E..0DO == PS1.GPCB -% 2009amos.confE..40T == PS1.GPCA -% 2012ApJ...756..158S == ubercal -The wide-field PS1 telescope optics \citep{2004SPIE.5489..667H} image -a 3.3 degree field of view on a 1.4 gigapixel camera -\citep[GPC1][]{2009amos.confE..40T}, with low distortion and generally -good image quality. The median seeing for the \TPS\ data vary -somewhat by filter: (\grizy) = (1.31, 1.19, 1.11, 1.07, 1.02) -arcseconds. Routine observations are conducted remotely from the -Advanced Technology Research Center in Kula, the main facility of the -University of Hawaii's Institute for Astronomy operations on Maui. - -GPC1 \citep{2009amos.confE..40T}, currently the largest astronomical -camera in terms of number of pixels, consists of a mosaic of 60 -edge-abutted $4800\times4800$ pixel detectors, with 10~$\mu$m pixels -subtending 0.258~arcsec. These CCID58 detectors, manufactured by -Lincoln Laboratory, are 75\micron-thick back-illuminated CCDs -\citep{2006amos.confE..47T,2008SPIE.7021E..05T}. Initial performance -assessments are presented in \cite{2008SPIE.7014E..0DO}. The active, -usable pixels cover \approx 80\% of the FOV. - -\subsection{Data Processing and Calibration} - -% PS1_IPP = \bibitem[Magnier(2006)]{PS1.IPP} Magnier, E.\ 2006, -% Proceedings of The Advanced Maui Optical and Space Surveillance -% Technologies Conference, Ed.: S. Ryan, The Maui Economic Development -% Board, p.E5 - -Images obtained by PS1 are processed by the Pan-STARRS Image -Processing Pipeline (IPP; -\citealp{2006amos.confE..50M,magnier2017.datasystem}). All -observations are processed nightly, with results sent to groups within -the science consortium (i.e., PS1SC during the \TPS) performing -short-term science projects (e.g., searching for transient and moving -objects). In addition, the \TPS\ dataset has been re-processed -several times with improved calibration and analysis techniques. To -date (2017 July), 3 re-processings starting from raw pixel data have -been performed. The labels PV0, PV1, PV2, PV3 are used identify the -nightly processing and successive re-processing versions. PV3 has -been used for the public release of the Pan-STARRS \TPS\ data via the -{\it Barbara A. Mikulski Archive for Space Telescopes} (MAST) at the -Space Telescope Science Institute.\footnote{http//panstarrs.stci.edu} - -The data processing and calibration operations are discussed in detail -in elsewhere -\citep{magnier2017.analysis,magnier2017.calibration,waters2017}. -We re-visit here a number of points that are of significance to this -study. Images are processed following a fairly standard sequence of -image detrending, source detection, and initial calibration -(astrometric and photometric) of those detected sources. Additional -standard processing critical to PS1 science operations includes -geometric transformation (`warping') and image combinations (summed -stacks and differences). For the purposes of this analysis, we are -only considering the sources detected in the individual exposures from -the initial analysis steps. - -% Magnier.belgium: -% \bibitem[Magnier(2007)]{PS1.photometry} Magnier, E.\ 2007, The Future -% of Photometric, Spectrophotometric and Polarimetric Standardization, ASP Conference Series {\bf 364}, 153 - -%IPP astrometry (NOT USED) -% \bibitem[Magnier {\it et al.}(2008)]{PS1.astrometry} Magnier, E.~A., Liu, -% M., Monet, D.~G., \& Chambers, K.~C.\ 2008, IAU Symposium, {\bf 248}, 553 - -As discussed in \cite{waters2017}, image detrending includes -flat-field processing with a single epoch flat-field image for each -filter. The flat-field image used for this analysis has been -generated by median-combining dome flat-field images (after -pre-processing and pixel outlier rejections) and then multiplying by a -photometric flat-field correction image generated by the analysis of a -grid of images of a dense stellar field. The purpose of this second -step is to correct the basic flat-field image for errors arising from -the non-uniformity of the illumination, from non-pixel uniformity due -to the varying optical distorition across the field, and any other -factors which may make the flat-field image inconsistent with stellar -photometry, e.g., SED, filter band-pass variations, etc -\citep[see][]{waters2017,2004PASP..116..449M,2007ASPC..364..153M}. -This correction was made on a relatively coarse grid across the focal -plane in order to accumulate sufficient statistics from the stars in -the relatively small number of images available at the time. We have -found that a single flat-field set can be used for all PS1 -observations to yield photometric systematic errors at the level of \approx -2\%. PS1 benefits in this regard from the stability of having a -single instrument which is rarely removed. - -Photometry of the PS1 images is performed using a -point-spread-function (PSF) model as well as multiple kinds of -apertures \citep{magnier2017.analysis}. In this analysis, we refer to -aperture photometry performed using an aperture defined based on the -image quality observed for a given chip. The aperture diameter is set -to be 3.75 times the FWHM for the image. - -To improve the photometric systematic errors beyond the level achieved -with a single (photometrically corrected) flat-field set, the PS1 -photometry is re-calibrated within the databasing system based on the -properties of the measured photometry. The calibration process is -discussed by -\cite{2012ApJ...756..158S,2013ApJS..205...20M,magnier2017.calibration}. -As part of this process, several flat-field corrections have been -determined. For the PV2 analysis discussed here, a flat-field -correction determined during the ubercal analysis -\citep[see][]{2012ApJ...756..158S} consisted of an $8\times 8$ grid of -corrections for each GPC1 chip, corresponding to a correction for each -OTA ``cell'' and filter for each of 4 seasons. The boundaries of -those seasons are tentatively identified with modifications to the -baffle structures or the system optics. The critical point here is -that the final effective flat-field image for the PV2 dataset is based -on a dome-flat at the highest resolution, with very low resolution -corrections based on photometry, resulting in photometric systematic -uncertainties in the range 7 - 12 millimagnitudes, depending on the -filter \citep{2013ApJS..205...20M}. - -For all objects, positions are measured from the PSF model for the -brighter sources (using a non-linear fitting process) and from a -simple centroid (1st moment) for the fainter source -\citep{magnier2017.analysis}. These position measurements are -used in the astrometric analysis. The astrometric calibration is -discussed by \cite{magnier2017.calibration}; for the PV2 -dataset, the typical systematic floor is \approx 15 - 20 -milliarcsecond for individual measurements of brighter stars. - -\section{Tree-Ring Patterns} -\label{sec:tree.rings} - -\begin{table} -% \tiny -\begin{center} -\caption{Systematic Trends : Standard deviation by filter\label{table:sigmas.by.filter}} -\begin{tabular}{|l|rrrrr|} -\hline -{\bf Filter} & {\bf psf mags} & {\bf ap mags} & {\bf astrom} & {\bf smear} & {\bf flat} \\ - & mmags & mmags & mas & pixels$^2$ & mmags \\ -\hline -\gps & 11.8 & 13 & 8.0 & 0.169 & 3.0 \\ -\rps & 10.9 & 12 & 6.7 & 0.133 & 2.2 \\ -\ips & 8.5 & 10 & 6.0 & 0.069 & 1.7 \\ -\zps & 8.7 & 12 & 5.5 & 0.052 & 3.2 \\ -\yps & 16.5 & 26 & 6.8 & 0.059 & 15.3 \\ -\hline -\end{tabular} -\end{center} -\end{table} - -For many of the GPC1 OTA CCDs, we observe a spatial pattern in the -photometric residuals for each device which is similar in appearence -to the tree rings described in the Dark Energy Camera (DECam) by -\cite{2014PASP..126..750P}. This pattern consists of systematic -deviations which are consistent in a set of circular arcs centered on -the corner of the CCD, as shown in Figure~\ref{fig:psfmags.by.filter}. -The details of the analysis used to generate -Figure~\ref{fig:psfmags.by.filter} are given below. For now, we note -that the GPC1 CCDs are constructed by dividing the circular silicon -wafer into 4 inscribed squares. Thus the corners of the CCDs lie in -the center of the silicon boule, just as the center of the circular -tree rings described by \cite{2014PASP..126..750P} match the center of -the boule from which they came. This gives the impression that a -similar mechanism is responsible for the pattern observed in the PS1 -photometry and the DECam photometry, namely the diffusive effects of -lateral electric field variations in the detectors. In the next -section, we will make the case that the patterns observed in the PS1 -photometry residuals are {\em not} caused by this mechanism, but are -instead caused by variations in the {\em vertical} electric field (the -field direction perpendicular to the CCD surface). - -First, in this section, we will describe how we have measured the -presence or absence of these tree-ring patterns in 5 types of data. -For all of these examples, we use a single GPC1 CCD (XY40) to -illustrate the effects in detail, but a similar set of effects are -seen in many of the GPC1 detectors. First, we show the residual PSF -photometry. Second, we show the residual aperture photometry. Third, -we show the astrometric residual patterns. Fourth, we show the -patterns observed in the flat-field images. Finally, we show -measurements derived from the second-moments of the stars. - -For all effects discussed below, we are measuring the mean value of -the effect in 10x10 pixel superpixels across the detector. The -resulting images are all constructed so that a given superpixel -represents the same range of true GPC1 XY40 pixels regardless of the -type of measurement. To generate the photometry, astrometry, or -second-moment plots, measurements were extracted from the PV0 DVO -database \citep{magnier2017.calibration} for observations covering -the region ($\alpha$,$\delta$) = (90\degree\ -- 150\degree, --25\degree\ -- 10\degree). This region of the sky provides a fairly -high density of stars, but avoids the Galactic Plane where confusion -may potentially contaminate the measurement. We limit the analysis to -good measurements (\ippmisc{PSF_QF} $>$ 0.85, see -\citealt{magnier2017.analysis}) of likely stars ($|m_{psf} - -m_{aper}| < 0.2$). Only measurements with instrumental magnitude $< --8.0$ ($-2.5\log \mbox{cts sec}^{-1} < -8.0$) are included to ensure -reasonable signal-to-noise per measurement. We require at least 2 -measurements in a given filter and at least 5 measurements total for -any star included in the analysis. - -\subsection{Photometric Residuals} - -% PSF Magnitudes -\def\figwidth{5.2in} -\def\jumpleft{-2.6in} -\def\capwidth{2.4in} -\begin{figure*}[htbp] -\begin{center} -\parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/dmag.\plotext}} -\hspace{\jumpleft} -\parbox[b]{\capwidth}{ -\caption{PSF Magnitude residuals by filter (\grizy). White boxes are - GPC1 cells which have been masked due to poor response. Superpixels - representing regions of $10\times10$ pixels are used to determine - the median deviation for measurements at the given chip pixel - location compared with the average photometry for the given - object.} \label{fig:psfmags.by.filter}} -\end{center} -\end{figure*} - -% Aperture Magnitudes -\begin{figure*}[htbp] -\begin{center} -\parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/dapmag.\plotext}} -\hspace{\jumpleft} -\parbox[b]{\capwidth}{ -\caption{Aperture Magnitude residuals by filter (\grizy). White boxes - are GPC1 cells which have been masked due to poor response. - Superpixels representing regions of $10\times10$ pixels are used to - determine the median deviation for measurements at the given chip - pixel location compared with the average photometry for the given - object. } \label{fig:apmags.by.filter}} -\end{center} -\end{figure*} - -Figure~\ref{fig:psfmags.by.filter} shows the 2D patterns of PSF -photometry residuals. In this case, we select PSF magnitude -measurements for detections of stars which fall in the given -superpixel. We subtract each measurement from the average magnitude -for the object in the selected filter ($\delta m_{psf} = -\overline{m}_{psf} - m_{psf}$) to determine the residual magnitude, -excluding as an outlier any measurement with $|\delta m_{psf}| > 0.5$. -For a given superpixel, we measure the median of the $\delta m_{psf}$ -distribution. The figure shows $\delta m_{psf}$ for each filter -(\grizy). The dynamic range of the color scale is from -20 to +20 -millimagnitudes for all 5 plots. - -The tree-ring pattern is clearly visible for the four blue filters, -but finging dominates the pattern for \yps. Small offsets of -individual cells are also apparent for \zps. While the patterns are -clear across the image, the signal-to-noise of the structures per -pixel is not very strong in these images. The per-pixel standard -deviations of these plots are listed in -Table~\ref{table:sigmas.by.filter}. The per-pixel standard deviation -is comparable to the amplitude of the correlated structures, so we -need to integrate along the radial structures to make stronger -statements about these patterns. - -Figure~\ref{fig:apmags.by.filter} shows the equivalent measurement for -aperture photometry instead of PSF photometry. The finging -pattern again dominates the plot for \yps, but the tree rings are not -seen in any of the filters. A diagonal pattern is visible in \gps -which is not observed in the PSF magnitudes. While the per-pixel -scatter is somewhat (10\% to 20\%) higher for these aperture -magnitudes than for the PSF magnitudes -(Table~\ref{table:sigmas.by.filter}), a structure with the amplitude -of the PSF magnitude tree-rings would certainly have been obvious. - -\subsection{Astrometric Residuals} - -% astrometry radial term -\begin{figure*}[htbp] -\begin{center} -\parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/drad.\plotext}} -\hspace{\jumpleft} -\parbox[b]{\capwidth}{ -\caption{Astrometric residuals of the displacement in the radial - direction, relative to the chip coordinate -5,4960 (upper left - corner), by filter (\grizy). White boxes are GPC1 cells which have - been masked due to poor response. Superpixels representing regions - of $10\times10$ pixels are used to determine the median deviation - for measurements at the given chip pixel location compared with the - average astrometry for the given - object. } \label{fig:astrom.by.filter}} -\end{center} -\end{figure*} - -Figure~\ref{fig:astrom.by.filter} shows a similar type of measurement -for astrometric residuals. To generate this plot, we use the same -selection of measurements for astrometry as for photometry. In this -case, we extract the residual in both the RA and DEC directions -($\delta RA = \overline{RA} - RA_i$, $\delta DEC = \overline{DEC} - -DEC_i$) and rotate these values to the chip coordinate system ($\delta -X,\delta Y$) using our knowledge of the chip orientation on the sky. -We again exclude as bad any measurement with $|\delta X|$ or $|\delta -Y| > 0.5$ arcsec before measuring the median values for each -superpixel. We have determined the approximate center of the circular -tree-ring pattern as (-5,4960) for this particular chip based on the -pattern of the X astrometry displacements. Using this coordinate as the center -of the pattern, we have converted the $\delta X,\delta Y$ offsets into -$\delta R,\delta \theta$ measurements ($\delta R$ : radial component -away from the center, $\delta \theta$ : tangential component). - -Figure~\ref{fig:astrom.by.filter} shows the 2D patterns of $\delta R$ -for each filter (\grizy). The dynamic range of the color scale is -from -20 to +20 milliarcseconds for all 5 plots. A tree-ring -pattern is visible for all five filters, with systematic structures -following a circular pattern centered on the chip corner; the finging -pattern is not apparent in the \yps\ astrometry. The per-pixel -standard deviations of these plots area listed in -Table~\ref{table:sigmas.by.filter}. The signal-to-noise of these -structures is again somewhat weak, but the pattern is clearly visible -in these figures. - -\subsection{Flat-field Structures} - -% flat-field residual -\begin{figure*}[htbp] -\begin{center} -\parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/dflat.\plotext}} -\hspace{\jumpleft} -\parbox[b]{\capwidth}{ -\caption{Flat-field high-frequency structues, by filter (\grizy). - White boxes are GPC1 cells which have been masked due to poor - response. Flat-field images generated using a tunable laser have - been combined (see text); a smoothed version has been subtracted to - high-pass the response. Flat-field pixels are averaged for - $10\times10$ superpixels. } \label{fig:flats.by.filter}} -\end{center} -\end{figure*} - -% 2012ApJ...750...99T = Tonry et al PS1 phot system -Figure~\ref{fig:flats.by.filter} shows the high-spatial-frequency -structures in the flat-field images. For this measurement, we have -used a set of monochromatic flat-field images obtained with a tunable -laser. The laser is used to illuminate our flat-field screen which is -then observed by the PS1 telescope. These flat-field images were -obtained 2011 Feb 09 as part of a campaign to study the PS1 system -response \citep{2012ApJ...750...99T}. Flats were obtain in a set of -4nm steps sampling the spectral response curve of each filter. To -enhance the signal-to-noise, we have median-combined a set of 6 flats -at the wavelength center of the corresponding filter. - -In order to mask pixels which do not flatten well, we generate a copy -of the image smoothed with a Gaussian kernel with $\sigma = 1.5$ -pixels. Any pixels in the smoothed image which deviate from the -median value in the image by more than 4 standard deviations are -masked. We generate the superpixel image by averaging the unmasked -pixels associated with each superpixel. - -Figure~\ref{fig:flats.by.filter} shows the superpixel images for the -flat-fields in the five filters. These flat-field images are -displayed as fractional deviations relative to the median flat-field -image and can thus be compared to the magnitude residuals. When -flattening an image, these flat-fields would be divided into the flux -of the raw image. The residuals are thus defined in the sense that a -positive feature in these flats which does {\em not} represent a real -quantum efficiency deviation would induce a {\em reduction} in the -measured flux in those pixels, and thus a {\em negative} deviation in -$\delta m_{psf}$ as defined above. The dynamic range of the color -scale in these plots is -0.01 to +0.01. The tree-ring pattern is -strong in the (\gps,\rps,\ips) images, but nearly swamped by fringing -in \zps, and completely lost to finging in \yps. A diagonal banding -pattern is seen in \gps: this features is thought to be due to the -lithography process used to generate the CCD. A blob can also been -seen covering 4 cells near the center of this chip; this is apparently -a deposit of some kind on the detector. Both of the latter two -effects behave like quantum efficiency variations and are removed well -by standard flat-field techniques. Note that a small amount of the -diagonal banding pattern remains in the aperture magnitude residuals -for \gps. For the rest of this article, we ignore these features and -concentrate on the tree ring features. - -In order to suppress the large-scale structures for a quantitative -analysis of the tree rings, we high-pass filter the superpixel image -by subtracting a copy smoothed with a Gaussian of $\sigma = 3.0$ -superpixels. - -\subsection{Second Moments} - -% Smear Images -\begin{figure*}[htbp] -\begin{center} -\parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/smear.\plotext}} -\hspace{\jumpleft} -\parbox[b]{\capwidth}{ -\caption{Average residual smear variations, by filter (\grizy). White - boxes are GPC1 cells which have been masked due to poor response. - The residual smear ($\sigma^2_{\mbox{major}} + \sigma^2_{\mbox{minor}}$) has been - determined after the after PSF second moments have been subtracted - for each image; these values are averaged for each $10\times10$ - superpixels. } \label{fig:smear.by.filter}} -\end{center} -\end{figure*} - -% Shear Images -\begin{figure*}[htbp] -\begin{center} -\parbox[b]{\figwidth}{\includegraphics[width=\figwidth]{\picdir/shear.\plotext}} -\hspace{\jumpleft} -\parbox[b]{\capwidth}{ -\caption{Average residual shear variations, by filter (\grizy). White - boxes are GPC1 cells which have been masked due to poor response. - The residual shear ($\sigma^2_{\mbox{major}} - \sigma^2_{\mbox{minor}}$) has been - determined after the after PSF second moments have been subtracted - for each image; these values are averaged for each $10\times10$ - superpixels. } \label{fig:shear.by.filter}} -\end{center} -\end{figure*} - -During the image analysis, the second moments are measured for all -stars. The values can be used to assess changes in the shape of stars -on the image. To measure changes in the shapes, we have extracted the -second moments for all stellar detections, subject to the same -selections as for the photometry and astrometry residuals (good stars, -multiple detections). The second moments are measured with a Gaussian -weighting function, with the $\sigma_{w}$ scaled by the PSF size so -that the $\sigma$ measured for PSF stars is \approx 65\% of -$\sigma_{w}$. (Note that, since the measured $\sigma$ of stellar -objects is biased down by the weighting function, this is not quite -the same as having $\sigma_{w} = 1.6$ times the true PSF $\sigma$, see -discussion in \citealt{magnier2017.analysis}). For each stellar -detection, we extract the values $M_{xx,xy,yy} = \sum F_i w_i (x^2, x -y, y^2) / \sum F_i w_i$. For each exposure, we find the median second -moments for PSF objects on this chip (XY40) and subtract those median -values from the instantaneous measurements of $M_{xx,xy,yy}$. We then -determine the median of the residual second moments for each -superpixel, resulting in 3 images ($\delta M_{xx,xy,yy}$) for each -filter. - -Using the second moment images, we can construct certain interesting -combinations, inspired by discussions of lensing measurements \citep{1995ApJ...449..460K}: -\begin{eqnarray} -e_0 & = & \delta M_{xx} + \delta M_{yy} \\ -e_1 & = & \delta M_{xx} - \delta M_{yy} \\ -e_2 & = & \sqrt{e_1^2 + 4 \delta M_{xy}} -\end{eqnarray} -For a 2D Gaussian profile with an elliptical contour, these values are -related to the shape of the elliptical contour as follows: -\begin{eqnarray} -e_0 & = & \sigma^2_{\mbox{major}} + \sigma^2_{\mbox{minor}} \\ -e_1 & = & (\sigma^2_{\mbox{major}} - \sigma^2_{\mbox{minor}}) \cos (2 \theta) \\ -e_2 & = & \sigma^2_{\mbox{major}} - \sigma^2_{\mbox{minor}} -\end{eqnarray} -Where $\sigma_{\mbox{major}}$ and $\sigma_{\mbox{minor}}$ are the -major and minor axis dimensions of the ellipse and $\theta$ is the -position angle. Thus, $e_0$ is a measurement of the change in the -size of the stellar PSFs as a function of position in the detector -(``smear''), $e_2$ is a measurement of the change in ellipticity of -the stellar PSFs (``shear''), and we can determine the angle of the -PSF ellipticity from the $e_1$ term. - -Figure~\ref{fig:smear.by.filter} shows the spatial trend of $e_0$, the {\em - smear}. This value corresponds to the increase or decrease in -the circularly-symmetric component of the image size. The dynamic -range of these images is -0.3 to +0.3 pixel$^2$. A tree-ring -pattern is visible for all 5 filters, though \yps is dominated by the -fringing pattern. Structures with relatively low spatial frequencies -can also be seen. - -Figure~\ref{fig:shear.by.filter} shows the spatial trend of $e_2$, the -{\em shear}. This value is positive definite and is plotted with a -color scale ranging from -0.02 to 0.22 pixel$^2$. We can also -determine the orientation of the corresponding ellipse. Overlayed on -Figure~\ref{fig:shear.by.filter} is a set of vectors representing the -ellipse orientation as a function of postion. The length of the -vectors corresponds to the value of $\sigma^2_{major} - -\sigma^2_{minor}$. The tree-ring structure is {\em not} apparent -in this figure for any filter. The spatial variations are -low-frequency and unrelated to the radial trend from the upper-left -corner. - -\subsection{Correlations Between Tree-Ring Patterns} - -% All Effects in r-band -\begin{figure*}[htbp] -\begin{center} -\parbox[b]{\figwidth}{\includegraphics[width=5.0in]{\picdir/all.effects.r.\plotext}} -\caption{All 6 measured effects for \rps. This figure illustrates the - different spatial structure observed for each of the 6 patterns - measured in this work. The PSF magnitude (upper-left) and smear - residuals (lower-left) have a very clear common tree-ring structure, - while the astrometric residual (middle-left) and flat-field - residuals (middle-right) have their own common tree-ring pattern with - much higher frequencies than the previous two effects. Aperture - magnitude (upper-right) and shear residuals (lower-right) do not - show a strong signal consistent with either of the two patterns.} \label{fig:all.effects.rband} -\end{center} -\end{figure*} - -\begin{table} -% \tiny -\begin{center} -\caption{Systematic Trends : Correlations by filter\label{table:correlation.by.filter}} -\begin{tabular}{|l|rrrr|} -\hline -{\bf Filter} & {\bf smear} & {\bf psf mags} & {\bf astrom} & {\bf flat} \\ -\hline -\gps & 1.00 & 1.00 & 1.00 & 1.00 \\ -\rps & 0.78 & 0.84 & 0.84 & 0.76 \\ -\ips & 0.40 & 0.50 & 0.66 & 0.64 \\ -\zps & 0.16 & 0.26 & 0.37 & 0.33 \\ -\yps & 0.10 & 0.10 & 0.25 & 0.30 \\ -\hline -\end{tabular} -\end{center} -\end{table} - -Tree-ring patterns are clearly seen in 4 of the measurement types -above: the PSF photometry, the astrometry, the flat-field, and the -smear terms. As discussed above, the signal-to-noise per pixel in the -plots of the systematic trends is relatively low (\approx 1.0). While -the tree-ring patterns are apparent in many of these figures, -there are also some other systematic structures which may degrade the -signal further. - -To quantatatively compare the tree-ring trends between -filters and between the types of measurements, we need to measure the -tree-ring structure explicitly and filter out the other effects if -possible. To do this, we have applied a high-pass filter to all of -the relevant images (PSF photometry residuals, astrometric residuals -in the radial direction, flat-field residuals, and second moment smear -terms) to remove unrelated spatial structures. We have then measured -the median of the signal in radial bins centered on (-5,4960) across -an arc from $\phi$ = -20\degrees\ to -50\degrees (as measured relative -to the top row of the images. We have selected a small fraction of -the arc to minimize the error associated with the choice of the -pattern center and to avoid several bad cells near the bottom of the -chip. - -% \note{include the arc on one of the figures?} - -% \note{do plots of all filter pairs in a triangle? is that interesting?} - -For a given type of measurement, the systematic effect is strongly -correlated between filters. The strongest correlation is the smear -term: Figure~\ref{fig:smear.trends} shows the correlation of the smear -pattern between \gps\ and the other four filters. Even \yps\ is -strongly correlated with \gps\ despite the presence of the fringe -pattern. PSF photometric residuals are also correlated between -filters, as shown in Figure~\ref{fig:psfmag.trends}. Here, the -\yps\ correlation with \gps\ is quite weak: the fringing pattern -dominates the tree rings for PSF photometry. The radial component of -the astrometric residual is also well correlated between filters, with -no loss of correlation due to fringing in \yps. Finally, the -flat-field residuals are generally correlated between filters, but -both \zps\ and \yps\ are affected by fringing. For \yps, the -correlation is completely washed out by the very strong fringing -pattern. - -For all four types of measurements, the slope of the fitted lines are -listed in Table~\ref{table:correlation.by.filter}. There is a -consistency in the trend from \gps, with the strongest systematic -tree-ring effects to \yps, with the weakest effects. Note that the -second moment smear and astrometry terms have different relative -strength in \yps\ compared with \gps. - -% smear trends by filter -\def\figwidth{6.5in} -\begin{figure*}[htbp] -\begin{center} -\includegraphics[width=\figwidth]{\picdir/smear.trends.\plotext} -\caption{Correlation of the smear ($\sigma^2_{\mbox{major}} + - \sigma^2_{\mbox{minor}}$) signal in \gps\ with the other 4 bands: - \rps\ (upper-left), \ips\ (upper-right), \zps\ (lower-left), \yps\ (lower-right). -} \label{fig:smear.trends} -\end{center} -\end{figure*} - -% psfmag trends by filter -\def\figwidth{6.5in} -\begin{figure*}[htbp] -\begin{center} -\includegraphics[width=\figwidth]{\picdir/psfmag.trends.\plotext} -\caption{Correlation of the PSF magnitude residuals ($\delta m_{psf}$) - in \gps\ with the other 4 bands: \rps\ (upper-left), \ips\ - (upper-right), \zps\ (lower-left), \yps\ (lower-right). -} \label{fig:psfmag.trends} -\end{center} -\end{figure*} - -% astrom trends by filter -\def\figwidth{6.5in} -\begin{figure*}[htbp] -\begin{center} -\includegraphics[width=\figwidth]{\picdir/astrom.trends.\plotext} -\caption{Correlation of the radial astrometric residual displacement ($\delta R$) - in \gps\ with the other 4 bands: \rps\ (upper-left), \ips\ - (upper-right), \zps\ (lower-left), \yps\ (lower-right). -} \label{fig:astrom.trends} -\end{center} -\end{figure*} - -% flat trends by filter -\def\figwidth{6.5in} -\begin{figure*}[htbp] -\begin{center} -\includegraphics[width=\figwidth]{\picdir/flat.trends.\plotext} -\caption{Correlation of the flat-field tree-ring structures in \gps\ - with the other 4 bands: \rps\ (upper-left), \ips\ (upper-right), \zps\ - (lower-left), \yps\ (lower-right). } \label{fig:flat.trends} -\end{center} -\end{figure*} - -An important question is the relationship of the tree-ring -pattern between the different types of measurements. Different models -for the tree-ring structures make different predictions about the -correlations between different effects. Note the very different -spatial structure between the different measurements in a given -filter: the radial variations do not all follow the same patterns. -Instead, we find the following relationships hold: - -First, the PSF magnitude residuals and the second-moment smear trends -are strongly anti-correlated: regions which have larger PSFs than the -mean tend to have smaller measured PSF fluxes than the mean (note that -$\delta m_{psf}$ is defined so that positive values correspond to -larger fluxes). These trends are shown in -Figure~\ref{fig:smear.vs.psfmag}. - -Second, the radial derivative of the smear is anti-correlated with the -radial component of the astrometric residuals: $\frac{\partial - (\sigma^2_{major} + \sigma^2_{minor})}{\partial radius} \sim \delta -R$ (see Figure~\ref{fig:dsmear.vs.astrom}). - -Finally, the radial derivative of the radial component of the -astrometric residual is anti-correlated with the flat-field residual -errors: $\frac{\partial \delta R}{\partial radius} \sim \delta flat$ -(see Figure~\ref{fig:dastrom.vs.flat}. This last relationship is -somewhat weakly measured. Because of the periodic nature of the Tree -Rings, it is also difficult to be completely certain that the -flat-field is proportional to the derivative of the astrometry -residual, rather than the astrometry residual being proportional to -the derivative of the flat-field. The correlation is somewhat weaker -for derivative of the flat-field vs astrometry residual. The -correlation is very weak between the flat-field and the astrometry -residual values without a derivative. We are convinced that we have -the sense of the derivative correct by examination of specific -features in each imaage. - -\begin{table} -% \tiny -\begin{center} -\caption{Systematic Trends : Correlations between trends\label{table:correlation.by.trend}} -\begin{tabular}{|l|rrr|} -\hline -{\bf Filter} & {\bf psf mags} & {\bf $\grad$ smear} & {\bf $\grad$ astrom} \\ - & {\bf vs smear} & {\bf vs astrom} & {\bf vs flat} \\ -\hline -\gps & -0.056 & -0.060 & -0.47 \\ -\rps & -0.071 & -0.073 & -0.45 \\ -\ips & -0.077 & -0.095 & -0.45 \\ -\zps & -0.082 & -0.078 & -0.17 \\ -\hline -\end{tabular} -\end{center} -\end{table} - -% smear vs psfmag -\def\figwidth{6.5in} -\begin{figure*}[htbp] -\begin{center} -\includegraphics[width=\figwidth]{\picdir/smear.vs.psfmag.\plotext} -\caption{Correlation of the PSF magnitude residuals ($\delta m_{PSF}$) - with the smear ($\sigma^2_{\mbox{major}} + \sigma^2_{\mbox{minor}}$) - signal for \gps\ (upper-left), \rps\ (upper-right), \ips\ (lower-left), - \zps\ (lower-right). -} \label{fig:smear.vs.psfmag} -\end{center} -\end{figure*} - -% dsmear vs astrom -\def\figwidth{6.5in} -\begin{figure*}[htbp] -\begin{center} -\includegraphics[width=\figwidth]{\picdir/dsmear.vs.astrom.\plotext} -\caption{ -Correlation of the radial astrometric residual displacement ($\delta -R$) with the derivative of the smear ($\partial -\sigma^2_{\mbox{major}} + \sigma^2_{\mbox{minor}}$) signal with -respect to the radial postion for \gps\ (upper-left), \rps\ -(upper-right), \ips\ (lower-left), \zps\ (lower-right). -} \label{fig:dsmear.vs.astrom} -\end{center} -\end{figure*} - -% dastrom vs flat -\def\figwidth{6.5in} -\begin{figure*}[htbp] -\begin{center} -\includegraphics[width=\figwidth]{\picdir/dastrom.vs.flat.\plotext} -\caption{ -Correlation of the derivative of the radial astrometric residual -displacement ($\delta R$) with respect to the radial position with the -flat-field tree-ring signal for \gps\ (upper-left), \rps\ (upper-right), -\ips\ (lower-left), \zps\ (lower-right). -} \label{fig:dastrom.vs.flat} -\end{center} -\end{figure*} - -\section{Discussion} -\label{sec:discussion} - -These trends measured above (Section~\ref{sec:tree.rings}) help to -illuminate the underlying causes of these different effects. - -First, if we consider the smear pattern -(Figure~\ref{fig:smear.by.filter}), the measurement shows that the -intrinsic sizes of the stellar images are varying in a radial sense -between the different tree-ring regions. Although images experience -an average image quality (due to seeing and focus) across the chip -which may vary substantially from exposure to exposure, stars landing -in the different tree-ring regions are consistently somewhat -larger or somewhat smaller than that average. - -Next, we can explain the correlation between the PSF photometry -residuals and the observed smear (Figure~\ref{fig:smear.vs.psfmag}). -In the photometry analysis, we model the PSF allowing for some spatial -variation in the shape. However, we have a limited number of stars to -measure any spatial variation. Thus the 2D variations are sampled on -a very coarse (e.g., $3 \times 3$) grid for each chip: the PSF -parameters may vary smoothly across the chip following the bilinear -interpolation between the $3 \times 3$ grid points. Thus, the spatial -scale on which we model PSF variations is much larger than the spatial -scale on which PSF variations are actually occuring, as illustrated -by the changes in the smear plot (Figure~\ref{fig:smear.by.filter}). -When the true PSF is larger than the model PSF, our model fits -systematically underestimate the amount of flux in a given object. -Conversely, when the true PSF is smaller, we overestimate the flux -- this -type of offset is a typical effect when mis-estimating the PSF size. -The slope of the trend depends on the mean typical seeing for the -given filter. For example, the \gps\ seeing is typically 1.3\arcsec, -corresponding to a Gaussian $\sigma$ of 2.15 pixels. A smearing of -$\sigma^2_{major} + \sigma^2_{minor} = 0.1$ pixels$^2$ would increase -the size by about 0.02 pixels, or 1\%, roughly consistent with the -observed photometric deviation of about 5 to 10 millimags for this -amount of smearing. - -The correlation between the flat-field structures and the radial -derivative of the astrometric residual displacements in the radial -direction (Figure~\ref{fig:dastrom.vs.flat}) is consistent with radial -variations in the plate-scale. The tree-rings observed by DES are -completely attributed to effective plate scale changes. Effective -plate scale changes result in flat-field deviations because the -flat-field illumination is a source of constant surface brightness. -Pixels see a varying amount of flux depending on their effective area. -This changing plate scale also affects the astrometry since these -variations occur on spatial scales much smaller than the astrometric -model. In this description of the tree rings, the flat-field -deviations are $-1 \times \frac{\partial \delta R}{\partial r}$. The -best-fit slopes of our correlations are \approx 0.5, but the -signal-to-noise is rather low. A slope of -1 appears to be consistent -with our measurements. - -The fact that the PSF ellipticity changes are {\em not} correlated -with the tree-ring structure (Figure~\ref{fig:shear.by.filter}) tells us -that, unlike the case for DES, the effective plate-scale changes seen -in the flat-field and astrometry signals are not the dominant cause of -the PSF photometry errors. Also, the fact that we do not measure -significant aperture photometry errors correlated with the tree rings -confirms this point. The amplitude of the flat-field errors are 1-2 -millimagnitudes, much smaller than the PSF photometry errors, and far -below the pixel-to-pixel noise in the aperture magnitude residuals. -It is likely in our opinion that the plate-scale changes causing the -flat-field and astrometry effects is affecting both the ellipticity -and the aperture magnitudes, but the level of the effect is too small -to see given the other systematic structures (in the shear plot) and -the noise level (in the aperture magnitudes). - -Finally, the correlation between the smear structures and the -astrometry residuals shows that these two effects are connected. -Although the correlation is weak in Figure~\ref{fig:dsmear.vs.astrom}, -careful inspection of the location of the these two tree ring patterns -shows that the locations of the rings in the radial astrometric -residual images occurs at the boundaries between regions with -substantially different values of the smear signal. - -We suggest that the underlying connection between all of these -tree-ring effects is the pattern of the doping variations in the -silicon. As discussed by \cite{2014PASP..126..750P}, the tree-ring -patterns seen by the DES team are caused by lateral electic fields in -the detector silicon (in the plane of the CCD wafer) generated by -variations in the space charges embedded in the silicon, in turn -coming from low-level changes in the doping as the silicon boule is -grown. We conclude that the astrometric and flat-field variations -seen in our detectors are caused by these same types of doping -variations. The changes in the smear (and thus the PSF magnitudes) -are apparently also related to the doping variations. The lateral -electric fields which introduce the astrometry and flat-field -variations occur at the boundary between regions with higher and lower -space charges from the dopant. Regions with high (or low) space -charge density thus correspond to regions with relatively high (or -low) amounts of smear; the astrometric deviations follow the gradient -between these regions. - -We interpret the changes in the {\em smear} term as changes in the -amount of charge diffusion as the photoelectrons travel to the bottom -of the pixel well. The blue filters exhibit the strongest changes in -the amount of smear. These are also the filters for which the -detected electrons have travelled the longest distance in the silicon, -and are thus most affected by diffusion effects. Charge diffusion (as -opposed to the charge drift caused by the lateral electric fields) -results in a Gaussian smearing of the stellar profile: as the -photoelectrons migrate from the site where they were generated by the -incoming photon to the bottom of the pixel well, they follow a random -walk in the plane of the detector. The longer the electrons take to -make the journey down to the bottom of the pixel, the further they are -able to wander from their creation coordinate in the detector. -Following the discussion in \cite{Holland.2003}, the amount of charge -diffusion is thus related to the velocity of the electrons in the -direction of the optical axis: $\sigma \sim \sqrt{2Dt}$ where $\sigma$ -is the size of the smearing kernel, $t$ is the time required for the -electrons to traverse the thickness of the silicon wafer, and $D$ is -the diffusion coefficient. The velocity of the photoelectron, and -thus the time to traverse the silicon, is related to the vertical -electric fields in the silicon, which are caused by a combination of -the applied voltages and the distribution of the space charges from -the dopant. As shown by \cite{Holland.2003}, the charge diffusion is -related to the space charge density by $\sigma \sim -\rho^{-\frac{1}{2}}$ (their equation 6). Regions with high space -charge densities increase the migration speed of the photoelectrons -and reduce the amount of charge diffusion smearing; and vice versa for -regions of low space-charge densities. - -In summary, the variations in the space-charge density caused by -variations in the dopant result in regions of higher and lower charge -diffusion, and in turn regions with PSF photometry systematic -residuals. The lateral gradients in the space-charge density induce -lateral electric fields which in turn cause lateral motions of the -photoelectrons, resulting in astrometric and flat-field deviations. - -The DES team did not detect these charge diffusion variations. In -that case, the amplitude of the photometric effects due to the lateral -field are dominant; these include both the modification of the -flat-field as well as PSF fitting errors due to the changing PSF sizes -introduced by the varying effective pixels sizes. If the smearing -effect reported here were as large for DES compared with the lateral -PSF size changes as they are for GPC1, then the reported PSF -photometry residuals for would have had very different -characteristics. We conclude that, for DES, the lateral effects are -much larger than the diffusion variations, compared with GPC1. The -relative amplitude of these two effects depends on the details of the -applied voltages, the amplitude of the space-charge density variations -compared with the typical space-charge density, and the detector -thicknesses. It is beyond the scope of this article to model these -effects in detail. - -% http://adsabs.harvard.edu/abs/2006NIMPA.568...41K - -\section{Conclusion} - -The tree rings observed in the Pan-STARRS GPC1 data show (at least) -two effects, though they are related. First, the images are -experiencing circularly-symmetric changes in the PSF size correlated -with the tree-ring pattern. These PSF size changes drive errors in -the PSF photometry on the scale of a few millimagnitudes, are also -correlated with the tree-ring pattern. These PSF size changes are -consistent with changes in the charge diffusion, which also introduces -a circularly symmetric smearing. - -In addition, there are radial plate-scale changes correlated with the -tree rings. These plate-scale changes introduce a flat-field errors -on the scale of \approx 1 millimagnitude and astrometric errors in the -scale of 2-3 milliarcseconds. The observed relationship between the -flat-field deviations and the radial derivative of the astrometric -deviations confirms this interpretation \citep[see also discussion - in][]{2014PASP..126..750P}. - -The spatial correlation of the gradient in the smear variations and -the astrometric variations imply that both of these two types of tree -ring effects are related, even though they manifest through different -mechanisms. We conclude that the variations in both the vertical charge -diffusion and the lateral charge migration are driven by changes -in the electric field structures in the silicon due to the same -variations in the doping structures in the silicon. - -% The small-scale variations in the charge diffusion observed in these -% devices has not been reported for DECam, Hypersuprime Cam, or -% prototype LSST sensors. - -The small-scale variations in the charge diffusion observed in the -Pan-STARRS detectors represents a new type of systematic effect in -deep depletion devices. This feature, if present in other detectors, -could manifest in systematic errors in several ways. Like in the -Pan-STARRS analysis example, the charge diffusion variations result in -fine-structure in the observed stellar point-spread functions. For -very precise photometry or morphological analysis, it will be -necessary for the PSF models to account for the extra charge -diffusion. Unlike the non-uniform pixel-size effects, correction of -the PSF photometry cannot simply be performed as an average flat-field -correction on the measurements after they have been processed. -The additional smearing acts as a convolution with a Gaussian kernel -of fixed size for a given filter. The photometry bias is a function -of the fractional change of the PSF size. Thus, the introduced error -depends on the average PSF for the image in question: an image with -good image quality will suffer larger PSF model errors than an image -with poor image quality. To account for this effect in a rigorous -way, the analysis should use the measured diffusion variations to -modify the model PSFs as a function of position before they are used -for the image analysis. - -The charge diffusion variations may also have an impact on -spectroscopic measurements. Modern, precise spectroscopic -measurements rely on precise measurements of the stellar line -profiles. If such an analysis ignores variations in the charge -diffusion, the measured line widths may be systematically biased. - -This analysis points to the importance of careful instrumental -characterization, especially for those instruments which are used for -large-scale surveys with largely automatic data analysis systems and -stringent precision goals. - -\acknowledgments - -The Pan-STARRS1 Surveys (PS1) have been made possible through -contributions of the Institute for Astronomy, the University of -Hawaii, the Pan-STARRS Project Office, the Max-Planck Society and its -participating institutes, the Max Planck Institute for Astronomy, -Heidelberg and the Max Planck Institute for Extraterrestrial Physics, -Garching, The Johns Hopkins University, Durham University, the -University of Edinburgh, Queen's University Belfast, the -Harvard-Smithsonian Center for Astrophysics, the Las Cumbres -Observatory Global Telescope Network Incorporated, the National -Central University of Taiwan, the Space Telescope Science Institute, -the National Aeronautics and Space Administration under Grant -No. NNX08AR22G issued through the Planetary Science Division of the -NASA Science Mission Directorate, the National Science Foundation -under Grant No. AST-1238877, the University of Maryland, and Eotvos -Lorand University (ELTE) and the Los Alamos National Laboratory. - -\note{Ken: please add NASA ops grants} - -\bibliographystyle{apj} -\bibliography{lib}{} -%\input{analysis.bbl} - -\end{document} - -%% Some refs to be added as appropriate: -% Bernstein DEC astrometry : arxiv 1703.01679 -% Baumer et al arxiv 1706.07400 (Flat-fielding) Index: /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20120509.sh =================================================================== --- /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20120509.sh (revision 40482) +++ /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20120509.sh (revision 40483) @@ -4,6 +4,6 @@ # general locations -set wwwroot = /export/ippc32.0/www/dvodist/www-root -set catroot = /data/ipp064.0/eugene/3pi.pv0.20120509 +set wwwroot = /export/ipp117.0/www/dvodist/www-root +set catroot = /data/ipp105.0/eugene/3pi.dvo.masters/3pi.pv0.20120509 set catdir = 3pi.pv0.20120509 Index: /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20120525.sh =================================================================== --- /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20120525.sh (revision 40482) +++ /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20120525.sh (revision 40483) @@ -4,6 +4,6 @@ # general locations -set wwwroot = /export/ippc32.0/www/dvodist/www-root -set catroot = /data/ipp064.0/eugene/3pi.pv0.20120525 +set wwwroot = /export/ipp117.0/www/dvodist/www-root +set catroot = /data/ipp105.0/eugene/3pi.dvo.masters/3pi.pv0.20120525 set catdir = 3pi.pv0.20120525 Index: /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20120606.sh =================================================================== --- /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20120606.sh (revision 40482) +++ /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20120606.sh (revision 40483) @@ -4,6 +4,6 @@ # general locations -set wwwroot = /export/ippc32.0/www/dvodist/www-root -set catroot = /data/ipp064.0/eugene/3pi.pv0.20120606 +set wwwroot = /export/ipp117.0/www/dvodist/www-root +set catroot = /data/ipp105.0/eugene/3pi.dvo.masters/3pi.pv0.20120606 set catdir = 3pi.pv0.20120606 Index: /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20130227.sh =================================================================== --- /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20130227.sh (revision 40482) +++ /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20130227.sh (revision 40483) @@ -4,6 +4,6 @@ # general locations -set wwwroot = /export/ippc32.0/www/dvodist/www-root -set catroot = /data/ipp064.0/eugene/3pi.pv0.20130227 +set wwwroot = /export/ipp117.0/www/dvodist/www-root +set catroot = /data/ipp105.0/eugene/3pi.dvo.masters/3pi.pv0.20130227 set catdir = 3pi.pv0.20130227 Index: /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20140621.sh =================================================================== --- /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20140621.sh (revision 40482) +++ /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20140621.sh (revision 40483) @@ -4,6 +4,6 @@ # general locations -set wwwroot = /export/ippc32.0/www/dvodist/www-root -set catroot = /data/ipp064.0/eugene/3pi.pv0.20140621 +set wwwroot = /export/ipp117.0/www/dvodist/www-root +set catroot = /data/ipp105.0/eugene/3pi.dvo.masters/3pi.pv0.20140621 set catdir = 3pi.pv0.20140621 Index: /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20140713.sh =================================================================== --- /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20140713.sh (revision 40482) +++ /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv0.20140713.sh (revision 40483) @@ -4,6 +4,6 @@ # general locations -set wwwroot = /export/ippc32.0/www/dvodist/www-root -set catroot = /data/ipp064.0/eugene/3pi.pv0.20140713 +set wwwroot = /export/ipp117.0/www/dvodist/www-root +set catroot = /data/ipp105.0/eugene/3pi.dvo.masters/3pi.pv0.20140713 set catdir = 3pi.pv0.20140713 Index: /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv1.20130708.sh =================================================================== --- /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv1.20130708.sh (revision 40482) +++ /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv1.20130708.sh (revision 40483) @@ -4,6 +4,6 @@ # general locations -set wwwroot = /export/ippc32.0/www/dvodist/www-root -set catroot = /data/ipp064.0/eugene/3pi.pv1.20130708 +set wwwroot = /export/ipp117.0/www/dvodist/www-root +set catroot = /data/ipp105.0/eugene/3pi.dvo.masters/3pi.pv1.20130708 set catdir = 3pi.pv1.20130708 Index: /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv2.20141215.sh =================================================================== --- /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv2.20141215.sh (revision 40482) +++ /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv2.20141215.sh (revision 40483) @@ -4,6 +4,6 @@ # general locations -set wwwroot = /export/ippc32.0/www/dvodist/www-root -set catroot = /data/ipp064.0/eugene/3pi.pv2.20141215 +set wwwroot = /export/ipp117.0/www/dvodist/www-root +set catroot = /data/ipp105.0/eugene/3pi.dvo.masters/3pi.pv2.20141215 set catdir = 3pi.pv2.20141215 Index: /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv3.20160422.sh =================================================================== --- /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv3.20160422.sh (revision 40482) +++ /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv3.20160422.sh (revision 40483) @@ -4,8 +4,9 @@ # general locations -set wwwroot = /export/ippc32.0/www/dvodist/www-root -set catroot = /data/ipp094.0/eugene/pv3.cam.20150607/catdir.master +set wwwroot = /export/ipp117.0/www/dvodist/www-root +set catroot = /data/ipp105.0/eugene/3pi.dvo.masters/3pi.pv3.20160806 -set catdir = 3pi.pv3.20160422 +# set catdir = 3pi.pv3.20160422 +set catdir = 3pi.pv3.20160806 set cattgt = $wwwroot/$catdir @@ -16,5 +17,5 @@ # top-level files -if (0) then +if (1) then mkdir -p $cattgt foreach file (HostTable.dat AstroMap.fits Images.dat Photcodes.dat SkyTable.fits flatcorr.fits flatfield.fits ) @@ -30,5 +31,5 @@ # link all files from all hostdirs into all DEC dirs # foreach hostdir (`grep -v "^#" $cattgt/HostTable.dat | awk '{print $3}'`) -foreach hostdir (`grep -v "^#" $cattgt/HostTable.dat | grep "#" | awk '{print $3}'`) +foreach hostdir (`grep -v "^#" $cattgt/HostTable.dat | awk '{print $3}'`) echo $hostdir foreach dir ($nlist $slist) Index: /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv3.20170216.sh =================================================================== --- /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv3.20170216.sh (revision 40483) +++ /branches/czw_branch/20170908/dvodist/scripts/mkdist.3pi.pv3.20170216.sh (revision 40483) @@ -0,0 +1,42 @@ +#!/bin/csh -f + +# create the links needed to expose a specific catalog to the distribution server + +# general locations +set wwwroot = /export/ipp117.0/www/dvodist/www-root +set catroot = /data/ipp105.0/eugene/3pi.dvo.masters/3pi.pv3.20170216 + +set catdir = 3pi.pv3.20170216 +set cattgt = $wwwroot/$catdir + +set nlist = (n0000 n0730 n1500 n2230 n3000 n3730 n4500 n5230 n6000 n6730 n7500 n8230) +set slist = (s0000 s0730 s1500 s2230 s3000 s3730 s4500 s5230 s6000 s6730 s7500 s8230) + +set extlist = (cpt cpm cps cpn cpx cpy cpz cpq) + +# top-level files +if (1) then + mkdir -p $cattgt + foreach file (HostTable.dat AstroMap.fits Images.dat Photcodes.dat SkyTable.fits flatcorr.fits flatfield.fits ) + ln -sf $catroot/$file $cattgt + end + + # make DEC directories + foreach dir ($nlist $slist) + mkdir -p $cattgt/$dir + end +endif + +# link all files from all hostdirs into all DEC dirs +# foreach hostdir (`grep -v "^#" $cattgt/HostTable.dat | grep "#" | awk '{print $3}'`) + +foreach hostdir (`grep -v "^#" $cattgt/HostTable.dat | awk '{print $3}'`) + echo $hostdir + foreach dir ($nlist $slist) + echo $hostdir : $dir + foreach ext ($extlist) + echo $hostdir : $dir : $ext + ln -sf $hostdir/$dir/*.$ext $cattgt/$dir + end + end +end Index: /branches/czw_branch/20170908/hardware/dotConsolerc =================================================================== --- /branches/czw_branch/20170908/hardware/dotConsolerc (revision 40482) +++ /branches/czw_branch/20170908/hardware/dotConsolerc (revision 40483) @@ -14,4 +14,10 @@ console: ippdevcon.ifa.hawaii.edu ippb05: + console: ippdevcon.ifa.hawaii.edu + ippc20: + console: ippdevcon.ifa.hawaii.edu + ippc21: + console: ippdevcon.ifa.hawaii.edu + ippc22: console: ippdevcon.ifa.hawaii.edu @@ -28,4 +34,8 @@ ippdb08: console: itc-i04-con.ifa.hawaii.edu + ippdb09: + console: itc-i04-con.ifa.hawaii.edu + ippdb10: + console: itc-i07-con.ifa.hawaii.edu ipp032: @@ -165,11 +175,19 @@ ipp122: console: itc-i12-con.ifa.hawaii.edu + ipp123: + console: itc-i12-con.ifa.hawaii.edu + ipp124: + console: itc-i12-con.ifa.hawaii.edu + ipp125: + console: itc-i12-con.ifa.hawaii.edu + ipp126: + console: itc-i12-con.ifa.hawaii.edu ippc17: - console: itc-i04-con.ifa.hawaii.edu + console: itc-i10-con.ifa.hawaii.edu ippc18: console: itc-i04-con.ifa.hawaii.edu ippc19: - console: itc-i11-con.ifa.hawaii.edu + console: itc-i10-con.ifa.hawaii.edu ippc23: console: itc-i12-con.ifa.hawaii.edu @@ -321,5 +339,5 @@ console: itc-i07-con.ifa.hawaii.edu ippc95: - console: itc-i07-con.ifa.hawaii.ed + console: itc-i07-con.ifa.hawaii.edu ippc96: console: itc-i05-con.ifa.hawaii.edu @@ -634,2 +652,18 @@ ippb15: console: ippbcon.ifa.hawaii.edu + ippb16: + console: ippbcon.ifa.hawaii.edu + ippb17: + console: ippbcon.ifa.hawaii.edu + ippb18: + console: ippbcon.ifa.hawaii.edu + ippb19: + console: ippbcon.ifa.hawaii.edu + ippb20: + console: ippbcon.ifa.hawaii.edu + ippb21: + console: ippbcon.ifa.hawaii.edu + ippb22: + console: ippbcon.ifa.hawaii.edu + ippb23: + console: ippbcon.ifa.hawaii.edu Index: /branches/czw_branch/20170908/ippScripts/Build.PL =================================================================== --- /branches/czw_branch/20170908/ippScripts/Build.PL (revision 40482) +++ /branches/czw_branch/20170908/ippScripts/Build.PL (revision 40483) @@ -115,4 +115,5 @@ scripts/automate_stacks.pl scripts/nightly_science.pl + scripts/night_report.pl scripts/lossy_compress_imfile.pl scripts/rawcheck.pl Index: /branches/czw_branch/20170908/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/czw_branch/20170908/ippScripts/scripts/camera_exp.pl (revision 40482) +++ /branches/czw_branch/20170908/ippScripts/scripts/camera_exp.pl (revision 40483) @@ -404,5 +404,5 @@ # Construct FPA continuity corrected background images - if (($camera =~ /ISP/)||($camera =~ /HSC/)) { + if (($camera =~ /ISP/)||($camera =~ /HSC/)||($camera =~ /gpc2/i)) { print "Skipping FPA continuity corrected background images for ISP\n"; } elsif ($do_bkg) { Index: /branches/czw_branch/20170908/ippScripts/scripts/chip_imfile.pl =================================================================== --- /branches/czw_branch/20170908/ippScripts/scripts/chip_imfile.pl (revision 40482) +++ /branches/czw_branch/20170908/ippScripts/scripts/chip_imfile.pl (revision 40483) @@ -632,4 +632,5 @@ if ($gone) { carp "WARNING: photometry sources storage object exists but all instances are permanently gone"; + $ipprc->file_rename($outputSources,$outputSources . ".gone"); $make_sources = 1; } @@ -657,4 +658,5 @@ if ($gone) { carp "WARNING: PSF storage object exists but all instances are permanently gone"; + $ipprc->file_rename($outputPsf,$outputPsf . ".gone"); $make_psf = 1; } @@ -742,4 +744,6 @@ print STDERR "$file is on $volume which is gone\n"; $numGone++; + } elsif ($answer eq 'permanently') { + $numGone++; } elsif ($answer eq 'available') { $numNotGone++; Index: /branches/czw_branch/20170908/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- /branches/czw_branch/20170908/ippScripts/scripts/ipp_cleanup.pl (revision 40482) +++ /branches/czw_branch/20170908/ippScripts/scripts/ipp_cleanup.pl (revision 40483) @@ -27,5 +27,5 @@ # turn this on the set the state to error_state when errors occur cleaning up individual components # We may stop doing this -my $set_error_state_for_run = 1; +my $set_error_state_for_run = 0; # this gets set to 1 the first time we set the corresponding destreak run to be cleaned Index: /branches/czw_branch/20170908/ippScripts/scripts/ipp_filename.pl =================================================================== --- /branches/czw_branch/20170908/ippScripts/scripts/ipp_filename.pl (revision 40482) +++ /branches/czw_branch/20170908/ippScripts/scripts/ipp_filename.pl (revision 40483) @@ -10,10 +10,15 @@ #print "$ENV{'PATH'}\n"; #print "$ENV{'PERL5LIB'}\n"; +#print "NEB_SERVER: $ENV{'NEB_SERVER'}
\n"; -# CZW: This is a horrible hack, but I don't want to have to debug all of ippMonitor to figure out why it's not working. +# EAM: check for missing NEB_SERVER and exit with an error unless (defined($ENV{'NEB_SERVER'})) { - $ENV{'NEB_SERVER'} = 'http://nebserver.ipp.ifa.hawaii.edu:80/nebulous'; + print "NEB_SERVER not defined in ipp_filename.pl\n"; + exit (5); } - +if ($ENV{'NEB_SERVER'} eq "") { + print "NEB_SERVER not set in ipp_filename.pl\n"; + exit (6); +} use PS::IPP::Config; @@ -52,4 +57,9 @@ my $realname = $ipprc->file_resolve( $filename, $touch ); +if (not defined $realname) { + print "nebulous file $filename not found\n"; + exit (1); +} + print "$realname\n"; Index: /branches/czw_branch/20170908/ippScripts/scripts/night_report.pl =================================================================== --- /branches/czw_branch/20170908/ippScripts/scripts/night_report.pl (revision 40483) +++ /branches/czw_branch/20170908/ippScripts/scripts/night_report.pl (revision 40483) @@ -0,0 +1,733 @@ +#!/usr/bin/env perl + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use DBI; +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; # Socket for mysql + +# this saves the original arguments for later reporting +my @ARGS = @ARGV; + +GetOptions( + 'date=s' => \$date, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'very-verbose' => \$verbose2, # Print to stdout + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option(s): @ARGV", -exitval => 2 ) if @ARGV; + +# example if I require some options +# pod2usage( -msg => "Required options: --exp_id --chip_id", -exitval => 3) unless defined $exp_id and defined $chip_id; + +$date = `date --utc --rfc-3339=date` unless defined $date; +chomp $date; +$date=~ s/\//-/g; +$date=~ s/\./-/g; + +# database connection information: +$dbhost = 'ippdb08'; +$dbname = 'gpc1' unless defined $dbname; +$dbuser = 'ippuser'; +$dbpass = 'ippuser'; + +# dataGroupDate : formatted to match data_group values (YYYYMMDD) +$dataGroupDate = $date; +$dataGroupDate=~ s/-//g; +$dataGroupDate=~ s/\.//g; + +print "\n\n\n"; +print "--- Report for $dbname : $date ---\n\n"; + +# set up the database connection: +my $dsn = "DBI:mysql:host=$dbhost;database=$dbname"; +my $dbh = DBI->connect( $dsn, $dbuser, $dbpass ) or die "Unable to connect to database: $DBI::errstr"; + +# find a useful exp_id +$refExpID = 0; +if (1) { + # print "Exposures 10 days ago:\n"; + + my $query = "SELECT MAX(exp_id)"; + $query .= " FROM rawExp"; + $query .= " WHERE dateobs >= date_sub('$date', interval 20 day)"; + $query .= " AND dateobs <= date_sub('$date', interval 10 day)"; + + my $result = &mysql_select ($query); + + @row = $result->fetchrow_array(); + $refExpID = $row[0]; + if ($refExpID eq "") { + $refExpID = 0; + print "*** WARNING: no data in period 10-20 days before requested date (query will be slower) **\n"; + } + # print "Reference exp_id: $refExpID\n"; +} + +# check the status of summit exposures: +if (1) { + if ($verbose) { print "Exposures at summit:\n"; } + + $summitSkipped = 0; + $summitPending = 0; + +# my $query = "SELECT exp_name, exp_type, fault, state, count(*)"; + my $query = "SELECT exp_type, fault, state, count(*)"; + $query .= " FROM summitExp"; + $query .= " LEFT JOIN pzDownloadExp USING (summit_id)"; + $query .= " WHERE dateobs like '$date%'"; + $query .= " AND (state != 'drop' or state IS NULL)"; + $query .= " AND fault != 1042"; + $query .= " GROUP BY exp_type, fault, state"; + +# print "$query\n"; +# die; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s\n", "exp_type", "fault", "state", "N(exp)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d\n", $row[0], $row[1], $row[2], $row[3]; } + if ($row[2] eq "run") { + $summitPending += $row[3]; + } + if ($row[2] eq "") { + $summitSkipped += $row[3]; + } + } + if ($summitSkipped || $summitPending) { print "*** WARNING: $summitSkipped skipped rawExp, $summitPending incomplete rawExp ***\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# check for exposures not sent to chip: +if ($verbose2) { + + my $query = "SELECT summitExp.exp_name, exp_type, fault, state"; + $query .= " FROM summitExp"; + $query .= " LEFT JOIN pzDownloadExp USING (summit_id)"; + $query .= " WHERE dateobs like '$date%'"; +# $query .= " AND (state != 'drop' or state IS NULL)"; + $query .= " AND fault != 1042"; + $query .= " AND state IS NULL"; + + print "$query\n"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "OBJECT Exposures NOT sent to chipRun:\n"; + printf "%11s %15s %14s %14s\n", "exp_name", "exp_type", "fault", "state"; + $doHeader = 0; + } + printf "%11s %15s %14s %14s\n", $row[0], $row[1], $row[2], $row[3]; + } + if (! $doHeader) { print "\n"; } +} + +# check the status of raw exposures: +if (1) { + if ($verbose) { print "Exposures in rawExp:\n"; } + + $expSkipped = 0; + $expPending = 0; + + my $query = "SELECT exp_type, obs_mode, state, count(*)"; + $query .= " FROM rawExp"; + $query .= " WHERE dateobs like '$date%'"; + $query .= " AND exp_id > $refExpID"; # REF +# $query .= " AND exp_type = 'OBJECT'"; + $query .= " GROUP BY exp_type, obs_mode, state"; + $query .= " ORDER BY exp_type, obs_mode"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s\n", "exp_type", "obs_mode", "state", "N(exp)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d\n", $row[0], $row[1], $row[2], $row[3]; } + if ($row[2] eq "new") { + $expPending += $row[3]; + } + if ($row[2] eq "") { + $expSkipped += $row[3]; + } + } + if ($expSkipped || $expPending) { print "*** WARNING: $expSkipped skipped rawExp, $expPending incomplete rawExp ***\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# check the status of chip runs: +if (1) { + if ($verbose) { print "Exposures in chipRun (OBJECT, obs_mode not like ENGINEERING, not NULL):\n"; } # the obs_mode not NULL is implicit given the obs_mode not like ENGINEERING + + $chipPending = 0; + $chipSkipped = 0; + + my $query = "SELECT exp_type, obs_mode, chipRun.state, count(*)"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " GROUP BY obs_mode, chipRun.state"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s", "exp_type", "obs_mode", "state", "N(exp)\n"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d\n", $row[0], $row[1], $row[2], $row[3]; } + if ($row[2] eq "new") { + $chipPending += $row[3]; + } + if ($row[2] eq "") { + $chipSkipped += $row[3]; + } + } + if ($chipPending || $chipSkipped) { print "*** WARNING: $chipSkipped skipped chipRun, $chipPending incomplete chipRun ***\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# check the status of camera runs: +$camPending = 0; +$camSkipped = 0; +{ + if ($verbose) { print "Exposures in camRun (OBJECT, not ENGINEERING)...\n"; } + + # my $result = &mysql_select (""); + my $query = "SELECT exp_type, obs_mode, camRun.state, camProcessedExp.fault, camProcessedExp.quality, count(*)"; + $query .= " FROM rawExp join chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " GROUP BY obs_mode, camRun.state, camProcessedExp.fault, camProcessedExp.quality"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s | %5s %5s\n", "exp_type", "obs_mode", "state", "N(exp)", "fault", "quality"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d | %5d %5d\n", $row[0], $row[1], $row[2], $row[5], $row[3], $row[4]; } # exp_type, obs_mode, camRun.state, camProcessedExp.fault, camProcessedExp.quality, count(*) + if ($row[2] eq "new") { + $camPending += $row[5]; + } + if ($row[2] eq "") { + $camSkipped += $row[5]; + } + } + if ($camSkipped || $camPending) { print "*** WARNING: $camSkipped skipped camRuns, $camPending pending camRuns ***\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# check the status of fake runs: +{ + if ($verbose) { print "Exposures in fakeRun:\n"; } + + $fakePending = 0; + $fakeSkipped = 0; + + my $query = "SELECT exp_type, obs_mode, fakeRun.state, count(*)"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun using (exp_id)"; + $query .= " JOIN camRun using (chip_id)"; + $query .= " JOIN camProcessedExp using (cam_id)"; + $query .= " LEFT JOIN fakeRun using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND (camProcessedExp.quality is null or camProcessedExp.quality = 0)"; + $query .= " GROUP BY obs_mode, fakeRun.state"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s\n", "exp_type", "obs_mode", "state", "N(exp)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d\n", $row[0], $row[1], $row[2], $row[3]; } # exp_type, obs_mode, fakeRun.state, count(*) + if ($row[2] eq "new") { + $fakePending += $row[3]; + } + if ($row[2] eq "") { + $fakeSkipped += $row[3]; + } + } + if ($fakeSkipped || $fakePending) { print "$fakeSkipped skipped fakeRuns, $fakePending pending fakeRuns\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# check the status of warp runs: +{ + if ($verbose) { print "Exposures in warpRuns:\n"; } + + $warpPending = 0; + $warpSkipped = 0; + + # my $result = &mysql_select (""); + my $query = "SELECT exp_type, obs_mode, warpRun.state, count(*)"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun using (exp_id)"; + $query .= " JOIN camRun using (chip_id)"; + $query .= " JOIN camProcessedExp using (cam_id)"; + $query .= " JOIN fakeRun using (cam_id)"; + $query .= " LEFT JOIN warpRun using (fake_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND (camProcessedExp.quality IS NULL OR camProcessedExp.quality = 0)"; + $query .= " GROUP BY obs_mode, warpRun.state"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%8s %15s %6s %4s\n", "exp_type", "obs_mode", "state", "N(exp)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%8s %15s %6s %4d\n", $row[0], $row[1], $row[2], $row[3]; } # exp_type, obs_mode, warpRun.state, count(*) + if ($row[2] eq "new") { + $warpPending += $row[3]; + } + if ($row[2] eq "") { + $warpSkipped += $row[3]; + } + } + if ($warpSkipped || $warpPending) { print "$warpSkipped skipped warpRuns, $warpPending pending warpRuns\n\n"; } else { if ($verbose) { print "\n"; }} +} + +# count the number of exposures per chunk, determine expected diffRuns: +{ + print "Expected number of WWdiffs excluding camRuns with bad quality:\n"; + print "NOTE: multiple exposures with same comment string (xyz visit a) are treated as 1 exposure\n"; + + # my $result = &mysql_select (""); + my $query = "SELECT Nvisit, count(*) FROM"; + $query .= " ("; + $query .= " SELECT object, filter, chunk, count(*) as Nvisit FROM"; + $query .= " ("; + $query .= " SELECT object, filter, substr(comment, 1, position(' ' in comment)) as chunk"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND obs_mode like '%SS%'"; + $query .= " AND rawExp.comment like '%visit%'"; + $query .= " AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)"; + $query .= " GROUP BY comment, filter, substr(comment, 1, position(' ' in comment))"; + $query .= " ) AS tableChunks GROUP BY object, filter, chunk"; + $query .= " ) AS tableVisits GROUP BY Nvisit"; + + my $result = &mysql_select ($query); + + $diffExpect = 0; + while (@row = $result->fetchrow_array()) { + # printf "%8s %15s : ", $row[0], $row[1]; # c, count(*) + if ($row[0] == 1) { + if ($row[1] == 1) { $verb = "quad has"; } else { $verb = "quads have"; } + printf "%3d %-11s 1 visit, no WWdiffs\n", $row[1], $verb; + } elsif ($row[0] == 2) { + if ($row[1] == 1) { $verb = "quad has"; } else { $verb = "quads have"; } + printf "%3d %-11s 2 visits, expect $row[1] WWdiff(s)\n", $row[1], $verb; + $diffExpect += $row[1]; + } elsif ($row[0] == 3) { + if ($row[1] == 1) { $verb = "quad has"; } else { $verb = "quads have"; } + printf "%3d %-11s 3 visits, expect %d WWdiffs (2 per quad)\n", $row[1], $verb, 2 * $row[1]; + $diffExpect += $row[1]*2; + } elsif ($row[0] == 4) { + if ($row[1] == 1) { $verb = "quad has"; } else { $verb = "quads have"; } + printf "%3d %-11s 4 visits, expect %d WWdiffs (2 per quad)\n", $row[1], $verb, 2 * $row[1]; + $diffExpect += $row[1]*2; + } else { + print "I don't understand, there are more visits than I expect\n"; + } + } + print "\n"; + print " * $diffExpect diffRuns expected *\n"; + if ($verbose) { print "\n"; } +} + +# List chunks with duplicate visits +{ + # my $result = &mysql_select (""); +# my $query = "SELECT Nvisit, count(*) FROM"; +# $query .= " ("; + $query .= " SELECT object, filter, chunk, Nexp_in_chunk FROM"; + $query .= " ("; + $query .= " SELECT object, filter, substr(comment, 1, position(' ' in comment)) as chunk, count(*) as Nexp_in_chunk"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND obs_mode like '%SS%'"; + $query .= " AND rawExp.comment like '%visit%'"; + $query .= " AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)"; + $query .= " GROUP BY comment, filter, substr(comment, 1, position(' ' in comment))"; + $query .= " ) AS tableChunks where (Nexp_in_chunk > 1)"; +# $query .= " ) AS tableVisits GROUP BY Nvisit"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "chunks with duplicate visits:\n"; + printf "%11s %15s %14s %5s\n", "object", "filter", "chunk", "Nexp in chunk"; + $doHeader = 0; + } + printf "%11s %15s %14s %5s\n", $row[0], $row[1], $row[2], $row[3]; + } + if (! $doHeader) { print "\n"; } +} + +# List exposures which may be used for WWdiffs +if (0) { + print "List exposures for WWdiffs excluding camRuns with bad quality:\n"; + + # my $result = &mysql_select (""); + my $query = ""; + $query .= " SELECT exp_name, object, filter, substr(comment, 1, position(' ' in comment)) as chunk"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND obs_mode like '%SS%'"; + $query .= " AND rawExp.comment like '%visit%'"; + $query .= " AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)"; + + my $result = &mysql_select ($query); + &mysql_dump_result ($result); +} + +# check the status of diff runs: +{ + if ($verbose) { print "diff processing status (not WS):\n"; } + + $diffDone = 0; + + my $query = "SELECT label, state, count(*)"; + $query .= " FROM diffRun"; + $query .= " WHERE data_group like '%$dataGroupDate'"; + $query .= " AND label NOT LIKE '%WS.nigh%'"; + $query .= " GROUP BY label, state"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%20s %6s | %8s\n", "label", "state", "N(diffs)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%20s %6s | %8d\n", $row[0], $row[1], $row[2]; } # label, state, count + if (($row[1] ne "new") && ($row[1] ne "")) { + $diffDone += $row[2]; + } + } + if ($verbose) { print "\n"; } + if ($diffDone != $diffExpect) { + print "*** WARNING: Expect $diffExpect diffRuns, complete $diffDone diffRuns: inconsistent ***\n"; + } else { + print " * $diffDone of $diffExpect diffRuns completed *\n"; + } +} + +# count diffRuns to be published: +{ + my $query = "SELECT count(*) FROM"; + $query .= " ("; + $query .= " SELECT label, state, count(*)"; + $query .= " FROM diffRun"; + $query .= " LEFT JOIN diffSkyfile using (diff_id)"; + $query .= " WHERE data_group LIKE '%$dataGroupDate%'"; + $query .= " AND label NOT LIKE '%WS.nigh%'"; + $query .= " AND (quality IS NULL OR quality = 0)"; + $query .= " GROUP BY label, state, diff_id"; + $query .= " ) as tableDiffs"; + + my $result = &mysql_select ($query); + + @row = $result->fetchrow_array(); + printf " * %d diffRuns can be published * (has at least 1 skycell with quality = 0 or quality is NULL)\n", $row[0]; + if ($verbose) { print "\n"; } + $diffRunsToBePublished = $row[0]; +} + +# check the status of publish runs: +{ + if ($verbose) { print "publishing status (not WS)...\n"; } + + $pubDone = 0; + + my $query = "SELECT diffRun.label, publishRun.state, count(*)"; + $query .= " FROM diffRun"; + $query .= " LEFT JOIN publishRun on (diff_id = stage_id)"; + $query .= " WHERE data_group LIKE '%$dataGroupDate'"; + $query .= " AND diffRun.label NOT LIKE '%WS.nigh%'"; + $query .= " GROUP BY diffRun.label, publishRun.state"; + + my $result = &mysql_select ($query); + + if ($verbose) { printf "%20s %6s | %8s\n", "label", "state", "N(diffs)"; } + while (@row = $result->fetchrow_array()) { + if ($verbose) { printf "%20s %6s | %8d\n", $row[0], $row[1], $row[2]; } # label, state, count + if (($row[1] ne "new") && ($row[1] ne "")) { + $pubDone += $row[2]; + } + } + if ($verbose) { print "\n"; } + if ($pubDone != $diffRunsToBePublished) { + print "*** WARNING: Expect $diffRunsToBePublished diffRuns to be published, only $pubDone pubRuns: inconsistent ***\n\n"; + } else { + print " * $pubDone of $diffRunsToBePublished diffRuns published **\n\n"; + } +} + +# check for exposures not sent to chip: +if ($verbose) { + + my $query = "SELECT exp_name, obs_mode, comment"; + $query .= " FROM rawExp left join chipRun using (exp_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND chip_id is null"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "OBJECT Exposures NOT sent to chipRun:\n"; + printf "%11s %15s %14s", "exp_name", "obs_mode", "comment\n"; + $doHeader = 0; + } + printf "%11s %15s %14s\n", $row[0], $row[1], $row[2]; + } + if (! $doHeader) { print "\n"; } +} + +# check for chipRuns not sent to camera +if ($camSkipped && $verbose) { + print "chipRuns NOT sent to camRun:\n"; + + my $query = "SELECT exp_name, obs_mode, comment, chipRun.state"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun USING (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND cam_id IS NULL"; + + my $result = &mysql_select ($query); + + printf "%11s %15s %10s %14s\n", "exp_name", "obs_mode", "chip.state", "comment"; + while (@row = $result->fetchrow_array()) { + printf "%11s %15s %10s %14s\n", $row[0], $row[1], $row[2], $row[3]; + } + print "\n"; +} + +# check for bad quality camera exposurs +if (1) { + my $query = "SELECT exp_name, camProcessedExp.quality, comment"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun USING (exp_id)"; + $query .= " JOIN camRun USING (chip_id)"; + $query .= " JOIN camProcessedExp USING (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND (camProcessedExp.quality > 0)"; + $query .= " ORDER BY comment"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + $NbadCam = 0; + + while (@row = $result->fetchrow_array()) { + if ($verbose && $doHeader) { + print "Bad quality camRun exposures:\n"; + printf "%11s %7s | %15s\n", "exp_name", "quality", "comment"; + $doHeader = 0; + } + if ($verbose) { printf "%11s %7d | %15s\n", $row[0], $row[1], $row[2]; } else { $NbadCam ++; } + } + if ($verbose && !$doHeader) { print "\n"; } + if (!$verbose) { print "$NbadCam bad quality camera exposures\n\n"; } +} + +# check the status of diff runs NOT for Solar System: +if ($verbose) { + # my $result = &mysql_select (""); + my $query = "SELECT * FROM"; + $query .= " ("; + $query .= " SELECT object, filter, chunk, count(*) as Nvisit FROM"; + $query .= " ("; + $query .= " SELECT object, filter, substr(comment, 1, position(' ' in comment)) as chunk"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND obs_mode NOT like '%SS%'"; + $query .= " AND rawExp.comment like '%visit%'"; + $query .= " AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)"; + $query .= " GROUP BY comment, filter, substr(comment, 1, position(' ' in comment))"; + $query .= " ) AS tableChunks GROUP BY object, filter, chunk"; + $query .= " ) AS tableVisits ORDER BY Nvisit, chunk, object"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "Chunks which should not generate WWdiffs (not Solar System diffs)\n"; + printf "%8s %8s %8s %8s\n", "Object", "Filter", "Chunk", "N(visit)"; + $doHeader = 0; + } + printf "%8s %8s %8s %4d\n", $row[0], $row[1], $row[2], $row[3]; + } + if (!$doHeader) { print "\n"; } +} + +# check for unpublished solar-system exposures +{ + my $query = "SELECT expall.exp_name, expall.dateobs, expall.comment, expall.warp_id, exppub.diff_id, exppub.pubstate from"; + $query .= " ("; + $query .= " SELECT exp_name, comment, dateobs, warp_id"; + $query .= " FROM rawExp"; + $query .= " JOIN chipRun using (exp_id)"; + $query .= " JOIN camRun using (chip_id)"; + $query .= " JOIN camProcessedExp using (cam_id)"; + $query .= " JOIN fakeRun using (cam_id)"; + $query .= " JOIN warpRun using (fake_id)"; + $query .= " WHERE dateobs like '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND obs_mode like '%SS%'"; + $query .= " AND (camProcessedExp.quality = 0)"; + $query .= " ) AS expall"; + $query .= " LEFT JOIN"; + $query .= " ("; + $query .= " SELECT exp_name, warpRun.warp_id, diffRun.diff_id, publishRun.state as pubstate"; + $query .= " FROM diffRun"; + $query .= " JOIN diffInputSkyfile using (diff_id)"; + $query .= " JOIN publishRun on (diffRun.diff_id = stage_id)"; + $query .= " JOIN warpRun ON ( warp1 = warp_id or warp2 = warp_id)"; + $query .= " JOIN fakeRun using (fake_id)"; + $query .= " JOIN camRun using (cam_id)"; + $query .= " JOIN camProcessedExp using (cam_id)"; + $query .= " JOIN chipRun using (chip_id)"; + $query .= " JOIN rawExp using (exp_id)"; + $query .= " WHERE diffRun.reduction like 'SWEETSPOT'"; +## $query .= " AND exp_id > $refExpID"; -- adding this restriction here slows the query way down1 + $query .= " AND stack2 IS NULL"; + $query .= " AND diffRun.data_group LIKE '%SS.$dataGroupDate'"; + $query .= " GROUP BY exp_name"; + $query .= " ) AS exppub"; + $query .= " ON expall.exp_name = exppub.exp_name"; + $query .= " WHERE exppub.exp_name is NULL"; + $query .= " OR exppub.pubstate = 'new'"; + + my $result = &mysql_select ($query); + + $doHeader = 1; + while (@row = $result->fetchrow_array()) { + if ($doHeader) { + print "Unpublished Solar System Exposures ( obs_mode like %SS% ) with good camera quality :\n"; + printf "%12s %12s %12s | %-19s | %-16s\n", "exp_name", "warp_id", "diff_id", "dateobs", "comment"; + $doHeader = 0; + } + if ($row[3] eq "") { $row[3] = "NULL"; } + if ($row[4] eq "") { $row[4] = "NULL"; } + printf "%12s %12s %12s | %-19s | %-16s\n", $row[0], $row[3], $row[4], $row[1], $row[2] + } +} +print "\n\n"; + +# count the number of exposures per chunk, determine expected diffRuns: +if ($verbose2) { + print "Number of exposures for each chunk:\n"; + + # my $result = &mysql_select (""); + my $query = "SELECT object, filter, chunk, count(*) as Nvisit FROM"; + $query .= " ("; + $query .= " SELECT object, filter, substr(comment, 1, position(' ' in comment)) as chunk"; + $query .= " FROM rawExp"; + $query .= " LEFT JOIN chipRun using (exp_id)"; + $query .= " LEFT JOIN camRun using (chip_id)"; + $query .= " LEFT JOIN camProcessedExp using (cam_id)"; + $query .= " WHERE dateobs LIKE '$date%'"; + $query .= " AND exp_id > $refExpID"; #REF + $query .= " AND exp_type = 'OBJECT'"; + $query .= " AND obs_mode NOT LIKE 'ENGINEERING'"; + $query .= " AND obs_mode like '%SS%'"; + $query .= " AND rawExp.comment like '%visit%'"; + $query .= " AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)"; + $query .= " GROUP BY comment, filter, substr(comment, 1, position(' ' in comment))"; + $query .= " ) AS tableChunks GROUP BY object, filter, chunk"; + + my $result = &mysql_select ($query); + + printf "%11s %7s %18s %-3s\n", "object", "filter", "chunk", "N(visit)"; + while (@row = $result->fetchrow_array()) { + printf "%11s %7s %18s %3d\n", $row[0], $row[1], $row[2], $row[3]; + } + print "\n"; +} + +exit 0; + +# global dbh? +sub mysql_select { + + my $query = $_[0]; + + my $statement = $dbh->prepare($query); + + $statement->execute(); + + return $statement; +} + +sub mysql_dump_result { + + my $result = $_[0]; + + $Nfields = $result->{NUM_OF_FIELDS}; + + print "Nfields: $Nfields\n"; + + for ($i = 0; $i < $Nfields; $i++) { + printf "Field %d : %s\n", $i, $result->{NAME}->[$i]; + } + + while (@row = $result->fetchrow_array()) { + for ($i = 0; $i < $Nfields; $i++) { + print "$row[$i] "; + } + print "\n"; + } + return; +} + +# example using the dbh prepare, execute, fetch commands: +if (0) { + + my $result = &mysql_select ("SELECT * from rawExp limit 2"); + + &mysql_dump_result ($result); + + exit 1; +} Index: /branches/czw_branch/20170908/ippScripts/scripts/nightly_science.pl =================================================================== --- /branches/czw_branch/20170908/ippScripts/scripts/nightly_science.pl (revision 40482) +++ /branches/czw_branch/20170908/ippScripts/scripts/nightly_science.pl (revision 40483) @@ -46,5 +46,5 @@ # Grab options -my ( $date, $datetime, $camera, $dbname, $logfile, $verbose, $manual); +my ( $date, $datetime, $now, $camera, $dbname, $logfile, $verbose, $manual); my ( $help, $isburning, $force_stack_count, $force_diff_count, $force_registration, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode); my ( $registration_status, $burntool_status, $observing_status, $old_date); @@ -95,5 +95,6 @@ --verbose --force_stack_count Force the chip/warp counts. - --force_diff_count Force the chip/warp counts. + --force_diff_count Force the chip/warp counts. + --force_registration Force registration counts. --this_target_only Process only a single target. --this_filter_only Process only a single filter. @@ -255,4 +256,5 @@ $date = $datetime->ymd(); } +$now = DateTime->now(time_zone => 'UTC'); if (defined($this_target_only)) { @@ -2117,8 +2119,8 @@ month => $datetime->month, day => $datetime->day, - hour => 6, + hour => 17, # 7, minute => 30, second => 0, - time_zone => 'Pacific/Honolulu'); + time_zone => 'UTC'); foreach my $eon (keys %eon_config) { @@ -2142,6 +2144,9 @@ } } -# print "$datetime $eon_dt\n"; - if (DateTime->compare($datetime,$eon_dt) < 1) { + if ($force_registration) { + return("END_OF_NIGHT"); + } +# print "$now $eon_dt " . DateTime->compare($now,$eon_dt) . "\n"; + if (DateTime->compare($now,$eon_dt) < 1) { return("OBSERVING"); } Index: /branches/czw_branch/20170908/ippScripts/scripts/rawcheck_mddb.pl =================================================================== --- /branches/czw_branch/20170908/ippScripts/scripts/rawcheck_mddb.pl (revision 40483) +++ /branches/czw_branch/20170908/ippScripts/scripts/rawcheck_mddb.pl (revision 40483) @@ -0,0 +1,720 @@ +#!/usr/bin/env perl + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Nebulous::Client; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Digest::MD5; +use URI; + +my $missing_tools = 0; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); + +my ($neb_server,$md_server,$dbname,$dbuser,$dbpass,$exp_id,$do_cull,$do_revalidate,$save_log); + + +my $do_ops = 1; +my $i; + +my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n"; +my $siteConfig = $ipprc->{_siteConfig}; +$dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER"); +$dbpass = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD"); + +$neb_server = $ENV{'NEB_SERVER'} unless $neb_server; + +# $dbname = 'gpc1'; +GetOptions( + 'neb_server|s=s' => \$neb_server, # This is the apache server + 'md_server|s=s' => \$md_server, # This is the actual database server + 'dbname=s' => \$dbname, + 'dbuser=s' => \$dbuser, + 'dbpass=s' => \$dbpass, + 'exp_id|x=s' => \$exp_id, + 'save_log' => \$save_log, + 'cull' => \$do_cull, + 'revalidate' => \$do_revalidate, +) || pod2usage( 2 ); + +unless(defined($do_cull)) { + $do_cull = 0; +} +unless(defined($do_revalidate)) { + $do_revalidate = 0; +} +if (($do_cull == 1)&&($do_revalidate == 1)) { + pod2usage( -msg => "Only one of --cull and --revalidate may be chosen.", -exitval => 2 ); +} + + +# Option parsing +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --neb_server (apache server)", -exitval => 2 ) + unless $neb_server; +pod2usage( -msg => "Required options: --md_server (database server)", -exitval => 2 ) + unless $md_server; +pod2usage( -msg => "Required options: --dbname", -exitval => 2 ) + unless defined $dbname; +pod2usage( -msg => "Required options: --dbuser", -exitval => 2 ) + unless defined $dbuser; +pod2usage( -msg => "Required options: --dbpass", -exitval => 2 ) + unless defined $dbpass; +pod2usage( -msg => "missing key", exitval => 2 ) + unless defined $exp_id; + +if ($save_log) { + my $logDest = "neb://any/raw_check_mddb/exp_${exp_id}"; + if ($do_cull) { + $logDest .= ".cull"; + } + elsif ($do_revalidate) { + $logDest .= ".revalidate"; + } + else { + $logDest .= ".check"; + } + + $ipprc->redirect_to_logfile($logDest) or die "Could not redirect output to logfile ${logDest}\n"; +} + +# Global options: +## Define the configuration. Ideally, this would be retrieved from the nebulous +## database, but there are some issues with that (such as grouping the b nodes +## into a less restrictive "offsite" location). + +## Set up nebulous db interface, and pull processing information to consider. +my $neb = Nebulous::Client->new( + proxy => "$neb_server", +); +die "can't connected to Nebulous Server: $neb_server" + unless defined $neb; + +## Set up md5 db interface +my $mddb = DBI->connect("DBI:mysql:database=otaMD5;host=${md_server};" . + "mysql_socket=/var/run/mysqld/mysqld.sock", + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + +## This new implementation is somewhat messy, but is more general and adaptable. +## First, we set up a requirement mapping, explaining where we want copies, and +## how many copies we want at each site. +my %requirement_map = (); +$requirement_map{ITC} = 1; +$requirement_map{OFFSITE} = 1; + +## Second, construct a list of volumes, mapped to their site location, using the +## same site locations as in the requirement map. +my %volume_map = (); +for ($i = 32; $i <= 32; $i++) { + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'ITC'; +} +for ($i = 54; $i <= 97; $i++) { + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'ITC'; +} +for ($i = 100; $i <= 126; $i++) { + my $vol = sprintf("ipp%03d.0",$i); + $volume_map{$vol} = 'ITC'; + $vol = sprintf("ipp%03d.1",$i); + $volume_map{$vol} = 'ITC'; +} + +for ($i = 0; $i <= 15; $i++) { + my $loc = 'OFFSITE'; +# if ($i == 6) { +# $loc = 'ITC'; +# } # This isn't "offsite", it's with the rest of the maui cluster. +# if ($i == 9) { next; } # Not online + + my $vol = sprintf("ippb%02d.0",$i); + $volume_map{$vol} = $loc; + $vol = sprintf("ippb%02d.1",$i); + $volume_map{$vol} = $loc; + if ($i <= 6) { + $vol = sprintf("ippb%02d.2",$i); + $volume_map{$vol} = $loc; + } +} + +## Next, get disk space values, and check which hosts are listed as available. +my %acceptable_volume = (); +my $mounts = $neb->mounts(); +my %volume_fill_factors = (); +my %down_volume = (); +foreach my $vol_row (@$mounts) { + my ($mount_point, $total, $used, $vol_id, $name, $host, $path, $allocate, $available, $xattr) = @{ $vol_row }; + if (($allocate == 1)&&($available == 1)&&( $used / $total < 0.98)) { + $acceptable_volume{$name} = 1; + } + else { +# print "## $name $allocate $available $used $total\n"; + $acceptable_volume{$name} = 0; + } + + if (($available != 1)&&($xattr != 5)) { + $down_volume{$name} = 1; + } + else { + $down_volume{$name} = 0; + } + + if (($name =~ /ippb05/)||($name =~ /ippb02/)) { + $acceptable_volume{$name} = 0; + } + + $volume_fill_factors{$name} = $used / $total; + + # This should prioritize hosts that we wish to deprecate. + if ($total < 52885172852) { + $volume_fill_factors{$name} = 1.0; + } +} + +## Finally, generate lists containing which volumes are located at which site. +## This allows us to randomly select a volume from the list for the site that +## we plan on replicating to. +my %volume_lists = (); +foreach my $vol_key (keys %requirement_map) { + @{ $volume_lists{$vol_key} } = grep { $acceptable_volume{$_} == 1 } ( + grep { $volume_map{$_} eq $vol_key } (keys %volume_map) + ); + print "#$vol_key " . join(' ', @{ $volume_lists{$vol_key} }) . "\n"; + + if ($#{ $volume_lists{$vol_key} } == -1) { + die "No acceptable volume found for site $vol_key!\n"; + } +} + +# die; +# Pull data from the gpc1 database about the exposure to consider + +my $verbose = 0; +my $mdcParser = PS::IPP::Metadata::Config->new; + +print("#$regtool -processedimfile -exp_id $exp_id -dbname $dbname\n"); +my $regtool_cmd = "$regtool -processedimfile -exp_id $exp_id -dbname $dbname"; +my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $regtool_cmd, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool -processedimfile: $error_code", $exp_id); +} +my $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from regtool -processedimfile", $exp_id); + +my $timer_start = time(); +my $timer = time(); +print "## rawcheck.pl: $exp_id $dbname $do_cull $do_ops $timer_start\n"; +# Do prescan checks. +if ($#{ $imfiles } > 0) { + my @to_do_imfiles = (); + my $prior_imfile; + my $append = 0; + + foreach my $imfile (@$imfiles) { + my $key = $imfile->{uri}; + my $data_state = $imfile->{data_state}; + my $md5sum = $imfile->{md5sum}; + my $hostname = $imfile->{hostname}; + my $class_id = $imfile->{class_id}; + + unless(defined($key)) { die "No database uri: $exp_id"; } + unless(defined($data_state)) { $data_state = 'no_data_state'; } + if ($data_state eq 'corrupt') { next; } # This has already been marked as bad, so we can move on. + unless(defined($md5sum)) { die "No database md5sum: $exp_id"; } + unless(defined($hostname)) { $hostname = 'ipp004'; } + unless(defined($class_id)) { die "No database class_id: $class_id"; } + + # This hash contains all the information about the instances, indexed by the full filename of the instance. + my %unified_list = (); + my %orphans = (); + + $timer = time(); + print ("\n# $key $data_state $md5sum $hostname $class_id T: $timer\n"); + + # Get nebulous results for this key, and parse them + my $stat = $neb->stat($key); + die "nebulous key: $key not found" unless $stat; + my $instances = $neb->find_instances($key, 'any'); + die "no instances found" unless $instances; + my @files = map {URI->new($_)->file if $_} @$instances; + + for (my $i = 0; $i <= $#files; $i++) { + my $f = $files[$i]; + $f =~ s%//%/%g; + my ($instance_exists,$instance_host,$instance_volume,$instance_site); + ($instance_host,$instance_volume) = parse_volume($files[$i]); + $instance_site = $volume_map{$instance_volume}; + if (-e $f) { + $instance_exists = 1; + } + else { + $instance_exists = 0; + } + $unified_list{$f}{neb_host} = $instance_host; + $unified_list{$f}{neb_vol} = $instance_volume; + $unified_list{$f}{neb_site} = $instance_site; + $unified_list{$f}{exists} = $instance_exists; + } + + # Get md database results for this exp_id/class_id + my $md_query = "select file_id,ins_id,mvol_name,site_name,file,disk_sum,(STRCMP(md5sum,disk_sum) = 0) AS is_valid FROM obj LEFT JOIN ins USING(file_id) LEFT JOIN vol USING(mvol_id) WHERE exp_id = $exp_id AND class_id = '${class_id}'"; + my $md_instances = $mddb->selectall_arrayref( $md_query ); + + $timer = time(); +# print "##NEB $timer\n"; + my $md_file_id = -1; + foreach my $r (@{ $md_instances }) { + my ($file_id,$ins_id,$vol_name,$site_name,$f,$disk_sum,$is_valid) = @{ $r }; + + if ((defined($f))&&(defined($ins_id))) { + $f =~ s%//%/%g; + unless (exists($unified_list{$f})) { + if (-e $f) { + $unified_list{$f}{exists} = 1; + } + else { + $unified_list{$f}{exists} = 0; + } + } + # This should only happen if the same scan was ingested twice. + # Be paranoid and refuse to cull if this is the case. + if (exists($unified_list{$f}{md_ins_id})) { + if (($file_id == $unified_list{$f}{md_file_id})&& + ($vol_name eq $unified_list{$f}{md_vol})&& + ($site_name eq $unified_list{$f}{md_site})&& + ($disk_sum eq $unified_list{$f}{disk_sum})&& + ($is_valid == $unified_list{$f}{is_valid})) { + print "#DUPLICATE INS: $ins_id have $unified_list{$f}{md_ins_id} already\n"; + mddb_delete_instance($mddb,$ins_id); + $do_cull = -1; + next; + } + else { + # The data is inconsistent. Make a human fix it. + print "INS ERROR: $f -> \n"; + print "\t($file_id $ins_id $vol_name $site_name $disk_sum $is_valid)\n"; + print "\t($unified_list{$f}{md_file_id} $unified_list{$f}{md_ins_id} $unified_list{$f}{md_vol} $unified_list{$f}{md_site} $unified_list{$f}{disk_sum} $unified_list{$f}{is_valid})\n"; + die; + } + } + + $unified_list{$f}{md_file_id} = $file_id; + $unified_list{$f}{md_ins_id} = $ins_id; + $unified_list{$f}{md_vol} = $vol_name; + $unified_list{$f}{md_site} = $site_name; + $unified_list{$f}{disk_sum} = $disk_sum; + $unified_list{$f}{is_valid} = $is_valid; + } + + if (defined($file_id)) { + if ($md_file_id == -1) { + $md_file_id = $file_id; + } + elsif ($file_id != $md_file_id) { + die "Instance $ins_id has bad file_id $file_id expected $md_file_id"; + } + } + } + if ($md_file_id == -1) { + # There is no md information for this exposure + $md_file_id = mddb_create_file($mddb,$key,$exp_id,$class_id,$md5sum); + $do_cull = -1; + } + + $timer = time(); +# print "##MD $timer\n"; + + # Check that evrything is consistent. + my $good_copies = 0; + my $all_copies = scalar(keys(%unified_list)); + my $a_good_copy = ''; + + my %instance_count = (); + foreach my $site (keys(%requirement_map)) { + $instance_count{$site} = 0; + } + + foreach my $f (keys(%unified_list)) { + $unified_list{$f}{good_instance} = 0; + print "$f "; + + # Identify orphans, and push their handling until later. + unless (exists($unified_list{$f}{neb_host})) { + if ($down_volume{$unified_list{$f}{md_vol}} != 1) { + # Handle this. + $orphans{$f} = $unified_list{$f}; + warn "No neb entry for $orphans{$f}{md_ins_id} $exp_id $class_id $f \n"; + } + else { + warn "No neb entry for likely due to host issue $exp_id $class_id $f \n"; + } + delete($unified_list{$f}); + next; + } + print "$unified_list{$f}{neb_host} $unified_list{$f}{neb_vol} $unified_list{$f}{neb_site} $unified_list{$f}{exists} "; + + + # Create an entry in the mddb database if that doesn't already exist. + unless (exists($unified_list{$f}{md_vol})) { + # Insert a new row in the mddb ins table for this instance. + if ($unified_list{$f}{exists} != 1) { + # Ensure that the file exists, so an md5sum can be performed. + system("touch $f"); + } + my $tmpmd5 = local_md5sum($f); + my ($ins_id, $site_name) = mddb_insert_instance($mddb,$md_file_id,$f, $unified_list{$f}{neb_vol}, $tmpmd5); + + $unified_list{$f}{md_vol} = $unified_list{$f}{neb_vol}; + $unified_list{$f}{md_site} = $site_name; + $unified_list{$f}{disk_sum} = $tmpmd5; + $unified_list{$f}{md_file_id} = $md_file_id; + $unified_list{$f}{md_ins_id} = $ins_id; + + if ($tmpmd5 ne $md5sum) { + # Insert even if we fail so we can try fixing next time. + # This may cause a problem if the first instance we checked has a bad md5sum. + warn "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + $unified_list{$f}{is_valid} = 0; + } + else { + $unified_list{$f}{is_valid} = 1; + } + } + + # This will force md5sum calculations to everything, so disk<->mddb consistency can be checked. + if ($do_revalidate) { + my $tmpmd5 = local_md5sum($f); + print "REVALIDATE: ($tmpmd5) "; + if ($tmpmd5 ne $unified_list{$f}{disk_sum}) { + mddb_update_instance($mddb,$unified_list{$f}{md_ins_id}, $tmpmd5); + warn "Updated instance $unified_list{$f}{md_ins_id} to $tmpmd5\n"; + } + if ($tmpmd5 ne $md5sum) { + warn "Revalidated md5sum does not match!"; + } + } + + print "$unified_list{$f}{md_file_id} $unified_list{$f}{md_ins_id} $unified_list{$f}{md_vol} $unified_list{$f}{md_site} $unified_list{$f}{disk_sum} $unified_list{$f}{is_valid} "; + + + if ($unified_list{$f}{exists} != 1) { + warn "Instance does not exist $exp_id $class_id $f\n"; + next; + } + if ($unified_list{$f}{neb_vol} ne $unified_list{$f}{md_vol}) { + die "Inconsistent volumes for $exp_id $class_id $f $unified_list{$f}{neb_vol} $unified_list{$f}{md_vol}\n"; + } + if ($unified_list{$f}{is_valid} != 1) { + warn "Instance is not valid $exp_id $class_id $f\n"; + next; + } + if ($unified_list{$f}{disk_sum} ne $md5sum) { + # This is a die, because if the gpc1 md5sum and the mddb md5sum don't match, there's a problem + die "Inconsistent md5sum values $exp_id $class_id $f $md5sum $unified_list{$f}{disk_sum}\n"; + } + + # If none of that stopped us, then this instance is probably fine. + $unified_list{$f}{good_instance} = 1; + $instance_count{$unified_list{$f}{neb_site}} ++; + $good_copies++; + if ($a_good_copy eq '') { + $a_good_copy = $f; + } + print "$unified_list{$f}{good_instance}"; + print "\n"; + } + + if ($good_copies == 0) { + my $orphan_copy = ''; + foreach my $orph (keys (%orphans)) { + if ($orphans{$orph}{is_valid} == 1) { + $orphan_copy = $orph; + } + } + + if ($orphan_copy eq '') { + die "No good copies available for $exp_id $class_id\n"; + } + else { + die "No good copies available for $exp_id $class_id (but $orphan_copy may exist)\n"; + } + } + + $timer = time(); +# print "##CHECK $timer\n"; + + # Attempt to repair any bad instance. + print "COUNT CHECK $good_copies $all_copies\n"; + if ($good_copies != $all_copies) { + foreach my $f (keys(%unified_list)) { + if ($unified_list{$f}{good_instance} != 1) { + system("cp $a_good_copy $f"); + my $tmpmd5 = local_md5sum($f); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + + mddb_update_instance($mddb,$unified_list{$f}{md_ins_id}, $tmpmd5); + + $do_cull = -1; + $unified_list{$f}{good_instance} = 2; + $unified_list{$f}{disk_sum} = $tmpmd5; + $instance_count{$unified_list{$f}{neb_site}} ++; + $good_copies++; + + # Print out the same information as before, but updated to show the repair. + print "$f "; + print "$unified_list{$f}{neb_host} $unified_list{$f}{neb_vol} $unified_list{$f}{neb_site} $unified_list{$f}{exists} "; + print "$unified_list{$f}{md_file_id} $unified_list{$f}{md_ins_id} $unified_list{$f}{md_vol} $unified_list{$f}{md_site} $unified_list{$f}{disk_sum} $unified_list{$f}{is_valid} "; + print "$unified_list{$f}{good_instance}"; + print "\n"; + } + } + } + + # We cannot reach this point if good_copies != all_copies. + + # Do replications + foreach my $site (keys %requirement_map) { + print "#$site $instance_count{$site} $requirement_map{$site}\n"; + if ($instance_count{$site} < $requirement_map{$site}) { + my $rep_vol = get_random_site_volume($site); + print "neb-replicate --volume $rep_vol $key\n"; + $do_cull = -1; + + if ($do_ops) { + $neb->replicate($key,$rep_vol) or die "failed to replicate $key to $rep_vol"; + if ($@) { die $@; } + + # Begin my best validation thought + system("sync") == 0 or die "Couldn't sync?"; + my $uris = $neb->find_instances($key,$rep_vol); + @$uris = map {URI->new($_)->file if $_} @$uris; + my $f = ${ $uris }[0]; + my $tmpmd5 = local_md5sum($f); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + # End my best validation thought. + + # Report on the replicated instance. + my ($ins_id, $site_name) = mddb_insert_instance($mddb,$md_file_id,$f, $rep_vol, $tmpmd5); + $unified_list{$f}{md_vol} = $rep_vol; + $unified_list{$f}{md_site} = $site_name; + $unified_list{$f}{disk_sum} = $tmpmd5; + $unified_list{$f}{is_valid} = 1; + $unified_list{$f}{md_file_id} = $md_file_id; + $unified_list{$f}{md_ins_id} = $ins_id; + + my ($instance_host,$instance_volume) = parse_volume($f); + my $instance_site = $volume_map{$instance_volume}; + $unified_list{$f}{neb_host} = $instance_host; + $unified_list{$f}{neb_vol} = $instance_volume; + $unified_list{$f}{neb_site} = $instance_site; + $unified_list{$f}{exists} = 1; + $unified_list{$f}{good_instance} = 1; + + print "$f "; + print "$unified_list{$f}{neb_host} $unified_list{$f}{neb_vol} $unified_list{$f}{neb_site} $unified_list{$f}{exists} "; + print "$unified_list{$f}{md_file_id} $unified_list{$f}{md_ins_id} $unified_list{$f}{md_vol} $unified_list{$f}{md_site} $unified_list{$f}{disk_sum} $unified_list{$f}{is_valid} "; + print "$unified_list{$f}{good_instance}"; + print "\n"; + } + } + } # End replicating + + # Do culls + if ($do_cull == 1) { + foreach my $site (keys %requirement_map) { + if ($instance_count{$site} > $requirement_map{$site}) { + my @cull_order = (); + my %sorting_hash = (); + for my $f (keys %unified_list) { + if ($unified_list{$f}{neb_site} eq $site) { + push @cull_order, $f; + $sorting_hash{$f} = $volume_fill_factors{$unified_list{$f}{neb_vol}}; + } + } + @cull_order = sort { $sorting_hash{$b} <=> $sorting_hash{$a} } @cull_order; + +# my $tmp = join ' ', @cull_order; +# print "$exp_id $site $tmp\n"; + for ($i = 0; $i < ($instance_count{$site} - $requirement_map{$site}); $i++) { + my $f = $cull_order[$i]; + my $instance_volume = $unified_list{$f}{neb_vol}; + print "neb-cull --volume $instance_volume $key\n"; + print "$f $unified_list{$f}{md_ins_id}\n"; + if ($do_ops) { + # The tilde here is to force hard volumes. Don't touch it. + # Also: the 2 is a "minimum number of copies" restriction. Let's not be crazy here. + $neb->cull($key,"~${instance_volume}",2) or die "failed to cull a superfluous instance"; + if ($@) { die "$@"; } + mddb_delete_instance($mddb,$unified_list{$f}{md_ins_id}); + } + } + } + } + + # If we're here, then we've sucessfully sorted out this instance. + foreach my $f (keys %orphans) { + print "ORPHAN $f $orphans{$f}{md_ins_id}\n"; + mddb_delete_instance($mddb,$orphans{$f}{md_ins_id}); +# unlink($f) or warn "Could not unlink $f: $!"; + } + + } # End culling + + } # End scan over imfiles +} # End check for gpc1 database entry +# End prescan checks. +if ($do_cull == -1) { + die "Could not cull after repairing a file.\n"; +} + +# mddb_create_file($mddb,$key,$exp_id,$class_id,$md5sum); +sub mddb_create_file { + my $db = shift; + my $key = shift; + my $exp_id = shift; + my $class_id = shift; + my $md5sum = shift; + + my $query = "INSERT INTO obj (neb_key, exp_id, class_id, md5sum) VALUES (?, ?, ?, ?)"; + $db->do("BEGIN"); + my $stmt = $db->prepare($query); + $stmt->execute($key,$exp_id,$class_id,$md5sum) or + &my_die("failed to insert obj with new file $key $exp_id $class_id $md5sum"); + $stmt->finish(); + $db->do("COMMIT"); + + $query = "SELECT file_id FROM obj WHERE exp_id = $exp_id AND class_id = '$class_id'"; + my $md_instances = $mddb->selectall_arrayref( $query ); + my $row = ${ $md_instances }[0]; + my $file_id = ${ $row }[0]; + return($file_id); +} + +sub mddb_update_instance { + my $db = shift; + my $ins_id = shift; + my $disk_sum = shift; + + my $query = "UPDATE ins SET disk_sum = ? WHERE ins_id = ?"; + $db->do("BEGIN"); + my $stmt = $db->prepare($query); + $stmt->execute($disk_sum,$ins_id) or + &my_die("failed to update ins with new sum $ins_id $disk_sum"); + $stmt->finish(); + $db->do("COMMIT"); +} + +sub mddb_insert_instance { + my $db = shift; + my $file_id = shift; + my $file = shift; + my $volume = shift; + my $disk_sum = shift; + + my $query = "INSERT INTO ins (file_id, file, mvol_id, disk_sum) SELECT ?, ?, mvol_id, ? FROM vol WHERE vol.mvol_name = ?"; + $db->do("BEGIN"); + my $stmt = $db->prepare($query); + $stmt->execute($file_id,$file,$disk_sum,$volume) or + &my_die("failed to insert ins with new instance $file_id $file $volume $disk_sum"); + $stmt->finish(); + $db->do("COMMIT"); + + $query = "SELECT ins_id, site_name FROM ins JOIN vol USING(mvol_id) WHERE file_id = $file_id AND file = '$file'"; + my $md_instances = $mddb->selectall_arrayref( $query ); + my $row = ${ $md_instances }[0]; + my $ins_id = ${ $row }[0]; + my $site_name = ${ $row }[1]; + return($ins_id, $site_name); +} + +sub mddb_delete_instance { + my $db = shift; + my $ins_id = shift; + + print "DELETE $ins_id\n"; + my $query = "DELETE FROM ins WHERE ins_id = ?"; + $db->do("BEGIN"); + my $stmt = $db->prepare($query); + $stmt->execute($ins_id) or + &my_die("failed to delete ins entry $ins_id"); + $stmt->finish(); + $db->do("COMMIT"); +} + +sub local_md5sum { + my $filename = shift; + my $volume = (split /\//, $filename)[2]; + my $host = $volume; + $host =~ s/\.\d//; +# print "$filename $host $volume\n"; + my $response = `ssh $host remote_md5sum.pl $filename`; + chomp($response); + my ($sum, undef) = split /\s+/, $response; + unless(defined($sum)) { + my_die("Failed to calculate md5sum locally. $filename $host $volume"); + } + return($sum); +} + +sub parse_volume { + my $filename = shift(@_); + my $full_volume = (split /\//, $filename)[2]; + my ($hostname,undef) = split /\./, $full_volume; # /; + return($hostname,$full_volume); +} + +sub get_random_site_volume { + my $site_key = shift(@_); + my $NN = scalar @{ $volume_lists{$site_key} }; + my $backup_volume = ${ $volume_lists{$site_key} }[int(rand($NN))]; + return($backup_volume); +} + +sub my_die { + my $msg = shift(@_); + print $msg . "\n"; + foreach my $a (@_) { + print "ARG: $a\n"; + } + die; +} + + + + + + + + + + + + + + + + + + + + +__END__ + + Index: /branches/czw_branch/20170908/ippTasks/ipphosts.mhpcc.config =================================================================== --- /branches/czw_branch/20170908/ippTasks/ipphosts.mhpcc.config (revision 40482) +++ /branches/czw_branch/20170908/ippTasks/ipphosts.mhpcc.config (revision 40483) @@ -18,11 +18,11 @@ ## ## PSNSC -- above is way out of date -- update 20170112 in anticipation of -## nodes = $PSTAMP_MAX_FAULT_COUNT $PSTAMP_MAX_FAULT_COUNT = 3 @@ -408,5 +418,8 @@ # limit number of requests in the queue to avoid blocking everything # when jobs take an infinite amount of time to parse - $run = $run -limit 10 + #$run = $run -limit 10 + # except parsing is not label/prio based, high prio can get stuck behind low prio, so bump up to 20 + # -- should be a config option + $run = $run -limit $PSTAMP_PARSE_LIMIT command $run end Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/README =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/README (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/README (revision 40483) @@ -0,0 +1,9 @@ +Instructions to parse/check stuff between ipptopsps and the paper + +copy main.tex from sharelatex here + +run 'perl parseipptopspsallflags.pl', this parses tables.IN.vot as well as main.tex into different txt files + +(ipptopsps.DetectionFlags.txt and latex.DetectionFlags.txt, for example) + +do diffs on the 8 different tables: are they as we expect? (note: some have funny whitespace characters i've been unable to diagnose). Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.DetectionFlags.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.DetectionFlags.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.DetectionFlags.txt (revision 40483) @@ -0,0 +1,33 @@ +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||PSFMODEL||0x00000001||1||Source fitted with a psf model (linear or non-linear).|| +||EXTMODEL||0x00000002||2||Source fitted with an extended-source model.|| +||FITTED||0x00000004||4||Source fitted with non-linear model (PSF or EXT; good or bad).|| +||FAIL||0x00000008||8||Fit (non-linear) failed (non-converge; off-edge; run to zero).|| +||POOR||0x00000010||16||Fit succeeds, but low-S/N, high-Chisq, or large (for PSF -- drop?).|| +||PAIR||0x00000020||32||Source fitted with a double PSF.|| +||PSFSTAR||0x00000040||64||Source used to define PSF model.|| +||SATSTAR||0x00000080||128||Source model peak is above saturation.|| +||BLEND||0x00000100||256||Source is a blend with other sources.|| +||EXTERNAL||0x00000200||512||Source based on supplied input position.|| +||BADPSF||0x00000400||1024||Failed to get good estimate of object's PSF.|| +||DEFECT||0x00000800||2048||Source is thought to be a defect.|| +||SATURATED||0x00001000||4096||Source is thought to be saturated pixels (bleed trail).|| +||CR_LIMIT||0x00002000||8192||Source has crNsigma above limit.|| +||EXT_LIMIT||0x00004000||16384||Source has extNsigma above limit.|| +||MOMENTS_FAILURE||0x00008000||32768||Could not measure the moments.|| +||SKY_FAILURE||0x00010000||65536||Could not measure the local sky.|| +||SKYVAR_FAILURE||0x00020000||131072||Could not measure the local sky variance.|| +||BELOW_MOMENTS_SN||0x00040000||262144||Moments not measured due to low S/N.|| +||UNDEF_1||0x00080000||524288||Unused bit value.|| +||BIG_RADIUS||0x00100000||1048576||Poor moments for small radius, try large radius.|| +||AP_MAGS||0x00200000||2097152||Source has an aperture magnitude.|| +||BLEND_FIT||0x00400000||4194304||Source was fitted as a blend.|| +||EXTENDED_FIT||0x00800000||8388608||Full extended fit was used.|| +||EXTENDED_STATS||0x01000000||16777216||Extended aperture stats calculated.|| +||LINEAR_FIT||0x02000000||33554432||Source fitted with the linear fit.|| +||NONLINEAR_FIT||0x04000000||67108864||Source fitted with the non-linear fit.|| +||RADIAL_FLUX||0x08000000||134217728||Radial flux measurements calculated.|| +||SIZE_SKIPPED||0x10000000||268435456||Size could not be determined.|| +||PEAK_ON_SPIKE||0x20000000||536870912||Peak lands on diffraction spike.|| +||PEAK_ON_GHOST||0x40000000||1073741824||Peak lands on ghost or glint.|| +||PEAK_OFF_CHIP||0x80000000||2147483648||Peak lands off edge of chip.|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.DetectionFlags2.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.DetectionFlags2.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.DetectionFlags2.txt (revision 40483) @@ -0,0 +1,24 @@ +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||DIFF_WITH_SINGLE||0x00000001||1||Difference source matched to a single positive detection.|| +||DIFF_WITH_DOUBLE||0x00000002||2||Difference source matched to positive detections in both images.|| +||MATCHED||0x00000004||4||Source generated based on another image (forced photometry at source location).|| +||ON_SPIKE||0x00000008||8||More than 25% of (PSF-weighted) pixels land on diffraction spike.|| +||ON_STARCORE||0x00000010||16||More than 25% of (PSF-weighted) pixels land on starcore.|| +||ON_BURNTOOL||0x00000020||32||More than 25% of (PSF-weighted) pixels land on burntool.|| +||ON_CONVPOOR||0x00000040||64||More than 25% of (PSF-weighted) pixels land on convpoor.|| +||PASS1_SRC||0x00000080||128||Source detected in first pass analysis|| +||HAS_BRIGHTER_NEIGHBOR||0x00000100||256||Peak is not the brightest in its footprint|| +||BRIGHT_NEIGHBOR_1||0x00000200||512||Flux_negative / (r^2 flux_positive) > 1.|| +||BRIGHT_NEIGHBOR_10||0x00000400||1024||Flux_negative / (r^2 flux_positive) > 10.|| +||DIFF_SELF_MATCH||0x00000800||2048||Positive detection match is probably this source.|| +||SATSTAR_PROFILE||0x00001000||4096||Saturated source is modeled with a radial profile.|| +||ECONTOUR_FEW_PTS||0x00002000||8192||Too few points to measure the elliptical contour.|| +||RADBIN_NAN_CENTER||0x00004000||16384||Radial bins failed with too many NaN center bin.|| +||PETRO_NAN_CENTER||0x00008000||32768||Petrosian (1976) radial bins failed with too many NaN center bin.|| +||PETRO_NO_PROFILE||0x00010000||65536||Petrosian (1976) not built becaues radial bins missing.|| +||PETRO_INSIG_RATIO||0x00020000||131072||Insignificant measurement of Petrosian (1976) ratio.|| +||PETRO_RATIO_ZEROBIN||0x00040000||262144||Petrosian (1976) ratio in the 0th bin (likely bad).|| +||EXT_FITS_RUN||0x00080000||524288||Attempted to run extended fits on this source.|| +||EXT_FITS_FAIL||0x00100000||1048576||At least one of the model fits failed.|| +||EXT_FITS_RETRY||0x00200000||2097152||One of the model fits was retried with new window.|| +||EXT_FITS_NONE||0x00400000||4194304||All of the model fits failed.|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.DetectionFlags3.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.DetectionFlags3.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.DetectionFlags3.txt (revision 40483) @@ -0,0 +1,30 @@ +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||NOCAL||0x00000001||1||Detection ignored for this analysis (photcode; time range) -- internal only.|| +||POOR_PHOTOM||0x00000002||2||Detection is photometry outlier.|| +||SKIP_PHOTOM||0x00000004||4||Detection was ignored for photometry measurement.|| +||AREA||0x00000008||8||Detection near image edge.|| +||POOR_ASTROM||0x00000010||16||Detection is astrometry outlier.|| +||SKIP_ASTROM||0x00000020||32||Detection was ignored for astrometry measurement.|| +||USED_OBJ||0x00000040||64||Detection was used during update objects|| +||USED_CHIP||0x00000080||128||Detection was used during update chips to measure astrometry with IRLS fit.|| +||BLEND_MEAS||0x00000100||256||Detection is within radius of multiple objects.|| +||BLEND_OBJ||0x00000200||512||Multiple detections within radius of object.|| +||WARP_USED||0x00000400||1024||Measurement used to find mean warp photometry.|| +||UNMASKED_ASTRO||0x00000800||2048||Detection was unmasked in update chips to determine astrometry parameter errors.|| +||BLEND_MEAS_X||0x00001000||4096||Detection is within radius of multiple objects across catalogs.|| +||ARTIFACT||0x00002000||8192||Detection is thought to be non-astronomical.|| +||SYNTH_MAG||0x00004000||16384||Magnitude is synthetic.|| +||PHOTOM_UBERCAL||0x00008000||32768||Externally-supplied zero point from ubercal analysis.|| +||STACK_PRIMARY||0x00010000||65536||This stack measurement is in the primary skycell.|| +||STACK_PHOT_SRC||0x00020000||131072||This measurement supplied the stack photometry.|| +||ICRF_QSO||0x00040000||262144||This measurement is an ICRF reference position.|| +||IMAGE_EPOCH||0x00080000||524288||This measurement is registered to the image epoch (not tied to the reference catalog epoch).|| +||PHOTOM_PSF||0x00100000||1048576||This measurement is used for the mean PSF magnitude.|| +||PHOTOM_APER||0x00200000||2097152||This measurement is used for the mean aperture magnitude.|| +||PHOTOM_KRON||0x00400000||4194304||This measurement is used for the mean Kron (1980) magnitude.|| +||MASKED_PSF||0x01000000||16777216||This measurement is masked based on IRLS weights for the mean PSF magnitude.|| +||MASKED_APER||0x02000000||33554432||This measurement is masked based on IRLS weights for the mean aperture magnitude.|| +||MASKED_KRON||0x04000000||67108864||This measurement is masked based on IRLS weights for the mean Kron (1980) magnitude.|| +||OBJECT_HAS_2MASS||0x10000000||268435456||This measurement comes from an object with 2MASS data.|| +||OBJECT_HAS_GAIA||0x20000000||536870912||This measurement comes from an object with Gaia data.|| +||OBJECT_HAS_TYCHO||0x40000000||1073741824||This measurement comes from an object with Tycho data.|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ForcedGalaxyShapeFlags.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ForcedGalaxyShapeFlags.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ForcedGalaxyShapeFlags.txt (revision 40483) @@ -0,0 +1,5 @@ +||NO_ERROR||0x00000000||0||No error condition raised.|| +||FAIL_FIT||0x00000001||1||Fit failed to converge or was degenerate|| +||TOO_FEW||0x00000002||2||Not enough points to fit the model|| +||OUT_OF_RANGE||0x00000004||4||Fit minimum too far outside data range|| +||BAD_ERROR||0x00000008||8||Invalid size error (nan or inf)|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ImageFlags.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ImageFlags.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ImageFlags.txt (revision 40483) @@ -0,0 +1,12 @@ +||NEW||0x00000000||0||No relphot / relastro attempted.|| +||PHOTOM_NOCAL||0x00000001||1||Used within relphot to mean 'don't apply fit'.|| +||PHOTOM_POOR||0x00000002||2||Relphot says image is bad (dMcal > limit).|| +||PHOTOM_SKIP||0x00000004||4||External information image has bad photometry.|| +||PHOTOM_FEW||0x00000008||8||Currently too few measurements for good value for photometry.|| +||ASTROM_NOCAL||0x00000010||16||User-set value used within relastro: ignore.|| +||ASTROM_POOR||0x00000020||32||Relastro says image is bad (dR,dD > limit).|| +||ASTROM_FAIL||0x00000040||64||Relastro fit diverged, fit not applied.|| +||ASTROM_SKIP||0x00000080||128||External information image has bad astrometry.|| +||ASTROM_FEW||0x00000100||256||Currently too few measurements for good value for astrometry.|| +||PHOTOM_UBERCAL||0x00000200||512||Externally-supplied photometry zero point from ubercal analysis.|| +||ASTROM_GMM||0x00000400||1024||Image was fitted to positions corrected by the galaxy motion model.|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ObjectFilterFlags.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ObjectFilterFlags.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ObjectFilterFlags.txt (revision 40483) @@ -0,0 +1,22 @@ +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||SECF_STAR_FEW||0x00000001||1||Used within relphot: skip star.|| +||SECF_STAR_POOR||0x00000002||2||Used within relphot: skip star.|| +||SECF_USE_SYNTH||0x00000004||4||Synthetic photometry used in average measurement.|| +||SECF_USE_UBERCAL||0x00000008||8||Ubercal photometry used in average measurement.|| +||SECF_HAS_PS1||0x00000010||16||PS1 photometry used in average measurement.|| +||SECF_HAS_PS1_STACK||0x00000020||32||PS1 stack photometry exists.|| +||SECF_HAS_TYCHO||0x00000040||64||Tycho photometry used for synthetic magnitudes.|| +||SECF_FIX_SYNTH||0x00000080||128||Synthetic magnitudes repaired with zeropoint map.|| +||SECF_RANK_0||0x00000100||256||Average magnitude calculated in 0th pass.|| +||SECF_RANK_1||0x00000200||512||Average magnitude calculated in 1st pass.|| +||SECF_RANK_2||0x00000400||1024||Average magnitude calculated in 2nd pass.|| +||SECF_RANK_3||0x00000800||2048||Average magnitude calculated in 3rd pass.|| +||SECF_RANK_4||0x00001000||4096||Average magnitude calculated in 4th pass.|| +||SECF_STACK_PRIMARY||0x00004000||16384||PS1 stack photometry comes from primary skycell.|| +||SECF_STACK_BESTDET||0x00008000||32768||PS1 stack best measurement is a detection (not forced).|| +||SECF_STACK_PRIMDET||0x00010000||65536||PS1 stack primary measurement is a detection (not forced).|| +||SECF_HAS_SDSS||0x00100000||1048576||This photcode has SDSS photometry.|| +||SECF_HAS_HSC||0x00200000||2097152||This photcode has HSC photometry.|| +||SECF_HAS_CFH||0x00400000||4194304||This photcode has CFH photometry (mostly Megacam).|| +||SECF_HAS_DES||0x00800000||8388608||This photcode has DES photometry.|| +||SECF_OBJ_EXT||0x01000000||16777216||Extended in this band.|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ObjectInfoFlags.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ObjectInfoFlags.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ObjectInfoFlags.txt (revision 40483) @@ -0,0 +1,32 @@ +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||FEW||0x00000001||1||Used within relphot; skip star.|| +||POOR||0x00000002||2||Used within relphot; skip star.|| +||ICRF_QSO||0x00000004||4||Object IDed with known ICRF quasar (may have ICRF position measurement)|| +||HERN_QSO_P60||0x00000008||8||Identified as likely QSO (Hernitschek et al 2015), P_QSO >= 0.60|| +||HERN_QSO_P05||0x00000010||16||Identified as possible QSO (Hernitschek et al 2015), P_QSO >= 0.05|| +||HERN_RRL_P60||0x00000020||32||Identified as likely RR Lyra (Hernitschek et al 2015), P_RRLyra >= 0.60|| +||HERN_RRL_P05||0x00000040||64||Identified as possible RR Lyra (Hernitschek et al 2015), P_RRLyra >= 0.05|| +||HERN_VARIABLE||0x00000080||128||Identified as a variable based on ChiSq? (Hernitschek et al 2015)|| +||TRANSIENT||0x00000100||256||Identified as a non-periodic (stationary) transient|| +||HAS_SOLSYS_DET||0x00000200||512||At least one detection identified with a known solar-system object (asteroid or other).|| +||MOST_SOLSYS_DET||0x00000400||1024||Most detections identified with a known solar-system object (asteroid or other).|| +||LARGE_PM||0x00000800||2048||Star with large proper motion|| +||RAW_AVE||0x00001000||4096||Simple weighted average position was used (no IRLS fitting)|| +||FIT_AVE||0x00002000||8192||Average position was fitted|| +||FIT_PM||0x00004000||16384||Proper motion model was fitted|| +||FIT_PAR||0x00008000||32768||Parallax model was fitted|| +||USE_AVE||0x00010000||65536||Average position used (not PM or PAR)|| +||USE_PM||0x00020000||131072||Proper motion used (not AVE or PAR)|| +||USE_PAR||0x00040000||262144||Parallax used (not AVE or PM)|| +||NO_MEAN_ASTROM||0x00080000||524288||Mean astrometry could not be measured|| +||STACK_FOR_MEAN||0x00100000||1048576||Stack position used for mean astrometry|| +||MEAN_FOR_STACK||0x00200000||2097152||Mean astrometry used for stack position|| +||BAD_PM||0x00400000||4194304||Failure to measure proper-motion model|| +||EXT||0x00800000||8388608||Extended in our data (eg; PS)|| +||EXT_ALT||0x01000000||16777216||Extended in external data (eg; 2MASS)|| +||GOOD||0x02000000||33554432||Good-quality measurement in our data (eg; PS)|| +||GOOD_ALT||0x04000000||67108864||Good-quality measurement in external data (eg; 2MASS)|| +||GOOD_STACK||0x08000000||134217728||Good-quality object in the stack (> 1 good stack measurement)|| +||BEST_STACK||0x10000000||268435456||The primary stack measurements are the best measurements|| +||SUSPECT_STACK||0x20000000||536870912||Suspect object in the stack (no more than 1 good measurement, 2 or more suspect or good stack measurement)|| +||BAD_STACK||0x40000000||1073741824||Poor-quality stack object (no more than 1 good or suspect measurement).|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ObjectQualityFlags.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ObjectQualityFlags.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/ipptopsps.ObjectQualityFlags.txt (revision 40483) @@ -0,0 +1,9 @@ +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||QF_OBJ_EXT||0x00000001||1||Extended in our data (eg; PS).|| +||QF_OBJ_EXT_ALT||0x00000002||2||Extended in external data (eg; 2MASS).|| +||QF_OBJ_GOOD||0x00000004||4||Good-quality measurement in our data (eg; PS).|| +||QF_OBJ_GOOD_ALT||0x00000008||8||Good-quality measurement in external data (eg; 2MASS).|| +||QF_OBJ_GOOD_STACK||0x00000010||16||Good-quality object in the stack (> 1 good stack measurement).|| +||QF_OBJ_BEST_STACK||0x00000020||32||The primary stack measurements are the best measurements.|| +||QF_OBJ_SUSPECT_STACK||0x00000040||64||Suspect object in the stack (no more than 1 good measurement, 2 or more suspect or good stack measurement).|| +||QF_OBJ_BAD_STACK||0x00000080||128||Poor-quality stack object (no more than 1 good or suspect measurement).|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.DetectionFlags.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.DetectionFlags.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.DetectionFlags.txt (revision 40483) @@ -0,0 +1,34 @@ +||Name||Hexadecimal||Value||Description|| +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||PSFMODEL||0x00000001||1||Source fitted with a psf model (linear or non-linear).|| +||EXTMODEL||0x00000002||2||Source fitted with an extended-source model.|| +||FITTED||0x00000004||4||Source fitted with non-linear model (PSF or EXT; good or bad).|| +||FAIL||0x00000008||8||Fit (non-linear) failed (non-converge; off-edge; run to zero).|| +||POOR||0x00000010||16||Fit succeeds; but low-S/N; high-Chisq; or large (for PSF -- drop?).|| +||PAIR||0x00000020||32||Source fitted with a double PSF.|| +||PSFSTAR||0x00000040||64||Source used to define PSF model.|| +||SATSTAR||0x00000080||128||Source model peak is above saturation.|| +||BLEND||0x00000100||256||Source is a blend with other sources.|| +||EXTERNAL||0x00000200||512||Source based on supplied input position.|| +||BADPSF||0x00000400||1024||Failed to get good estimate of object's PSF.|| +||DEFECT||0x00000800||2048||Source is thought to be a defect.|| +||SATURATED||0x00001000||4096||Source is thought to be saturated pixels (bleed trail).|| +||CR_LIMIT||0x00002000||8192||Source has crNsigma above limit.|| +||EXT_LIMIT||0x00004000||16384||Source has extNsigma above limit.|| +||MOMENTS_FAILURE||0x00008000||32768||Could not measure the moments.|| +||SKY_FAILURE||0x00010000||65536||Could not measure the local sky.|| +||SKYVAR_FAILURE||0x00020000||131072||Could not measure the local sky variance.|| +||BELOW_MOMENTS_SN||0x00040000||262144||Moments not measured due to low S/N.|| +||UNDEF_1||0x00080000||524288||Unused bit value.|| +||BIG_RADIUS||0x00100000||1048576||Poor moments for small radius; try large radius.|| +||AP_MAGS||0x00200000||2097152||Source has an aperture magnitude.|| +||BLEND_FIT||0x00400000||4194304||Source was fitted as a blend.|| +||EXTENDED_FIT||0x00800000||8388608||Full extended fit was used.|| +||EXTENDED_STATS||0x01000000||16777216||Extended aperture stats calculated.|| +||LINEAR_FIT||0x02000000||33554432||Source fitted with the linear fit.|| +||NONLINEAR_FIT||0x04000000||67108864||Source fitted with the non-linear fit.|| +||RADIAL_FLUX||0x08000000||134217728||Radial flux measurements calculated.|| +||SIZE_SKIPPED||0x10000000||268435456||Size could not be determined.|| +||PEAK_ON_SPIKE||0x20000000||536870912||Peak lands on diffraction spike.|| +||PEAK_ON_GHOST||0x40000000||1073741824||Peak lands on ghost or glint.|| +||PEAK_OFF_CHIP||0x80000000||2147483648||Peak lands off edge of chip.|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.DetectionFlags2.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.DetectionFlags2.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.DetectionFlags2.txt (revision 40483) @@ -0,0 +1,25 @@ +||Name||Hexadecimal||Value||Description|| +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||DIFF_WITH_SINGLE||0x00000001||1||Difference source matched to a single positive detection.|| +||DIFF_WITH_DOUBLE||0x00000002||2||Difference source matched to positive detections in both images.|| +||MATCHED||0x00000004||4||Source generated based on another image (forced photometry at source location).|| +||ON_SPIKE||0x00000008||8||More than 25% of (PSF-weighted) pixels land on diffraction spike.|| +||ON_STARCORE||0x00000010||16||More than 25% of (PSF-weighted) pixels land on starcore.|| +||ON_BURNTOOL||0x00000020||32||More than 25% of (PSF-weighted) pixels land on burntool.|| +||ON_CONVPOOR||0x00000040||64||More than 25% of (PSF-weighted) pixels land on convpoor.|| +||PASS1_SRC||0x00000080||128||Source detected in first pass analysis|| +||HAS_BRIGHTER_NEIGHBOR||0x00000100||256||Peak is not the brightest in its footprint|| +||BRIGHT_NEIGHBOR_1||0x00000200||512||Flux_negative / (r^2 flux_positive) > 1.|| +||BRIGHT_NEIGHBOR_10||0x00000400||1024||Flux_negative / (r^2 flux_positive) > 10.|| +||DIFF_SELF_MATCH||0x00000800||2048||Positive detection match is probably this source.|| +||SATSTAR_PROFILE||0x00001000||4096||Saturated source is modeled with a radial profile.|| +||ECONTOUR_FEW_PTS||0x00002000||8192||Too few points to measure the elliptical contour.|| +||RADBIN_NAN_CENTER||0x00004000||16384||Radial bins failed with too many NaN center bin.|| +||PETRO_NAN_CENTER||0x00008000||32768||Petrosian (1976) radial bins failed with too many NaN center bin.|| +||PETRO_NO_PROFILE||0x00010000||65536||Petrosian (1976) not built becaues radial bins missing.|| +||PETRO_INSIG_RATIO||0x00020000||131072||Insignificant measurement of Petrosian (1976) ratio.|| +||PETRO_RATIO_ZEROBIN||0x00040000||262144||Petrosian (1976) ratio in the 0th bin (likely bad).|| +||EXT_FITS_RUN||0x00080000||524288||Attempted to run extended fits on this source.|| +||EXT_FITS_FAIL||0x00100000||1048576||At least one of the model fits failed.|| +||EXT_FITS_RETRY||0x00200000||2097152||One of the model fits was retried with new window.|| +||EXT_FITS_NONE||0x00400000||4194304||All of the model fits failed.|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.DetectionFlags3.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.DetectionFlags3.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.DetectionFlags3.txt (revision 40483) @@ -0,0 +1,32 @@ +||Name||Hexadecimal||Value||Description|| +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||NOCAL||0x00000001||1||Detection ignored for this analysis (photcode; time range) -- internal only.|| +||POOR_PHOTOM||0x00000002||2||Detection is photometry outlier.|| +||SKIP_PHOTOM||0x00000004||4||Detection was ignored for photometry measurement.|| +||AREA||0x00000008||8||Detection near image edge.|| +||POOR_ASTROM||0x00000010||16||Detection is astrometry outlier.|| +||SKIP_ASTROM||0x00000020||32||Detection was ignored for astrometry measurement.|| +||USED_OBJ||0x00000040||64||Detection was used during update objects|| +||USED_CHIP||0x00000080||128||Detection was used during update chips to measure astrometry with IRLS fit.|| +||BLEND_MEAS||0x00000100||256||Detection is within radius of multiple objects.|| +||BLEND_OBJ||0x00000200||512||Multiple detections within radius of object.|| +||WARP_USED||0x00000400||1024||Measurement used to find mean warp photometry.|| +||UNMASKED_ASTRO||0x00000800||2048||Detection was unmasked in update chips to determine astrometry parameter errors.|| +||BLEND_MEAS_X||0x00001000||4096||Detection is within radius of multiple objects across catalogs.|| +||ARTIFACT||0x00002000||8192||Detection is thought to be non-astronomical.|| +||SYNTH_MAG||0x00004000||16384||Magnitude is synthetic.|| +||PHOTOM_UBERCAL||0x00008000||32768||Externally-supplied zero point from ubercal analysis.|| +||STACK_PRIMARY||0x00010000||65536||This stack measurement is in the primary skycell.|| +||STACK_PHOT_SRC||0x00020000||131072||This measurement supplied the stack photometry.|| +||ICRF_QSO||0x00040000||262144||This measurement is an ICRF reference position.|| +||IMAGE_EPOCH||0x00080000||524288||This measurement is registered to the image epoch|| +||||||||(not tied to the reference catalog epoch).|| +||PHOTOM_PSF||0x00100000||1048576||This measurement is used for the mean PSF magnitude.|| +||PHOTOM_APER||0x00200000||2097152||This measurement is used for the mean aperture magnitude.|| +||PHOTOM_KRON||0x00400000||4194304||This measurement is used for the mean Kron (1980) magnitude.|| +||MASKED_PSF||0x01000000||16777216||This measurement is masked based on IRLS weights for the mean PSF magnitude.|| +||MASKED_APER||0x02000000||33554432||This measurement is masked based on IRLS weights for the mean aperture magnitude.|| +||MASKED_KRON||0x04000000||67108864||This measurement is masked based on IRLS weights for the mean Kron (1980) magnitude.|| +||OBJECT_HAS_2MASS||0x10000000||268435456||This measurement comes from an object with 2MASS data.|| +||OBJECT_HAS_GAIA||0x20000000||536870912||This measurement comes from an object with Gaia data.|| +||OBJECT_HAS_TYCHO||0x40000000||1073741824||This measurement comes from an object with Tycho data.|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ImageFlags.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ImageFlags.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ImageFlags.txt (revision 40483) @@ -0,0 +1,13 @@ +||Name||Hexadecimal||Value||Description|| +||NEW||0x00000000||0||No relphot / relastro attempted.|| +||PHOTOM_NOCAL||0x00000001||1||Used within relphot to mean 'don't apply fit'.|| +||PHOTOM_POOR||0x00000002||2||Relphot says image is bad (dMcal > limit).|| +||PHOTOM_SKIP||0x00000004||4||External information image has bad photometry.|| +||PHOTOM_FEW||0x00000008||8||Currently too few measurements for good value for photometry.|| +||ASTROM_NOCAL||0x00000010||16||User-set value used within relastro: ignore.|| +||ASTROM_POOR||0x00000020||32||Relastro says image is bad (dR,dD > limit).|| +||ASTROM_FAIL||0x00000040||64||Relastro fit diverged, fit not applied.|| +||ASTROM_SKIP||0x00000080||128||External information image has bad astrometry.|| +||ASTROM_FEW||0x00000100||256||Currently too few measurements for good value for astrometry.|| +||PHOTOM_UBERCAL||0x00000200||512||Externally-supplied photometry zero point from ubercal analysis.|| +||ASTROM_GMM||0x00000400||1024||Image was fitted to positions corrected by the galaxy motion model.|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ObjectFilterFlags.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ObjectFilterFlags.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ObjectFilterFlags.txt (revision 40483) @@ -0,0 +1,23 @@ +||Name||Hexadecimal||Value||Description|| +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||SECF_STAR_FEW||0x00000001||1||Used within relphot: skip star.|| +||SECF_STAR_POOR||0x00000002||2||Used within relphot: skip star.|| +||SECF_USE_SYNTH||0x00000004||4||Synthetic photometry used in average measurement.|| +||SECF_USE_UBERCAL||0x00000008||8||Ubercal photometry used in average measurement.|| +||SECF_HAS_PS1||0x00000010||16||PS1 photometry used in average measurement.|| +||SECF_HAS_PS1_STACK||0x00000020||32||PS1 stack photometry exists.|| +||SECF_HAS_TYCHO||0x00000040||64||Tycho photometry used for synthetic magnitudes.|| +||SECF_FIX_SYNTH||0x00000080||128||Synthetic magnitudes repaired with zeropoint map.|| +||SECF_RANK_0||0x00000100||256||Average magnitude calculated in 0th pass.|| +||SECF_RANK_1||0x00000200||512||Average magnitude calculated in 1st pass.|| +||SECF_RANK_2||0x00000400||1024||Average magnitude calculated in 2nd pass.|| +||SECF_RANK_3||0x00000800||2048||Average magnitude calculated in 3rd pass.|| +||SECF_RANK_4||0x00001000||4096||Average magnitude calculated in 4th pass.|| +||SECF_STACK_PRIMARY||0x00004000||16384||PS1 stack photometry comes from primary skycell.|| +||SECF_STACK_BESTDET||0x00008000||32768||PS1 stack best measurement is a detection (not forced).|| +||SECF_STACK_PRIMDET||0x00010000||65536||PS1 stack primary measurement is a detection (not forced).|| +||SECF_HAS_SDSS||0x00100000||1048576||This photcode has SDSS photometry.|| +||SECF_HAS_HSC||0x00200000||2097152||This photcode has HSC photometry.|| +||SECF_HAS_CFH||0x00400000||4194304||This photcode has CFH photometry (mostly megacam).|| +||SECF_HAS_DES||0x00800000||8388608||This photcode has DES photometry.|| +||SECF_OBJ_EXT||0x01000000||16777216||Extended in this band.|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ObjectInfoFlags.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ObjectInfoFlags.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ObjectInfoFlags.txt (revision 40483) @@ -0,0 +1,36 @@ +||Name||Hexadecimal||Value||Description|| +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||FEW||0x00000001||1||Used within relphot; skip star.|| +||POOR||0x00000002||2||Used within relphot; skip star.|| +||ICRF_QSO||0x00000004||4||Object IDed with known ICRF quasar|| +||||||||(may have ICRF position measurement)|| +||HERN_QSO_P60||0x00000008||8||Identified as likely QSO (Hernitschek et al 2015);|| +||||||||P_QSO >= 0.60|| +||HERN_QSO_P05||0x00000010||16||Identified as possible QSO (Hernitschek et al 2015), P_QSO >= 0.05|| +||HERN_RRL_P60||0x00000020||32||Identified as likely RR Lyra (Hernitschek et al 2015), P_RRLyra >= 0.60|| +||HERN_RRL_P05||0x00000040||64||Identified as possible RR Lyra (Hernitschek et al 2015), P_RRLyra >= 0.05|| +||HERN_VARIABLE||0x00000080||128||Identified as a variable based on ChiSq? (Hernitschek et al 2015)|| +||TRANSIENT||0x00000100||256||Identified as a non-periodic (stationary) transient|| +||HAS_SOLSYS_DET||0x00000200||512||At least one detection identified with a known solar-system object (asteroid or other).|| +||MOST_SOLSYS_DET||0x00000400||1024||Most detections identified with a known solar-system object (asteroid or other).|| +||LARGE_PM||0x00000800||2048||Star with large proper motion|| +||RAW_AVE||0x00001000||4096||Simple weighted average position was used (no IRLS fitting)|| +||FIT_AVE||0x00002000||8192||Average position was fitted|| +||FIT_PM||0x00004000||16384||Proper motion model was fitted|| +||FIT_PAR||0x00008000||32768||Parallax model was fitted|| +||USE_AVE||0x00010000||65536||Average position used (not PM or PAR)|| +||USE_PM||0x00020000||131072||Proper motion used (not AVE or PAR)|| +||USE_PAR||0x00040000||262144||Parallax used (not AVE or PM)|| +||NO_MEAN_ASTROM||0x00080000||524288||Mean astrometry could not be measured|| +||STACK_FOR_MEAN||0x00100000||1048576||Stack position used for mean astrometry|| +||MEAN_FOR_STACK||0x00200000||2097152||Mean astrometry used for stack position|| +||BAD_PM||0x00400000||4194304||Failure to measure proper-motion model|| +||EXT||0x00800000||8388608||Extended in our data (eg; PS)|| +||EXT_ALT||0x01000000||16777216||Extended in external data (eg; 2MASS)|| +||GOOD||0x02000000||33554432||Good-quality measurement in our data (eg; PS)|| +||GOOD_ALT||0x04000000||67108864||Good-quality measurement in external data (eg; 2MASS)|| +||GOOD_STACK||0x08000000||134217728||Good-quality object in the stack (> 1 good stack measurement)|| +||BEST_STACK||0x10000000||268435456||The primary stack measurements are the best measurements|| +||SUSPECT_STACK||0x20000000||536870912||Suspect object in the stack|| +||||||||(no more than 1 good measurement, 2 or more suspect or good stack measurement)|| +||BAD_STACK||0x40000000||1073741824||Poor-quality stack object (no more than 1 good or suspect measurement)|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ObjectQualtiyFlags.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ObjectQualtiyFlags.txt (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/latex.ObjectQualtiyFlags.txt (revision 40483) @@ -0,0 +1,10 @@ +||Name||Hexadecimal||Value||Description|| +||DEFAULT||0x00000000||0||Initial value; resets all bits.|| +||QF_OBJ_EXT||0x00000001||1||Extended in our data (eg; PS).|| +||QF_OBJ_EXT_ALT||0x00000002||2||Extended in external data (eg; 2MASS).|| +||QF_OBJ_GOOD||0x00000004||4||Good-quality measurement in our data (eg; PS).|| +||QF_OBJ_GOOD_ALT||0x00000008||8||Good-quality measurement in external data (eg; 2MASS).|| +||QF_OBJ_GOOD_STACK||0x00000010||16||Good-quality object in the stack (> 1 good stack measurement).|| +||QF_OBJ_BEST_STACK||0x00000020||32||The primary stack measurements are the best measurements.|| +||QF_OBJ_SUSPECT_STACK||0x00000040||64||Suspect object in the stack (no more than 1 good measurement, 2 or more suspect or good stack measurement).|| +||QF_OBJ_BAD_STACK||0x00000080||128||Poor-quality stack object (no more than 1 good or suspect measurement).|| Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/parse_tableIN_sub.pl =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/parse_tableIN_sub.pl (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/parse_tableIN_sub.pl (revision 40483) @@ -0,0 +1,45 @@ +#$table = 'ObjectInfoFlags'; +$table='ObjectQualityFlags'; +$table=$ARGV[1]; +#$table = 'ImageFlags'; +# +#
+#
+#
+#
+ +#
+ +$go = 0; +while (<>) { + + + + +if ($_ =~ /\/|/g; + $_ =~s /\s+\/|/g; + $_ =~s /\/|/g; + $_ =~s /\<\/TR\>/|/g; + $_ =~s /\s+\<\/TD\>/|/g; + $_ =~s /\<\/TD\>/|/g; + $_ =~s /^\s+//; + + print $_; + } + } +} + +} Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/parseipptopspsallflags.pl =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/parseipptopspsallflags.pl (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/parseipptopspsallflags.pl (revision 40483) @@ -0,0 +1,19 @@ +#`rm ipptopsps.*lags.txt`; +`perl parse_tableIN_sub.pl ../tables.IN.vot ObjectInfoFlags > ipptopsps.ObjectInfoFlags.txt`; + `perl parse_tableIN_sub.pl ../tables.IN.vot ObjectQualityFlags > ipptopsps.ObjectQualityFlags.txt`; + `perl parse_tableIN_sub.pl ../tables.IN.vot ObjectFilterFlags > ipptopsps.ObjectFilterFlags.txt`; + `perl parse_tableIN_sub.pl ../tables.IN.vot DetectionFlags > ipptopsps.DetectionFlags.txt`; + + `perl parse_tableIN_sub.pl ../tables.IN.vot DetectionFlags2 > ipptopsps.DetectionFlags2.txt`; + `perl parse_tableIN_sub.pl ../tables.IN.vot DetectionFlags3 > ipptopsps.DetectionFlags3.txt`; + `perl parse_tableIN_sub.pl ../tables.IN.vot ImageFlags > ipptopsps.ImageFlags.txt`; + `perl parse_tableIN_sub.pl ../tables.IN.vot ForcedGalaxyShapeFlags > ipptopsps.ForcedGalaxyShapeFlags.txt`; + +`perl parselatex.pl main.tex ObjectInfoFlags > latex.ObjectInfoFlags.txt`; +`perl parselatex.pl main.tex ObjectQualityFlags > latex.ObjectQualtiyFlags.txt`; +`perl parselatex.pl main.tex ObjectFilterFlags > latex.ObjectFilterFlags.txt`; +`perl parselatex.pl main.tex DetectionFlags > latex.DetectionFlags.txt`; +`perl parselatex.pl main.tex DetectionFlags2 > latex.DetectionFlags2.txt`; +`perl parselatex.pl main.tex DetectionFlags3 > latex.DetectionFlags3.txt`; +`perl parselatex.pl main.tex ImageFlags > latex.ImageFlags.txt`; +`perl parselatex.pl main.tex ForceGalaxyShapeFlags > latex.ForcedGalaxyShapeFlags.txt`; Index: /branches/czw_branch/20170908/ippToPsps/config/latextables/parselatex.pl =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/latextables/parselatex.pl (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/config/latextables/parselatex.pl (revision 40483) @@ -0,0 +1,28 @@ +$table = $ARGV[1]; +$go = 0; +while (<>) { + if ($_ =~ /caption/) { + if ($_ =~ /$table\}/) { + $go = 1; + } else { + $go = 0; + } + + } + if ($go==1) { + if ($_ =~ /\&/) { + #this is a good line (?) +# print $_; + chomp $_; + $_ =~ s/\\//g; + $_ =~ s/\s+&/&/g; + $_ =~ s/&\s+/&/g; + $_ =~ s/\{em //g; + $_ =~ s/\}//g; + $_ =~ s/\$//g; + my @split = split ('&', $_); + $split[3] =~ s/\s+$//; + print "||$split[0]||$split[1]||$split[2]||$split[3]||\n"; + } + } +} Index: /branches/czw_branch/20170908/ippToPsps/config/tables.DO.vot =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/tables.DO.vot (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/config/tables.DO.vot (revision 40483) @@ -7,8 +7,8 @@ - IAU name for this object. + IAU-approved name for this object. - - Alternate Pan-STARRS name for this object. + + Alternate Sexigesimal name for this object. Index: /branches/czw_branch/20170908/ippToPsps/config/tables.IN.vot =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/tables.IN.vot (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/config/tables.IN.vot (revision 40483) @@ -576,12 +576,12 @@ - - - + + + - + - + @@ -614,5 +614,5 @@ - + @@ -669,5 +669,5 @@ - + @@ -715,5 +715,5 @@ - + @@ -770,33 +770,33 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -822,28 +822,29 @@ - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -869,12 +870,12 @@ - - - - - - - - + + + + + + + + @@ -900,8 +901,8 @@ - - - - + + + + Index: /branches/czw_branch/20170908/ippToPsps/config/tables.OB.vot =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/tables.OB.vot (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/config/tables.OB.vot (revision 40483) @@ -9,8 +9,8 @@ - IAU name for this object. - - - Alternate Pan-STARRS name for this object. + IAU-approved name for this object. + + + Alternate Sexigesimal name for this object. Index: /branches/czw_branch/20170908/ippToPsps/config/tables.ST.vot =================================================================== --- /branches/czw_branch/20170908/ippToPsps/config/tables.ST.vot (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/config/tables.ST.vot (revision 40483) @@ -93,5 +93,8 @@ - Locally derived photometric zero point for this stack. + Locally derived photometric zero point for this stack (PSF-like measurements only). + + + Locally derived photometric zero point for this stack (Aperture-like measurements only). @@ -231,4 +234,7 @@ Information flag bitmask indicating details of the g filter stack photometry. Values listed in DetectionFlags3. + + Information flag bitmask indicating details of the g filter stack photometry. Values listed in ObjectFilterFlags. + Number of input frames/exposures contributing to the g filter stack detection. @@ -287,4 +293,7 @@ Information flag bitmask indicating details of the r filter stack photometry. Values listed in DetectionFlags3. + + Information flag bitmask indicating details of the r filter stack photometry. Values listed in ObjectFilterFlags. + Number of input frames/exposures contributing to the r filter stack detection. @@ -343,4 +352,7 @@ Information flag bitmask indicating details of the i filter stack photometry. Values listed in DetectionFlags3. + + Information flag bitmask indicating details of the i filter stack photometry. Values listed in ObjectFilterFlags. + Number of input frames/exposures contributing to the i filter stack detection. @@ -399,4 +411,7 @@ Information flag bitmask indicating details of the z filter stack photometry. Values listed in DetectionFlags3. + + Information flag bitmask indicating details of the z filter stack photometry. Values listed in ObjectFilterFlags. + Number of input frames/exposures contributing to the z filter stack detection. @@ -454,4 +469,7 @@ Information flag bitmask indicating details of the y filter stack photometry. Values listed in DetectionFlags3. + + + Information flag bitmask indicating details of the y filter stack photometry. Values listed in ObjectFilterFlags. @@ -463,5 +481,5 @@
NEW 0x000000000No relphot / relastro attempted.
PHOTOM_NOCAL 0x000000011Used within relphot to mean 'don't apply fit'. | user-set value used withing relphot: ignore (EAM!)
PHOTOM_POOR 0x000000022Relphot says image is bad (dMcal > limit). (EAM!)
PHOTOM_SKIP 0x000000044External information image has bad photometry (EAM).
PHOTOM_NOCAL 0x000000011Used within relphot to mean 'don't apply fit'.
PHOTOM_POOR 0x000000022Relphot says image is bad (dMcal > limit).
PHOTOM_SKIP 0x000000044External information image has bad photometry.
PHOTOM_FEW 0x000000088Currently too few measurements for good value for photometry.
ASTROM_NOCAL 0x0000001016User-set value used within relastro: ignore (EAM!).
ASTROM_NOCAL 0x0000001016User-set value used within relastro: ignore.
ASTROM_POOR 0x0000002032Relastro says image is bad (dR,dD > limit).
ASTROM_FAIL 0x0000004064Relastro fit diverged, fit not applied.
ASTROM_SKIP 0x00000080128External information image has bad astrometry. (EAM)
ASTROM_SKIP 0x00000080128External information image has bad astrometry.
ASTROM_FEW 0x00000100256Currently too few measurements for good value for astrometry.
PHOTOM_UBERCAL0x00000200512Externally-supplied photometry zero point from ubercal analysis.
EXTMODEL 0x000000022 Source fitted with an extended-source model.
FITTED 0x000000044 Source fitted with non-linear model (PSF or EXT; good or bad).
FAIL 0x000000088 Fit (non-linear) failed (non-converge, off-edge, run to zero).
FAIL 0x000000088 Fit (non-linear) failed (non-converge; off-edge; run to zero).
POOR 0x0000001016 Fit succeeds, but low-S/N, high-Chisq, or large (for PSF -- drop?).
PAIR 0x0000002032 Source fitted with a double PSF.
DIFF_WITH_SINGLE 0x000000011 Difference source matched to a single positive detection.
DIFF_WITH_DOUBLE 0x000000022 Difference source matched to positive detections in both images.
MATCHED 0x000000044 source generated based on another image (forced photometry at source location).
MATCHED 0x000000044 Source generated based on another image (forced photometry at source location).
ON_SPIKE 0x000000088 More than 25% of (PSF-weighted) pixels land on diffraction spike.
ON_STARCORE 0x0000001016 More than 25% of (PSF-weighted) pixels land on starcore.
DEFAULT 0x000000000 Initial value; resets all bits.
NOCAL 0x000000011 Detection ignored for this analysis (photcode, time range) -- internal only.
NOCAL 0x000000011 Detection ignored for this analysis (photcode; time range) -- internal only.
POOR_PHOTOM 0x000000022 Detection is photometry outlier.
SKIP_PHOTOM 0x000000044 Detection was ignored for photometry measurement.
FEW 0x000000011 Used within relphot; skip star.
POOR 0x000000022 Used within relphot; skip star.
ICRF_QSO 0x000000044 object IDed with known ICRF quasar (may have ICRF position measurement)
HERN_QSO_P60 0x000000088 identified as likely QSO (Hernitschek et al 2015), P_QSO >= 0.60
HERN_QSO_P05 0x0000001016 identified as possible QSO (Hernitschek et al 2015), P_QSO >= 0.05
HERN_RRL_P60 0x0000002032 identified as likely RR Lyra (Hernitschek et al 2015), P_RRLyra >= 0.60
HERN_RRL_P05 0x0000004064 identified as possible RR Lyra (Hernitschek et al 2015), P_RRLyra >= 0.05
HERN_VARIABLE 0x00000080128 identified as a variable based on ChiSq (Hernitschek et al 2015)
TRANSIENT 0x00000100256 identified as a non-periodic (stationary) transient
HAS_SOLSYS_DET 0x00000200512 at least one detection identified with a known solar-system object (asteroid or other).
MOST_SOLSYS_DET 0x000004001024 most detections identified with a known solar-system object (asteroid or other).
LARGE_PM 0x000008002048 star with large proper motion
RAW_AVE 0x000010004096 simple weighted average position was used (no IRLS fitting)
FIT_AVE 0x000020008192 average position was fitted
FIT_PM 0x0000400016384 proper motion model was fitted
FIT_PAR 0x0000800032768 parallax model was fitted
USE_AVE 0x0001000065536 average position used (not PM or PAR)
USE_PM 0x00020000131072 proper motion used (not AVE or PAR)
USE_PAR 0x00040000262144 parallax used (not AVE or PM)
NO_MEAN_ASTROM 0x00080000524288 mean astrometry could not be measured
STACK_FOR_MEAN 0x001000001048576 stack position used for mean astrometry
MEAN_FOR_STACK 0x002000002097152 mean astrometry used for stack position
BAD_PM 0x004000004194304 failure to measure proper-motion model
EXT 0x008000008388608 extended in our data (eg, PS)
EXT_ALT 0x0100000016777216 extended in external data (eg, 2MASS)
GOOD 0x0200000033554432 good-quality measurement in our data (eg,PS)
GOOD_ALT 0x0400000067108864 good-quality measurement in external data (eg, 2MASS)
GOOD_STACK 0x08000000134217728 good-quality object in the stack (> 1 good stack measurement)
BEST_STACK 0x10000000268435456 the primary stack measurements are the best measurements
SUSPECT_STACK 0x20000000536870912 suspect object in the stack (no more than 1 good measurement, 2 or more suspect or good stack measurement)
BAD_STACK 0x400000001073741824 poor-quality stack object (no more than 1 good or suspect measurement)
ICRF_QSO 0x000000044 Object IDed with known ICRF quasar (may have ICRF position measurement)
HERN_QSO_P60 0x000000088 Identified as likely QSO (Hernitschek et al 2015), P_QSO >= 0.60
HERN_QSO_P05 0x0000001016 Identified as possible QSO (Hernitschek et al 2015), P_QSO >= 0.05
HERN_RRL_P60 0x0000002032 Identified as likely RR Lyra (Hernitschek et al 2015), P_RRLyra >= 0.60
HERN_RRL_P05 0x0000004064 Identified as possible RR Lyra (Hernitschek et al 2015), P_RRLyra >= 0.05
HERN_VARIABLE 0x00000080128 Identified as a variable based on ChiSq? (Hernitschek et al 2015)
TRANSIENT 0x00000100256 Identified as a non-periodic (stationary) transient
HAS_SOLSYS_DET 0x00000200512 At least one detection identified with a known solar-system object (asteroid or other).
MOST_SOLSYS_DET 0x000004001024 Most detections identified with a known solar-system object (asteroid or other).
LARGE_PM 0x000008002048 Star with large proper motion
RAW_AVE 0x000010004096 Simple weighted average position was used (no IRLS fitting)
FIT_AVE 0x000020008192 Average position was fitted
FIT_PM 0x0000400016384 Proper motion model was fitted
FIT_PAR 0x0000800032768 Parallax model was fitted
USE_AVE 0x0001000065536 Average position used (not PM or PAR)
USE_PM 0x00020000131072 Proper motion used (not AVE or PAR)
USE_PAR 0x00040000262144 Parallax used (not AVE or PM)
NO_MEAN_ASTROM 0x00080000524288 Mean astrometry could not be measured
STACK_FOR_MEAN 0x001000001048576 Stack position used for mean astrometry
MEAN_FOR_STACK 0x002000002097152 Mean astrometry used for stack position
BAD_PM 0x004000004194304 Failure to measure proper-motion model
EXT 0x008000008388608 Extended in our data (eg; PS)
EXT_ALT 0x0100000016777216 Extended in external data (eg; 2MASS)
GOOD 0x0200000033554432 Good-quality measurement in our data (eg; PS)
GOOD_ALT 0x0400000067108864 Good-quality measurement in external data (eg; 2MASS)
GOOD_STACK 0x08000000134217728 Good-quality object in the stack (> 1 good stack measurement)
BEST_STACK 0x10000000268435456 The primary stack measurements are the best measurements
SUSPECT_STACK 0x20000000536870912 Suspect object in the stack (no more than 1 good measurement, 2 or more suspect or good stack measurement)
BAD_STACK 0x400000001073741824 Poor-quality stack object (no more than 1 good or suspect measurement).
DEFAULT 0x000000000 Initial value; resets all bits.
SECF_STAR_FEW 0x000000011 Used within relphot: skip star.
SECF_STAR_POOR 0x000000022 Used within relphot: skip star.
SECF_USE_SYNTH 0x000000044 Synthetic photometry used in average measurement.
SECF_USE_UBERCAL 0x000000088 Ubercal photometry used in average measurement.
SECF_HAS_PS1 0x0000001016 PS1 photometry used in average measurement.
SECF_HAS_PS1_STACK0x0000002032 PS1 stack photometry exists.
SECF_HAS_TYCHO 0x0000004064 Tycho photometry used for synthetic magnitudes.
SECF_FIX_SYNTH 0x00000080128 Synthetic magnitudes repaired with zeropoint map.
SECF_RANK_0 0x00000100256 Average magnitude uses only rank 0 detections.
SECF_RANK_1 0x00000200512 Average magnitude uses only rank 1 detections.
SECF_RANK_2 0x000004001024 Average magnitude uses only rank 2 detections.
SECF_RANK_3 0x000008002048 Average magnitude uses only rank 3 detections.
SECF_RANK_4 0x000010004096 Average magnitude uses only rank 4 detections.
SECF_STACK_PRIMARY0x0000400016384 PS1 stack photometry comes from primary skycell.
SECF_STACK_BESTDET0x0000800032768 PS1 stack best measurement is a detection (not forced).
SECF_STACK_PRIMDET0x0001000065536 PS1 stack primary measurement is a detection (not forced).
SECF_HAS_SDSS 0x001000001048576 This photcode has SDSS photometry (EAM??).
SECF_HAS_HSC 0x002000002097152 This photcode has HSC photometry (EAM??).
SECF_HAS_CFH 0x004000004194304 This photcode has CFH photometry (mostly Megacam) (EAM??).
SECF_HAS_DES 0x008000008388608 This photcode has DES photometry (EAM??).
SECF_OBJ_EXT 0x0100000016777216 Extended in this band.
DEFAULT 0x000000000 Initial value; resets all bits.
SECF_STAR_FEW 0x000000011 Used within relphot: skip star.
SECF_STAR_POOR 0x000000022 Used within relphot: skip star.
SECF_USE_SYNTH 0x000000044 Synthetic photometry used in average measurement.
SECF_USE_UBERCAL 0x000000088 Ubercal photometry used in average measurement.
SECF_HAS_PS1 0x0000001016 PS1 photometry used in average measurement.
SECF_HAS_PS1_STACK 0x0000002032 PS1 stack photometry exists.
SECF_HAS_TYCHO 0x0000004064 Tycho photometry used for synthetic magnitudes.
SECF_FIX_SYNTH 0x00000080128 Synthetic magnitudes repaired with zeropoint map.
SECF_RANK_0 0x00000100256 Average magnitude calculated in 0th pass.
SECF_RANK_1 0x00000200512 Average magnitude calculated in 1st pass.
SECF_RANK_2 0x000004001024 Average magnitude calculated in 2nd pass.
SECF_RANK_3 0x000008002048 Average magnitude calculated in 3rd pass.
SECF_RANK_4 0x000010004096 Average magnitude calculated in 4th pass.
SECF_STACK_PRIMARY 0x0000400016384 PS1 stack photometry includes a primary skycell.
SECF_STACK_BESTDET 0x0000800032768 PS1 stack best measurement is a detection (not forced).
SECF_STACK_PRIMDET 0x0001000065536 PS1 stack primary measurement is a detection (not forced).
SECF_STACK_PRIMARY_MULTIPLE0x0001000065536 PS1 stack object has multiple primary measurements.
SECF_HAS_SDSS 0x001000001048576 This photcode has SDSS photometry.
SECF_HAS_HSC 0x002000002097152 This photcode has HSC photometry.
SECF_HAS_CFH 0x004000004194304 This photcode has CFH photometry (mostly Megacam).
SECF_HAS_DES 0x008000008388608 This photcode has DES photometry.
SECF_OBJ_EXT 0x0100000016777216 Extended in this band.
DEFAULT 0x000000000 Initial value; resets all bits.
QF_OBJ_EXT 0x000000011 Extended in our data (eg, PS).
QF_OBJ_EXT_ALT 0x000000022 Extended in external data (eg, 2MASS).
QF_OBJ_GOOD 0x000000044 Good-quality measurement in our data (eg,PS).
QF_OBJ_GOOD_ALT 0x000000088 Good-quality measurement in external data (eg, 2MASS).
QF_OBJ_GOOD_STACK 0x0000001016 good-quality object in the stack (> 1 good stack measurement)
QF_OBJ_BEST_STACK 0x0000002032 the primary stack measurements are the best measurements.
QF_OBJ_SUSPECT_STACK0x0000004064 suspect object in the stack (no more than 1 good measurement, 2 or more suspect or good stack measurement).
QF_OBJ_BAD_STACK 0x00000080128 poor-quality stack object (no more than 1 good or suspect measurement).
QF_OBJ_EXT 0x000000011 Extended in our data (eg; PS).
QF_OBJ_EXT_ALT 0x000000022 Extended in external data (eg; 2MASS).
QF_OBJ_GOOD 0x000000044 Good-quality measurement in our data (eg; PS).
QF_OBJ_GOOD_ALT 0x000000088 Good-quality measurement in external data (eg; 2MASS).
QF_OBJ_GOOD_STACK 0x0000001016 Good-quality object in the stack (> 1 good stack measurement).
QF_OBJ_BEST_STACK 0x0000002032 The primary stack measurements are the best measurements.
QF_OBJ_SUSPECT_STACK0x0000004064 Suspect object in the stack (no more than 1 good measurement, 2 or more suspect or good stack measurement).
QF_OBJ_BAD_STACK 0x00000080128 Poor-quality stack object (no more than 1 good or suspect measurement).
NO_ERROR 0x000000000 No error condition raised.
FAIL_FIT 0x000000011 fit failed to converge or was degenerate
TOO_FEW 0x000000022 not enough points to fit the model
OUT_OF_RANGE 0x000000044 fit minimum too far outside data range
BAD_ERROR 0x000000088 invalid size error (nan or inf)
FAIL_FIT 0x000000011 Fit failed to converge or was degenerate
TOO_FEW 0x000000022 Not enough points to fit the model
OUT_OF_RANGE 0x000000044 Fit minimum too far outside data range
BAD_ERROR 0x000000088 Invalid size error (nan or inf)
- + Contains the PSF, Kron (1980), and aperture fluxes for all filters in a single row, along with point-source object shape parameters. See StackObjectThin table for discussion of primary, secondary, and best detections. References: Kron, R. G. 1980, ApJS, 43, 305. @@ -587,5 +605,8 @@ - Photometric zeropoint for the g filter stack. Necessary for converting listed fluxes and magnitudes back to measured ADU counts. + Photometric zeropoint for the g filter stack (PSF-like magnitudes only). Necessary for converting listed fluxes and magnitudes back to measured ADU counts. + + + Photometric zeropoint for the g filter stack (APERTURE-like magnitudes only). Necessary for converting listed fluxes and magnitudes back to measured ADU counts. @@ -694,5 +715,8 @@ - Photometric zeropoint for the r filter stack. Necessary for converting listed fluxes and magnitudes back to measured ADU counts. + Photometric zeropoint for the r filter stack (PSF-like magnitudes only). Necessary for converting listed fluxes and magnitudes back to measured ADU counts. + + + Photometric zeropoint for the r filter stack (APERTURE-like magnitudes only). Necessary for converting listed fluxes and magnitudes back to measured ADU counts. @@ -801,5 +825,8 @@ - Photometric zeropoint for the i filter stack. Necessary for converting listed fluxes and magnitudes back to measured ADU counts. + Photometric zeropoint for the i filter stack (PSF-like magnitudes only). Necessary for converting listed fluxes and magnitudes back to measured ADU counts. + + + Photometric zeropoint for the i filter stack (APERTURE-like magnitudes only). Necessary for converting listed fluxes and magnitudes back to measured ADU counts. @@ -908,5 +935,8 @@ - Photometric zeropoint for the z filter stack. Necessary for converting listed fluxes and magnitudes back to measured ADU counts. + Photometric zeropoint for the z filter stack (PSF-like magnitudes only). Necessary for converting listed fluxes and magnitudes back to measured ADU counts. + + + Photometric zeropoint for the z filter stack (APERTURE-like magnitudes only). Necessary for converting listed fluxes and magnitudes back to measured ADU counts. @@ -1015,5 +1045,8 @@ - Photometric zeropoint for the y filter stack. Necessary for converting listed fluxes and magnitudes back to measured ADU counts. + Photometric zeropoint for the y filter stack (PSF-like magnitudes only). Necessary for converting listed fluxes and magnitudes back to measured ADU counts. + + + Photometric zeropoint for the y filter stack (APERTURE-like magnitudes only). Necessary for converting listed fluxes and magnitudes back to measured ADU counts. @@ -4790,5 +4823,5 @@ - Photometric zeropoint. Necessary for converting listed fluxes and magnitudes back to measured ADU counts. + Photometric zeropoint of frame/exposure. Necessary for converting listed fluxes and magnitudes back to measured ADU counts. Index: /branches/czw_branch/20170908/ippToPsps/jython/batch.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/batch.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/batch.py (revision 40483) @@ -68,5 +68,5 @@ raise else: - logger.errorPair("missing fits item?", str(id)) + logger.errorPair("missing fits item for batch ID" + str(id), "continue") # define the dvo table names Index: /branches/czw_branch/20170908/ippToPsps/jython/detectionbatch.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/detectionbatch.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/detectionbatch.py (revision 40483) @@ -60,6 +60,10 @@ meta = self.gpc1Db.getCameraStageMeta(self.id) if not meta: - self.logger.errorPair("Could not get", "camera metadata") - raise + self.logger.infoPair("Could not get camera metadata", "trying with no warp id") + meta = self.gpc1Db.getCameraStageMetaNoWarpID(self.id) + if not meta: + self.logger.errorPair("Could not get camera metadata", "now what?") + raise + # set a flag indicating that we have the new columns added in version 5 of the @@ -80,5 +84,6 @@ self.chipID = meta[6]; self.warpID = meta[7]; - + + if not self.analysisVer: self.analysisVer = -999 @@ -815,24 +820,24 @@ sqlLine.group("a.ra", "b.ra") sqlLine.group("a.dec", "b.dec_") - sqlLine.group("a.zp", "b.zp") + sqlLine.group("a.zp", "b.zpPSF") sqlLine.group("a.telluricExt", "b.telluricExt") sqlLine.group("a.airmass", "b.airmass") sqlLine.group("a.infoFlag3", "b.flags") sqlLine.group("a.expTime", "b.expTime") - sqlLine.group("a.psfFlux", "a.psfFlux * b.zpFactor") - sqlLine.group("a.psfFluxErr", "a.psfFluxErr * b.zpFactor") - sqlLine.group("a.apFlux", "a.apFlux * b.zpFactor") - sqlLine.group("a.apFluxErr", "a.apFluxErr * b.zpFactor") - sqlLine.group("a.kronFlux", "a.kronFlux * b.zpFactor") - sqlLine.group("a.kronFluxErr", "a.kronFluxErr * b.zpFactor") - sqlLine.group("a.sky", "a.sky * b.zpFactor") - sqlLine.group("a.skyErr", "a.skyErr * b.zpFactor") + sqlLine.group("a.psfFlux", "a.psfFlux * b.zpFactorPSF") + sqlLine.group("a.psfFluxErr", "a.psfFluxErr * b.zpFactorPSF") + sqlLine.group("a.apFlux", "a.apFlux * b.zpFactorAPER") + sqlLine.group("a.apFluxErr", "a.apFluxErr * b.zpFactorAPER") + sqlLine.group("a.kronFlux", "a.kronFlux * b.zpFactorAPER") + sqlLine.group("a.kronFluxErr", "a.kronFluxErr * b.zpFactorAPER") + sqlLine.group("a.sky", "a.sky * b.zpFactorAPER") + sqlLine.group("a.skyErr", "a.skyErr * b.zpFactorAPER") sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " + str(imageID)) - ## a.psfFlux = a.psfFlux * 3630.78 * POW(10.0, -0.4*b.zp), \ + ## a.psfFlux = a.psfFlux * 3630.78 * POW(10.0, -0.4*b.zpPSF), \ # instrumental flux vs Janskies: - # b.ZP is defined to include the airmass term and zero point offset from relative calibration or ubercal + # b.zpPSF is defined to include the airmass term and zero point offset from relative calibration or ubercal # (see dvopsps/src/insert_detections_dvopsps_catalog.c:311 # mag_AB = -2.5*log(flux_DN/sec) + ZP Index: /branches/czw_branch/20170908/ippToPsps/jython/diffbatch.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/diffbatch.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/diffbatch.py (revision 40483) @@ -457,5 +457,5 @@ sqlLine.group("a.raErr", "b.raErr") # NOTE: This is supplied above as well, is this needed?? sqlLine.group("a.decErr", "b.decErr") # NOTE: This is supplied above as well, is this needed?? - sqlLine.group("a.zp", "b.zp") + sqlLine.group("a.zp", "b.zpPSF") sqlLine.group("a.telluricExt", "b.telluricExt") sqlLine.group("a.airmass", "b.airmass") @@ -633,17 +633,17 @@ a.ra = b.ra, \ a.dec = b.dec_, \ - a.zp = b.zp, \ + a.zp = b.zpPSF, \ a.telluricExt = b.telluricExt, \ a.airmass = b.airmass, \ a.expTime = b.expTime, \ a.DinfoFlag3 = b.flags \ - a.DpsfFlux = a.DpsfFlux * b.zpFactor, \ - a.DpsfFluxErr = a.DpsfFluxErr * b.zpFactor, \ - a.apFlux = a.apFlux * b.zpFactor, \ - a.apFluxErr = a.apFluxErr * b.zpFactor, \ - a.kronFlux = a.kronFlux * b.zpFactor, \ - a.kronFluxErr = a.kronFluxErr * b.zpFactor \ - a.sky = a.sky * b.zpFactor, \ - a.skyErr = a.skyErr * b.zpFactor \ + a.DpsfFlux = a.DpsfFlux * b.zpFactorPSF, \ + a.DpsfFluxErr = a.DpsfFluxErr * b.zpFactorPSF, \ + a.apFlux = a.apFlux * b.zpFactorAPER, \ + a.apFluxErr = a.apFluxErr * b.zpFactorAPER, \ + a.kronFlux = a.kronFlux * b.zpFactorAPER, \ + a.kronFluxErr = a.kronFluxErr * b.zpFactorAPER \ + a.sky = a.sky * b.zpFactorAPER, \ + a.skyErr = a.skyErr * b.zpFactorAPER \ WHERE a.ippDetectID = b.ippDetectID \ AND b.imageID = " + str(imageID) Index: /branches/czw_branch/20170908/ippToPsps/jython/diffobjectbatch.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/diffobjectbatch.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/diffobjectbatch.py (revision 40483) @@ -265,6 +265,6 @@ sqlLine.group("ra", "RA_MEAN") sqlLine.group("dec_", "DEC_MEAN") - sqlLine.group("diffObjName", "IAU_NAME") - sqlLine.group("diffObjPSOName", "PSO_NAME") + sqlLine.group("diffObjName", "PSO_NAME") + sqlLine.group("diffObjNameHMS", "PSX_NAME") sqlLine.group("nDetections", "'0'") sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName) Index: /branches/czw_branch/20170908/ippToPsps/jython/dvo.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/dvo.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/dvo.py (revision 40483) @@ -101,6 +101,8 @@ if not self.loadImages(): + self.logger.debugPair("FAILED to load images", "exiting") os._exit(1) + print "DONE load images (resetAllTables)" ''' @@ -145,7 +147,28 @@ self.logger.infoPair("DVO Images.dat file", "NOT up-to-date") + # check on the format: + try: + imagesHeaderPHU = self.readFitsHeader("PHU", path) + except Exception,e: + # print the error, wait a few secs then try again + print str(e) + self.logger.infoPair("Failed to find PHU for Images.dat", path) + raise + + if "FORMAT" in imagesHeaderPHU: + fileFormat = imagesHeaderPHU["FORMAT"] + else: + self.logger.infoPair("Missing FORMAT for Images.dat", path) + raise + + self.logger.infoPair("got FORMAT for Images.dat", fileFormat) + + if not (fileFormat == "PS1_V6"): + self.logger.infoPair("Invalid FORMAT for Images.dat (PS1_V6)", fileFormat) + return False + self.importFits( path, - "SOURCE_ID IMAGE_ID EXTERN_ID FLAGS MCAL SECZ PHOTCODE XPIX_SYS_ERR YPIX_SYS_ERR N_FIT_ASTROM MAG_SYS_ERR N_FIT_PHOTOM", + "SOURCE_ID IMAGE_ID EXTERN_ID FLAGS MCAL_PSF MCAL_APER SECZ PHOTCODE XPIX_SYS_ERR YPIX_SYS_ERR N_FIT_ASTROM MAG_SYS_ERR N_FIT_PHOTOM", self.scratchDb.dvoImagesTable) self.logger.infoPair("Adding primary key to", self.scratchDb.dvoImagesTable) @@ -559,6 +582,7 @@ self.logger.debugPair("Reading IPP table", table.name) table = stilts.tpipe(table, cmd='explodeall') - #adds an index to all the tables - # table = stilts.tpipe(table, cmd='addcol table_index $0') + + # adds an index to all the tables + # table = stilts.tpipe(table, cmd='addcol table_index $0') # IPP FITS files are littered with infinity values. Remove them @@ -566,4 +590,15 @@ table = stilts.tpipe(table, cmd='replaceval -Infinity null *') table = stilts.tpipe(table, cmd='replaceval Infinity null *') + + # verify that all columns in 'columns' exist in the tabl + ## myColumns = table.columns() + ## for wcol in columns: + ## foundCol = False + ## for tcol in myColumns: + ## if (wcol == tcol.name): + ## print "found wcol " + wcol + ## foundCol = True; + ## break + ## if not foundCol: attempts = 0 @@ -575,6 +610,7 @@ table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName) break - except: - # wait a few secs then try again + except Exception,e: + # print the error, wait a few secs then try again + print str(e) time.sleep(2) attempts = attempts + 1 @@ -663,4 +699,30 @@ ''' + Find and read a header extension. + ''' + def readFitsHeader(self, extname, filename): + + cmd = "fhead -n %s %s" % (extname, filename) + p = Popen(cmd, shell=True, stdout=PIPE) + output = p.communicate()[0] + + header = {} + + # split the output header bytes into key/value pairs for each 80-byte line + for i in range(0, len(output), 80): + # print "line %d : %s" % (i, output[i:i+80]) + record = output[i:i+80] + + match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-+_\.:\s@#\(\)\,]+)\'*\\/*', record) + + if match: + param = match.group(2) + value = match.group(3).strip() + if value == "NaN": value = "NULL" + header[param] = value + + return header + + ''' ingest skyregion into MySQL database using the native DVO program dvopsps Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this @@ -688,5 +750,9 @@ # make sure we have an up-to-date Images table - self.loadImages() # this is in nativeIngestDetections (remove?) + if not self.loadImages(): # this is in nativeIngestDetections (remove?) + self.logger.debugPair("FAILED to load images", "exiting") + os._exit(1) + + print "DONE load images (nativeIngestDetections)" # the box dimensions are the area used to select the items of Index: /branches/czw_branch/20170908/ippToPsps/jython/dvodetections.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/dvodetections.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/dvodetections.py (revision 40483) @@ -24,4 +24,5 @@ super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db) + print "----- DvoDetections is not used, is it?? -----" os._exit(1) Index: /branches/czw_branch/20170908/ippToPsps/jython/dvodiffdetections.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/dvodiffdetections.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/dvodiffdetections.py (revision 40483) @@ -14,4 +14,5 @@ ''' A class for ingesting DVO detections into MySQL +XXX EAM 20151029 : I do not believe this class is used at all. I am poisoning it with an exit ''' class DvoDetections(Dvo): @@ -25,4 +26,7 @@ super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db) # super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) + + print "----- DvoDetections is not used, is it?? -----" + os._exit(1) # declare DVO file types of interest Index: /branches/czw_branch/20170908/ippToPsps/jython/dvoforcedobjects.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/dvoforcedobjects.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/dvoforcedobjects.py (revision 40483) @@ -48,4 +48,7 @@ cmd += " -cpt " + region + if self.config.test: + cmd += " -test-mode" + if self.skychunk.parallel: if self.scratchDb.dbHost == "localhost": Index: /branches/czw_branch/20170908/ippToPsps/jython/forcedwarpbatch.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/forcedwarpbatch.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/forcedwarpbatch.py (revision 40483) @@ -424,6 +424,17 @@ # self.selectDvoObjIDs() self.logger.infoPair("inserting dvo info: find imageID", "from externID") - imageID = self.scratchDb.getImageIDFromExternIDandPhotcodeRange(self.warpSkyFileID[num],12000,12500) - self.logger.infoPair("I don't like this", "it is hardwired to gpc1") + + # get the imageID: this code has cases for simtest vs gpc1 cameras + # if self.config.test: + self.logger.infoPair("current camera is", self.config.camera) + if self.config.camera == "gpc1": + self.logger.infoPair("I don't like this", "it is hardwired to gpc1") + imageID = self.scratchDb.getImageIDFromExternIDandPhotcodeRange(self.warpSkyFileID[num],12000,12500) + if self.config.camera == "simtest": + imageID = self.scratchDb.getImageIDFromExternIDandPhotcodeRange(self.warpSkyFileID[num],15100,15500) + if imageID == 0: + self.logger.infoPair ("failed to get imageID for", str(num)) + raise + self.logger.infoPair("updating", pspsTableName) @@ -433,17 +444,17 @@ sqlLine.group("a.ippObjID", "b.ippObjID") sqlLine.group("a.dvoRegionID","b.catID") - sqlLine.group("a.zp", "b.zp") + sqlLine.group("a.zp", "b.zpPSF") sqlLine.group("a.telluricExt","b.telluricExt") sqlLine.group("a.airmass", "b.airmass") sqlLine.group("a.FinfoFlag3", "b.flags") - sqlLine.group("a.FpsfFlux", "a.FpsfFlux * b.zpFactor") - sqlLine.group("a.FpsfFluxErr", "a.FpsfFluxErr * b.zpFactor") - sqlLine.group("a.FapFlux", "a.FapFlux * b.zpFactor") - sqlLine.group("a.FapFluxErr", "a.FapFluxErr * b.zpFactor") - sqlLine.group("a.FkronFlux", "a.FkronFlux * b.zpFactor") - sqlLine.group("a.FkronFluxErr", "a.FkronFluxErr * b.zpFactor") - sqlLine.group("a.Fsky", "a.Fsky * b.zpFactor") - sqlLine.group("a.FskyErr", "a.FskyErr * b.zpFactor") + sqlLine.group("a.FpsfFlux", "a.FpsfFlux * b.zpFactorPSF") + sqlLine.group("a.FpsfFluxErr", "a.FpsfFluxErr * b.zpFactorPSF") + sqlLine.group("a.FapFlux", "a.FapFlux * b.zpFactorAPER") + sqlLine.group("a.FapFluxErr", "a.FapFluxErr * b.zpFactorAPER") + sqlLine.group("a.FkronFlux", "a.FkronFlux * b.zpFactorAPER") + sqlLine.group("a.FkronFluxErr", "a.FkronFluxErr * b.zpFactorAPER") + sqlLine.group("a.Fsky", "a.Fsky * b.zpFactorAPER") + sqlLine.group("a.FskyErr", "a.FskyErr * b.zpFactorAPER") sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " +str( imageID)) @@ -457,7 +468,15 @@ self.logger.infoPair("row count of ForcedWarpMeasurement after cull of null objID ", rowCountAfter) if rowCountAfter == 0: - self.skipBatch = True - self.logger.infoPair ("skip this batch", "it is bad") - return False + if self.haveLensPSF[num] ==0: + self.logger.infoPair("skipping this table", "no lenspsf and no dvo, run this:") + self.logger.infoPair("update addRun join addProcessedExp using (add_id) set fault = 43 where add_id = ",str(self.number[num])) + self.skipBatch = True + return False + else: + self.skipBatch = True + self.logger.infoPair ("skip this batch", "it is bad") + return False + + self.logger.infoPair("Adding 'row' columns to", pspsTableName) @@ -557,15 +576,15 @@ sqlLine = sqlUtility("UPDATE " + pspsTableName + " as a, " + self.scratchDb.dvoDetectionTable + " as b SET ") - sqlLine.group("a.flxR5", "a.flxR5 * b.zpFactor") - sqlLine.group("a.flxR5Err", "a.flxR5Err * b.zpFactor") - sqlLine.group("a.flxR5Std", "a.flxR5Std * b.zpFactor") - - sqlLine.group("a.flxR6", "a.flxR6 * b.zpFactor") - sqlLine.group("a.flxR6Err", "a.flxR6Err * b.zpFactor") - sqlLine.group("a.flxR6Std", "a.flxR6Std * b.zpFactor") - - sqlLine.group("a.flxR7", "a.flxR7 * b.zpFactor") - sqlLine.group("a.flxR7Err", "a.flxR7Err * b.zpFactor") - sqlLine.group("a.flxR7Std", "a.flxR7Std * b.zpFactor") + sqlLine.group("a.flxR5", "a.flxR5 * b.zpFactorAPER") + sqlLine.group("a.flxR5Err", "a.flxR5Err * b.zpFactorAPER") + sqlLine.group("a.flxR5Std", "a.flxR5Std * b.zpFactorAPER") + + sqlLine.group("a.flxR6", "a.flxR6 * b.zpFactorAPER") + sqlLine.group("a.flxR6Err", "a.flxR6Err * b.zpFactorAPER") + sqlLine.group("a.flxR6Std", "a.flxR6Std * b.zpFactorAPER") + + sqlLine.group("a.flxR7", "a.flxR7 * b.zpFactorAPER") + sqlLine.group("a.flxR7Err", "a.flxR7Err * b.zpFactorAPER") + sqlLine.group("a.flxR7Std", "a.flxR7Std * b.zpFactorAPER") sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " +str( imageID)) @@ -657,4 +676,5 @@ # each of the "populate*" methods below add their tables to the list: self.tablesToExport=[] + tables = [] # list of tables to calculate min/max objid on... for num in self.number: @@ -671,11 +691,12 @@ self.logger.infoPair("Populating", myTable) if not self.populateForcedWarpMeasurement(num): - self.logger.infoPair ("skippping ForcedWarpMeasurement", str(num)) + self.logger.infoPair ("failed on ForcedWarpMeasurement", str(num)) #continue raise self.tablesToExport.append(myTable) - - self.logger.infoPair("setting min/max objid from", myTable) - self.setMinMaxObjID([myTable]) + tables.append(myTable) + #self.logger.infoPair("setting min/max objid from", myTable) + + #self.setMinMaxObjID([myTable]) # populateForcedWarpMeasurement also populates ForcedWarpMasked (from ForcedWarpMeasurement) @@ -697,7 +718,9 @@ self.populateForcedWarpToImage(num) self.tablesToExport.append(myTable) + self.logger.infoPair("setting min/max objid from", "all the fw tables") + self.setMinMaxObjID(tables) + self.logger.infoPair("finishing","populatePspsTables") + - self.logger.infoPair("finishing","populatePspsTables") - return True @@ -762,4 +785,5 @@ self.haveLensPSF[num] = 0 self.haveLensing[num] = 0 + for table in tables: # check for AP_NPIX columns: missing for SAS39 Index: /branches/czw_branch/20170908/ippToPsps/jython/gpc1db.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/gpc1db.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/gpc1db.py (revision 40483) @@ -511,6 +511,8 @@ files = glob.glob(pathBase + ".cmf") + if len(files) < 1: + self.logger.errorPair("No files found for forced warp path", pathBase) + return None - if len(files) < 1: return None fits = Fits(self.logger, self.config, files[0]) self.logger.infoPair("cmf file",files[0]) @@ -760,4 +762,6 @@ meta = [] + + sql = "SELECT exp_id, exp_name, camRun.dist_group, camRun.software_ver, rawExp.bg, rawExp.bg_stdev, chip_id, warp_id \ FROM camRun \ @@ -773,6 +777,5 @@ rs.first() except: - self.logger.errorPair("Can't query for", "camera meta data") - + self.logger.errorPair("Can't query for", "camera meta data") try: meta.append(rs.getInt(1)) @@ -786,6 +789,42 @@ except: self.logger.errorPair("getCameraStageMeta()", "empty meta data") - return meta + + def getCameraStageMetaNoWarpID(self, camID): + + self.logger.debug("Querying GPC1 for camera meta data") + + meta = [] + + + sql = "SELECT exp_id, exp_name, camRun.dist_group, camRun.software_ver, rawExp.bg, rawExp.bg_stdev, chip_id, -999 \ + FROM camRun \ + JOIN chipRun USING(chip_id) \ + JOIN rawExp USING(exp_id) \ + LEFT JOIN fakeRun USING(cam_id) \ + LEFT JOIN warpRun USING(fake_id) \ + WHERE camRun.cam_id = %d" % camID + + try: + rs = self.executeQuery(sql) + rs.first() + except: + self.logger.errorPair("Can't query for", "camera meta data") + try: + meta.append(rs.getInt(1)) + meta.append(rs.getString(2)) + meta.append(rs.getString(3)) + meta.append(rs.getString(4)) + meta.append(rs.getString(5)) + meta.append(rs.getString(6)) + meta.append(rs.getInt(7)) + meta.append(rs.getInt(8)) + except: + self.logger.errorPair("getCameraStageMeta()", "empty meta data") + + return meta + + + ''' @@ -818,5 +857,4 @@ files = [] - #there are a couple of states for smfversion @@ -829,5 +867,4 @@ # use (file).smf if camRun.state = 'full' and if there is only a .smf # fault out if camRun.state != 'full' -- that is a race condition we don't want. - if (smfversion == "not_reproc") or (smfversion == "use_new"): @@ -904,5 +941,7 @@ self.logger.infoPair("smf files:", files) - if len(files) < 1: return None + if len(files) < 1: + self.logger.errorPair("No files found for cam path", path) + return None return Fits(self.logger, self.config, files[0]) # TODO just returning first file - check @@ -974,5 +1013,7 @@ # print "stack cmf files:", files - if len(files) < 1: return None + if len(files) < 1: + self.logger.errorPair("No files found for stack path", pathBase) + return None # if we get here, then the cmf is readable, now check the stack_id Index: /branches/czw_branch/20170908/ippToPsps/jython/ippjytest =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/ippjytest (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/ippjytest (revision 40483) @@ -14,5 +14,5 @@ set JARDIR = $datadir/jar set JYDIR = $datadir/jython -# set JYDIR = . +set JYDIR = . setenv IPPTOPSPS_DATA $datadir/ipptopsps Index: /branches/czw_branch/20170908/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/ipptopspsdb.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/ipptopspsdb.py (revision 40483) @@ -27,5 +27,5 @@ self.skychunk = skychunk - self.MAX_FAILS = 5 + self.MAX_FAILS = 15 ''' Index: /branches/czw_branch/20170908/ippToPsps/jython/objectbatch.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/objectbatch.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/objectbatch.py (revision 40483) @@ -312,6 +312,6 @@ sqlLine.group("raMean", "RA_MEAN") sqlLine.group("decMean", "DEC_MEAN") - sqlLine.group("objName", "IAU_NAME") - sqlLine.group("objPSOName", "PSO_NAME") + sqlLine.group("objName", "PSO_NAME") + sqlLine.group("objNameHMS", "PSX_NAME") sqlLine.group("epochMean", "EPOCH_MEAN") sqlLine.group("raMeanErr", "RA_ERR") Index: /branches/czw_branch/20170908/ippToPsps/jython/queue.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/queue.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/queue.py (revision 40483) @@ -134,5 +134,5 @@ #if (dec > 85): raise # this may be a problem for poles # HAF XXX - if (dec > 90): raise + ####if (dec > 90): raise # print "dec, dD: ", dec, dD ,self.skychunk.maxDec @@ -178,5 +178,5 @@ dec = dec + 2*dD #if (dec > 85): raise - if (dec > 90): raise + ###if (dec > 90): raise #dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec)) dR = 0.5*self.skychunk.boxSize Index: /branches/czw_branch/20170908/ippToPsps/jython/scratchdb.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/scratchdb.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/scratchdb.py (revision 40483) @@ -271,5 +271,7 @@ zeroPoint = -999 - sql = "SELECT MCAL, SECZ, C_LAM, K FROM %s AS a JOIN %s as b WHERE EXTERN_ID = %s AND a.PHOTCODE = b.CODE" % (self.dvoImagesTable, self.dvoPhotcodesTable, (externID)) + # MCAL_PSF, MCAL_APER, SECZ are from dvoImagesTabl + # C_LAM, K are from dvoPhotcodeTable + sql = "SELECT MCAL_PSF, SECZ, C_LAM, K FROM %s AS a JOIN %s as b WHERE EXTERN_ID = %s AND a.PHOTCODE = b.CODE" % (self.dvoImagesTable, self.dvoPhotcodesTable, (externID)) rs = self.executeQuery(sql) if not rs: @@ -288,4 +290,34 @@ zeroPoint = 0.001*Clam + Klam*(airmass - 1.0) - Mcal return zeroPoint + + ''' + Gets zero point for image (PSF and APER) + ''' + def getStackZeroPoint(self, externID): + + zeroPointPSF = -999 + zeroPointAPER = -999 + + # MCAL_PSF, MCAL_APER, SECZ are from dvoImagesTabl + # C_LAM, K are from dvoPhotcodeTable + sql = "SELECT MCAL_PSF, MCAL_APER, SECZ, C_LAM, K FROM %s AS a JOIN %s as b WHERE EXTERN_ID = %s AND a.PHOTCODE = b.CODE" % (self.dvoImagesTable, self.dvoPhotcodesTable, (externID)) + rs = self.executeQuery(sql) + if not rs: + print "missing result set for imageID query" + os._exit(2) + + if not rs.first(): + self.logger.infoPair("no zero point; image is missing, externID: ", externID) + return zeroPoint + + McalPSF = rs.getFloat(1) + McalAPER = rs.getFloat(2) + airmass = rs.getFloat(3) + Clam = rs.getFloat(4) + Klam = rs.getFloat(5) + + zeroPointPSF = 0.001*Clam + Klam*(airmass - 1.0) - McalPSF + zeroPointAPER = 0.001*Clam + Klam*(airmass - 1.0) - McalAPER + return (zeroPointPSF, zeroPointAPER) ''' @@ -576,5 +608,6 @@ EXTERN_ID INT, \ FLAGS INT, \ - MCAL FLOAT, \ + MCAL_PSF FLOAT, \ + MCAL_APER FLOAT, \ SECZ FLOAT, \ PHOTCODE SMALLINT, \ @@ -607,6 +640,8 @@ raErr FLOAT, \ decErr FLOAT, \ - zp FLOAT, \ - zpFactor FLOAT, \ + zpPSF FLOAT, \ + zpFactorPSF FLOAT, \ + zpAPER FLOAT, \ + zpFactorAPER FLOAT, \ telluricExt FLOAT, \ airmass FLOAT, \ @@ -620,4 +655,5 @@ flags INT, \ objflags INT, \ + filtflags INT, \ PRIMARY KEY (imageID, ippDetectID), \ KEY (objID, detectID), \ Index: /branches/czw_branch/20170908/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/stackbatch.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/stackbatch.py (revision 40483) @@ -307,12 +307,13 @@ # zp correction should comes from DVO (unless image is not in DVO) - zpImage = self.scratchDb.getImageZeroPoint(stackID) - - if (zpImage == "NULL"): - zpImage = zp + (zpImagePSF, zpImageAPER) = self.scratchDb.getStackZeroPoint(stackID) + + # zp is the value in the header, zpImage includes the McalPSF value from DVO + if (zpImagePSF == "NULL"): zpImagePSF = zp + if (zpImageAPER == "NULL"): zpImageAPER = zpImagePSF detectionThreshold = "NULL" - if (magref != "NULL") and (zpImage != "NULL") and (expTime != "NULL"): - detectionThreshold = magref + zpImage + 2.5 * math.log10(expTime) + if (magref != "NULL") and (zpImagePSF != "NULL") and (expTime != "NULL"): + detectionThreshold = magref + zpImagePSF + 2.5 * math.log10(expTime) # insert stack metadata into table @@ -340,5 +341,6 @@ # sqlLine.group("psfFwhm_max", fwhm_maj_uq) sqlLine.group("astroScat", astroscat) - sqlLine.group("photoZero", zpImage) + sqlLine.group("photoZero", zpImagePSF) + sqlLine.group("photoZeroAperture", zpImageAPER) sqlLine.group("photoScat", zpErr) sqlLine.group("nAstroRef", header['NASTRO']) @@ -429,5 +431,6 @@ sqlLine.group("a."+filter+"Epoch", str(stackEpoch)) - sqlLine.group("a."+filter+"PSFMag", "b.Mpsf") + # the Mag values below are set in dvopsps/insert_detections_dvopsps_catalog.c using either zpPSF (PSFMag) or zpAPER (KronMag, ApMag) + sqlLine.group("a."+filter+"PSFMag", "b.Mpsf") sqlLine.group("a."+filter+"PSFMagErr", "b.dMpsf") sqlLine.group("a."+filter+"KronMag", "b.Mkron") @@ -440,4 +443,5 @@ sqlLine.group("a."+filter+"infoFlag3", "b.flags") + sqlLine.group("a."+filter+"infoFlag4", "b.filtflags") sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID AND b.imageID = " + str(imageID)) @@ -557,14 +561,15 @@ sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + self.scratchDb.dvoDetectionTable + " as b SET") sqlLine.group("a."+filter+"expTime", "b.expTime") - sqlLine.group("a."+filter+"zp", "b.zp") - sqlLine.group("a."+filter+"PSFFlux", "a."+filter+"PSFFlux * b.zpFactor") - sqlLine.group("a."+filter+"PSFFluxErr", "a."+filter+"PSFFluxErr * b.zpFactor") - sqlLine.group("a."+filter+"ApFlux", "a."+filter+"ApFlux * b.zpFactor") - sqlLine.group("a."+filter+"ApFluxErr", "a."+filter+"ApFluxErr * b.zpFactor") - sqlLine.group("a."+filter+"KronFlux", "a."+filter+"KronFlux * b.zpFactor") - sqlLine.group("a."+filter+"KronFluxErr", "a."+filter+"KronFluxErr * b.zpFactor") - - sqlLine.group("a."+filter+"sky", "a."+filter+"sky * b.zpFactor") - sqlLine.group("a."+filter+"skyErr", "a."+filter+"skyErr * b.zpFactor") + sqlLine.group("a."+filter+"zp", "b.zpPSF") + sqlLine.group("a."+filter+"zpAPER", "b.zpAPER") + sqlLine.group("a."+filter+"PSFFlux", "a."+filter+"PSFFlux * b.zpFactorPSF") + sqlLine.group("a."+filter+"PSFFluxErr", "a."+filter+"PSFFluxErr * b.zpFactorPSF") + sqlLine.group("a."+filter+"ApFlux", "a."+filter+"ApFlux * b.zpFactorAPER") + sqlLine.group("a."+filter+"ApFluxErr", "a."+filter+"ApFluxErr * b.zpFactorAPER") + sqlLine.group("a."+filter+"KronFlux", "a."+filter+"KronFlux * b.zpFactorAPER") + sqlLine.group("a."+filter+"KronFluxErr", "a."+filter+"KronFluxErr * b.zpFactorAPER") + + sqlLine.group("a."+filter+"sky", "a."+filter+"sky * b.zpFactorAPER") + sqlLine.group("a."+filter+"skyErr", "a."+filter+"skyErr * b.zpFactorAPER") # where should these go? @@ -700,4 +705,7 @@ imageID = self.imageIDs[filter] self.logger.infoPair("selecting imageID", imageID) + + # NOTE: for model magnitudes, we use zpPSF not zpAPER since they are PSF-matched + # (maybe that is the wrong solution?) # model calibrated magnitude = instrumental magnitude + 2.5*log10(exptime) + ZP (added below from dvoDetectionTable) @@ -765,5 +773,5 @@ sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + self.scratchDb.dvoDetectionTable + " as b SET") - sqlLine.group("a." + filter + model + "Mag", "a." + filter + model + "Mag + b.zp") + sqlLine.group("a." + filter + model + "Mag", "a." + filter + model + "Mag + b.zpPSF") sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID AND b.imageID = " + str(imageID)) @@ -884,5 +892,4 @@ # PETRO_MAG is instrumental + 2.5log(exptime) + FPA.ZP # we want to apply the DVO zero point, so remove FPA.ZP: - ## XXX is this statement true: is FPA.ZP + 2.5log(exptime) applied to PETRO_MAG? hdrZP = str(self.zpImage[filter]) @@ -897,5 +904,5 @@ sqlLine.group("a."+filter+"haveData", "'1'") - sqlLine.group("a." + filter + "petMag", "b.PETRO_MAG - " + hdrZP) + sqlLine.group("a." + filter + "petMag", "b.PETRO_MAG - " + hdrZP) # note that this is **subtracting** the hdrZP (zpAPER is added in below) sqlLine.group("a." + filter + "petMagErr", "1.0 / b.PETRO_MAG_ERR") # still inverted? (yes) @@ -913,5 +920,5 @@ # modify petMag to apply the zero point sqlLine = sqlUtility("UPDATE " + tablename + " AS a, " + self.scratchDb.dvoDetectionTable + " as b SET") - sqlLine.group("a." + filter + "petMag", "a." + filter + "petMag + b.zp") + sqlLine.group("a." + filter + "petMag", "a." + filter + "petMag + b.zpAPER") sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID AND b.imageID = " + str(imageID)) @@ -1019,20 +1026,22 @@ # care of this, but does not? - sqlLine = sqlUtility("UPDATE " + tablename + " SET") - for radius in [1, 2, 3, 4, 5, 6, 7, 8, 9]: - if radius < minRadius: continue - if radius > maxRadius: continue - pspsRadius = radius + 2 - sqlLine.group(filter + prefix + "flxR" + str(pspsRadius), "-999") - sqlLine.group(filter + prefix + "flxR" + str(pspsRadius) + "Err", "-999") - sqlLine.group(filter + prefix + "flxR" + str(pspsRadius) + "Std", "-999") - sqlLine.group(filter + prefix + "flxR" + str(pspsRadius) + "Fill", "-999") - - sql = sqlLine.makeEquals("") - - try: self.scratchDb.execute(sql) - except: - self.logger.errorPair('failed sql',sql) - raise + # chopping this out to see if it still works + + #sqlLine = sqlUtility("UPDATE " + tablename + " SET") + #for radius in [1, 2, 3, 4, 5, 6, 7, 8, 9]: + # if radius < minRadius: continue + # if radius > maxRadius: continue + # pspsRadius = radius + 2 + # sqlLine.group(filter + prefix + "flxR" + str(pspsRadius), "-999") + # sqlLine.group(filter + prefix + "flxR" + str(pspsRadius) + "Err", "-999") + # sqlLine.group(filter + prefix + "flxR" + str(pspsRadius) + "Std", "-999") + # sqlLine.group(filter + prefix + "flxR" + str(pspsRadius) + "Fill", "-999")# + + #sql = sqlLine.makeEquals("") + + # try: self.scratchDb.execute(sql) + # except: + # self.logger.errorPair('failed sql',sql) + # raise # set the flux values from the cmf file @@ -1110,7 +1119,7 @@ pspsNum = str(int(number) + 2) field = filter + prefix + "flxR" + pspsNum - sqlLine.group(field, field + " * b.zpFactor") - sqlLine.group(field + "Err", field + "err" + " * b.zpFactor") - sqlLine.group(field + "Std", field + "Std" + " * b.zpFactor") + sqlLine.group(field, field + " * b.zpFactorAPER") + sqlLine.group(field + "Err", field + "err" + " * b.zpFactorAPER") + sqlLine.group(field + "Std", field + "Std" + " * b.zpFactorAPER") sql = sqlLine.makeEquals("WHERE a.objID = b.objID AND a." + filter + "ippDetectID = b.ippDetectID AND b.imageID = " + str(imageID)) @@ -1120,4 +1129,5 @@ self.logger.infoPair("failed sql",sql) raise + ''' Populates the StackToImage table Index: /branches/czw_branch/20170908/ippToPsps/jython/testCode.py =================================================================== --- /branches/czw_branch/20170908/ippToPsps/jython/testCode.py (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/jython/testCode.py (revision 40483) @@ -46,4 +46,10 @@ def importIppTables(self): + if len(sys.argv) != 2: + print "USAGE: ippjython testCode.py (filename)" + os._exit(2) + + self.filename = sys.argv[1] + try: tables = stilts.treads(self.filename) @@ -54,11 +60,33 @@ for table in tables: - print "import smf table " + table.name + print "import table " + table.name # need to generate an index on the IPP_IDET column # table = stilts.tpipe(table, cmd='addcol table_index $0') - table = stilts.tpipe(table, cmd='explodeall') + try: + table = stilts.tpipe(table, cmd='explodeall') + except Exception,e: + # print the error, wait a few secs then try again + print str(e) + return False - print "read smf table " + table.name + try: + myColumns = table.columns() + except Exception,e: + # print the error, wait a few secs then try again + print str(e) + return False + + print "got myColumns" + + try: + for column in myColumns: + print "got column " + column.name + except Exception,e: + # print the error, wait a few secs then try again + print str(e) + return False + + print "read table " + table.name # drop any previous tables before import @@ -69,13 +97,13 @@ # IPP FITS files are littered with infinities, so remove these - print "Removing Infinity values from all columns" - table = stilts.tpipe(table, cmd='replaceval -Infinity null *') - table = stilts.tpipe(table, cmd='replaceval Infinity null *') - - try: - table.write(self.url + '#' + table.name) - except: - print "Problem writing table '" + table.name + "' to the database" - os._exit(4) + ## print "Removing Infinity values from all columns" + ## table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + ## table = stilts.tpipe(table, cmd='replaceval Infinity null *') + ## + ## try: + ## table.write(self.url + '#' + table.name) + ## except: + ## print "Problem writing table '" + table.name + "' to the database" + ## os._exit(4) return True @@ -103,4 +131,5 @@ print "alt word1" + self.importIppTables() # self.connectMysql(argv) Index: /branches/czw_branch/20170908/ippToPsps/test/fulltest.sh =================================================================== --- /branches/czw_branch/20170908/ippToPsps/test/fulltest.sh (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/test/fulltest.sh (revision 40483) @@ -16,4 +16,27 @@ end setenv IPPTOPSPS_DATA $datadir/ipptopsps + +set DRversion = "DR2" +set myScript = "none" +if ($DRversion == "DR1") then + set myScript = mkgpc1data.dvo + set CATDIR_CAM = catdir.cam + set CATDIR_STK = catdir.stk + set CATDIR_WRP = catdir.wrp + set CATDIR_DIF = catdir.dif +endif +if ($DRversion == "DR2") then + set myScript = mkgpc1data.dr2.dvo + set CATDIR_CAM = catdir.mrg + set CATDIR_STK = catdir.mrg + set CATDIR_WRP = catdir.mrg + set CATDIR_DIF = catdir.dif +endif +if ($DRversion == "none") then + echo "invalid DRversion $DRversion" + exit 2 +endif + +set coords = `$myScript -dump-region` # before running this test, you must have a mysql user: @@ -217,6 +240,6 @@ if ($mkgpc1) then echo "" - echo " ---- mkgpc1data.dvo : creating the dvo & gpc1 database entries ----" - mkgpc1data.dvo + echo " ---- $myScript : creating the dvo & gpc1 database entries ----" + $myScript if ($status) then echo "failed to generate the data & dvo, quitting" @@ -283,4 +306,6 @@ echo "0 " >> initbatch.dat # queue_FW echo "0 " >> initbatch.dat # queue_FO + echo "0 " >> initbatch.dat # queue_FG + echo "0 " >> initbatch.dat # queue_GO echo "1 " >> initbatch.dat # active echo "0 " >> initbatch.dat # parallel @@ -313,9 +338,9 @@ echo "0 " >> cambatch.dat # datastore_publish echo "catdir.cam " >> cambatch.dat # dvo_label - echo "$OUTDIR/catdir.cam " >> cambatch.dat # dvo_location - echo "9 " >> cambatch.dat # min_ra - echo "11 " >> cambatch.dat # max_ra - echo "19 " >> cambatch.dat # min_dec - echo "21 " >> cambatch.dat # max_dec + echo "$OUTDIR/$CATDIR_CAM " >> cambatch.dat # dvo_location + echo "$coords[1] " >> cambatch.dat # min_ra + echo "$coords[2] " >> cambatch.dat # max_ra + echo "$coords[3] " >> cambatch.dat # min_dec + echo "$coords[4] " >> cambatch.dat # max_dec echo "2 " >> cambatch.dat # box_size echo "$OUTDIR " >> cambatch.dat # base_path @@ -334,4 +359,6 @@ echo "0 " >> cambatch.dat # queue_FW echo "0 " >> cambatch.dat # queue_FO + echo "0 " >> cambatch.dat # queue_FG + echo "0 " >> cambatch.dat # queue_GO echo "1 " >> cambatch.dat # active echo "0 " >> cambatch.dat # parallel @@ -366,9 +393,9 @@ echo "0 " >> objectbatch.dat # datastore_publish echo "catdir.cam " >> objectbatch.dat # dvo_label - echo "$OUTDIR/catdir.cam " >> objectbatch.dat # dvo_location - echo "9 " >> objectbatch.dat # min_ra - echo "11 " >> objectbatch.dat # max_ra - echo "19 " >> objectbatch.dat # min_dec - echo "21 " >> objectbatch.dat # max_dec + echo "$OUTDIR/$CATDIR_CAM " >> objectbatch.dat # dvo_location + echo "$coords[1] " >> objectbatch.dat # min_ra + echo "$coords[2] " >> objectbatch.dat # max_ra + echo "$coords[3] " >> objectbatch.dat # min_dec + echo "$coords[4] " >> objectbatch.dat # max_dec echo "2 " >> objectbatch.dat # box_size echo "$OUTDIR " >> objectbatch.dat # base_path @@ -387,4 +414,6 @@ echo "0 " >> objectbatch.dat # queue_FW echo "0 " >> objectbatch.dat # queue_FO + echo "0 " >> objectbatch.dat # queue_FG + echo "0 " >> objectbatch.dat # queue_GO echo "1 " >> objectbatch.dat # active echo "0 " >> objectbatch.dat # parallel @@ -419,9 +448,9 @@ echo "0 " >> stackbatch.dat # datastore_publish echo "catdir.stk " >> stackbatch.dat # dvo_label - echo "$OUTDIR/catdir.stk " >> stackbatch.dat # dvo_location - echo "9 " >> stackbatch.dat # min_ra - echo "11 " >> stackbatch.dat # max_ra - echo "19 " >> stackbatch.dat # min_dec - echo "21 " >> stackbatch.dat # max_dec + echo "$OUTDIR/$CATDIR_STK " >> stackbatch.dat # dvo_location + echo "$coords[1] " >> stackbatch.dat # min_ra + echo "$coords[2] " >> stackbatch.dat # max_ra + echo "$coords[3] " >> stackbatch.dat # min_dec + echo "$coords[4] " >> stackbatch.dat # max_dec echo "2 " >> stackbatch.dat # box_size echo "$OUTDIR " >> stackbatch.dat # base_path @@ -440,4 +469,6 @@ echo "0 " >> stackbatch.dat # queue_FW echo "0 " >> stackbatch.dat # queue_FO + echo "0 " >> stackbatch.dat # queue_FG + echo "0 " >> stackbatch.dat # queue_GO echo "1 " >> stackbatch.dat # active echo "0 " >> stackbatch.dat # parallel @@ -472,9 +503,9 @@ echo "0 " >> forcebatch.dat # datastore_publish echo "catdir.wrp " >> forcebatch.dat # dvo_label - echo "$OUTDIR/catdir.wrp " >> forcebatch.dat # dvo_location - echo "9 " >> forcebatch.dat # min_ra - echo "11 " >> forcebatch.dat # max_ra - echo "19 " >> forcebatch.dat # min_dec - echo "21 " >> forcebatch.dat # max_dec + echo "$OUTDIR/$CATDIR_WRP " >> forcebatch.dat # dvo_location + echo "$coords[1] " >> forcebatch.dat # min_ra + echo "$coords[2] " >> forcebatch.dat # max_ra + echo "$coords[3] " >> forcebatch.dat # min_dec + echo "$coords[4] " >> forcebatch.dat # max_dec echo "2 " >> forcebatch.dat # box_size echo "$OUTDIR " >> forcebatch.dat # base_path @@ -493,4 +524,6 @@ echo "1 " >> forcebatch.dat # queue_FW echo "0 " >> forcebatch.dat # queue_FO + echo "0 " >> forcebatch.dat # queue_FG + echo "0 " >> forcebatch.dat # queue_GO echo "1 " >> forcebatch.dat # active echo "0 " >> forcebatch.dat # parallel @@ -525,9 +558,9 @@ echo "0 " >> forcedobjectbatch.dat # datastore_publish echo "catdir.wrp " >> forcedobjectbatch.dat # dvo_label - echo "$OUTDIR/catdir.wrp " >> forcedobjectbatch.dat # dvo_location - echo "9 " >> forcedobjectbatch.dat # min_ra - echo "11 " >> forcedobjectbatch.dat # max_ra - echo "19 " >> forcedobjectbatch.dat # min_dec - echo "21 " >> forcedobjectbatch.dat # max_dec + echo "$OUTDIR/$CATDIR_WRP " >> forcedobjectbatch.dat # dvo_location + echo "$coords[1] " >> forcedobjectbatch.dat # min_ra + echo "$coords[2] " >> forcedobjectbatch.dat # max_ra + echo "$coords[3] " >> forcedobjectbatch.dat # min_dec + echo "$coords[4] " >> forcedobjectbatch.dat # max_dec echo "2 " >> forcedobjectbatch.dat # box_size echo "$OUTDIR " >> forcedobjectbatch.dat # base_path @@ -546,4 +579,6 @@ echo "0 " >> forcedobjectbatch.dat # queue_FW echo "1 " >> forcedobjectbatch.dat # queue_FO + echo "0 " >> forcedobjectbatch.dat # queue_FG + echo "0 " >> forcedobjectbatch.dat # queue_GO echo "1 " >> forcedobjectbatch.dat # active echo "0 " >> forcedobjectbatch.dat # parallel @@ -579,9 +614,9 @@ echo "0 " >> diffbatch.dat # datastore_publish echo "catdir.dif " >> diffbatch.dat # dvo_label - echo "$OUTDIR/catdir.dif " >> diffbatch.dat # dvo_location - echo "9 " >> diffbatch.dat # min_ra - echo "11 " >> diffbatch.dat # max_ra - echo "19 " >> diffbatch.dat # min_dec - echo "21 " >> diffbatch.dat # max_dec + echo "$OUTDIR/$CATDIR_DIF " >> diffbatch.dat # dvo_location + echo "$coords[1] " >> diffbatch.dat # min_ra + echo "$coords[2] " >> diffbatch.dat # max_ra + echo "$coords[3] " >> diffbatch.dat # min_dec + echo "$coords[4] " >> diffbatch.dat # max_dec echo "2 " >> diffbatch.dat # box_size echo "$OUTDIR " >> diffbatch.dat # base_path @@ -600,4 +635,6 @@ echo "0 " >> diffbatch.dat # queue_FW echo "0 " >> diffbatch.dat # queue_FO + echo "0 " >> diffbatch.dat # queue_FG + echo "0 " >> diffbatch.dat # queue_GO echo "1 " >> diffbatch.dat # active echo "0 " >> diffbatch.dat # parallel @@ -631,9 +668,9 @@ echo "0 " >> diffobjbatch.dat # datastore_publish echo "catdir.dif " >> diffobjbatch.dat # dvo_label - echo "$OUTDIR/catdir.dif " >> diffobjbatch.dat # dvo_location - echo "9 " >> diffobjbatch.dat # min_ra - echo "11 " >> diffobjbatch.dat # max_ra - echo "19 " >> diffobjbatch.dat # min_dec - echo "21 " >> diffobjbatch.dat # max_dec + echo "$OUTDIR/$CATDIR_DIF " >> diffobjbatch.dat # dvo_location + echo "$coords[1] " >> diffobjbatch.dat # min_ra + echo "$coords[2] " >> diffobjbatch.dat # max_ra + echo "$coords[3] " >> diffobjbatch.dat # min_dec + echo "$coords[4] " >> diffobjbatch.dat # max_dec echo "2 " >> diffobjbatch.dat # box_size echo "$OUTDIR " >> diffobjbatch.dat # base_path @@ -652,4 +689,6 @@ echo "0 " >> diffobjbatch.dat # queue_FW echo "0 " >> diffobjbatch.dat # queue_FO + echo "0 " >> diffobjbatch.dat # queue_FG + echo "0 " >> diffobjbatch.dat # queue_GO echo "1 " >> diffobjbatch.dat # active echo "0 " >> diffobjbatch.dat # parallel Index: /branches/czw_branch/20170908/ippToPsps/test/gdb.sh =================================================================== --- /branches/czw_branch/20170908/ippToPsps/test/gdb.sh (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/test/gdb.sh (revision 40483) @@ -0,0 +1,21 @@ + +define t1 + set $i = 0 + while ($i < 50) + printf "%d %d %f\n", $i, secfilt[$i].stackBestOff, secfilt[$i].MpsfStk + set $i = $i + 1 + end +end + +define t2 + set $i = 0 + while ($i < Ncstack) + if (cstack[$i]) + printf "%d %s\n", $i, cstack[$i] + else + printf "%d %s\n", $i, "NULL" + end + set $i = $i + 1 + end +end + Index: /branches/czw_branch/20170908/ippToPsps/test/mkgpc1.sh =================================================================== --- /branches/czw_branch/20170908/ippToPsps/test/mkgpc1.sh (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/test/mkgpc1.sh (revision 40483) @@ -15,4 +15,14 @@ endif +if ("$1" == "user.sudo") then + if ($#argv != 4) goto usage; + set dbhost = $2 + set dbuser = $3 + set dbpass = $4 + + sudo mysql -h $dbhost -u root -p -e "grant all on *.* to $dbuser@"$dbhost" identified by '$dbpass'" + exit 0; +endif + if ("$1" == "create") then if ($#argv < 4) goto usage; @@ -28,4 +38,10 @@ # create a database with completely minimal tables needed to test mysql -h $dbhost -u $dbuser $dbpass -e "CREATE DATABASE $dbname" + if ($status) then + echo "ERROR: failed to create mysql database $dbname ($0)" + echo " NOTE: before running this test, you must have a mysql user:" + echo " mkgpc1.sh user localhost dvo dvo (requires root password)" + exit 1 + endif mysql -h $dbhost -u $dbuser $dbpass $dbname < gpc1schema.sql exit 0; @@ -45,9 +61,10 @@ (mysql -B -h $dbhost -u $dbuser $dbpass -e "show databases" | grep ^$dbname\$) >& /dev/null if ($status) then - echo "database $dbname does not yet exist" + echo "INFO: database $dbname does not yet exist ($0)" exit 0 endif - # mysql -h $dbhost -u $dbuser $dbpass $dbname -e "show tables" + echo mysql -B -h $dbhost -u $dbuser $dbpass $dbname -e "show tables" + mysql -h $dbhost -u $dbuser $dbpass $dbname -e "show tables" mysql -h $dbhost -u $dbuser $dbpass $dbname -e "describe ippToPspsFake" >& /dev/null if ($status) then @@ -91,4 +108,5 @@ endif + echo mysql -B -h $dbhost -u $dbuser $dbpass $dbname -e "show tables" set ntable = `mysql -B -h $dbhost -u $dbuser $dbpass $dbname -e "show tables" | wc -l` echo ntable: $ntable @@ -112,6 +130,13 @@ echo " delete an existing test db (enter pass 3x)" echo "" + echo " mkgpc1.sh forcedelete (dbhost) (dbname) (dbuser) [dbpass]" + echo " delete an existing test db (enter pass 3x), even if the number of tables is wrong" + echo "" echo " mkgpc1.sh user (dbhost) (dbuser) (password)" echo " create a new user and password for the test db" echo "" + echo " mkgpc1.sh user.sudo (dbhost) (dbuser) (password)" + echo " create a new user and password for the test db, using sudo mysql" + echo " (this option is needed if your mysql installation uses the auth_socket plugin" + echo "" exit 2 Index: /branches/czw_branch/20170908/ippToPsps/test/mkgpc1data.dr2.dvo =================================================================== --- /branches/czw_branch/20170908/ippToPsps/test/mkgpc1data.dr2.dvo (revision 40483) +++ /branches/czw_branch/20170908/ippToPsps/test/mkgpc1data.dr2.dvo (revision 40483) @@ -0,0 +1,823 @@ +#!/usr/bin/env dvo +# -*-sh-*- +input tap.dvo + +if (not($?VERBOSE)) set VERBOSE = 0 +if (not($?PARALLEL)) set PARALLEL = 0 + +$SMALLTEST = 1 + +## this script must be run in this directory, output data goes to 'testdata' +$OUTDIR = testdata + +# note that ipptopsps and gpc1db need the path ex extension +# I'm setting the base filename without the .smf and add it in mkcatdir.* + +# some global value for ease of access: +$RA = 9.93 +$DEC = -0.03 +$dRA = 1.0 +$dDEC = 1.0 + +$SKYCELL = skycell.1133.081 +$TESS_ID = RINGS.V3 +$BAD_PSFQF_FRAC = 0.0 +$ADDNOISE = "-no-noise" +$ADDNOISE = "" + +$CATFORMAT = PS1_V5 +$CMFFORMAT = PS1_V5 +$DIFFORMAT = PS1_DV5 + +# a note on double quotes and TESS_ID: +# since the variables TESS_ID and SKYCELL are used below to construct the list, their values should NOT be wrapped in double quotes +# or those double quotes are embedded in the strings in the list +# when TESS_ID and SKYCELL appear in mysql commands as values they need to be protected with double quotes so mysql does not +# interpret them as db table fields + +### XXX some of the ipptopsps functions try to find an image based on EXTERN_ID alone; this should be in combination with SOURCE_ID + +# imagedata describes the full fake exposure set. the fields are +list imagedata +# exp_id stk_id filebase M_off date time warpfile filter difffile + 1 101 $OUTDIR/test.cam 0.000 2010/01/01 01:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 2 101 $OUTDIR/test.cam -0.025 2010/01/01 02:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 3 101 $OUTDIR/test.cam 0.025 2010/01/01 03:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 4 101 $OUTDIR/test.cam 0.010 2010/01/01 04:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 5 102 $OUTDIR/test.cam 0.000 2010/01/01 05:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 6 102 $OUTDIR/test.cam -0.025 2010/01/01 06:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 7 102 $OUTDIR/test.cam 0.025 2010/01/01 07:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 8 102 $OUTDIR/test.cam 0.010 2010/01/01 08:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 9 103 $OUTDIR/test.cam 0.000 2010/01/01 09:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 10 103 $OUTDIR/test.cam -0.025 2010/01/01 10:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 11 103 $OUTDIR/test.cam 0.025 2010/01/01 11:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 12 103 $OUTDIR/test.cam 0.010 2010/01/01 12:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 13 104 $OUTDIR/test.cam -0.025 2010/01/02 01:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 14 104 $OUTDIR/test.cam 0.025 2010/01/02 02:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 15 104 $OUTDIR/test.cam 0.010 2010/01/02 03:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 16 104 $OUTDIR/test.cam 0.000 2010/01/02 04:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 17 105 $OUTDIR/test.cam -0.025 2010/01/02 05:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 18 105 $OUTDIR/test.cam 0.025 2010/01/02 06:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 19 105 $OUTDIR/test.cam 0.010 2010/01/02 07:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 20 105 $OUTDIR/test.cam 0.000 2010/01/02 08:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 21 106 $OUTDIR/test.cam -0.025 2010/01/02 09:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 22 106 $OUTDIR/test.cam 0.025 2010/01/02 10:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 23 106 $OUTDIR/test.cam 0.010 2010/01/02 11:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 24 106 $OUTDIR/test.cam -0.020 2010/01/02 12:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff +end + +list stackdata +# stk_id sky_id filename M_off filter forcedgalaxy + 101 1 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.010 g $OUTDIR/$TESS_ID.$SKYCELL.fgal + 102 2 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.020 g $OUTDIR/$TESS_ID.$SKYCELL.fgal + 103 3 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.030 g $OUTDIR/$TESS_ID.$SKYCELL.fgal + 104 1 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.025 r $OUTDIR/$TESS_ID.$SKYCELL.fgal + 105 2 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.035 r $OUTDIR/$TESS_ID.$SKYCELL.fgal + 106 3 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.045 r $OUTDIR/$TESS_ID.$SKYCELL.fgal +end + +macro mkfull + init.db + + mkcatdir.cam $CMFFORMAT $CATFORMAT + mkcatdir.stk $CMFFORMAT $CATFORMAT + mkcatdir.wrp $CMFFORMAT\_Lensing $CATFORMAT + mkcatdir.dif $DIFFORMAT PS1_V6 + insert.stack.set + + exec rm -rf testdata/catdir.mrg + + # if CATFORMAT is not specified, dvomerge will default to the .ptolemyrc version + # CATFORMAT, SKY_DEPTH here must match the addstar calls + # (or else I should copy the first catdir instead of using dvomerge) + # (or else dvomerge should be smart enough to inherit these from the input if the output does not exist) + exec dvomerge -matched-tables testdata/catdir.cam into testdata/catdir.mrg -D CATFORMAT $CATFORMAT -D SKY_DEPTH 4 + exec dvomerge -matched-tables testdata/catdir.stk into testdata/catdir.mrg + exec dvomerge -matched-tables testdata/catdir.wrp into testdata/catdir.mrg + + # basic relphot calibration, imitating the original PV3.3 calibration + echo relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits -cloud-limit 0.1 -use-mcal-psf-for-stack-aper + exec relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits -cloud-limit 0.1 -use-mcal-psf-for-stack-aper + + # test parallel-regions version, updating only stacks and warps, and applying the McalPSF vs McalAPER + exec cp RegionHost.dat testdata/catdir.mrg + + # parallel version is now setting the format in my local build + echo relphot -parallel-regions -region-hosts RegionHost.dat -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -vv g,r -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits -cloud-limit 0.1 -imfreeze -mosfreeze -only-stacks-and-warps -update-catformat PS1_V6 -nloop 0 + exec relphot -parallel-regions -region-hosts RegionHost.dat -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -vv g,r -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits -cloud-limit 0.1 -imfreeze -mosfreeze -only-stacks-and-warps -update-catformat PS1_V6 -nloop 0 + + # echo relphot -images -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -vv g,r -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits -cloud-limit 0.1 -imfreeze -mosfreeze -only-stacks-and-warps -update-catformat PS1_V6 -nloop 0 + # exec relphot -images -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -vv g,r -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits -cloud-limit 0.1 -imfreeze -mosfreeze -only-stacks-and-warps -update-catformat PS1_V6 -nloop 0 +end + +macro init.db + + $dbhost = localhost + $dbname = gpc1test + $dbuser = dvo + $dbpass = dvo + + $gscfile = `gconfig GSCFILE` + if ("$gscfile" == "not found") + echo "problem with dvo / ptolemy configuration. check ~/.ptolemyrc" + break + end + + exec mkgpc1.sh delete $dbhost $dbname $dbuser $dbpass + exec mkgpc1.sh create $dbhost $dbname $dbuser $dbpass + + dbconnect $dbhost $dbuser $dbname -p $dbpass + + # system-wide info + dbinsert ippToPspsFake (my_id) values (2) + dbinsert skycell (radeg, decdeg, tess_id, skycell_id) values ($RA, $DEC, "$TESS_ID", "$SKYCELL") +end + +# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp +macro mkcatdir.cam + if ($0 != 3) + echo "mkcatdir.cam (cmftype) (dvotype)" + break + end + + # tapPLAN 4 + + mkdir $OUTDIR + + local i catdir ID stkID rawfile cmffile offset + $catdir = $OUTDIR/catdir.cam + exec rm -rf $catdir + + $TIMEFORMAT = mjd + $TIMEREF = 2001/01/01,00:00:00 + + for i 0 $imagedata:n + list word -split $imagedata:$i + $ID = $word:0 + $stkID = $word:1 + sprintf rawfile "%s.%02d.txt" $word:2 $ID + sprintf cmffile "%s.%02d.smf" $word:2 $ID + $offset = $word:3 + sprintf myDATE "%s" $word:4 + sprintf myTIME "%s" $word:5 + $filter = $word:7 + + if ($SMALLTEST && ($stkID != 101) && ($stkID != 104)) continue + + echo ctimes -abs $myDATE,$myTIME -var mjd + ctimes -abs $myDATE,$myTIME -var mjd + + # XXX for a simple test of ippToPsps, i need to generate smf files + # with some correspondence to gpc1 exposure smfs + + # this means: + # 1) a PHU with some basic header data + # 2) add extensions with EXTNAME of XYnn + + # NOTE: mkcmf does not populate a PHU, so we have to fake it + + # create an empty header and populate with the desired keywords + mcreate dummy 0 0 + keyword dummy MJD-OBS -wf $mjd + keyword dummy FILTERID -w $filter.00000 + keyword dummy EXPTIME -wf 1.0 + + keyword dummy ZPT_ERR -wf 0.01 + keyword dummy EXPREQ -wf 1.0 + keyword dummy AIRMASS -wf 1.0 + keyword dummy RA -wf $RA + keyword dummy DEC -wf $DEC + keyword dummy CTYPE1 -w "RA---DIS" + keyword dummy CTYPE2 -w "DEC--DIS" + keyword dummy CRVAL1 -wf $RA + keyword dummy CRVAL2 -wf $DEC + keyword dummy CRPIX1 -wf 0.0 + keyword dummy CRPIX2 -wf 0.0 + keyword dummy CDELT1 -wf {1.0/3600} + keyword dummy CDELT2 -wf {1.0/3600} + keyword dummy PC001001 -wf 1.0 + keyword dummy PC001002 -wf 0.0 + keyword dummy PC002001 -wf 0.0 + keyword dummy PC002002 -wf 1.0 + keyword dummy NPLYTERM -wd 0 + keyword dummy PCA1X3Y0 -wf 0.0 + keyword dummy PCA1X2Y1 -wf 0.0 + keyword dummy PCA1X1Y2 -wf 0.0 + keyword dummy PCA1X0Y3 -wf 0.0 + keyword dummy PCA1X2Y0 -wf 0.0 + keyword dummy PCA1X1Y1 -wf 0.0 + keyword dummy PCA1X0Y2 -wf 0.0 + keyword dummy PCA2X3Y0 -wf 0.0 + keyword dummy PCA2X2Y1 -wf 0.0 + keyword dummy PCA2X1Y2 -wf 0.0 + keyword dummy PCA2X0Y3 -wf 0.0 + keyword dummy PCA2X2Y0 -wf 0.0 + keyword dummy PCA2X1Y1 -wf 0.0 + keyword dummy PCA2X0Y2 -wf 0.0 + wd dummy $cmffile + + mkinput $offset $rawfile + echo mkcmf -photcode SIMTEST.$filter.Chip $ADDNOISE -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 -coords + exec mkcmf -photcode SIMTEST.$filter.Chip $ADDNOISE -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 -coords + + if ($i == 3) + echo "NOTE: not adding image $i to DVO : this simulates images with poor astrometry" + continue + end + + if (0 && ($i == 5)) + echo "NOTE: adding image $i 2x to DVO, but without detections : this simulates a failure in dvo contruction" + + echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -image -dup-images + exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -image -dup-images + end + + if (0 && ($i == 7)) + echo "NOTE: adding detections from image $i 2x to DVO, but with wrong photcode and duplicate image ID : this simulates a failure in dvo construction" + echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -dup-images -image-id-override 7 -photcode 2MASS_J + exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -dup-images -image-id-override 7 -photcode 2MASS_J + end + + echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass + exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass + end + + if ($SMALLTEST) + echo relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update + + echo relastro -v -D CATDIR $catdir -D CAMERA simtest -images -update-chips -update-all-cameras -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -apply-offsets + exec relastro -v -D CATDIR $catdir -D CAMERA simtest -images -update-chips -update-all-cameras -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -apply-offsets + else + echo relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -update + end + + if ($PARALLEL) + $hostname = `hostname` + exec rm -f $catdir/HostTable.dat + output $catdir/HostTable.dat + echo "# ID Hostname Catdir" + echo " 1 $hostname $catdir.p1" + echo " 2 $hostname $catdir.p2" + echo " 3 $hostname $catdir.p3" + output stdout + exec dvodist -out $catdir + end + + # for i 0 $offset:n + # tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i" + # end + + # tapDONE +end + +# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp +macro mkcatdir.stk + if ($0 != 3) + echo "mkcatdir.stk (cmftype) (dvotype)" + break + end + + # tapPLAN 4 + + mkdir $OUTDIR + + local i catdir stkID skyID rawfile cmffile offset + $catdir = $OUTDIR/catdir.stk + exec rm -rf $catdir + + for i 0 $stackdata:n + list word -split $stackdata:$i + $stkID = $word:0 + $skyID = $word:1 + sprintf rawfile "%s.%02d.txt" $word:2 $stkID + sprintf cmffile "%s.%02d.cmf" $word:2 $stkID + $offset = $word:3 + $filter = $word:4 + + ## this is not generating photcodes recognized as stack by relphot + if ($SMALLTEST && ($stkID != 101) && ($stkID != 104)) continue + + mkinput $offset $rawfile + echo mkcmf -stack -photcode SIMTEST.$filter.SkyChip $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords -aper-offset 0.15 + exec mkcmf -stack -photcode SIMTEST.$filter.SkyChip $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords -aper-offset 0.15 + + # add some required header keywords: + # exec echo "HIERARCH FPA.ZP = {25.+$offset} / Magnitude zero point" > tmp.hdr + # exec fits_insert $cmffile tmp.hdr + + echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -D ZERO_POINT_OPTION CHIP_HEADER + exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -D ZERO_POINT_OPTION CHIP_HEADER + end + + if ($SMALLTEST) + echo relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -averages -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -averages -update + else + echo relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -update + end + + if ($PARALLEL) + $hostname = `hostname` + exec rm -f $catdir/HostTable.dat + output $catdir/HostTable.dat + echo "# ID Hostname Catdir" + echo " 1 $hostname $catdir.p1" + echo " 2 $hostname $catdir.p2" + echo " 3 $hostname $catdir.p3" + output stdout + exec dvodist -out $catdir + end + + # for i 0 $offset:n + # tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i" + # end + + # tapDONE +end + +# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp +macro mkcatdir.wrp + if ($0 != 3) + echo "mkcatdir.wrp (cmftype) (dvotype)" + break + end + + # tapPLAN 4 + + mkdir $OUTDIR + + local i catdir ID stkID rawfile cmffile offset + $catdir = $OUTDIR/catdir.wrp + exec rm -rf $catdir + + $TIMEFORMAT = mjd + $TIMEREF = 2001/01/01,00:00:00 + + for i 0 $imagedata:n + list word -split $imagedata:$i + $ID = $word:0 + $stkID = $word:1 + sprintf rawfile "%s.%02d.txt" $word:2 $ID; # use the cam-stage rawfile + sprintf cmffile "%s.%02d.cmf" $word:6 $ID + $offset = $word:3 + sprintf myDATE "%s" $word:4 + sprintf myTIME "%s" $word:5 + $filter = $word:7 + + if ($SMALLTEST && ($stkID != 101) && ($stkID != 104)) continue + + echo ctimes -abs $myDATE,$myTIME -var mjd + ctimes -abs $myDATE,$myTIME -var mjd + + ## this is not generating photcodes recognized as stack by relphot + + mkinput $offset $rawfile + echo mkcmf -forcedwarp -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 2 -tess_id $TESS_ID -skycell $SKYCELL -bad-psfqf-frac $BAD_PSFQF_FRAC -coords + exec mkcmf -forcedwarp -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 2 -tess_id $TESS_ID -skycell $SKYCELL -bad-psfqf-frac $BAD_PSFQF_FRAC -coords + + echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -xrad + exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -xrad + end + + # add the forcedgalaxy files for the above warps: + for i 0 $stackdata:n + list word -split $stackdata:$i + $stkID = $word:0 + sprintf rawfile "%s.%02d.txt" $word:2 $stkID + sprintf cmffile "%s.%02d.smf" $word:5 $stkID + $offset = $word:3 + $filter = $word:4 + + ## this is not generating photcodes recognized as stack by relphot + if ($SMALLTEST && ($stkID != 101) && ($stkID != 104)) continue + + echo mkcmf -forcedgalaxy -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords + exec mkcmf -forcedgalaxy -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords + + echo loadgalphot -D CATDIR $catdir -D CAMERA simtest -photcode SIMTEST.$filter.ForcedWarp -image-id $stkID $cmffile + exec loadgalphot -D CATDIR $catdir -D CAMERA simtest -photcode SIMTEST.$filter.ForcedWarp -image-id $stkID $cmffile + end + + if ($SMALLTEST) + echo relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -averages -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -averages -update + else + echo relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -update + end + + echo dvolens -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update-objects -update + exec dvolens -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update-objects -update + + if ($PARALLEL) + $hostname = `hostname` + exec rm -f $catdir/HostTable.dat + output $catdir/HostTable.dat + echo "# ID Hostname Catdir" + echo " 1 $hostname $catdir.p1" + echo " 2 $hostname $catdir.p2" + echo " 3 $hostname $catdir.p3" + output stdout + exec dvodist -out $catdir + end + + # for i 0 $offset:n + # tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i" + # end + + # tapDONE +end + +macro insert.exp + if ($0 != 4) + echo "USAGE: insert.exp (ID) (filename) (filter)" + break + end + + local myID filename filter + $myID = $1 + $filename = $2 + $filter = $3 + + # a single exposure + echo $myDATE + echo $myTIME + + dbinsert rawExp (exp_id, exp_name, exp_time, filter, bg, bg_stdev, ra, decl, dateobs) values ($myID, "test.$myID", 10.0, "$filter.00000", 100.0, 2.0, {$RA*3.1415/180.0}, {$DEC*3.1415/180.0}, '$myDATE $myTIME') + dbinsert chipRun (exp_id, chip_id) values ($myID, $myID) + dbinsert camRun (chip_id, cam_id, dist_group, software_ver) values ($myID, $myID, "testgroup", "38000M") + dbinsert camProcessedExp (cam_id, path_base, fault) values ($myID, "$filename", 0) + dbinsert fakeRun (fake_id, cam_id) values ($myID, $myID) + dbinsert warpRun (warp_id, fake_id, tess_id, state) values ($myID, $myID, "$TESS_ID", "full") + dbinsert warpSkyCellMap (warp_id, skycell_id, class_id) values ($myID, "$SKYCELL", "XY00") + dbinsert warpImfile (warp_id, skycell_id, warp_skyfile_id) values ($myID, "$SKYCELL", $myID) + + dbinsert addRun (add_id, stage, state, stage_id, minidvodb_name) values ($myID, "cam", "full", $myID, "catdir.cam") + dbinsert addProcessedExp (add_id, fault) values ($myID, 0) +end + +macro insert.wrp + if ($0 != 4) + echo "USAGE: insert.wrp (warpID) (stackID) (filename)" + break + end + + local wrpID stkID filename + $wrpID = $1 + $stkID = $2 + $filename = $3 + + # a single exposure + echo $myDATE + echo $myTIME + + dbinsert addRun (add_id, stage, state, stage_id, stage_extra1, minidvodb_name) values ($wrpID, "fullforce", "full", $stkID, $wrpID, "catdir.wrp") + + # we have a single fullForceRun for each stack, so we can match fullForceRun ids to stackRun ids + dbinsert fullForceInput (ff_id, warp_id) values ($stkID, $wrpID) + dbinsert fullForceResult (ff_id, warp_id, software_ver, path_base) values ($stkID, $wrpID, "38000M", "$filename") +end + +# this inserts a single stack, adding the exposures associated with this stack +macro insert.stack + if ($0 != 5) + echo "USAGE: insert.stack (stkID) (skyID) (stkfile) (filter)" + break + end + + local i stkID skyID expID expfile wrpfile stkfile filter + + $stkID = $1 + $skyID = $2 + $stkfile = $3 + $filter = $4 + + $skycalID = $stkID + 20 + + dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "$SKYCELL", "$filter.00000", "37500") + + # we have a single fullForceRun for each stack, so we can match fullForceRun ids to stackRun ids + dbinsert fullForceRun (ff_id, skycal_id) values ($stkID, $skycalID) + + $TIMEFORMAT = mjd + $TIMEREF = 2001/01/01,00:00:00 + + $mjdsum = 0.0 + $Nstk = 0 + + for i 0 $imagedata:n + echo $imagedata:$i + list tmp -split $imagedata:$i + if ($stkID != $tmp:1) continue + + $expID = $tmp:0 + sprintf myDATE "%s" $tmp:4 + sprintf myTIME "%s" $tmp:5 + +# sprintf rawfile "%s.%02d.txt" $tmp:2 $ID + sprintf expfile "%s.%02d" $tmp:2 $expID + sprintf wrpfile "%s.%02d" $tmp:6 $expID + + $myFilter = $tmp:7 + insert.exp $expID $expfile $myFilter + if ($myFilter != $filter) + echo "*** WARNING: filter mis-match between exp and stk: $myFilter vs $filter" + break + end + + insert.wrp $expID $stkID $wrpfile + + dbinsert stackInputSkyfile (stack_id, warp_id) values ($stkID, $expID) + + echo ctimes -abs $myDATE,$myTIME -var mjd + ctimes -abs $myDATE,$myTIME -var mjd + $mjdsum += $mjd + $Nstk ++ + end + $mjd = $mjdsum / $Nstk + + dbinsert stackSumSkyfile (stack_id, mjd_obs) values ($stkID, $mjd) + dbinsert skycalRun (sky_id, skycal_id, stack_id) values ($skyID, $skycalID, $stkID) + dbinsert skycalResult (skycal_id, path_base) values ($skycalID, "$stkfile") + + dbinsert addRun (add_id, stage, state, stage_id, minidvodb_name) values ($stkID, "skycal", "full", $skycalID, "catdir.stk") +end + +macro mkcatdir.dif + if ($0 != 3) + echo "mkcatdir.cam (cmftype) (dvotype)" + break + end + + # tapPLAN 4 + + mkdir $OUTDIR + + local i catdir ID stkID rawfile cmffile offset + $catdir = $OUTDIR/catdir.dif + exec rm -rf $catdir + + $TIMEFORMAT = mjd + $TIMEREF = 2001/01/01,00:00:00 + + for i 0 $imagedata:n + list word -split $imagedata:$i + $ID = $word:0 + $stkID = $word:1 + sprintf rawfile "%s.%02d.txt" $word:2 $stkID + sprintf cmffile "%s.%02d.smf" $word:8 $stkID + sprintf invfile "%s.%02d.inv.smf" $word:8 $stkID + $offset = $word:3 + sprintf myDATE "%s" $word:4 + sprintf myTIME "%s" $word:5 + $filter = $word:7 + + if ($SMALLTEST && ($stkID != 101) && ($stkID != 104)) continue + + echo ctimes -abs $myDATE,$myTIME -var mjd + ctimes -abs $myDATE,$myTIME -var mjd + + mkinput $offset $rawfile + $diff_skyfile_id = $ID + 25 + echo mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords + exec mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords + + echo mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $invfile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords + exec mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $invfile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords + + insert.dif $ID $ID $stkID $word:8 + + if ($i == 3) + echo "NOTE: not adding image $i to DVO : this simulates images with poor astrometry" + continue + end + + exec rm -rf addstar.txt + output addstar.txt + echo $cmffile + echo $invfile + output stdout + echo addstar -diff-inv -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest -list addstar.txt -D CATFORMAT $2 -quick-airmass + exec addstar -diff-inv -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest -list addstar.txt -D CATFORMAT $2 -quick-airmass + end + if ($SMALLTEST) + echo relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update -boundary-tree tess.3pi.fits -is-diff-db -statmode WT_MEAN + exec relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update -boundary-tree tess.3pi.fits -is-diff-db -statmode WT_MEAN + else + echo relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -images g,r,i,z,y -update + end + + if ($PARALLEL) + $hostname = `hostname` + exec rm -f $catdir/HostTable.dat + output $catdir/HostTable.dat + echo "# ID Hostname Catdir" + echo " 1 $hostname $catdir.p1" + echo " 2 $hostname $catdir.p2" + echo " 3 $hostname $catdir.p3" + output stdout + exec dvodist -out $catdir + end + + # for i 0 $offset:n + # tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i" + # end + + # tapDONE +end + + +macro insert.stack.set + + local i stkID skyID cmffile filter + + for i 0 $stackdata:n + list word -split $stackdata:$i + $stkID = $word:0 + $skyID = $word:1 + sprintf cmffile "%s.%02d" $word:2 $stkID + $filter = $word:4 + + if ($SMALLTEST && ($stkID != 101) && ($stkID != 104)) continue + + insert.stack $stkID $skyID $cmffile $filter + end + + # I am setting these databases in a very fake way, with same in and out names! + + # catdirs for the exposures + dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.cam", 1, "merged") + dbinsert minidvodbProcessed (minidvodb_id, fault) values (1, 0) + + dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (1, 1, "full", "catdir.cam") + dbinsert mergedvodbProcessed (merge_id, fault) values (1, 0) + + # catdirs for the stack + dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.stk", 2, "merged") + dbinsert minidvodbProcessed (minidvodb_id, fault) values (2, 0) + + dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (2, 2, "full", "catdir.stk") + dbinsert mergedvodbProcessed (merge_id, fault) values (2, 0) + + # catdirs for the forced warps + dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.wrp", 3, "merged") + dbinsert minidvodbProcessed (minidvodb_id, fault) values (3, 0) + + dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (3, 3, "full", "catdir.wrp") + dbinsert mergedvodbProcessed (merge_id, fault) values (3, 0) + + # catdirs for the diffs + dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.dif", 4, "merged") + dbinsert minidvodbProcessed (minidvodb_id, fault) values (4, 0) + + dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (4, 4, "full", "catdir.dif") + dbinsert mergedvodbProcessed (merge_id, fault) values (4, 0) + +end + +macro insert.dif + if ($0 != 5) + echo "USAGE: insert.dif (diffID) (warpID) (stackID) (filename)" + break + end + + local difID wrpID stkID filename + $difID = $1 + $wrpID = $2 + $stkID = $3 + $filename = $4 + + echo $myDATE + echo $myTIME + + $addID = $difID + 100 + $diffSkyfileID = $difID + 25; + dbinsert addRun (add_id, stage, state, stage_id, stage_extra1, minidvodb_name) values ($addID, "diff", "full", $difID, $diffSkyfileID, "catdir.dif") + dbinsert addProcessedExp (add_id, fault) values ($addID, 0) + + dbinsert diffRun (diff_id, tess_id, state, diff_mode, software_ver) values ($difID, "$TESS_ID", "full", 2, "38000M") + dbinsert diffInputSkyfile (diff_id, warp1, stack2, diff_skyfile_id, tess_id, skycell_id) values ($difID, $wrpID, $stkID, $diffSkyfileID, "$TESS_ID", "$SKYCELL" ) + dbinsert diffSkyfile (diff_id, skycell_id, path_base) values ($difID, "$SKYCELL", "$filename") +end + +# make a simple input file for mkcmf +macro mkinput + if ($0 != 3) + echo "mkinput (offset) (output)" + break + end + + local output + $output = $2 + + exec rm -f $output + + local i j ra dec + output $output + for i 10 1024 100 + for j 10 1024 100 + $ra = $RA + $i*0.25/3600 + $dec = $DEC + $j*0.25/3600 + fprintf " %10.6f %10.6f %7.3f %6.1f %6.1f %7.3f" $ra $dec {10.0 + $1 + 2.5*($i + $j)/1000.0} $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} + end + end + # XXX add 2 duplicate detections + $i = 110; $j = 210 + # fprintf " %10.6f %10.6f %7.3f %6.1f %6.1f %7.3f" $ra $dec {10.0 + $1 + 2.5*($i + $j)/1000.0} $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} + # fprintf " %6.1f %6.1f %7.3f" $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} + $i = 210; $j = 110 + # fprintf " %10.6f %10.6f %7.3f %6.1f %6.1f %7.3f" $ra $dec {10.0 + $1 + 2.5*($i + $j)/1000.0} $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} + # fprintf " %6.1f %6.1f %7.3f" $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} + output stdout +end + +macro check.mags + if ($0 != 3) + echo "USAGE: check.mags (catdir) (ver)" + break + end + + # NOTE: relphot refuses to set the warp averages if there is no boundary tree: otherwise it cannot choose the correct subset of measurements + # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -vv -averages -boundary-tree tess.3pi.fits + # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits + + catdir $1 + skyregion {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} + avextract ra dec g:psf:chp g:kron:chp g:psf:stk g:kron:stk g:psf:wrp g:kron:wrp + + set dg:psf:chp:stk = g:psf:chp - g:psf:stk + set dg:psf:chp:wrp = g:psf:chp - g:psf:wrp + + set dg:kron:chp:stk = g:kron:chp - g:kron:stk + set dg:kron:chp:wrp = g:kron:chp - g:kron:wrp + + set dg:psf:kron:chp = g:psf:chp - g:kron:chp + set dg:psf:kron:stk = g:psf:stk - g:kron:stk + set dg:psf:kron:wrp = g:psf:wrp - g:kron:wrp + + lim -n 0$2 g:psf:chp -0.2 0.2; clear; box; plot g:psf:chp dg:psf:chp:stk -c blue -pt 7; plot g:psf:chp dg:psf:chp:wrp -c red -pt 2 + label -x "g:psf:chp" -y "g:psf:chp - g:psf:stk(blue),wrp(red)" + resize 600 400 + + lim -n 1$2 g:kron:chp -0.2 0.2; clear; box; plot g:kron:chp dg:kron:chp:stk -c blue -pt 7; plot g:kron:chp dg:kron:chp:wrp -c red -pt 2 + label -x "g:kron:chp" -y "g:kron:chp - g:kron:stk(blue),wrp(red)" + resize 600 400 + + lim -n 2$2 g:psf:chp -0.2 0.2; clear; box; plot g:psf:chp dg:psf:kron:chp -c green -pt 7; plot g:psf:chp dg:psf:kron:stk -c blue -pt 2; plot g:psf:chp dg:psf:kron:wrp -c red -pt 2 + label -x "g:psf:chp" -y "g:psf:chp - g:kron:chp(green),stk(blue),wrp(red)" + resize 600 400 + +# for i 0 10 +# echo ra[$i] dec[$i] g:psf:chp[$i] g:kron:chp[$i] g:psf:stk[$i] g:psf:wrp[$i] +# end +end + +macro check.mags.meas + if ($0 != 3) + echo "USAGE: check.mags.meas (catdir) (ver)" + break + end + + # NOTE: relphot refuses to set the warp averages if there is no boundary tree: otherwise it cannot choose the correct subset of measurements + # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -vv -averages -boundary-tree tess.3pi.fits + # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits + + catdir $1 + skyregion {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} + + mextract ra dec mag:rel:psf mag:rel:kron mag:ave:psf mag:ave:kron photcode + set dM:psf = mag:ave:psf - mag:rel:psf + lim -n 0$2 mag:ave:psf dM:psf; clear; box; plot mag:ave:psf dM:psf + label -x "mag:ave:psf" -y "mag:ave:psf - mag:rel:psf" + resize 600 400 + + set dM:kron = mag:ave:kron - mag:rel:kron + lim -n 1$2 mag:ave:kron dM:kron; clear; box; plot mag:ave:kron dM:kron + label -x "mag:ave:kron" -y "mag:ave:kron - mag:rel:kron" + resize 600 400 + + set dM:psf:kron = mag:rel:psf - mag:rel:kron + lim -n 2$2 mag:rel:kron dM:psf:kron; clear; box; plot mag:rel:kron dM:psf:kron + label -x "mag:rel:kron" -y "mag:rel:psf - mag:rel:kron" + resize 600 400 +end + +if ($SCRIPT) + if ($argv:n > 0) + if ("$argv:0" == "-parallel") + $PARALLEL = 1 + end + if ("$argv:0" == "-dump-region") + echo {$RA - $dRA} {$RA + $dRA} {$DEC - $dDEC} {$DEC + $dDEC} + exit 0 + end + end + mkfull + if (not($STATUS)) exit 1 + exit 0 +end + Index: /branches/czw_branch/20170908/ippToPsps/test/mkgpc1data.dvo =================================================================== --- /branches/czw_branch/20170908/ippToPsps/test/mkgpc1data.dvo (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/test/mkgpc1data.dvo (revision 40483) @@ -17,45 +17,52 @@ $RA = 10.0 $DEC = 20.0 -$SKYCELL = "skycell.1133.081" -$TESS_ID = "RINGS.V3" -$BAD_PSFQF_FRAC = 0.2 +$SKYCELL = skycell.1133.081 +$TESS_ID = RINGS.V3 +$BAD_PSFQF_FRAC = 0.0 +$ADDNOISE = "-no-noise" + +# a note on double quotes and TESS_ID: +# since the variables TESS_ID and SKYCELL are used below to construct the list, their values should NOT be wrapped in double quotes +# or those double quotes are embedded in the strings in the list +# when TESS_ID and SKYCELL appear in mysql commands as values they need to be protected with double quotes so mysql does not +# interpret them as db table fields # imagedata describes the full fake exposure set. the fields are list imagedata -# exp_id stk_id filebase M_off date time warpfile filter difffile - 1 1 $OUTDIR/test.01 0.000 2010/01/01 01:00:00 $OUTDIR/warp.01 g $OUTDIR/diff.01 - 2 1 $OUTDIR/test.02 -0.025 2010/01/01 02:00:00 $OUTDIR/warp.02 g $OUTDIR/diff.02 - 3 1 $OUTDIR/test.03 0.025 2010/01/01 03:00:00 $OUTDIR/warp.03 g $OUTDIR/diff.03 - 4 1 $OUTDIR/test.04 0.010 2010/01/01 04:00:00 $OUTDIR/warp.04 g $OUTDIR/diff.04 - 5 2 $OUTDIR/test.05 0.000 2010/01/01 05:00:00 $OUTDIR/warp.05 g $OUTDIR/diff.05 - 6 2 $OUTDIR/test.06 -0.025 2010/01/01 06:00:00 $OUTDIR/warp.06 g $OUTDIR/diff.06 - 7 2 $OUTDIR/test.07 0.025 2010/01/01 07:00:00 $OUTDIR/warp.07 g $OUTDIR/diff.07 - 8 2 $OUTDIR/test.08 0.010 2010/01/01 08:00:00 $OUTDIR/warp.08 g $OUTDIR/diff.08 - 9 3 $OUTDIR/test.09 0.000 2010/01/01 09:00:00 $OUTDIR/warp.09 g $OUTDIR/diff.09 - 10 3 $OUTDIR/test.10 -0.025 2010/01/01 10:00:00 $OUTDIR/warp.10 g $OUTDIR/diff.10 - 11 3 $OUTDIR/test.11 0.025 2010/01/01 11:00:00 $OUTDIR/warp.11 g $OUTDIR/diff.11 - 12 3 $OUTDIR/test.12 0.010 2010/01/01 12:00:00 $OUTDIR/warp.12 g $OUTDIR/diff.12 - 13 4 $OUTDIR/test.13 -0.025 2010/01/02 01:00:00 $OUTDIR/warp.13 r $OUTDIR/diff.13 - 14 4 $OUTDIR/test.14 0.025 2010/01/02 02:00:00 $OUTDIR/warp.14 r $OUTDIR/diff.14 - 15 4 $OUTDIR/test.15 0.010 2010/01/02 03:00:00 $OUTDIR/warp.15 r $OUTDIR/diff.15 - 16 4 $OUTDIR/test.16 0.000 2010/01/02 04:00:00 $OUTDIR/warp.16 r $OUTDIR/diff.16 - 17 5 $OUTDIR/test.17 -0.025 2010/01/02 05:00:00 $OUTDIR/warp.17 r $OUTDIR/diff.17 - 18 5 $OUTDIR/test.18 0.025 2010/01/02 06:00:00 $OUTDIR/warp.18 r $OUTDIR/diff.18 - 19 5 $OUTDIR/test.19 0.010 2010/01/02 07:00:00 $OUTDIR/warp.19 r $OUTDIR/diff.19 - 20 5 $OUTDIR/test.20 0.000 2010/01/02 08:00:00 $OUTDIR/warp.20 r $OUTDIR/diff.20 - 21 6 $OUTDIR/test.21 -0.025 2010/01/02 09:00:00 $OUTDIR/warp.21 r $OUTDIR/diff.21 - 22 6 $OUTDIR/test.22 0.025 2010/01/02 10:00:00 $OUTDIR/warp.22 r $OUTDIR/diff.22 - 23 6 $OUTDIR/test.23 0.010 2010/01/02 11:00:00 $OUTDIR/warp.23 r $OUTDIR/diff.23 - 24 6 $OUTDIR/test.24 -0.020 2010/01/02 12:00:00 $OUTDIR/warp.24 r $OUTDIR/diff.24 +# exp_id stk_id filebase M_off date time warpfile filter difffile + 1 1 $OUTDIR/test.cam 0.000 2010/01/01 01:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 2 1 $OUTDIR/test.cam -0.025 2010/01/01 02:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 3 1 $OUTDIR/test.cam 0.025 2010/01/01 03:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 4 1 $OUTDIR/test.cam 0.010 2010/01/01 04:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 5 2 $OUTDIR/test.cam 0.000 2010/01/01 05:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 6 2 $OUTDIR/test.cam -0.025 2010/01/01 06:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 7 2 $OUTDIR/test.cam 0.025 2010/01/01 07:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 8 2 $OUTDIR/test.cam 0.010 2010/01/01 08:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 9 3 $OUTDIR/test.cam 0.000 2010/01/01 09:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 10 3 $OUTDIR/test.cam -0.025 2010/01/01 10:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 11 3 $OUTDIR/test.cam 0.025 2010/01/01 11:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 12 3 $OUTDIR/test.cam 0.010 2010/01/01 12:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp g $OUTDIR/$TESS_ID.$SKYCELL.diff + 13 4 $OUTDIR/test.cam -0.025 2010/01/02 01:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 14 4 $OUTDIR/test.cam 0.025 2010/01/02 02:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 15 4 $OUTDIR/test.cam 0.010 2010/01/02 03:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 16 4 $OUTDIR/test.cam 0.000 2010/01/02 04:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 17 5 $OUTDIR/test.cam -0.025 2010/01/02 05:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 18 5 $OUTDIR/test.cam 0.025 2010/01/02 06:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 19 5 $OUTDIR/test.cam 0.010 2010/01/02 07:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 20 5 $OUTDIR/test.cam 0.000 2010/01/02 08:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 21 6 $OUTDIR/test.cam -0.025 2010/01/02 09:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 22 6 $OUTDIR/test.cam 0.025 2010/01/02 10:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 23 6 $OUTDIR/test.cam 0.010 2010/01/02 11:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff + 24 6 $OUTDIR/test.cam -0.020 2010/01/02 12:00:00 $OUTDIR/$TESS_ID.$SKYCELL.wrp r $OUTDIR/$TESS_ID.$SKYCELL.diff end list stackdata -# stk_id sky_id filename M_off filter forcedgalaxy - 1 1 $OUTDIR/stack.01 0.010 g $OUTDIR/fgal.01 - 2 2 $OUTDIR/stack.02 0.020 g $OUTDIR/fgal.02 - 3 3 $OUTDIR/stack.03 0.030 g $OUTDIR/fgal.03 - 4 1 $OUTDIR/stack.04 0.025 r $OUTDIR/fgal.04 - 5 2 $OUTDIR/stack.05 0.035 r $OUTDIR/fgal.05 - 6 3 $OUTDIR/stack.06 0.045 r $OUTDIR/fgal.06 +# stk_id sky_id filename M_off filter forcedgalaxy + 1 1 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.010 g $OUTDIR/$TESS_ID.$SKYCELL.fgal + 2 2 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.020 g $OUTDIR/$TESS_ID.$SKYCELL.fgal + 3 3 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.030 g $OUTDIR/$TESS_ID.$SKYCELL.fgal + 4 1 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.025 r $OUTDIR/$TESS_ID.$SKYCELL.fgal + 5 2 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.035 r $OUTDIR/$TESS_ID.$SKYCELL.fgal + 6 3 $OUTDIR/$TESS_ID.$SKYCELL.stk 0.045 r $OUTDIR/$TESS_ID.$SKYCELL.fgal end @@ -67,4 +74,12 @@ mkcatdir.dif PS1_DV5 PS1_V5 insert.stack.set + + exec rm -rf testdata/catdir.mrg + exec dvomerge -matched-tables testdata/catdir.cam into testdata/catdir.mrg + exec dvomerge -matched-tables testdata/catdir.stk into testdata/catdir.mrg + exec dvomerge -matched-tables testdata/catdir.wrp into testdata/catdir.mrg + + # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -averages -boundary-tree tess.3pi.fits + exec relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits -cloud-limit 0.1 end @@ -75,7 +90,11 @@ $dbuser = dvo $dbpass = dvo -# $dbuser = czw -# $dbpass = czw - + + $gscfile = `gconfig GSCFILE` + if ("$gscfile" == "not found") + echo "problem with dvo / ptolemy configuration. check ~/.ptolemyrc" + break + end + exec mkgpc1.sh delete $dbhost $dbname $dbuser $dbpass exec mkgpc1.sh create $dbhost $dbname $dbuser $dbpass @@ -85,5 +104,5 @@ # system-wide info dbinsert ippToPspsFake (my_id) values (2) - dbinsert skycell (radeg, decdeg, tess_id, skycell_id) values ($RA, $DEC, $TESS_ID, $SKYCELL) + dbinsert skycell (radeg, decdeg, tess_id, skycell_id) values ($RA, $DEC, "$TESS_ID", "$SKYCELL") end @@ -110,6 +129,6 @@ $ID = $word:0 $stkID = $word:1 - $rawfile = $word:2\.txt - $cmffile = $word:2\.smf + sprintf rawfile "%s.%02d.txt" $word:2 $ID + sprintf cmffile "%s.%02d.smf" $word:2 $ID $offset = $word:3 sprintf myDATE "%s" $word:4 @@ -172,6 +191,6 @@ mkinput $offset $rawfile - echo mkcmf -photcode SIMTEST.$filter.Chip -no-noise -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 - exec mkcmf -photcode SIMTEST.$filter.Chip -no-noise -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 + echo mkcmf -photcode SIMTEST.$filter.Chip $ADDNOISE -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 -coords + exec mkcmf -photcode SIMTEST.$filter.Chip $ADDNOISE -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 -coords if ($i == 3) @@ -198,12 +217,12 @@ if ($SMALLTEST) - echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update - exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update - - echo relastro -v -D CATDIR $catdir -images -update-chips -region 8 12 18 22 -update -apply-offsets - exec relastro -v -D CATDIR $catdir -images -update-chips -region 8 12 18 22 -update -apply-offsets + echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update + + echo relastro -v -D CATDIR $catdir -D CAMERA simtest -images -update-chips -update-all-cameras -region 8 12 18 22 -update -apply-offsets + exec relastro -v -D CATDIR $catdir -D CAMERA simtest -images -update-chips -update-all-cameras -region 8 12 18 22 -update -apply-offsets else - echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update - exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update + echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update end @@ -246,6 +265,6 @@ $stkID = $word:0 $skyID = $word:1 - $rawfile = $word:2\.txt - $cmffile = $word:2\.cmf + sprintf rawfile "%s.%02d.txt" $word:2 $stkID + sprintf cmffile "%s.%02d.cmf" $word:2 $stkID $offset = $word:3 $filter = $word:4 @@ -255,6 +274,6 @@ mkinput $offset $rawfile - echo mkcmf -stack -photcode SIMTEST.$filter.SkyChip $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL - exec mkcmf -stack -photcode SIMTEST.$filter.SkyChip $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL + echo mkcmf -stack -photcode SIMTEST.$filter.SkyChip $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords + exec mkcmf -stack -photcode SIMTEST.$filter.SkyChip $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords # add some required header keywords: @@ -267,9 +286,9 @@ if ($SMALLTEST) - echo relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update - exec relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update + echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -averages -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -averages -update else - echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update - exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update + echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update end @@ -315,6 +334,6 @@ $ID = $word:0 $stkID = $word:1 - $rawfile = $word:2\.txt; # use the cam-stage rawfile - $cmffile = $word:6\.cmf + sprintf rawfile "%s.%02d.txt" $word:2 $ID; # use the cam-stage rawfile + sprintf cmffile "%s.%02d.smf" $word:6 $ID $offset = $word:3 sprintf myDATE "%s" $word:4 @@ -330,6 +349,6 @@ mkinput $offset $rawfile - echo mkcmf -forcedwarp -photcode SIMTEST.$filter.ForcedWarp $rawfile $cmffile -extroot SkyChip -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 2 -tess_id $TESS_ID -skycell $SKYCELL -bad-psfqf-frac $BAD_PSFQF_FRAC - exec mkcmf -forcedwarp -photcode SIMTEST.$filter.ForcedWarp $rawfile $cmffile -extroot SkyChip -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 2 -tess_id $TESS_ID -skycell $SKYCELL -bad-psfqf-frac $BAD_PSFQF_FRAC + echo mkcmf -forcedwarp -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 2 -tess_id $TESS_ID -skycell $SKYCELL -bad-psfqf-frac $BAD_PSFQF_FRAC -coords + exec mkcmf -forcedwarp -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 2 -tess_id $TESS_ID -skycell $SKYCELL -bad-psfqf-frac $BAD_PSFQF_FRAC -coords echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -xrad @@ -341,6 +360,6 @@ list word -split $stackdata:$i $stkID = $word:0 - $rawfile = $word:2\.txt - $cmffile = $word:5\.cmf + sprintf rawfile "%s.%02d.txt" $word:2 $stkID + sprintf cmffile "%s.%02d.smf" $word:5 $stkID $offset = $word:3 $filter = $word:4 @@ -349,21 +368,21 @@ if ($SMALLTEST && ($stkID != 1) && ($stkID != 4)) continue - echo mkcmf -forcedgalaxy -photcode SIMTEST.$filter.ForcedWarp $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL - exec mkcmf -forcedgalaxy -photcode SIMTEST.$filter.ForcedWarp $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL - - echo loadgalphot -D CATDIR $catdir -photcode SIMTEST.$filter.ForcedWarp -image-id $stkID $cmffile - exec loadgalphot -D CATDIR $catdir -photcode SIMTEST.$filter.ForcedWarp -image-id $stkID $cmffile + echo mkcmf -forcedgalaxy -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords + exec mkcmf -forcedgalaxy -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords + + echo loadgalphot -D CATDIR $catdir -D CAMERA simtest -photcode SIMTEST.$filter.ForcedWarp -image-id $stkID $cmffile + exec loadgalphot -D CATDIR $catdir -D CAMERA simtest -photcode SIMTEST.$filter.ForcedWarp -image-id $stkID $cmffile end if ($SMALLTEST) - echo relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update - exec relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update + echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -averages -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -averages -update else - echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update - exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update - end - - echo dvolens -D CATDIR $catdir -region 8 12 18 22 -update-objects -update - exec dvolens -D CATDIR $catdir -region 8 12 18 22 -update-objects -update + echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update + end + + echo dvolens -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -update-objects -update + exec dvolens -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -update-objects -update if ($PARALLEL) @@ -406,7 +425,7 @@ dbinsert camProcessedExp (cam_id, path_base, fault) values ($myID, "$filename", 0) dbinsert fakeRun (fake_id, cam_id) values ($myID, $myID) - dbinsert warpRun (warp_id, fake_id, tess_id, state) values ($myID, $myID, $TESS_ID, "full") - dbinsert warpSkyCellMap (warp_id, skycell_id, class_id) values ($myID, $SKYCELL, "XY00") - dbinsert warpImfile (warp_id, skycell_id, warp_skyfile_id) values ($myID, $SKYCELL, $myID) + dbinsert warpRun (warp_id, fake_id, tess_id, state) values ($myID, $myID, "$TESS_ID", "full") + dbinsert warpSkyCellMap (warp_id, skycell_id, class_id) values ($myID, "$SKYCELL", "XY00") + dbinsert warpImfile (warp_id, skycell_id, warp_skyfile_id) values ($myID, "$SKYCELL", $myID) dbinsert addRun (add_id, stage, state, stage_id, minidvodb_name) values ($myID, "cam", "full", $myID, "catdir.cam") @@ -452,5 +471,5 @@ $skycalID = $stkID + 20 - dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, $SKYCELL, "$filter.00000", "37500") + dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "$SKYCELL", "$filter.00000", "37500") # we have a single fullForceRun for each stack, so we can match fullForceRun ids to stackRun ids @@ -520,7 +539,7 @@ $ID = $word:0 $stkID = $word:1 - $rawfile = $word:2\.txt - $cmffile = $word:8\.cmf - $invfile = $word:8\.inv.cmf + sprintf rawfile "%s.%02d.txt" $word:2 $stkID + sprintf cmffile "%s.%02d.smf" $word:8 $stkID + sprintf invfile "%s.%02d.inv.smf" $word:8 $stkID $offset = $word:3 sprintf myDATE "%s" $word:4 @@ -535,9 +554,9 @@ mkinput $offset $rawfile $diff_skyfile_id = $ID + 25 - echo mkcmf -diff -photcode SIMTEST.$filter.SkyChip -no-noise -extroot SkyChip $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC - exec mkcmf -diff -photcode SIMTEST.$filter.SkyChip -no-noise -extroot SkyChip $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC - - echo mkcmf -diff -photcode SIMTEST.$filter.SkyChip -no-noise -extroot SkyChip $rawfile $invfile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC - exec mkcmf -diff -photcode SIMTEST.$filter.SkyChip -no-noise -extroot SkyChip $rawfile $invfile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC + echo mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords + exec mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords + + echo mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $invfile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords + exec mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $invfile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords insert.dif $ID $ID $stkID $word:8 @@ -557,9 +576,9 @@ end if ($SMALLTEST) - echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update -boundary-tree tess.3pi.fits -is-diff-db -statmode WT_MEAN - exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update -boundary-tree tess.3pi.fits -is-diff-db -statmode WT_MEAN + echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update -boundary-tree tess.3pi.fits -is-diff-db -statmode WT_MEAN + exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update -boundary-tree tess.3pi.fits -is-diff-db -statmode WT_MEAN else - echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update - exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update + echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update + exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update end @@ -592,5 +611,5 @@ $stkID = $word:0 $skyID = $word:1 - $cmffile = $word:2 + sprintf cmffile "%s.%02d.cmf" $word:2 $stkID $filter = $word:4 @@ -652,7 +671,7 @@ dbinsert addProcessedExp (add_id, fault) values ($addID, 0) - dbinsert diffRun (diff_id, tess_id, state, diff_mode, software_ver) values ($difID, $TESS_ID, "full", 2, "38000M") - dbinsert diffInputSkyfile (diff_id, warp1, stack2, diff_skyfile_id, tess_id, skycell_id) values ($difID, $wrpID, $stkID, $diffSkyfileID, $TESS_ID, $SKYCELL ) - dbinsert diffSkyfile (diff_id, skycell_id, path_base) values ($difID, $SKYCELL, "$filename") + dbinsert diffRun (diff_id, tess_id, state, diff_mode, software_ver) values ($difID, "$TESS_ID", "full", 2, "38000M") + dbinsert diffInputSkyfile (diff_id, warp1, stack2, diff_skyfile_id, tess_id, skycell_id) values ($difID, $wrpID, $stkID, $diffSkyfileID, "$TESS_ID", "$SKYCELL" ) + dbinsert diffSkyfile (diff_id, skycell_id, path_base) values ($difID, "$SKYCELL", "$filename") end @@ -669,17 +688,57 @@ exec rm -f $output - local i j + local i j ra dec output $output for i 10 1024 100 for j 10 1024 100 - fprintf " %6.1f %6.1f %7.3f" $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} + $ra = 10.0 + $i*0.25/3600 + $dec = 20.0 + $j*0.25/3600 + fprintf " %10.6f %10.6f %7.3f %6.1f %6.1f %7.3f" $ra $dec {10.0 + $1 + 2.5*($i + $j)/1000.0} $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} end end # XXX add 2 duplicate detections $i = 110; $j = 210 - fprintf " %6.1f %6.1f %7.3f" $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} + # fprintf " %10.6f %10.6f %7.3f %6.1f %6.1f %7.3f" $ra $dec {10.0 + $1 + 2.5*($i + $j)/1000.0} $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} + # fprintf " %6.1f %6.1f %7.3f" $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} $i = 210; $j = 110 - fprintf " %6.1f %6.1f %7.3f" $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} + # fprintf " %10.6f %10.6f %7.3f %6.1f %6.1f %7.3f" $ra $dec {10.0 + $1 + 2.5*($i + $j)/1000.0} $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} + # fprintf " %6.1f %6.1f %7.3f" $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0} output stdout +end + +macro check.mags + + # NOTE: relphot refuses to set the warp averages if there is no boundary tree: otherwise it cannot choose the correct subset of measurements + # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -averages -boundary-tree tess.3pi.fits + # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits + + catdir testdata/catdir.mrg + skyregion 8 12 18 22 + avextract ra dec g:psf:chp g:kron:chp g:psf:stk g:psf:wrp + + set dg:psf:chp:stk = g:psf:chp - g:psf:stk + set dg:psf:chp:wrp = g:psf:chp - g:psf:wrp + lim g:psf:chp dg:psf:chp:stk; clear; box; plot g:psf:chp dg:psf:chp:stk -c blue -pt 7; plot g:psf:chp dg:psf:chp:wrp -c red -pt 2 + +# for i 0 10 +# echo ra[$i] dec[$i] g:psf:chp[$i] g:kron:chp[$i] g:psf:stk[$i] g:psf:wrp[$i] +# end +end + +macro check.mags.meas + + # NOTE: relphot refuses to set the warp averages if there is no boundary tree: otherwise it cannot choose the correct subset of measurements + # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -averages -boundary-tree tess.3pi.fits + # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits + + catdir testdata/catdir.mrg + skyregion 8 12 18 22 + + mextract ra dec mag:rel:psf mag:rel:kron mag:ave:psf mag:ave:kron photcode + set dM:psf = mag:ave:psf - mag:rel:psf + lim -n 0 mag:ave:psf dM:psf; clear; box; plot mag:ave:psf dM:psf + + set dM:kron = mag:ave:kron - mag:rel:kron + lim -n 1 mag:ave:kron dM:kron; clear; box; plot mag:ave:kron dM:kron end Index: /branches/czw_branch/20170908/ippToPsps/test/notes.txt =================================================================== --- /branches/czw_branch/20170908/ippToPsps/test/notes.txt (revision 40482) +++ /branches/czw_branch/20170908/ippToPsps/test/notes.txt (revision 40483) @@ -1,2 +1,8 @@ + +2017.11.28 EAM: + + updating the test suite (using my desktop kukui -- ubuntu 14.04). + + * need to install xmllint to run ipptopsps In order to have a reasonably complete test, I need to have a bunch of things that mimic the IPP infrastructure Index: /branches/czw_branch/20170908/ippTools/src/dettool.c =================================================================== --- /branches/czw_branch/20170908/ippTools/src/dettool.c (revision 40482) +++ /branches/czw_branch/20170908/ippTools/src/dettool.c (revision 40483) @@ -1513,4 +1513,30 @@ psList *exp_id_list = item->data.list; psMetadata *where = psMetadataAlloc(); + + // make sure that -exp_id was parsed correctly + if (item->type == PS_DATA_METADATA_MULTI) { + psListIterator *iter = psListIteratorAlloc(item->data.list, 0, false); + psMetadataItem *mItem = NULL; + while ((mItem = psListGetAndIncrement(iter))) { + psS64 exp_id = mItem->data.S64; + if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(iter); + psFree(where); + return false; + } + } + psFree(iter); + } + if (item->type == PS_DATA_S64) { + psS64 exp_id = item->data.S64; + if (!psMetadataAddS64(where, PS_LIST_TAIL, "exp_id", PS_META_DUPLICATE_OK, "==", exp_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id"); + psFree(where); + return false; + } + } + +# if (0) // make sure that -exp_id was parsed correctly // XXX this can be removed someday @@ -1541,4 +1567,5 @@ psAbort("-exp_id was not parsed correctly (this should not happen"); } +# endif // check that the specified exp_ids actually exist in the iteration zero @@ -1599,7 +1626,7 @@ psArray *newInputExps = psArrayAllocEmpty(psListLength(exp_id_list)); while ((mItem = psListGetAndIncrement(iter))) { - detInputExpRow *inputExp = psHashLookup(valid_exp_ids, - (char *)mItem->data.V); - if (!inputExp) { + psString exp_idStr = psDBIntToString(mItem->data.S64); + detInputExpRow *inputExp = psHashLookup(valid_exp_ids, exp_idStr); + if (!inputExp) { // rollback if (!psDBRollback(config->dbh)) { @@ -1607,6 +1634,5 @@ } // invalid exp_id - psError(PS_ERR_UNKNOWN, false, "exp_id %s is invalid for det_id %" PRId64, - (char *)mItem->data.V, det_id); + psError(PS_ERR_UNKNOWN, false, "exp_id %" PRId64 " is invalid for det_id %" PRId64, mItem->data.S64, det_id); psFree(iter); psFree(valid_exp_ids); @@ -2452,4 +2478,5 @@ psMetadata *where = psMetadataAlloc(); for (long i = 0; i < psArrayLength(inputExps); i++) { + XXX note that exp_id is not a STRING if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", PS_META_DUPLICATE_OK, "==", Index: /branches/czw_branch/20170908/ippTools/src/pxchip.c =================================================================== --- /branches/czw_branch/20170908/ippTools/src/pxchip.c (revision 40482) +++ /branches/czw_branch/20170908/ippTools/src/pxchip.c (revision 40483) @@ -43,4 +43,5 @@ psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-data_state", 0, "search by data_state", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); @@ -73,4 +74,5 @@ psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object (LIKE comparison)", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-comment-skip", 0, "exclude by comment field (NOT LIKE comparison)", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); return true; @@ -95,4 +97,5 @@ PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-data_state", "rawExp.state", "=="); PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); @@ -125,4 +128,5 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-comment-skip", "rawExp.comment", "NOTLKE"); PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); return true; Index: /branches/czw_branch/20170908/ppImage/configure.ac =================================================================== --- /branches/czw_branch/20170908/ppImage/configure.ac (revision 40482) +++ /branches/czw_branch/20170908/ppImage/configure.ac (revision 40483) @@ -16,4 +16,11 @@ AC_PROG_LIBTOOL AC_SYS_LARGEFILE + +dnl ------------------------------------------------------------ + +AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing]) +if test "$ERRORCODES" = "missing" ; then + AC_MSG_ERROR([psParseErrorCodes is required]) +fi PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0]) Index: /branches/czw_branch/20170908/ppImage/src/Makefile.am =================================================================== --- /branches/czw_branch/20170908/ppImage/src/Makefile.am (revision 40482) +++ /branches/czw_branch/20170908/ppImage/src/Makefile.am (revision 40483) @@ -1,6 +1,5 @@ bin_PROGRAMS = ppImage -noinst_HEADERS = \ - ppImage.h +noinst_HEADERS = ppImage.h ppImageErrorCodes.h # Force recompilation of ppImageVersion.c, since it gets the version information @@ -10,5 +9,5 @@ FORCE: ; -BUILT_SOURCES = ppImageVersionDefinitions.h +BUILT_SOURCES = ppImageErrorCodes.h ppImageErrorCodes.c ppImageVersionDefinitions.h ppImage_CFLAGS = $(PPIMAGE_CFLAGS) $(PPSTATS_CFLAGS) $(PSASTRO_CFLAGS) $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) @@ -27,4 +26,5 @@ ppImageDetrendFree.c \ ppImageDetrendPattern.c \ + ppImageErrorCodes.c \ ppImageRebinReadout.c \ ppImageMosaic.c \ @@ -55,5 +55,8 @@ ppImageBurntoolApply.c -CLEANFILES = *~ +CLEANFILES = *~ ppImageErrorCodes.h ppImageErrorCodes.c + +EXTRA_DIST = ppImageErrorCodes.dat ppImageErrorCodes.c.in ppImageErrorCodes.h.in + clean-local: @@ -62,2 +65,8 @@ tags: etags `find . -name \*.[ch] -print` + +ppImageErrorCodes.h : ppImageErrorCodes.dat ppImageErrorCodes.h.in + $(ERRORCODES) --data=ppImageErrorCodes.dat --outdir=. ppImageErrorCodes.h + +ppImageErrorCodes.c : ppImageErrorCodes.dat ppImageErrorCodes.c.in ppImageErrorCodes.h + $(ERRORCODES) --data=ppImageErrorCodes.dat --outdir=. ppImageErrorCodes.c Index: /branches/czw_branch/20170908/ppImage/src/ppImage.h =================================================================== --- /branches/czw_branch/20170908/ppImage/src/ppImage.h (revision 40482) +++ /branches/czw_branch/20170908/ppImage/src/ppImage.h (revision 40483) @@ -15,4 +15,5 @@ #include "psastro.h" #include "ppStats.h" +#include "ppImageErrorCodes.h" #define RECIPE_NAME "PPIMAGE" // Name of the recipe to use Index: /branches/czw_branch/20170908/ppImage/src/ppImageErrorCodes.c.in =================================================================== --- /branches/czw_branch/20170908/ppImage/src/ppImageErrorCodes.c.in (revision 40482) +++ /branches/czw_branch/20170908/ppImage/src/ppImageErrorCodes.c.in (revision 40483) @@ -18,5 +18,5 @@ for (int i = 0; i < nerror; i++) { psErrorDescription *tmp = psAlloc(sizeof(psErrorDescription)); - p_psMemSetPersistent(tmp, true); + psMemSetPersistent(tmp, true); *tmp = errors[i]; psErrorRegister(tmp, 1); Index: /branches/czw_branch/20170908/ppImage/src/ppImageErrorCodes.dat =================================================================== --- /branches/czw_branch/20170908/ppImage/src/ppImageErrorCodes.dat (revision 40482) +++ /branches/czw_branch/20170908/ppImage/src/ppImageErrorCodes.dat (revision 40483) @@ -9,2 +9,4 @@ PROG Programming error DATA invalid data +UNKNOWN unspecified error, not defined +NO_PIXELS No good pixels in image Index: /branches/czw_branch/20170908/ppImage/src/ppImageMaskStats.c =================================================================== --- /branches/czw_branch/20170908/ppImage/src/ppImageMaskStats.c (revision 40482) +++ /branches/czw_branch/20170908/ppImage/src/ppImageMaskStats.c (revision 40483) @@ -89,4 +89,11 @@ (float) Npix_dynamic / Npix_valid, (float) Npix_magic / Npix_valid, (float) Npix_advisory / Npix_valid); + + if ((Npix_valid == 0)||(Npix_static + Npix_dynamic >= Npix_valid)) { + if (psMetadataLookupS32(NULL, stats, "QUALITY") == 0) { + psMetadataAddS32(stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE, "No good pixels in image.", PPIMAGE_ERR_NO_PIXELS); + } + } + return(true); } Index: /branches/czw_branch/20170908/ppSub/src/ppSubDefineOutput.c =================================================================== --- /branches/czw_branch/20170908/ppSub/src/ppSubDefineOutput.c (revision 40482) +++ /branches/czw_branch/20170908/ppSub/src/ppSubDefineOutput.c (revision 40483) @@ -198,4 +198,12 @@ psMetadataAddStr(tgtHeader, PS_LIST_TAIL, keyword, PS_META_REPLACE, "input image", string); } + + // Copy PSREFCAT from the source to the target as well + snprintf(keyword, 80, "PSREFCAT"); + char *string = psMetadataLookupStr(&status, srcHeader, keyword); + if (status) { + psMetadataAddStr(tgtHeader, PS_LIST_TAIL, keyword, PS_META_REPLACE, "input image reference catalog", string); + } + return true; } Index: /branches/czw_branch/20170908/ppTranslate/src/ppMops.h =================================================================== --- /branches/czw_branch/20170908/ppTranslate/src/ppMops.h (revision 40482) +++ /branches/czw_branch/20170908/ppTranslate/src/ppMops.h (revision 40483) @@ -11,7 +11,5 @@ PM_SOURCE_MODE_CR_LIMIT | PM_SOURCE_MODE_SKY_FAILURE | PM_SOURCE_MODE_DEFECT) // Flags to exclude //#define SOURCE_MASK2 (PM_SOURCE_MODE2_DIFF_WITH_DOUBLE | PM_SOURCE_MODE2_ON_SPIKE | PM_SOURCE_MODE2_ON_STARCORE | PM_SOURCE_MODE2_ON_BURNTOOL | PM_SOURCE_MODE2_ON_CONVPOOR) // Flags2 to exclude - -#define SOURCE_MASK2 (PM_SOURCE_MODE2_DIFF_WITH_DOUBLE | PM_SOURCE_MODE2_ON_BURNTOOL | PM_SOURCE_MODE2_ON_STARCORE | \ - PM_SOURCE_MODE2_ON_SPIKE) +#define SOURCE_MASK2 (PM_SOURCE_MODE2_ON_STARCORE) //Flags2 to exclude #define PS1_DV_FORMAT "PS1_DV%d" @@ -68,4 +66,5 @@ psString fpashutoutc; // FPA shutoutc psString fpashutcutc; // FPA shutcutc + psString refcat; // Reference catalog used } ppMopsDetections; Index: /branches/czw_branch/20170908/ppTranslate/src/ppMopsGetSkyChipPsfVersion.c =================================================================== --- /branches/czw_branch/20170908/ppTranslate/src/ppMopsGetSkyChipPsfVersion.c (revision 40482) +++ /branches/czw_branch/20170908/ppTranslate/src/ppMopsGetSkyChipPsfVersion.c (revision 40483) @@ -17,4 +17,10 @@ psFree(headerSkyChip); return 3; + } else if (strcmp(version, "PS1_DV4") == 0) { + psFree(headerSkyChip); + return 4; + } else if (strcmp(version, "PS1_DV5") == 0) { + psFree(headerSkyChip); + return 5; } psWarning("Unsupported EXTTYPE in SkyChip.psf table: [%s]", version); Index: /branches/czw_branch/20170908/ppTranslate/src/ppMopsRead.c =================================================================== --- /branches/czw_branch/20170908/ppTranslate/src/ppMopsRead.c (revision 40482) +++ /branches/czw_branch/20170908/ppTranslate/src/ppMopsRead.c (revision 40483) @@ -107,4 +107,5 @@ det->fpashutoutc = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "FPA.SHUTOUTC")); det->fpashutcutc = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "FPA.SHUTCUTC")); + det->refcat = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "PSREFCAT")); psFree(header); @@ -242,5 +243,6 @@ //MEH -- diff params, may need skyChipPsfVersion - psVector *npos = psMetadataLookupVector(NULL, table, "DIFF_NPOS"); + // -- unclear for npos, remove for now + //psVector *npos = psMetadataLookupVector(NULL, table, "DIFF_NPOS"); psVector *fpos = psMetadataLookupVector(NULL, table, "DIFF_FRATIO"); psVector *rbad = psMetadataLookupVector(NULL, table, "DIFF_NRATIO_BAD"); @@ -270,5 +272,5 @@ } - // add diff params to discard -- needs to be in config.. -- WSdiff only? + // MEH -- add diff params to discard -- needs to be in config.. -- WSdiff only? // -- needs to be finite to test but unclear if should reject (probably) // -- using Napoli 2013 to start with @@ -278,14 +280,15 @@ // DIFF_NRATIO_MASK>0.4 // DIFF_NRATIO_ALL>0.3 - // -- unclear about npos still -- + // -- unclear about npos still -- so remove from check //if (npos->data.S32[row]<=3 || if (!isfinite(fpos->data.F32[row]) || !isfinite(rbad->data.F32[row]) || !isfinite(rmask->data.F32[row]) || !isfinite(rall->data.F32[row])) { - psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of NAN diff params: 3, 0.6, 0.4, 0.4, 0.3: %d %g %f %f %f ", row, i, npos->data.S32[row],fpos->data.F32[row],rbad->data.F32[row],rmask->data.F32[row],rall->data.F32[row]); + //psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of NAN diff params: 3, 0.6, 0.4, 0.4, 0.3: %d %g %f %f %f ", row, i, npos->data.S32[row],fpos->data.F32[row],rbad->data.F32[row],rmask->data.F32[row],rall->data.F32[row]); + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of NAN diff params: 3, 0.6, 0.4, 0.4, 0.3: %g %f %f %f ", row, i, fpos->data.F32[row],rbad->data.F32[row],rmask->data.F32[row],rall->data.F32[row]); det->mask->data.U8[row] = 0xFF; continue; } if (fpos->data.F32[row]<=0.6 || rbad->data.F32[row]<=0.4 || rmask->data.F32[row]<=0.4 || rall->data.F32[row]<=0.3) { - psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of diff params: 3, 0.6, 0.4, 0.4, 0.3: %d %g %f %f %f ", - row, i, npos->data.S32[row], fpos->data.F32[row], rbad->data.F32[row], rmask->data.F32[row], rall->data.F32[row]); + //psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of diff params: 3, 0.6, 0.4, 0.4, 0.3: %d %g %f %f %f ", row, i, npos->data.S32[row], fpos->data.F32[row], rbad->data.F32[row], rmask->data.F32[row], rall->data.F32[row]); + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of diff params: 3, 0.6, 0.4, 0.4, 0.3: %g %f %f %f ", row, i, fpos->data.F32[row], rbad->data.F32[row], rmask->data.F32[row], rall->data.F32[row]); det->mask->data.U8[row] = 0xFF; continue; Index: /branches/czw_branch/20170908/ppTranslate/src/ppMopsWrite.c =================================================================== --- /branches/czw_branch/20170908/ppTranslate/src/ppMopsWrite.c (revision 40482) +++ /branches/czw_branch/20170908/ppTranslate/src/ppMopsWrite.c (revision 40483) @@ -104,4 +104,5 @@ psMetadataAddStr(header, PS_LIST_TAIL, "FPA.SHUTOUTC", 0, "Time of exposure open (CHIP mod)", det->fpashutoutc); psMetadataAddStr(header, PS_LIST_TAIL, "FPA.SHUTCUTC", 0, "Time of exposure close (RAW)", det->fpashutcutc); + psMetadataAddStr(header, PS_LIST_TAIL, "PSREFCAT", 0, "Reference catalog used", det->refcat); //field in header that tells about the CMF version Index: /branches/czw_branch/20170908/psLib/src/db/psDB.c =================================================================== --- /branches/czw_branch/20170908/psLib/src/db/psDB.c (revision 40482) +++ /branches/czw_branch/20170908/psLib/src/db/psDB.c (revision 40483) @@ -2425,4 +2425,10 @@ // used in a where clause... psStringAppend(&query, "%s LIKE '%s'", itemName, item->data.str); + } else if (item->comment && psStrcasestr(item->comment, "NOTLKE")) { + // XXX ASC NOTE: we should have a better match for + // char & varchar columns than this. LIKE is OK for + // very large TEXT columns that really shouldn't be + // used in a where clause... + psStringAppend(&query, "%s NOT LIKE '%s'", itemName, item->data.str); } else { psStringAppend(&query, "%s %s '%s'", itemName, opStr, item->data.str); Index: /branches/czw_branch/20170908/pstamp/scripts/psmkreq =================================================================== --- /branches/czw_branch/20170908/pstamp/scripts/psmkreq (revision 40482) +++ /branches/czw_branch/20170908/pstamp/scripts/psmkreq (revision 40483) @@ -29,5 +29,5 @@ my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base); -my ($image, $mask, $variance, $jpeg, $cmf, $psf, $backmdl, $inverse, $restorebackground); +my ($image, $mask, $variance, $jpeg, $allroi, $cmf, $psf, $backmdl, $inverse, $restorebackground); my ($exptime, $expnum, $exptimejpeg, $expnumjpeg); my ($convolved, $unconvolved, $uncompressed, $use_imfile_id, $no_wait); @@ -113,4 +113,5 @@ 'mask' => \$mask, 'jpeg' => \$jpeg, + 'allroi' => \$allroi, 'variance' => \$variance, 'cmf' => \$cmf, @@ -218,4 +219,5 @@ $option_mask |= $PSTAMP_SELECT_VARIANCE if $variance; $option_mask |= $PSTAMP_SELECT_JPEG if $jpeg; + $option_mask |= $PSTAMP_MULTI_OVERLAP_IMAGE if $allroi; # if no image was requested make a stamp of the image Index: /branches/czw_branch/20170908/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /branches/czw_branch/20170908/pstamp/scripts/pstamp_job_run.pl (revision 40482) +++ /branches/czw_branch/20170908/pstamp/scripts/pstamp_job_run.pl (revision 40483) @@ -248,4 +248,7 @@ $command .= " -stage $stage"; $command .= " -forheader $calibfile" if $calibfile; + ## MEH hack for centeroffchip -- needs constraint for coord_mask 0 all in sky values.. + $command .= " -centeroffchip" if ($options & $PSTAMP_MULTI_OVERLAP_IMAGE); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: /branches/czw_branch/20170908/pstamp/src/ppstampArguments.c =================================================================== --- /branches/czw_branch/20170908/pstamp/src/ppstampArguments.c (revision 40482) +++ /branches/czw_branch/20170908/pstamp/src/ppstampArguments.c (revision 40483) @@ -31,4 +31,5 @@ fprintf(stderr, " [-write_cmf] : create an output cmf with the sources overlapping the stamp\n"); fprintf(stderr, " [-wholefile] : ignore the region of interest and process the entire input image\n"); + fprintf(stderr, " [-centeroffchip] : allow center to be off chip boundary and include any pixels in ROI (testing) \n"); // fprintf(stderr, " [-no_censor_masked] : do not set masked pixels to NAN\n"); fprintf(stderr, "\n"); @@ -66,4 +67,9 @@ *pOptions = options; + if ((argnum = psArgumentGet(argc, argv, "-centeroffchip"))) { + psArgumentRemove(argnum, &argc, argv); + options->centeroffchip = true; + } + if ((argnum = psArgumentGet(argc, argv, "-wholefile"))) { psArgumentRemove(argnum, &argc, argv); Index: /branches/czw_branch/20170908/pstamp/src/ppstampMakeStamp.c =================================================================== --- /branches/czw_branch/20170908/pstamp/src/ppstampMakeStamp.c (revision 40482) +++ /branches/czw_branch/20170908/pstamp/src/ppstampMakeStamp.c (revision 40483) @@ -592,5 +592,6 @@ } - if (onChip) { + //MEH add hack for ROI not to include center on chip + if (onChip || options->centeroffchip ) { if (options->roip.celestialRange) { findBoundingBox(options, input->fpa, chip, center); Index: /branches/czw_branch/20170908/pstamp/src/ppstampOptions.h =================================================================== --- /branches/czw_branch/20170908/pstamp/src/ppstampOptions.h (revision 40482) +++ /branches/czw_branch/20170908/pstamp/src/ppstampOptions.h (revision 40483) @@ -8,4 +8,5 @@ // input arguments pstampROI roip; + bool centeroffchip; bool wholeFile; psString chipName; Index: /branches/czw_branch/20170908/pstamp/src/pstamp.h =================================================================== --- /branches/czw_branch/20170908/pstamp/src/pstamp.h (revision 40482) +++ /branches/czw_branch/20170908/pstamp/src/pstamp.h (revision 40483) @@ -43,5 +43,6 @@ #define PSTAMP_SELECT_UNCONV 2048 #define PSTAMP_RESTORE_BACKGROUND 4096 -// unused 8192 +// MEH -- previously unused 8192 +#define PSTAMP_MULTI_OVERLAP_IMAGE 8192 #define PSTAMP_USE_IMFILE_ID 16384 Index: /branches/czw_branch/20170908/pswarp/src/pswarpLoop.c =================================================================== --- /branches/czw_branch/20170908/pswarp/src/pswarpLoop.c (revision 40482) +++ /branches/czw_branch/20170908/pswarp/src/pswarpLoop.c (revision 40483) @@ -58,4 +58,5 @@ } + psString refcat = NULL; // loop over this section once per input group for (int i = 0; i < nInputs; i++) { @@ -121,4 +122,17 @@ psMetadataAddPtr(readout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_DATA_ARRAY, "Sources from input astrometry", detections); } + + // Determine the reference catalog used + if ((!refcat)&&(astromRO)) { + if ((astromRO->parent->parent->hdu->header)&&(output->fpa->analysis)) { + psMetadataItem *refItem = psMetadataLookup(astromRO->parent->parent->hdu->header, "PSREFCAT"); + if (refItem) { + refcat = psMetadataLookupStr(NULL, astromRO->parent->parent->hdu->header, "PSREFCAT"); + psMetadataAddStr(output->fpa->analysis, PS_LIST_TAIL, "REFERENCE_CATALOG", PS_META_REPLACE, + "Reference catalog used for calibration.", refcat); + } + } + } + } Index: /branches/czw_branch/20170908/pswarp/src/pswarpLoopBackground.c =================================================================== --- /branches/czw_branch/20170908/pswarp/src/pswarpLoopBackground.c (revision 40482) +++ /branches/czw_branch/20170908/pswarp/src/pswarpLoopBackground.c (revision 40483) @@ -62,4 +62,5 @@ pmFPAfileActivate(config->files, true, "PSWARP.BKGMODEL"); + psString refcat = NULL; // loop over this section once per input group for (int i = 0; i < nInputs; i++) { @@ -134,4 +135,18 @@ } + if (astrom != input) { + pmReadout *astromRO = pmFPAviewThisReadout(view, astrom->fpa); // Readout for astrometry + if ((!refcat)&&(astromRO)) { + if ((astromRO->parent->parent->hdu->header)&&(output->fpa->analysis)) { + psMetadataItem *refItem = psMetadataLookup(astromRO->parent->parent->hdu->header, "PSREFCAT"); + if (refItem) { + refcat = psMetadataLookupStr(NULL, astromRO->parent->parent->hdu->header, "PSREFCAT"); + psMetadataAddStr(output->fpa->analysis, PS_LIST_TAIL, "REFERENCE_CATALOG", PS_META_REPLACE, + "Reference catalog used for calibration.", refcat); + } + } + } + } + // re-normalize the BKGMODEL pixels by modified astrometry for (int x = 0; x < readout->image->numCols; x++) { Index: /branches/czw_branch/20170908/pswarp/src/pswarpUpdateMetadata.c =================================================================== --- /branches/czw_branch/20170908/pswarp/src/pswarpUpdateMetadata.c (revision 40482) +++ /branches/czw_branch/20170908/pswarp/src/pswarpUpdateMetadata.c (revision 40483) @@ -20,4 +20,12 @@ bool bilevelAstrometry = psMetadataLookupBool (NULL, skycell->analysis, "ASTROMETRY.BILEVEL"); + psString refcat = NULL; + + if ((output)&&(output->analysis)) { + psMetadataItem *refItem = psMetadataLookup(output->analysis, "REFERENCE_CATALOG"); + if (refItem) { + refcat = psMetadataLookupStr (NULL, output->analysis, "REFERENCE_CATALOG"); + } + } pmChip *chip; while ((chip = pmFPAviewNextChip (view, output, 1)) != NULL) { @@ -165,4 +173,11 @@ } + if (refcat) { + if ((output)&&(output->hdu)&&(output->hdu->header)) { + psMetadataAddStr(output->hdu->header, PS_LIST_TAIL, "PSREFCAT", PS_META_REPLACE, + "Reference catalog used for calibration", refcat); + } + } + // apply the bilevel astrometry elements to the target if (bilevelAstrometry) {