Index: /trunk/doc/config/config.tex =================================================================== --- /trunk/doc/config/config.tex (revision 11568) +++ /trunk/doc/config/config.tex (revision 11569) @@ -1,9 +1,9 @@ -%%% $Id: config.tex,v 1.4 2006-11-01 02:54:16 price Exp $ +%%% $Id: config.tex,v 1.5 2007-02-02 04:46:06 price Exp $ \documentclass[panstarrs,spec]{panstarrs} % Basic document variables -\title{Pan-STARRS PS-1 Image Processing Pipeline Configurations} -\subtitle{Guide} -\shorttitle{IPP Configurations} +\title{Pan-STARRS Image Processing Pipeline} +\subtitle{Configuration Guide} +\shorttitle{IPP CG} \author{Paul Price} \audience{Pan-STARRS PMO} @@ -11,5 +11,5 @@ \project{Pan-STARRS Image Processing Pipeline} \organization{Institute for Astronomy} -\version{11} +\version{DR} \docnumber{PSDC-430-???} @@ -63,7 +63,7 @@ using a human-readable syntax. We therefore use these ``metadata configuration'' (MDC) files as the basis for our configuration files. -When referring to entries in an MDC file, we use the convention that -\code{NAME(TYPE)} refers to the item called \code{NAME}, with type -\code{TYPE}. +When referring to entries in an MDC file, we use the convention in +this document that \code{NAME(TYPE)} refers to the item called +\code{NAME}, with type \code{TYPE}. The IPP uses configuration parameters on four levels: @@ -97,8 +97,8 @@ \item The \code{-site} option\footnote{\code{-site} is used for C programs. For Perl programs, we use the \code{Getopt::Long} module -which requires us to use: \code{--site}} on the command line if +which requires us to use \code{--site}} on the command line if provided; \item The environment variable \code{PS_SITE}, if defined; or -\item \code{$HOME/.ipprc} otherwise. %$ +\item \code{$HOME/.ipprc} otherwise. %$ --- Balancing the former dollar sign for Emacs \end{enumerate} @@ -116,8 +116,12 @@ and camera configuration files. -The \code{WORKDIR(STR)} entry gives a top-level working directory that -is prepended to all filenames used by the \code{pXtools}. This allows -the database to be moved to a different system (with different -directory structure) without having to search and replace all paths. +The \code{DATAPATH(METADATA)} entry contains a series of symbolic +links (of tyoe \code{STR}) to data directories. This allows data to +be moved to a different system (with different directory structure) +without having to search and replace all paths within the database; +and also to juggle multiple projects using the same configuration +file. In the Perl components of the IPP, we use \code{path://DIR} to +mean ``look up \code{DIR} in the \code{DATAPATH} to get the +directory''. \subsubsection{Database setup} @@ -132,7 +136,7 @@ \code{DBUSER(STR)} specifies the database user name. -\code{DBPASSWORD(STR)} specifies the database password. \tbd{This is -an insecure method of storing what might be sensitive information. -This is to be revised in the future.} +\code{DBPASSWORD(STR)} specifies the database password. +\tbd{DBPASSWORD is an insecure method of storing what might be +sensitive information. This is to be revised in the future.} \subsubsection{Cameras} @@ -156,5 +160,6 @@ \code{TIME(STR)} specifies the location of the time configuration file -for psLib. +for psLib. This is not too important, since the original installation +location is known by psLib. \code{LOGLEVEL(S32)} specifies the logging level for \code{psLogMsg}. @@ -164,5 +169,8 @@ \code{LOGDEST(STR)} specifies the log destination (see -\code{psMessageDestination} for acceptable formats). +\code{psMessageDestination} for acceptable formats). We recommend +setting this to \code{STDERR} to avoid problems associated with +higher-level programs attempting to parse unintended input from +\code{stdout}. \code{TRACEFORMAT(STR)} specifies the trace format (see @@ -170,11 +178,16 @@ \code{TRACEDEST(STR)} specifies the trace destination (see -\code{psMessageDestination} for acceptable formats). +\code{psMessageDestination} for acceptable formats). We recommend +setting this to \code{STDERR} to avoid problems associated with +higher-level programs attempting to parse unintended input from +\code{stdout}. \code{TRACE(METADATA)} gives a list of trace facilities and their accompanying levels (of type \code{S32}); see \code{psTraceSetLevel}. -It is useful to include at least \code{err}, with level \code{10}, -since this will print all error messages. - +We recommend including at least \code{err}, with level \code{10}, +since this will print all error messages. Other useful traces to set +are \code{psModules.camera} for camera (and especially +\code{pmFPAfile}) operations; and \code{psLib.db} for database +lookups. \subsection{Example} @@ -184,5 +197,10 @@ PATH STR .:/my/home/.ipp # Default search path for configuration files -WORKDIR STR /my/data/disk/ # Top-level working directory + +# Place your data directories here and refer to as path://PATH/remainder +DATAPATH METADATA + HERE STR /data/my/host/ + THERE STR /data/other/host/ +END ### Database configuration @@ -194,18 +212,9 @@ ### Setups for each camera system CAMERAS METADATA - MCSHORT STR mcshort/camera.config - MCSHORT_CHIP STR mcshort_chip/camera.config - MCSHORT_FPA STR mcshort_fpa/camera.config - MEGACAM STR megacam/camera.config - MEGACAM_CHIP STR megacam_chipmosaic/camera.config - MEGACAM_FPA STR megacam_fpamosaic/camera.config - MEGACAM_DET STR megacam_detrended/camera.config - UCAM STR ucam/camera.config - UCAM_MOSAIC STR ucam_mosaic/camera.config - GPC1 STR gpc1/camera.config - LRIS_BLUE STR lris_blue/camera.config - LRIS_RED STR lris_red/camera.config - ISP STR isp/camera.config - SIMPLE STR simple/camera.config + CTIO_MOSAIC2 STR ctio_mosaic2/camera.config # CTIO MOSAIC2 camera, for ESSENCE + MEGACAM STR megacam/camera.config # Megacam, on CFHT + GPC1 STR gpc1/camera.config # Pan-STARRS GigaPixel Camera 1 + ISP STR isp/camera.config # Pan-STARRS Imaging Sky Probe + SIMPLE STR simple/camera.config # Simple single-chip camera END @@ -218,10 +227,16 @@ TRACEFORMAT STR THLNM # Trace format TRACE METADATA # Trace levels - err S32 10 + err S32 10 +# psLib.db S32 10 +# psModules.camera S32 10 END RECIPES METADATA # Site-level recipes - PPMERGE STR ppMerge_template.config # Recipe for combination - PPSTATS_PHASE0 STR ppStats_phase0.config # Recipe for phase 0 processing + PPIMAGE STR recipes/ppImage.config # Image reduction + PPMERGE STR recipes/ppMerge.config # Image combination + PPSTATS STR recipes/ppStats.config # Image statistics + PPSTATS_PHASE0 STR recipes/ppStats_phase0.config # Image statistics for Phase 0 + PSPHOT STR recipes/psphot.config # Photometry + PSASTRO STR recipes/psastro.config # Astrometry END \end{verbatim} @@ -249,5 +264,5 @@ the \code{CAMERAS(METADATA)}, which lists cameras by name, with their corresponding configuration file. Note that the \code{PATH(STR)} in -the site configuration defines the search paths for these files. +the site configuration defines the search path for these files. \subsection{Contents} @@ -257,6 +272,7 @@ stored in different formats (e.g., one amplifier per extension, vs all amplifiers spliced together in the PHU). The camera configuration -contains the formats, the camera description, filter translation table, -recipes, rejection levels and file rules. +contains the formats, the camera description, filter translation +table, observation type translation table, recipes, rejection levels +and file rules. \subsubsection{Formats} @@ -280,9 +296,19 @@ \subsubsection{Filter translation table} -\tbd{EAM: This needs a better description of the purpose.} - -\code{FILTER.ID(METADATA)} contains a list of filter identifications -(generally those found within the header), with a general term to -describe the filter, of type \code{STR}. +\code{FILTER.ID(METADATA)} contains a list of filter names (generally +those found within the FITS header; e.g., \code{r.MP9601}), with an +abstract name to describe the filter (e.g., \code{r}), of type +\code{STR}. This allows multiple descriptions of the same filter that +may exist in the FITS headers to be resolved as the same thing. + +\subsubsection{Observation type translation table} + +\code{OBSTYPE.TABLE(METADATA)} contains a list of observation types +(generally those found within the FITS header; e.g., \code{ZERO}), +with an abstract name to describe the observation type (e.g., +\code{BIAS}). This allows multiple descriptions of the same +observation type that may exist in the FITS headers to be resolved as +the same thing (e.g., \code{BIAS}, \code{ZERO} and \code{PEDESTAL} can +all be set to \code{BIAS}). \subsubsection{Recipes} @@ -335,4 +361,8 @@ the mean to the mean standard deviation of the mean; in terms of standard deviations. \tbd{Confusing enough?} +\item \code{IMFILE.SN}: rejection level for the signal-to-noise at the + imfile level. +\item \code{EXP.SN}: rejection level for the signal-to-noise at the + exposure level. \end{itemize} Apart from \code{FILTER}, values that are set to zero are ignored. @@ -341,10 +371,164 @@ be useful to define a type: \begin{verbatim} -TYPE LIMITS FILTER EXPECTED IMFILE.MEAN IMFILE.STDEV EXP.MEAN EXP.STDEV EXP.MEANSTDEV ENSEMBLE.MEAN ENSEMBLE.STDEV ENSEMBLE.MEANSTDEV +TYPE LIMITS FILTER EXPECTED IMFILE.MEAN IMFILE.STDEV EXP.MEAN EXP.STDEV EXP.MEANSTDEV ENSEMBLE.MEAN ENSEMBLE.STDEV ENSEMBLE.MEANSTDEV IMFILE.SN EXP.SN \end{verbatim} \subsubsection{File rules} -\tbd{EAM to fill this in} +\tbd{EAM to check and supplement this description.} + +The file rules are one of the most important aspects of the camera +configuration, and one of the easiest to get wrong. When setting up a +new camera configuration and getting errors (or worse, segmentation +faults), check the file rules first. Try turning up the +\code{psModules.camera} trace level to see what's going on. + +\code{FILERULES(METADATA)} lists the different types of files used in +the image processing, which specify how and when a file is read in and +written out. The files usually are of two or three components, +separated by a period (not for any particular reason except that's +what's been adopted); the first part specifies the program the file +will be used in, the second and third parts identify its role. For +example, \code{PPIMAGE.INPUT} specifies the input file for +\code{ppImage}; \code{PPIMAGE.OUTPUT.MASK} specifies the output mask +file from \code{ppImage}. + +\subsubsubsection{Replacements} + +Throughout the file rules, a syntax for defining strings from +variables is used: curly brackets \verb|{}| around an abstract name +are replaced by the program to obtain the proper value. Supported +abstract names are: +\begin{itemize} +\item \verb|{OUTPUT}| --- replaced with the output file root; +\item \verb|{CHIP.NAME}| --- replaced with the chip name; +\item \verb|{CHIP.N}| --- replaced with the chip number (printed \code{%02d}); +\item \verb|{CELL.NAME}| --- replaced with the cell name; +\item \verb|{CELL.N}| --- replaced with the chip number (printed \code{%02d}); +\item \verb|{EXTNAME}| --- replaced with the extension name; +\item \verb|{FILTER}| --- replaced with the filter name (without + applying the \code{FILTER.ID} translation table); +\item \verb|{FILTER.ID}| --- replaced with the filter identifier (after + applying the \code{FILTER.ID} translation table); +\item \verb|{CAMERA}| --- replaced with the instrument name (from \code{FPA.INSTRUMENT}); +\item \verb|{INSTRUMENT}| --- replaced with the instrument name (from \code{FPA.INSTRUMENT}); +\item \verb|{DETECTOR}| --- replaced with the detector name (from \code{FPA.DETECTOR}); and +\item \verb|{TELESCOPE}| --- replaced with the telescope name (from \code{FPA.TELESCOPE}). +\end{itemize} + +(More could potentially be added. If one you greatly desire is +missing, please ask!) + + +\subsubsubsection{Redirections} + +Entries with type \code{STR} are treated as symbolic links to another +line. For example, specifying: +\begin{verbatim} + PPIMAGE.OUTPUT STR PPIMAGE.OUTPUT.SPLIT +\end{verbatim} +means that the program will look up \code{PPIMAGE.OUTPUT.SPLIT} in the +place of \code{PPIMAGE.OUTPUT}. This allows a quick replacement if a +different output format is desired (e.g., \code{PPIMAGE.OUTPUT.MEF} +instead of \code{PPIMAGE.OUTPUT.SPLIT}). + +\subsubsubsection{File types} +\label{sec:camera-filerules-types} + +Both the input and output file rules use file types. The currently +supported file types are: +\begin{itemize} +\item \code{IMAGE} --- image data in FITS image format (treated as \code{F32}); +\item \code{MASK} --- mask data in FITS image format (treated as \code{U8}); +\item \code{WEIGHT} --- weight data in FITS image format (treated as \code{F32}) +\item \code{FRINGE} --- fringe image with fringe tables (one for each + cell) in FITS image format (image treated as \code{F32}); +\item \code{JPEG} --- image data in JPEG format (output only); +\item \code{CMP} --- object data in CMP format; +\item \code{CMF} --- object data in CMF format; +\item \code{RAW} --- object data in RAW format; +\item \code{SX} --- object data in SX format; and +\item \code{OBJ} --- object data in OBJ format. +\end{itemize} + +\tbd{EAM to fill in details on the object formats.} + + +\subsubsubsection{Inputs} + +It is useful to make the following \code{TYPE} declaration, which can +be used for all input files: +\begin{verbatim} +TYPE INPUT FILENAME.RULE FILENAME.XTRA EXTNAME.RULE EXTNAME.XTRA DATA.LEVEL FILE.TYPE +\end{verbatim} +The components are: +\begin{itemize} +\item \code{FILENAME.RULE} --- this specifies the rule for + constructing the filename. Options for doing so are: + \begin{itemize} + \item A simple filename, perhaps using the replacement syntax + defined above; + \item \code{@DETDB} to look up the appropriate file using the + detrend database (see \S\ref{sec:detrend-database}); or + \item \code{@FILES} to indicate that the input file(s) will be + specified on the command-line of the program. + \end{itemize} +\item \code{FILENAME.XTRA} --- \tbd{PAP is not entirely sure what this + is for; it may be unnecessary.} +\item \code{EXTNAME.RULE} --- This defines the extension name. + \tbd{Is this true? PAP thinks the camera format does that; this may + be unnecessary, or it may have to be tied into the camera format.} +\item \code{EXTNAME.XTRA} --- \tbd{PAP is not entirely sure what this + is for; it may be unnecessary.} +\item \code{DATA.LEVEL} --- the level of the hierarchy at which the + data is to be opened for reading. This should correspond to the + level of the extension in the FITS file, or higher. There are some + checks against the camera format that this is sensical, but don't + bet your life on it just yet. This is an important setting to check + if you're having problems. +\item \code{FILE.TYPE} --- the type of file, from the above list + (\S\ref{sec:camera-filerules-types}). +\end{itemize} + +\subsubsubsection{Outputs} + +It is useful to make the following \code{TYPE} declaration, which +can be used for all output files: +\begin{verbatim} +TYPE OUTPUT FILENAME.RULE FILENAME.XTRA EXTNAME.RULE EXTNAME.XTRA FILE.LEVEL DATA.LEVEL FILE.TYPE FILE.SAVE FILE.FORMAT +\end{verbatim} +The components are: +\begin{itemize} +\item \code{FILENAME.RULE} --- this specifies the rule for + constructing the filename. You most likely want to include + \verb|{OUTPUT}| somewhere here; Pan-STARRS convention is that + it goes at the front. +\item \code{FILENAME.XTRA} --- \tbd{PAP is not entirely sure what this + is for; it may be unnecessary.} +\item \code{EXTNAME.RULE} --- This defines the extension name. + \tbd{Is this true? PAP thinks the camera format does that; this may + be unnecessary, or it may have to be ties into the camera format.} +\item \code{EXTNAME.XTRA} --- \tbd{PAP is not entirely sure what this + is for; it may be unnecessary.} +\item \code{FILE.LEVEL} --- the level of the hierarchy at which a file + should be opened and the PHU written. This should correspond to the + level of the PHU. There are some checks against the camera format + that this is sensical, but don't bet your life on it just yet. This + is an important setting to check if you're having problems. +\item \code{DATA.LEVEL} --- the level of the hierarchy at which an + extension should be written. This should correspond to the level of + the extensions in the FITS file, or higher. There are some checks + against the camera format that this is sensical, but don't bet your + life on it just yet. This is an important setting to check if + you're having problems. +\item \code{FILE.TYPE} --- the type of file, from the above list + (\S\ref{sec:camera-filerules-types}). +\item \code{FILE.SAVE} --- whether this type of file should be saved + (\code{TRUE}) or not (\code{FALSE}). +\item \code{FILE.FORMAT} --- if the file format is to be changed, this + is the name of the file format (from the \code{FORMATS} metadata). + Otherwise, it is \code{NONE}. +\end{itemize} + \subsection{Example} @@ -372,5 +556,5 @@ -# Lookup table to go from filter ID to abstract name +# Lookup table to go from FPA.FILTER to abstract name for the filter FILTER.ID METADATA u.MP9301 STR u @@ -387,49 +571,56 @@ END +# Lookup table to go from FPA.OBSTYPE values to abstract name for the exposure type +OBSTYPE.TABLE METADATA + bias STR BIAS + zero STR BIAS + dark STR DARK + flat STR SKYFLAT + skyflat STR SKYFLAT + domeflat STR DOMEFLAT + object STR OBJECT + science STR OBJECT +END # Recipe options -RECIPES METADATA - # Recipes for ppImage - PPIMAGE STR megacam/ppImage.config # Default: all (normal) options on - PPIMAGE_O STR megacam/ppImage_o.config # Overscan only - PPIMAGE_OB STR megacam/ppImage_ob.config # Overscan, bias only - PPIMAGE_OBD STR megacam/ppImage_obd.config # Overscan, bias, dark only - PPIMAGE_OBDF STR megacam/ppImage_obdf.config # Overscan, bias, dark, flat only - PPIMAGE_B STR megacam/ppImage_b.config # Bias only - PPIMAGE_D STR megacam/ppImage_d.config # Dark only - PPIMAGE_F STR megacam/ppImage_f.config # Flat only - PPIMAGE_J1 STR megacam/ppImage_j1.config # JPEG only; binning 1 - PPIMAGE_J2 STR megacam/ppImage_j2.config # JPEG only; binning 2 - PPIMAGE_N STR megacam/ppImage_n.config # Nothing significant; binning only - - # Recipes for ppMerge - PPMERGE STR ppMerge_template.config # ppMerge recipe - PPMERGE_BIAS STR megacam/ppMerge_bias.config - PPMERGE_DARK STR megacam/ppMerge_dark.config - PPMERGE_FLAT STR megacam/ppMerge_flat.config - - # Other recipes +RECIPES METADATA + # Other recipes PSPHOT STR megacam/psphot.config # psphot details PSASTRO STR megacam/psastro.config # psastro details - PPSTATS STR megacam/ppStats.config # ppStats recipe -END - + PPSTATS STR megacam/ppStats.config # ppStats recipe + PPIMAGE STR megacam/ppImage.config # ppImage recipe +END # Rejection levels for detrend creation -REJECTION METADATA - TYPE LIMITS FILTER EXPECTED IMFILE.MEAN IMFILE.STDEV EXP.MEAN EXP.STDEV EXP.MEANSTDEV ENSEMBLE.MEAN ENSEMBLE.STDEV ENSEMBLE.MEANSTDEV - FLAT MULTI - - BIAS LIMITS * 0 0 15 0 15 0 0 0 0 - DARK LIMITS * 0 0 0 0 0 0 0 0 0 - FLAT LIMITS * 0 0 0 0 0 0 0 0 0 - FLAT LIMITS u 0 0 0 0 0 0 0 0 0 - FLAT LIMITS g 0 0 0 0 0 0 0 0 0 - FLAT LIMITS r 0 0 0 0 0 0 0 0 0 - FLAT LIMITS i 0 0 0 0 0 0 0 0 0 - FLAT LIMITS z 0 0 0 0 0 0 0 0 0 - -END - +REJECTION METADATA + TYPE LIMITS FILTER EXPECTED IMFILE.MEAN IMFILE.STDEV EXP.MEAN EXP.STDEV EXP.MEANSTDEV ENSEMBLE.MEAN ENSEMBLE.STDEV ENSEMBLE.MEANSTDEV IMFILE.SN EXP.SN + FLAT MULTI + + BIAS LIMITS * 0 1 5 0.5 3 0.5 3 3 0 0 0 + DARK LIMITS * 0 1 5 0.5 3 0.5 3 3 0 0 0 + FLAT LIMITS * 0 0 0 0 0 0 0 0 3 0 0 +# FLAT LIMITS u 0 0 0 0 0 0 0 0 3 0 0 +# FLAT LIMITS g 0 0 0 0 0 0 0 0 3 0 0 +# FLAT LIMITS r 0 0 0 0 0 0 0 0 3 0 0 +# FLAT LIMITS i 0 0 0 0 0 0 0 0 3 0 0 +# FLAT LIMITS z 0 0 0 0 0 0 0 0 3 0 0 + FRINGE LIMITS * 0 0 0 0 0 0 0 0 0 0 0 + +# FILTER is an additional qualifier, and may be "*" (or absent!), in which case it matches everything +# EXPECTED is the expected mean value +# IMFILE.MEAN is the maximum permitted mean value for an imfile, relative to the standard deviation +# IMFILE.STDEV is the maximum permitted standard deviation for an imfile +# EXP.MEAN is the maximum permitted mean value for an exposure, relative to the standard deviation +# EXP.STDEV is the maximum permitted standard deviation for an exposure +# EXP.MEANSTDEV is the maximum permitted mean standard deviation for an exposure relative to the mean +# ENSEMBLE.MEAN is the maximum permitted mean for an ensemble of exposures +# ENSEMBLE.STDEV is the maximum permitted standard deviation for an ensemble of exposures +# ENSEMBLE.MEANSTDEV is the maximum permitted mean standard deviation for an ensemble of exposures +# IMFILE.SN is the minimum permitted signal-to-noise for an imfile +# EXP.SN is the minimum permitted signal-to-noise for an exposure +# These values (all except FILTER) may be zero, in which case no clipping is applied. + +END + FILERULES METADATA @@ -496,9 +687,9 @@ \section{Camera format configuration} -The FITS data storage formation is a standard in the astronomical -community for storing astronomical images. A FITS file consists of an -arbitrary number of coupled human readable \code{ASCII} header -segments and binary data segments. The headers describe the format -and layout of the data segments. The first of these groups is +The FITS (Flexible Image Transport System) format is a standard in the +astronomical community for storing astronomical images. A FITS file +consists of an arbitrary number of coupled human readable \code{ASCII} +header segments and binary data segments. The headers describe the +format and layout of the data segments. The first of these groups is traditionally called the ``primary header unit'' (PHU) and the rest are referred to as ``extensions''. The header segments may contain @@ -507,6 +698,6 @@ the data, the header metadata is not so consistently defined within the astronomical community. Also, the flexibility of the data format -means that different representations are possible for the same -fundamental collection of data. +means that it is possible to construct a variety of different +representations for the same fundamental collection of data. The purpose of the camera format file is to define how FITS files are @@ -518,5 +709,5 @@ The camera formats for a particular camera are listed in the \code{FORMATS} metadata of the camera configuration file. Note that -the \code{PATH(STR)} in the site configuration defines the search +the \code{PATH} in the site configuration defines the search paths for these files. @@ -525,23 +716,24 @@ The camera format specifies how a FITS file from a particular camera is to be read. Different formats may be defined for a single camera -(e.g., one amplifier per extension, vs all amplifiers spliced together -in the PHU). The camera format configuration file contains the rules -for recognising the format, how to read the file, the contents of a -FITS file, data appropriate to different types of cells, information -on how to determine the concepts from the headers, default values, or -database, and expected formats for certain concepts. +(e.g., one amplifier per extension, or all amplifiers spliced together +in the PHU, or anything in between). The camera format configuration +file contains the rules for recognising the format, how to read the +file, the contents of a FITS file, data appropriate to different types +of cells, information on how to determine the concepts from the +headers, default values, or database, and expected formats for certain +concepts. \subsubsection{Rules for recognising} -\code{RULE(METADATA)} contains a list of telescope headers with -expected values (of the appropriate type) for this particular -combination of the camera and format. It is often useful to include -\code{TELESCOP} and \code{DETECTOR}, if possible, along with any other -headers that uniquely identify the camera and format. Note that all -of the headers must match exactly (modulo leading and trailing spaces -for strings), including the data type and value, for the rule to -match, and that the first format's rule to match is accepted. If a -rule doesn't match the header, try adjusting the types (especially for -numerical types; try S32 for integers, F32 and F64 for floats). +\code{RULE(METADATA)} contains a list of FITS headers with expected +values (of the appropriate type) for this particular combination of +the camera and format. It is often useful to include \code{TELESCOP} +and \code{DETECTOR}, if possible, along with any other headers that +uniquely identify the camera and format. Note that all of the headers +must match exactly (modulo leading and trailing spaces for strings), +including the data type and value, for the rule to match, and that the +first format's rule to match is accepted. If a rule doesn't match the +header, try adjusting the types (especially for numerical types --- +use S32 for integers, F32 and F64 for floats). \subsubsection{How to read the file} @@ -551,13 +743,14 @@ In the simplest case, the camera consists of a single chip consisting of a single cell always read with a single readout. In this case, the -image data could be written as part of the primary header unit. In a -more complex case with multiple chips and multiple cells, the data may -be organized in several ways. The data may be distributed into -multiple files or in multiple FITS data extensions. A single camera -image may be written as a collection of files for individual chips -with separate extensions for each cell (CFH12K.split, GPC). Another -camera may write a single file with multiple extensions for each cell -(Megacam.raw), or multiple extensions per chip, with each cell -representing portions of the chip image (Megacam.splice, CFHT-IR). +image data is generally written as part of the primary header unit. +However, in a more complex case with multiple chips and multiple +cells, the data may be organized in several ways. The data may be +distributed into multiple files or in multiple FITS data extensions +within a single file.. A single camera image may be written as a +collection of files for individual chips with separate extensions for +each cell (CFH12K.split, GPC). Another camera may write a single file +with multiple extensions for each cell (Megacam.raw), or multiple +extensions per chip, with each cell representing portions of the chip +image (Megacam.splice, CFHT-IR). In all of these representations, there are only two basic distinctions @@ -565,6 +758,6 @@ entire FITS file corresponds to (FPA, chip, or cell), and what level the extensions correspond to (chip, cell or no extensions at all). -Knowing these, and having a list of the extensions, we can construct -the Focal Plane hierarchy. +Knowing these, and having a list of the contents of each extension, we +can construct the Focal Plane hierarchy. \code{FILE(METADATA)} contains information on how to read the FITS @@ -583,13 +776,13 @@ with the same value of \code{FPA.NAME} can be admitted to the same FPA structure. -\item \code{CHIP.NAME(STR)} (necessary if \code{PHU} is \code{CHIP} or +\item \code{CHIP.NAME(STR)} (only required if \code{PHU} is \code{CHIP} or \code{CELL}) specifies a PHU header keyword that identifies the name of the chip. The purpose is to identify to which chip in the hierarchy the file belongs. -\item \code{CELL.NAME(STR)} (necessary if \code{PHU} is \code{CELL}) +\item \code{CELL.NAME(STR)} (only required if \code{PHU} is \code{CELL}) specifies a PHU header keyword that identifies the name of the cell within the chip. The purpose is to identify to which cell in the hierarchy the file belongs. -\item \code{CONTENT(STR)} (necessary if \code{EXTENSIONS} is +\item \code{CONTENT(STR)} (only required if \code{EXTENSIONS} is \code{NONE} and \code{PHU} is \code{CHIP} or \code{CELL}) specifies a key to the \code{CONTENTS} menu (see below). The purpose is to @@ -599,5 +792,5 @@ \code{CELL.NAME} only; \tbd{future concepts may be permitted in the future if there exists sufficient demand}. This allows such a - construct as \code{\{CHIP.NAME\}_\{CELL.NAME\}} to identify a + construct as \verb|{CHIP.NAME}_{CELL.NAME}| to identify a combination of chip and cell. \end{itemize} @@ -611,7 +804,7 @@ contents of an extension: the chip and cell to which a component belongs, and the type of the cell (see \S\ref{sec:cell_data} for cell -types), with the symbolic names separated by colons. The triplets may -be listed one after the other, separated by whitespace, where an -extension contains more than one cell. +types), with the symbolic names separated by colons. Where an +extension contains more than one cell, the triplets are listed one +after the other, separated by whitespace. \begin{itemize} @@ -642,5 +835,6 @@ here, since these differ according to the cell type. Since there is ambiguity in what the values here refer to (if the concept is of type -\code{STR}), we also require an additional entry with \code{.SOURCE} +\code{STR}, then the value could be a header name or the actual value +to use), we also require an additional entry with \code{.SOURCE} suffixed to the concept name, with the value (of type \code{STR}) being \code{VALUE} to indicate that the concept is specified by value, @@ -652,5 +846,5 @@ name may vary depending on the cell type. For example, the Megacam spliced format uses \code{TSECA} and \code{TSECB} to specify the trim -sections for the left and right amplifiers.] +sections for the left and right amplifiers, respectively.] \subsubsection{Concepts from headers} @@ -661,9 +855,9 @@ the concept is ingested. No distinction is made between the PHU and extension headers, but inheritance (look at the PHU if it's not in the -extension header) should be the normal behaviour. Multiple headers -may be given for certain concepts: +extension header) should be the normal behaviour. Multiple header +keywords (separated by whitespace) may be given for certain concepts: \begin{itemize} \item \code{FPA.TIME} and \code{CELL.TIME} to specify the date and - time in separate headers + time (in that order) are contained in separate header keywords. \item \code{CELL.BIASSEC} to specify multiple bias regions (e.g., a prescan and an overscan). @@ -719,6 +913,7 @@ indicates the date is a julian date. \item \code{CELL.X0}, \code{CELL.Y0}, \code{CHIP.X0} and - \code{CHIP.Y0}: \code{FORTRAN} indicates that the corner corresponds - to corner (1,1); if missing, assumes that the corner is at (0,0). + \code{CHIP.Y0}: \code{FORTRAN} indicates that the corner lower + left-hand pixel corresponds to coordinates (1,1); if missing, + assumes that the corner is at (0,0). \end{itemize} @@ -729,8 +924,13 @@ \code{DATABASE}: \begin{itemize} -\item \code{FPA.CAMERA}: Camera used +\item \code{FPA.TELESCOPE}: Telescope used +\item \code{FPA.INSTRUMENT}: Instrument used +\item \code{FPA.DETECTOR}: Detector used +\item \code{FPA.CAMERA}: Camera used; \tbd{To be deprecated?} \item \code{FPA.FOCUS}: Telescope focus \item \code{FPA.AIRMASS}: Airmass at boresight \item \code{FPA.FILTER}: Filter used +\item \code{FPA.FILTERID}: Filter identifier (parsed through the + \code{FILTER.ID} translation table in the camera configuration). \item \code{FPA.POSANGLE}: Position angle of instrument \item \code{FPA.RADECSYS}: Celestial coordinate system @@ -743,4 +943,6 @@ \item \code{FPA.TIMESYS}: Time system \item \code{FPA.TIME}: Time of exposure +\item \code{FPA.TEMP}: Temperature of the focal plane +\item \code{FPA.EXPOSURE}: Exposure time for the focal plane \item \code{CHIP.XPARITY}: Orientation in x compared to the rest of the FPA \item \code{CHIP.YPARITY}: Orientation in y compared to the rest of the FPA @@ -1174,29 +1376,81 @@ to. In addition, they may be specified in the site configuration and the camera configuration under the \code{RECIPES} metadata. Note that -the \code{PATH(STR)} in the site configuration defines the search +the \code{PATH} in the site configuration defines the search paths for these files. -\subsubsection{Symbolic links} - -Symbolic links to another recipe may be specified on the command line, -removing the need for the user to memorise a file name: e.g., -\code{-recipe PPIMAGE PPIMAGE_BIAS} might perform a bias subtraction -only. A symbolic link is recognised as such if the value is the name -of a recipe that has already been read (and the link is immediately -resolved), or if no filename of that name exists (in which case the -link is to be resolved later, as further sources become available). - -\subsubsection{Precedence} - -If multiple recipes have the same name, the precedence order is: -\begin{enumerate} -\item Command-line -\item Camera configuration -\item Site configuration -\end{enumerate} -with sources higher in this list having greater precedence. This -allows the user to override any recipes using the command-line, and to -specify bottom-level defaults in the site configuration while also -having camera-specific recipes in the camera configurations. +\subsubsection{Recipe combination} + +A single recipes are defined at multiple levels (site, camera, and +command-line), so it's important to know how these are loaded. The +site configuration recipes serve as the default recipes. Once the +particular camera is known, the values contained within its recipes +(provided either as a filename or as a symbolic link; see below for +symbolic links) override those defined in the site configuration +(unless the value has been declared as \code{MULTI}, in which case it +supplements). This is useful because recipes often depend on the +camera from which the data being processed originated; for example, +not all cameras require a dark to be subtracted. + +Finally, the command line can be used to provide further refinement. +A recipe can be defined on the command line using \code{-recipe +RECIPE_NAME filename.config} to specify a file containing the recipe, +or \code{-recipe RECIPE_NAME ALTERNATE_RECIPE_NAME} to specify an +symbolic link from which to inget values for the original recipe. + +Symbolic links offer the ability to override the default recipe values +by specifying a name, rather than a filename. A symbolic link can +refer to a recipe of a different name that has already been defined, +or it can refer to a \code{METADATA} within the recipe of that same +name. + +A few examples are useful here. Say the site configuration contains: +\begin{verbatim} +RECIPES METADATA + RECIPE STR recipe_default.config + RECIPE_EXOTIC STR recipe_exotic.config +END +\end{verbatim} + +The camera configuration has: +\begin{verbatim} +RECIPES METADATA + RECIPE STR recipe_camera.config +END +\end{verbatim} + +\code{recipe_default.config} has: +\begin{verbatim} +VALUE STR Default + +RECIPE_DULL METADATA + VALUE STR Dull +END +\end{verbatim} + + +\code{recipe_exotic.config} has: +\begin{verbatim} +VALUE STR Exotic +\end{verbatim} + +And \code{recipe_camera.config} has: +\begin{verbatim} +VALUE STR Camera +\end{verbatim} + +Then: +\begin{itemize} +\item If the recipe is examined without knowing the camera, + \code{VALUE} will be \code{Default}. +\item If the recipe is examined once the camera is known, \code{VALUE} + will be \code{Camera}. +\item If the command-line contains \code{-recipe RECIPE recipe_exotic.config}, + \code{VALUE} will be \code{Exotic}. +\item If the command-line contains \code{-recipe RECIPE RECIPE_EXOTIC}, + \code{VALUE} will be \code{Exotic}. +\item If the command-line contains \code{-recipe RECIPE RECIPE_DULL}, + \code{VALUE} will be \code{Dull}. +\end{itemize} + \subsection{Contents} @@ -1243,4 +1497,5 @@ good choice). \end{itemize} + \tbd{Non-linearity correction is implemented but not tested.} \item \code{OVERSCAN.SINGLE(BOOL)} indicates if the entire overscan is to be reduced to a single value. @@ -1253,4 +1508,7 @@ overscan: \code{MEAN} or \code{MEDIAN}. \tbd{Would like to change this to allow the full range of statistics.} +\item \code{FRINGE.ITER(S32)} specifies the number of rejection iterations for fringe solution. +\item \code{FRINGE.REJ(F32)} specifies the rejection threshold (in standard deviations) for fringe solution. +\item \code{FRINGE.KEEP(F32)} specifies the minimum fraction of points to keep in the fringe solution. \item \code{BIN1.XBIN(S32)} gives the level 1 binning in x \item \code{BIN2.YBIN(S32)} gives the level 1 binning in y @@ -1258,5 +1516,15 @@ \item \code{BIN2.YBIN(S32)} gives the level 2 binning in y: \item \code{PHOTCODE.RULE(STR)} gives a rule for producing a - photometry code, with values in curly brackets interpolated. + photometry code, with values in curly brackets interpolated in the + same manner as the file rules in the camera configuration. +\item \code{PPIMAGE.JPEG1(METADATA)} and \code{PPIMAGE.JPEG2(METADATA)} give parameters for JPEG scaling, and contains: + \begin{itemize} + \item \code{COLORMAP(STR)} specifies the colormap to use: + \code{greyscale}, \code{-greyscale} (inverse greyscale), + \code{rainbow}, \code{heat}. + \item \code{SCALE.MODE(STR)} specifies how the scaling is performed: \code{RANGE} or \code{FRACTION}. + \item \code{SCALE.MIN(F32)} specifies the minimum scale. + \item \code{SCALE.MAX(F32)} specifies the maximum scale. + \end{itemize} \end{itemize} @@ -1305,4 +1573,9 @@ OVERSCAN.STAT STR MEAN # MEAN | MEDIAN +# Fringe subtraction options +FRINGE.ITER S32 10 # Number of rejection iterations for fringe solution +FRINGE.REJ F32 2.0 # Rejection threshold for fringe solution +FRINGE.KEEP F32 0.5 # Minimum fraction to keep in fringe solution + # binned output image options BIN1.XBIN S32 8 @@ -1310,4 +1583,18 @@ BIN2.XBIN S32 64 BIN2.YBIN S32 64 + +PPIMAGE.JPEG1 METADATA + COLORMAP STR -greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 20.0 +END + +PPIMAGE.JPEG2 METADATA + COLORMAP STR greyscale + SCALE.MODE STR FRACTION + SCALE.MIN STR 0.50 + SCALE.MAX STR 2.00 +END PHOTCODE.RULE STR {CAMERA}.{FILTER.ID}.{CHIP.N} @@ -1332,10 +1619,35 @@ \item \code{MASKVAL(S32)} gives the mask value for input data. \item \code{COMBINE(STR)} gives the statistic to use for combination. -\item \code{BACKGROUND(STR)} gives the statistic to use to measure the background. +\item \code{MEAN(STR)} gives the statistic to use to measure the mean. +\item \code{STDEV(STR)} gives the statistic to use to measure the standard deviation. +\item \code{WEIGHTS(BOOL)} specifies whether image (Poisson) weights should be used in the combination. +\item For combining a fringe: + \begin{itemize} + \item \code{FRINGE.NUM(S32)} specifies the number of fringe regions for fringe combination. + \item \code{FRINGE.SIZE(S32)} specifies the half-size of the fringe regions. + \item \code{FRINGE.XSMOOTH(S32)} specifies the number of smoothing regions in x. + \item \code{FRINGE.YSMOOTH(S32)} specifies the number of smoothing regions in y. + \end{itemize} +\item For generating a shutter correction: + \begin{itemize} + \item \code{SHUTTER.SIZE(S32)} specifies the size for shutter measurement regions. + \item \code{SHUTTER.ITER(S32)} specifies the number of iterations for performing the shutter measurement. + \item \code{SHUTTER.REJECT(F32)} specifies the rejection limit for shutter measurement. + \end{itemize} +\item For generating a bad pixel mask: + \begin{itemize} + \item \code{MASK.SUSPECT(F32)} specifies the threshold for suspect pixels (in standard deviations). + \item \code{MASK.BAD(F32)} specifies the threshold for bad pixels + (in standard deviations); if negative, assume it's something like + a Poisson distribution. + \end{itemize} \end{itemize} -Statistics specified by a string (for \code{COMBINE} and -\code{BACKGROUND}) may be one of \code{MEAN}, \code{MEDIAN}, -\code{ROBUST}, \code{FITTED} or \code{CLIPPED}. +Mean statistics specified by a string (for \code{COMBINE}, +\code{MEAN}) may be one of \code{MEAN}, \code{MEDIAN}, \code{ROBUST}, +\code{FITTED} or \code{CLIPPED}. The standard deviation statistic +(\code{STDEV}) may be one of \code{STDEV}, \code{ROBUST_STDEV}, +\code{FITTED_STDEV}, or \code{CLIPPED_STDEV}. + \subsubsubsection{Example} @@ -1344,16 +1656,26 @@ # Recipe configuration for ppMerge -ROWS S32 128 # Number of rows to read at once +ROWS S32 512 # Number of rows to read at once ELECTRONS F32 100.0 # Minimum number of electrons for useful signal SAMPLE S32 100 # Sampling factor for measuring the background -REJ F32 3.0 # Rejection threshold (sigma) -ITER S32 1 # Number of rejection iterations -FRACHIGH F32 0.3 # Fraction of high pixels to reject immediately -FRACLOW F32 0.1 # Fraction of low pixels to reject immediately -NKEEP S32 5 # Minimum number of pixels in stack to keep -MASKVAL S32 0xff # Mask value for input data -### Statistics options: MEAN | MEDIAN | ROBUST | FITTED | CLIPPED -COMBINE STR MEAN # Statistic to use for combination: -BACKGROUND STR MEDIAN # Statistic to use to measure the background +REJ F32 3.0 # Rejection threshold (sigma) +ITER S32 1 # Number of rejection iterations +FRACHIGH F32 0.0 # Fraction of high pixels to reject immediately +FRACLOW F32 0.0 # Fraction of low pixels to reject immediately +NKEEP S32 5 # Minimum number of pixels in stack to keep +FRINGE.NUM S32 10000 # Number of fringe regions +FRINGE.SIZE S32 5 # Half-size of fringe regions +FRINGE.XSMOOTH S32 5 # Number of smoothing regions in x +FRINGE.YSMOOTH S32 11 # Number of smoothing regions in y +SHUTTER.SIZE S32 128 # Size for shutter measurement regions +SHUTTER.ITER S32 1 # Number of iterations for shutter measurement +SHUTTER.REJECT F32 2 # Rejection limit for shutter measurement +MASK.SUSPECT F32 5.0 # Threshold for suspect pixels (sigma) +MASK.BAD F32 -4.0 # Threshold for bad pixels (sigma) +MASKVAL S32 0xff # Mask value for input data +COMBINE STR CLIPPED # Statistic to use for combination +MEAN STR ROBUST_MEDIAN # Statistic to use to measure the mean +STDEV STR ROBUST_STDEV # Statistic to use to measure the stdev +WEIGHTS BOOL FALSE # Use image weights? \end{verbatim}