Index: /trunk/doc/design/design.tex
===================================================================
--- /trunk/doc/design/design.tex	(revision 552)
+++ /trunk/doc/design/design.tex	(revision 553)
@@ -1,3 +1,3 @@
-%%% $Id: design.tex,v 1.9 2004-04-29 21:30:37 price Exp $
+%%% $Id: design.tex,v 1.10 2004-04-30 03:41:45 price Exp $
 \documentclass[panstarrs]{panstarrs}
 
@@ -22,8 +22,9 @@
 \RevisionsStart
 % version     Date         Description
-DR.01     & 2003.01.01 & First draft  \\ \hline
-DR.02     & 2003.03.05 & Second draft \\ \hline
-DR.03     & 2003.03.25 & Section reorganization \\ 
-DR.04     & 2003.04.13 & Most sections fleshed out \\ 
+DR.01     & 2004.01.01 & First draft  \\ \hline
+DR.02     & 2004.03.05 & Second draft \\ \hline
+DR.03     & 2004.03.25 & Section reorganization \\ \hline
+DR.04     & 2004.04.13 & Most sections fleshed out \\ \hline
+DR.05     & 2004.04.29 & Reorganisation for consistency --- PAP. \\ \hline
 \RevisionsEnd
 
@@ -88,10 +89,4 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\section{Referenced Documents}
-
-This section lists documents referred to by this specification.\\
-
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
@@ -235,5 +230,5 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{OATS}
+\paragraph{OATS}
 
 The Observatory And Telescope System (OATS) is not a part of the IPP,
@@ -245,5 +240,5 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{Pollster}
+\paragraph{Pollster}
 
 The Pollster is a program that polls OATS at regular intervals for the
@@ -263,5 +258,5 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{Metadata DB}
+\paragraph{Metadata DB}
 
 The Metadata DB stores and maintains the metadata\footnote{Note that
@@ -277,5 +272,5 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{Scheduler}
+\paragraph{Scheduler}
 
 The Scheduler is responsible for determining the processing stages
@@ -297,5 +292,6 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{Localiser}
+\paragraph{Localiser}
+\label{sec:localiser}
 
 It is the duty of the Localiser to assign processing stages to
@@ -316,5 +312,5 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{Controller}
+\paragraph{Controller}
 
 The Controller's job is to control the execution of the processing
@@ -328,5 +324,6 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{Pixel DB}
+\paragraph{Pixel DB}
+\label{sec:pixeldb}
 
 The Pixel DB is responsible for storing and maintaining the location
@@ -345,5 +342,5 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{Nodes}
+\paragraph{Nodes}
 
 The Nodes perform the grunt work of executing the processing stages as
@@ -367,5 +364,5 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{Object DB}
+\paragraph{Object DB}
 
 The Object DB is a facility to store all of the information about
@@ -387,5 +384,5 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{CSPs and MOPS}
+\paragraph{CSPs and MOPS}
 
 The Client Science Programs (CSPs) and the Moving Object Processing
@@ -418,5 +415,5 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{Related/Connected components}
+\paragraph{Related/Connected components}
 
 The Pollster may be contained within the Scheduler (i.e., the
@@ -431,5 +428,5 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsubsection{Responsibility}
+\paragraph{Responsibility}
 
 The IPP team will develop and have responsibility for maintaining
@@ -440,4 +437,5 @@
 
 \subsubsection{Processing Stages}
+\label{sec:processingStages}
 
 We now consider the collection of IPP processing stages which are
@@ -472,14 +470,19 @@
 \item Calibration Image Processing Stages
   \begin{enumerate}
-  \item Calibration 1: Basic master-detrend creation --- combination
-    of simple detrend images (e.g., bias, dome flat etc).
-  \item Calibration 2: Sky-model/fringe-mode generation ---
-    combination of more-complicated detrend images (e.g., fringe,
-    scattered light etc).
-  \item Calibration 3: Flat-field correction image creation ---
-    analysis of photometry from multiple dithered FPAs.
+  \item Cal 1: Basic master-detrend creation --- combination of simple
+    detrend images (e.g., bias, dome flat etc).
+  \item Cal 2: Sky-model/fringe-mode generation --- combination of
+    more-complicated detrend images (e.g., fringe, scattered light
+    etc).
+  \item Cal 3: Flat-field correction image creation --- analysis of
+    photometry from multiple dithered FPAs.
   \end{enumerate}
-\item Calibration Test Processing Stage --- tests whether new
-  calibration data are required.
+\item Calibration Test Processing Stage
+  \begin{enumerate}
+    \item CalTest 1: Detrend frame testing --- tests whether new
+      calibration frames are required.
+    \item CalTest 2: Photometric float correction testing --- tests
+      whether a new photometric flat correction is required.
+  \end{enumerate}
 \item Reference Catalog Processing Stages
   \begin{enumerate}
@@ -582,33 +585,21 @@
 \subsubsection{Stages}
 
-The major IPP tasks are organized into stages, which consist of
-multiple modules.  Each stage represents a collection of complex
-operations performed on a single data entity.  Each stage therefore
-represents the maximum amount of effort which can be performed in
-serial without interaction between parallel threads.  The stages will
-be written in \tbd{Python}, linking the modules together.  Examples of
-stages are Phase 2 (detrend images) and Phase 4 (combine images from
-multiple telescopes and search for transients).
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subsubsection{Controllers}
-
-The stages are parallelized by a controller, which initiates the
-stages on separate machines and monitors their progress.  An example
-of the controller functionality is ``Run the phase 2 processing on
-exposure number 1234 using machines 1,3,5,7,9''.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subsubsection{Scheduler}
-
-The scheduler is responsible for interacting with \PS{} systems
-external to the IPP, and for initiating the reduction appropriate for
-images as they are received.  An example of the scheduler
-functionality is ``Retrieve exposure number 1234; run phase 1--4
-controllers on exposure 1234''.
+The major IPP processing tasks are organized into stages, which
+consist of multiple modules.  Each stage represents a collection of
+complex operations performed on a single data entity.  Each stage
+therefore represents the maximum amount of effort which can be
+performed in serial without interaction between parallel threads.  The
+stages will be written in \tbd{Python}, linking the modules together.
+Examples of stages are Phase 2 (detrend images) and Phase 4 (combine
+images from multiple telescopes and search for transients).
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subsubsection{Orchestration}
+
+High-level components such as the Scheduler, the Controller and the
+Localiser are for process control.  As such, they shall be written in
+\tbd{Python} in order to maintain flexibility.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -634,20 +625,35 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
+\subsubsection{Pollster}
+
+The Pollster simply polls OATS on a regular basis for metadata
+(including telescope exposures) which is not known by the IPP (i.e.,
+already written in the Metadata DB).  On the discovery of such metadata,
+it is written to the Metadata DB.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
 \subsubsection{Pixel Server}
 
 The IPP Pixel Server (IPS) is a repository for all image pixel data
-required by the IPP.  Images may reside in the IPS for different
-periods depending on their use and type.  Data stored by the IPS
-include the raw images, the calibration images, intermediate
-processing stage images as needed, final processed images, difference
-images, and image subsections, \tbd{along with the associated
-metadata}.  The IPS must retain images as long as they are needed, up
-to the lifetime of the project.  In order to achieve the I/O
-requirements, the IPS may maintain the pixel data distributed across
-the processor nodes in an organized fashion, i.e.\ associating
-specific machines with specific detectors.  The IPS interacts with the
-IPP Metadata Database to allow other systems or subsystems to identify
-the available images meeting specified criteria.  IPS specifications
-are described in the IPS subsystem specification.
+required by the IPP, and fulfills the roles of the Pixel DB
+(\S\ref{sec:pixeldb}) and the Localiser (\S\ref{sec:localiser}).  In
+addition, it also provides components for managing the distribution of
+data, and accessing the data.
+
+Images may reside in the IPS for different periods depending on their
+use and type.  Data stored by the IPS include the raw images, the
+calibration images, intermediate processing stage images as needed,
+final processed images, difference images, and image subsections,
+\tbd{along with the associated metadata}.  The IPS must retain images
+as long as they are needed, up to the lifetime of the project.  In
+order to achieve the I/O requirements, the IPS may maintain the pixel
+data distributed across the processor nodes in an organized fashion,
+i.e.\ associating specific machines with specific detectors.  The IPS
+interacts with the IPP Metadata Database to allow other systems or
+subsystems to identify the available images meeting specified
+criteria.  IPS specifications are described in the IPS subsystem
+specification.
 
 In addition to storing the pixel data, the IPS is responsible for
@@ -657,200 +663,184 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\paragraph{Pixel Server Components}
-
-The IPP Pixel Server consists of the following components:
+\paragraph{IPP Pixel Server Components}
+
+The IPP Pixel Server (IPS) fulfills the roles of the Pixel DB
+(\S\ref{sec:pixeldb}) and the Localiser (\S\ref{sec:localiser}), and
+consists of the following components:
 
 \begin{enumerate}
-\item IPP Pixel Server Scheduler (IPSS)
 \item IPP Pixel Server Data Locality Optimizer (IPSDLO)
 \item IPP Pixel Server Database (IPSD)
-\item IPP Pixel Server Node Agent (IPSNA)
+\item IPP Pixel Server Maintainance (IPSM)
 \item IPP Pixel Server I/O Library (IPSIOL)
 \end{enumerate}
 
+This assumes that the pixel data will be stored on the nodes.  Each
+image shall have a unique Universal Resource Identifier (URI) which
+specifies the location of the pixel data.  As an example, consider a
+cluster with cross-mounted disks --- in this case, the filename
+incorporating the full path would serve as the URI.
+
+The components of the IPS and their relation to other components (both
+within the IPS and without) are showin in Figure~\ref{fig:ips}.
+
+\begin{figure}
+\psfig{file=pics/IPS,width=15cm,angle=0}
+\caption{The components of the IPS.  In addition to the IPSDLO, IPSD
+and IPSM, the IPSIOL is also a component of the IPS; use of the IPSIOL
+is shown as dotted arrows in the interactions.  Note that the nodes use
+the IPSIOL to pass pixel data between each other.}
+\label{fig:ips}
+\end{figure}
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subparagraph{IPP Pixel Server Scheduler (IPSS)}
-
-The IPSS coordinates the movement of image data and executes batch
-image data management tasks.  The IPSS has four basic modes of
-operation:
-
+\subparagraph{IPP Pixel Server Data Locality Optimizer (IPPDLO)}
+
+Processing stages generated by the Scheduler are passed through the
+IPSDLO which does the following:
+\begin{enumerate}
+\item assigns tasks to specific nodes;
+\item identifies the URI of the required input data; and
+\item identifies the URI the output data should be written to.
+\end{enumerate}
+
+This allows the choice of processing node to be optimized so that it
+resides on the node which will process it, as well as allowing the
+output to be written to the node which requires it for the next
+processing stage.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{IPP Pixel Server Database (IPSD)}
+\label{sec:ipsd}
+
+The IPSD maintains a database of URIs for the pixel data on the nodes.
+It should be able to return the URI of the pixel data given one of:
+\begin{enumerate}
+\item an exposure identifier and a chip identifier (raw and processed
+  pixel data from the telescope);
+\item a calibration identifier (detrend pixel data); and
+\item a sky cell identifier (summed static sky, reduced and difference
+  pixel data).
+\end{enumerate}
+
+The IPSD will also contain a history of data management commands and
+actions.
+
+\tbd{Is there a reason why this is not a part of the Metadata DB?}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{IPP Pixel Server Maintenance (IPSM)}
+
+The IPSM initiates the execution of bulk data management processing
+stages.  It may have an automated component which, e.g., monitors the
+disk space on each of the nodes and redistributes them if they become
+unbalanced.  However, the main intent is that it is used by a human
+operator to reorgainise the data, e.g., after a new data optimisation
+plan has been formulated, or to delete old data.
+
+The IPSM passes processing stages to the Controller which executes
+them on the specified nodes.
+
+The IPSM allows four types of operation:
 \begin{itemize}
-\item Retrieve external data: The IPSS generates {\em retrieve data}
-  tasks which are executed by the IPSNAs on nodes specified by the
-  IPSDLO.  This mode will be used frequently to copy data from the
-  Summit Pixel Server to the IPP nodes for processing.
-\item Delete data: The IPSS looks up the location of the data in the
-  IPP Pixel Data Database and generates {\em delete data} tasks which
-  are executed by the IPSNAs on the appropriate nodes.  This mode will
-  be used on a regular basis to clean old data that is no longer
-  required.
-\item Replicate data: The IPSS generates {\em copy data} tasks which
-  are executed by the IPSNAs on nodes specified either by the
-  ``replicate data'' command, or by the IPPDLO.  This mode differs
-  from the ``copy external data'' mode in that it copies data already
-  within the IPSS.  This mode will be used to backup and rearrange
-  data.
-\item Move data: the IPSS executes a replication followed by a
-  deletion.  This mode will be used to reorganise the storage.
+\item Retrieve external data --- to manually trigger the copying of
+  external data (routine copying of the pixel data from OATS is
+  handled by the Scheduler).  The IPSM generates {\em retrieve data}
+  stages which are passed to the Controller for execution.
+\item Delete data --- to delete old data.  The IPSM looks up the
+  location of the data in the IPSD and generates {\em delete data}
+  stages which are passed to the Controller for execution.
+\item Replicate data --- to backup and rearrange data.  The IPSM
+  generates {\em copy data} stages which are passed to the Controller
+  for execution.  Note that this mode differs from the ``copy external
+  data'' mode in that it copies data already within the IPS.
+\item Move data --- to reorganise storage.  The IPSM executes a
+  replication followed by a deletion.
 \end{itemize}
 
-It is not intended that the IPSS will be used by the nodes in the
-course of processing --- it is only for bulk data management.  ``Copy
-external data'' mode will be used frequently to retrieve data from the
-Summit Pixel Server.  ``Delete data'' mode will be used on a regular
-basis to flush the system of stale files.  It is expected that the
-other modes will be used only occassionally, and initiated by a human
-operator.
-
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subparagraph{IPP Pixel Server Data Locality Optimizer (IPPDLO)}
-
-Data tasks generated by the IPSS are passed through the IPSDLO which
-assigns write tasks to specific nodes.  This allows the location of
-the data to be optimized so that it resides on the node which will
-process it.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subparagraph{IPP Pixel Server Database (IPSD)}
-
-The IPSD contains image data locations \tbd{and the associated
-metadata}.  The IPSD will contain at least:
+\subparagraph{IPP Pixel Server I/O Library (IPSIOL)}
+
+The IPSIOL provides a mechanism for reading and writing pixel data to
+the IPS.  The existence of the IPSIOL insulates the processing stages
+from the details of how the pixel data are stored (i.e., the
+processing stages need not worry whether the data is stored locally or
+remotely).  It will generally be used on the nodes and the IPSDLO,
+although other components will also make use of it.
+
+The IPSIOL will be able to:
 \begin{itemize}
-\item The location of image data and its associated metadata that is
-  available for retrieval from the Summit Pixel Server.
-\item The location of image data and its associated metadata that is
-  yet to be processed by the IPP System.
-\item The location of calibration data and its associated metadata for
-  processing within the IPP System.
-\item The location of reduced image data and its associated metadata as
-  generated by the IPP System.
-\item The location of difference image data and its associated metadata as
-  generated by the IPP System.
-\item The location of stacked image data and its associated metadata as
-  generated by the IPP System.
-\item A history of data management commands and actions.
+\item Open a file specified by a URI --- it may simply open the file
+  if it exists on the particular node, or it may retrieve the file
+  over the network.
+\item Write a file specified by a URI --- it may simply write the file
+  if it exists on the particular node, or it may copy the file over
+  the network.  It should also register with the IPSD that a file
+  specified by a URI has been written.
+\item Delete a file specified by a URI --- it may simply delete the
+  file if it exists on the particular node, or it may delete the file
+  over the network.
+\item Interface with the IPSD to return a URI given one of the
+  identifiers in \S\ref{sec:ipsd}.
 \end{itemize}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subparagraph{IPP Pixel Server Node Agent (IPSNA)}
-
-The IPSNA runs on a node to perform the operations required by the IPSS
-and IPSIOL.  This includes:
-\begin{itemize}
-\item Retrieve data from an external source (e.g.\ the Summit Pixel
-  Server) to a local disk as requested by the IPSS.
-\item Copy data from one of the other nodes to a local disk as
-requested by the IPPS.
-\item Delete data from a local disk as requested by the IPSS or
-  through the IPSIOL.
-\item Respond to requests for data made by nodes through the IPSIOL.
-\end{itemize}
-
-\tbd{The Agent does not wear a suit, nor does it know kung fu.}
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subparagraph{IPP Pixel Server I/O Library (IPSIOL)}
-
-The IPSIOL is the workhorse of the IPP Pixel Server system.  It is a
-library for reading and writing pixel data to the IPP Pixel Server.
-It will generally be used on the nodes, although the IPSS will also
-make use of it.  The IPSIOL will be able to:
-\begin{itemize}
-\item Lookup the location of new and reduced data for an exposure.
-\item Lookup the location of the appropriate calibration data for an
-  exposure.
-\item Open a file at the location returned by a lookup.
-\item Write new data and metadata to a specified location.
-\item Update the storage location and/or metadata of any data.
-\item Remove the storage location of data and metadata that has been
-deleted.
-\end{itemize}
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Pixel Data Flow}
-
-Below we sketch out the intended sequence of events for common
-operations.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subparagraph{Acquisition of data from the Summit Pixel Server}
-
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Pixel Data Flow Examples}
+
+For examples of the operation of the IPS, below we sketch out the
+intended sequence of events for common operations.
+
+Reads during processing:
 \begin{enumerate}
-\item The Summit Pixel Server sends a ``new data notification'' to the
-IPSS.
-\item The IPSS generates the {\em retrieve data} tasks which are to be
-executed on specific nodes (i.e.\ those which will reduce the raw
-data).
-\item Each specified node spawn IPSDRAs which downloads the image data
-from the Summit Pixel Server to the disk physically mounted on the
-node.
-\item The node reports the finished task to the IPSS.
-\item The IPSS updates the IPSD to the new storage location.
-\item The IPSS notifies the IPP Scheduler that new
-data is available.
+\item A processing stage has been passed (from the Scheduler) the URI
+  for an image that it needs to load into memory.
+\item The processing stage uses the IPSIOL to open the image.
+\item The processing stage reads the image into local memory in the
+  usual manner.
+\item The processing stage closes the image using the IPSIOL.
 \end{enumerate}
 
-\begin{figure}
-\begin{center}
-%\resizebox{!}{20cm}{\includegraphics{data_stack8.epsi}}
-\caption{ \label{acquisition} Pixel Data Flow: Acquisition}
-\end{center}
-\end{figure}
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subparagraph{Processing Reads}
-
+Writes during processing:
 \begin{enumerate}
-\item A processing stage needs pixel data, e.g.\ the appropriate
-flat-field for an image being processed.
-\item The processing stage uses the IPSIOL to look up the location of
-the appropriate image.
-\item The processing stage retrieves the required pixel data using the
-IPSIOL and loads it into local memory.
+\item A processing stage has been passed (from the Scheduler) the URI
+  for an image that needs to be saved, e.g., a subtracted image.
+\item The processing stage uses the IPSIOL to open the image.
+\item The processing stage writes the image in the usual manner.
+\item The processing stage closes the image using the IPSIOL.
 \end{enumerate}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subparagraph{Processing Writes}
-
+Note how the IPSIOL has insulated the processing stage from the details
+of the reading and writing.
+
+Maintenance:
 \begin{enumerate}
-\item A processing stage has produced pixel data which should be saved, e.g.\ the
-subtracted image.
-\item The processing stage uses the IPSIOL to look up the location the
-image should be written to.
-\item The processing stage uses the IPSIOL to write the image.
+\item A human operator decides that all the pixel data for chip 12
+  should be stored on node 3.
+\item Operator plugs this into the IPSM.
+\item The IPSM queries the IPSD using the IPSIOL.
+\item The IPSD returns the URIs for all the pixel data for chip 12.
+\item The IPSM generates processing tasks to be executed on the nodes
+  that will copy the data from the old URIs to a new URI which
+  specifies node 3.
+\item The IPSM generates processing tasks to be executed on the nodes
+  that deletes the data pointed to by the old URIs.
+\item The IPSM reports success to the operator.
 \end{enumerate}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subparagraph{Processing Updates}
-
+Client Science Pipelines:
 \begin{enumerate}
-\item A processing stage needs to update pixel data, e.g.\ the
-static sky image.
-\item The processing stage uses the IPSIOL to look up the location of
-the appropriate image.
-\item The processing stage retrieves the required pixel data using the
-IPSIOL and loads it into local memory.
-\item The processing stage modifies the pixel data in local memory.
-\item The processing stage uses the IPSIOL to write the image to the
-previous location with an overwrite flag.
+\item A CSP wants some pixel data.
+\item The CSP queries the IPSD using the IPSIOL (e.g., asking for a
+  particular exposure or sky cell).
+\item The IPSD returns the URI for the pixel data.
+\item The CSP opens the image using the IPSIOL and the URI.
+\item The CSP reads the pixel data into memory in the usual manner.
+\item The CSP closes the image using the IPSIOL.
 \end{enumerate}
-
-\begin{figure}
-\begin{center}
-%\resizebox{!}{20cm}{\includegraphics{data_processing1.epsi}}
-\caption{ \label{processing} Pixel Data Flow: Processing}
-\end{center}
-\end{figure}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -921,5 +911,6 @@
 \multicolumn{2}{l}{\bf Weather} \\
 Time & The time the weather information was measured. \\
-Temperature & The temperature at \tbd{some place.  Will likely want temperatures for a range of locations: external, dome, secondary, primary for starters.} \\
+Temperature & The temperature at \tbd{some place.  Will likely want temperatures for a range of locations:
+external, dome, secondary, primary for starters.} \\
 Humidity & The relative humidity. \\
 Pressure & The (external) atmospheric pressure. \\
@@ -1269,4 +1260,6 @@
 \paragraph{Metadata Queries}
 
+\tbd{How is the Metadata DB queried?}
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -1279,12 +1272,5 @@
 associated with specific input images, moving objects associated with
 specific chips.  Detailed requirements for the IOD are described in
-the IOD subsystem specification document xxx-xxx-xxxx.
-
-Reference Astrometry Catalogs:
-USNO-B
-2MASS
-HST-GSC
-Tycho
-etc?
+\tbd{the IOD subsystem specification document xxx-xxx-xxxx}.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -1313,8 +1299,12 @@
 \paragraph{Object DB Table Contents}
 
+\tbd{Dunno yet}
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 \paragraph{Object DB Queries}
 
+\tbd{Dunno yet}
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -1322,109 +1312,118 @@
 \subsubsection{Controller}
 
-The IPP Controller is responsible for connecting the low-level modules
-together to define the various processing subsystems.  The Controller
-manages the parallel processing of these subsystems in the IPP
-computer hardware environment and reports the processing status to the
-IMD.  The Controller must be able to manage more than a single
+The IPP Controller is responsible for managing the processing stages.
+The Controller manages the parallel processing of these stages in the
+IPP computer hardware environment and reports the completion to the
+Scheduler.  The Controller must be able to manage more than a single
 processing thread to make maximum use of available processor
-resources.  Some analysis jobs, such as operations on the chips, must
-be allocated preferentially to specified processors, while others must
-be distributed to the available machines in the cluster.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Components}
-
-The Controller consists of the following components: the Controller
-daemon, the remote clients, and the user clients.
-
-The Controller daemon maintains a table of processing nodes available
-to it and the status of those nodes.  When the controller daemon
-starts, it attempts to launch a remote client on each of the available
-processing nodes.  Processing nodes which are not responsive are
-placed into an inactive state and retried occasionally.  
-
-The Controller daemon also maintains three tables of processing jobs:
-pending jobs, active jobs, and completed jobs.  The pending jobs are
-those which have not yet been performed.  The active jobs are those
+resources.
+
+The Controller must honour demands that a processing stage run on a
+particular Node.  Requests that a processing stage run on a particular
+node should be honoured if possible.  Where no restriction is placed
+on the choice of Node choice by the Scheduler, the processing stage
+may be run on any available Node.
+
+The Controller maintains a table of processing nodes available to it
+and the status of these Nodes.  When the Controller starts, it
+attempts to launch a Node Agent on each of the available processing
+nodes.  Modes which are not responsive are placed into an inactive
+state and retried occasionally.
+
+The Controller also maintains three tables of processing jobs: pending
+stages, active stages, and completed stages.  The pending stages are
+those which have not yet been performed.  The active stages are those
 currently being performed on one of the remote nodes.  The completed
-jobs are those which have finished, either successfully or with an
+stages are those which have finished, either successfully or with an
 error state.  The Controller daemon monitors the collection of remote
-clients and sends them new pending jobs when they become free.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Remote Clients}
-
-The remote clients communicate with the Controller daemon via a socket
-connection.  They execute jobs upon request by the controller.  A job
-is executed in the UNIX user space, and is run as a fork by the remote
-client.  The remote client must monitor the standard error and
-standard output of the job and save them in separate buffers.  If the
-process dies, the remote client must detect the crash.  The remote
-client must respond to various commands from the controller daemon.
-The commands include:
-
-{\bf \em report status}: Return the state of the client (idle, busy,
-done), the state of the current job\footnote{Note that a job is
-considered ``current'' until it is cleared with {\em clear job} ---
-even if it has crashed or completed.} (`none', `busy', `crash',
-`done'), and the exit status of the current job (`none', 0--256).  The
-three states of the client indicate that the client has no current job
-(`idle'), that it has a job which is still running (`busy'), and that
-it has a job which has completed.  The job states indicate the there
-is no current job (`none'), that the current job is running (`busy'),
-that the current job has crashed (`crash'), and that the current job
-has exited gracefully (`done').  The exit state is the exit state
-reported by the job (0--256 with 0 indicating a successful completion)
-or is an indication that there is no current job (`none').
-
-{\bf \em report stdout}: Send and flush the current stdout buffer.  The
-remote client will return the complete contents of the stdout buffer
-via a buffered write and flush the buffer when it is finished.  The
-remote client will not accept more data on the stdout buffer from the
-current job until the send is complete and the buffer is flushed.  The
-daemon must accept all of the buffer output.
-
-{\bf \em report stderr}: Identical to `report stdout' for stderr.  
-
-{\bf \em kill job}: remote client should send a kill signal to the
-current job.  When the job has exited, the remote client should set
-the job status to `crash' and the client status to `done'.
-
-{\bf \em clear job}: The remote client should set the current job state
-to `none' and the client state to `idle'.  If a job is currently
-running, it should be killed before the job is cleared.
-
-{\bf \em start job [command]}: execute the given command.  The command
-should be a standard unix command without command line redirection or
-backgrounding.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{User Clients}
-
-The user clients send commands and jobs to the controller.  The user
-clients interact with the Controller daemon via a socket.  The user
-clients, which may be subsystems external to the Controller, interact
-with the Controller daemon via the socket connection using a defined
-set of commands.  The user clients can send new jobs to the controller
-daemon, monitor the current job tables, obtain status information on
-the completed jobs, change the list of available processing nodes, and
-send kill commands for specific jobs to the remote clients.
-
-{\bf \em new job} The new jobs are sent to the controller in the form
-of UNIX commands, along with optional specified processing nodes.  If
-the processing node is not specified, then the controller will select
-a node as one becomes available.  
-
-{\bf \em kill job} The user client may kill an existing
-job. \tbd{allow clients to kill jobs sent by other clients? how does
-the client specify the job to be killed?  is this a necessary
-function?}
-
-{\bf \em get status} The user client may request the current status of
-the controller, including the list of pending, active, and completed
-jobs and the status of the individual jobs.
+clients and sends them new pending stages when they become free.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subsubsection{Node Agents}
+
+A Node Agent runs on each of the individual nodes to perform the
+processing stages as directed by the Controller.  The Node Agents
+communicate with the Controller via a socket connection.
+
+A processing stage is executed in the UNIX user space, and is run as a fork by the
+Node Agent.  The Node Agent must monitor the standard error and
+standard output of the processing stage and save them in separate buffers.  If the
+process dies, the Node Agent must detect the crash.  The Node Agent
+must respond to various commands from the Controller.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Report status}
+
+The Node Agent returns the state of the Node (idle, busy, done), the
+state of the current processing stage\footnote{Note that a processing
+stage is considered ``current'' until it is cleared with {\em clear
+processing stage} --- even if it has crashed or completed.} (`none',
+`busy', `crash', `done'), and the exit status of the current
+processing stage (`none', 0--256).
+
+The three states of the Node indicate that the client has no current
+processing stage (`idle'), that it has a processing stage which is
+still running (`busy'), or that it has a processing stage which has
+completed.
+
+The processing stage states indicate the there is no current
+processing stage (`none'), that the current processing stage is
+running (`busy'), that the current processing stage has crashed
+(`crash'), or that the current processing stage has exited gracefully
+(`done').  The exit state is the exit state reported by the processing
+stage (0--256 with 0 indicating a successful completion) or is an
+indication that there is no current processing stage (`none').
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Report stdout}
+
+Send and flush the current stdout buffer.  The Node Agent will return
+the complete contents of the stdout buffer via a buffered write and
+flush the buffer when it is finished.  The Node Agent will not accept
+more data on the stdout buffer from the current processing stage until
+the send is complete and the buffer is flushed.  The daemon must
+accept all of the buffer output.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Report stderr}
+
+Identical to `report stdout', but for stderr.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Kill processing stage}
+
+The Node Agent should send a kill signal to the current processing
+stage.  When the processing stage has exited, the Node Agent should
+set the processing stage status to `crash' and the Node status to
+`done'.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Clear processing stage}
+
+The Node Agent should set the current processing stage state to `none'
+and the Node state to `idle'.  If a processing stage is currently
+running, it should be killed before the processing stage is cleared.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Start processing stage}
+
+The Node Agent forks a specified command.  The command should be a
+standard UNIX command without command line redirection or
+backgrounding.  For this reason, the Node Agent must provide a layer
+of security, for example, by employing SSL authentication.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Matrix}
+
+\tbd{The Node Agent does not wear a suit, nor does it know kung fu.}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -1433,23 +1432,93 @@
 \subsubsection{Scheduler}
 
-The IPP Scheduler is responsible for coordinating the IPP subsystems
-and for initiating the various processing systems, executed by the IPP
-Controller, based on the state of the survey as reflected by the IPP
-Metadata Database (IMD).  The Scheduler must send calibration data
-requests to the PTS, including required flat-field images, flat-field
-correction observations, or other specialized observations needed to
-improve the calibrations.  The Scheduler must balance the need for
-improved calibrations with the need to process the science images in a
-timely manner given the capabilities of the science pipelines.
-
-\tbd{how are the schedules defined? how are dependencies between jobs
-defined? scheduler must communicate with the controller (as a user
-client) to send new jobs}.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subsection{Analysis Stages}
+The IPP Scheduler is responsible for initiating the various processing
+stages (which are executed by the IPP Controller), based on the state
+of the survey as reflected by the IPP Metadata Database (IMD).
+
+The Scheduler shall maintain a list of processing stages, as well as
+the required input and dependencies for each of the processing stagesFor example, the
+dependencies for copying pixel data from OATS may be:
+\begin{itemize}
+\item OATS has new pixel data available;
+\item The new pixel data has not been copied.
+\end{itemize}
+Similarly, the dependencies for executing Phase 2 processing on a chip
+may be:
+\begin{itemize}
+\item The chip pixel data has been copied.
+\item Phase 1 has run successfully on the metadata for the FPA to which
+  the chip belongs.
+\item A reduced image (i.e., output from Phase 2) does not already
+  exist.
+\end{itemize}
+
+When the dependencies are satisfied, the Scheduler shall prepare for
+execution the particular processing stage on the appropriate data.
+The Scheduler must query the Metdata DB for the most appropriate
+calibration data, if required.  The processing stage should be
+filtered through the IPSDLO in order to assign the processing stage to
+a particular Node (if desired) and to determine the URIs for the
+required inputs.  The processing stage is then passed to the
+Controller.
+
+The Scheduler must also be able to send requests for new calibration
+data to OATS, including required flat-fields, flat-field correction
+observations, or other specialized observations needed to improve the
+calibrations.  The Scheduler must balance the need for improved
+calibrations with the need to process the science images in a timely
+manner given the capabilities of the science pipelines.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subsubsection{System UI}
+
+A user interface allows a human operator to monitor the Controller and
+Scheduler through some user interface (UI).  The System UI may
+interact with the Controller and Scheduler via a socket connection
+using a defined set of commands.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Execute processing stage}
+
+A new processing stages is sent to the Scheduler.  The Scheduler may
+filter the processing stages through the IPSDLO, or it may be
+prevented from doing so by the user.  The Scheduler then passes the
+processing stages to the Controller for execution.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Kill processing stage}
+
+The user may kill an existing processing stage.  The Controller is
+commanded to kill the particular processing stage.
+
+\tbd{Should we allow a System UI to kill processing stages sent by
+other System UIs?}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Get status}
+
+The System UI may request the current status of the Controller,
+including the list of pending, active, and completed processing stages
+and the status of the individual processing stages.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Available Nodes}
+
+The System UI may view and configure the list of Nodes available to
+the Controller (e.g., to remove a Node temporarily for maintenance).
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subsection{Processing Stages}
+
+In this section, we review the processing stages which are executed on
+the Nodes.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -1458,47 +1527,43 @@
 \subsubsection{Overview}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Science Image Pipelines}
-
-The IPP science image pipelines perform analyses on the night-sky
-science images to extract the science data from these images.  These
-consist of: Phase 1, the image processing preparation stage; Phase 2,
-the image reduction stage; Phase 3, the exposure analysis stage; and
-Phase 4, the image combination stage.  These pipelines must process
-the images in a timely manner so that the incoming data stream will
-not overload the IPS.  The decision to execute a specific pipeline for
-a specific dataset is made by the Scheduler, which sends the
-infomation to the Controller.  The Controller executes the pipeline
-for the data on an appropriate machine and monitors the success or
-failure of the job.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Calibration Image Pipelines}
-
-The IPP Calibration Image Pipelines perform the tasks needed to
-generate high-quality calibration images from the input image
-dataset.  These operations may be performed on whatever timescales are
-appropriate and necessary to maintain the quality and relevance of the
-calibration images.  There are four distinct types of calibration
-image pipelines:  the basic detrend creation pipeline, the photometric
-correction image creation pipeline, the fringe pattern generation
-pipeline, and the sky foreground pattern generation pipeline.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Reference Catalog Pipelines}
-
-The IPP reference catalog pipelines use the data in the IPP Metadata
-Database and the IPP Object Database to determined improved
-astrometric and photometric calibration references.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subsubsection{Phase 1 : image processing preparation}
-
-Phase 1 : image processing preparation
+The processing stages are the software that process data.  These
+processing stages are divided into five categories which are
+summarised in \S\ref{sec:processingStages}.  Each of the processing
+stages are described below.
+
+The processing stages are initiated by the Scheduler, parallized and
+managed by the Controller, and executed through the Node Agents on the
+nodes.  Processing stages are purely serial, and so they may be run on
+a single node at once without the need for interprocess communication.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subsubsection{Retrieval}
+
+The retrieval stages simply retrieve pixel data from an external
+source (ordinarily OATS at the Summit, but it could conceivably be
+some other external source) and store it on the nodes.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subsubsection{Science Image Processing}
+
+The IPP science image processing stages perform analyses on the
+night-sky science images to extract the science data from these
+images.  These consist of: Phase 1, the image processing preparation
+stage; Phase 2, the image reduction stage; Phase 3, the exposure
+analysis stage; and Phase 4, the image combination stage.  These
+pipelines must process the images in a timely manner so that the
+incoming data stream will not overload the IPS.  The decision to
+execute a specific pipeline for a specific dataset is made by the
+Scheduler, which sends the infomation to the Controller.  The
+Controller executes the pipeline for the data on an appropriate
+machine and monitors the success or failure of the processing stage.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Phase 1: image processing preparation}
 
 The Phase 1 system operates on data from each FPA to calculate basic
@@ -1536,7 +1601,6 @@
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subsubsection{Phase 2 : image reduction : new version}
+
+\paragraph{Phase 2 : image reduction : new version}
 
 \tbd{how long are processed images kept?}
@@ -1548,4 +1612,12 @@
 \tbd{what is the absolute astrometry accuracy at phase 2? 0.1 arcsec
 == 0.33 pix?}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Concept}
+
+Phase~2 processing within the \PS{} image processing pipeline is
+the de-trend stage, where the images from the detector are processed
+to remove instrumental signatures.
 
 \begin{figure}
@@ -1555,13 +1627,4 @@
 \end{center}
 \end{figure}
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Phase 2 Concept}
-
-Phase~2 processing within the \PS{} image processing pipeline is
-the de-trend stage, where the images from the detector are processed
-to remove instrumental signatures.  Phase~2 processing is purely serial,
-and so each can be run on a single node from start to finish.
 
 Prior to Phase~2, the Phase~1 process operates on an entire telescope
@@ -1588,7 +1651,7 @@
 These modules are each explained below.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Form OT Kernel}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Form OT Kernel}
 
 The first module for Phase~2 is to form the OT kernel from the image
@@ -1597,7 +1660,7 @@
 used to convolve by.  The output is the OT convolution kernel.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Convolve de-trend images}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Convolve de-trend images}
 
 This module convolves the de-trend images with the OT convolution kernel
@@ -1623,7 +1686,7 @@
 Each of these will be used for a later module.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Overscan Subtraction}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Overscan Subtraction}
 
 This module corrects the object exposures for the electronic pedestal
@@ -1651,7 +1714,7 @@
 These will be used for a subsequent module.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Trim}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Trim}
 
 This module trims the object image and each of the calibration frames to
@@ -1672,7 +1735,7 @@
 modules.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Non-Linearity Correction}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Non-Linearity Correction}
 
 This module corrects images for non-linearity in the detector.  The
@@ -1688,7 +1751,7 @@
 is the corrected object image, which is used for a later module.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Flat field}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Flat field}
 
 This module corrects the object image for variations in sensitivity over
@@ -1709,7 +1772,7 @@
 Both of these will be used in later modules.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Subtract sky}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Subtract sky}
 
 This module subtracts the sky background from the object image.  The
@@ -1731,7 +1794,7 @@
 which is used for the next module.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Identify CRs by morphology}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Identify CRs by morphology}
 
 This module identifies cosmic rays (or other hot pixels missed in the
@@ -1751,7 +1814,7 @@
 which is sent to the IPP Pixel Server.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Find objects}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Find objects}
 
 This module finds objects on the object image.  The inputs are:
@@ -1768,7 +1831,7 @@
 object image.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Bright object postage stamps}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Bright object postage stamps}
 
 This module saves postage stamps of bright objects, so that extra care
@@ -1786,7 +1849,7 @@
 the IPP Pixel Server.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Metadata}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Metadata Required}
 
 The following metadata associated with the images are required for
@@ -1806,7 +1869,7 @@
 \end{itemize}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Pixel Masks}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Pixel Masks}
 \label{ap:masks}
 
@@ -1829,7 +1892,7 @@
 affect the flux in neighbouring pixels
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Object Catalogs}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Object Catalogs}
 \label{ap:catalogs}
 
@@ -1854,14 +1917,6 @@
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subsubsection{Phase 3 : exposure analysis}
-
-\begin{figure}
-\begin{center}
-\resizebox{8cm}{!}{\includegraphics{pics/phase3}}
-\caption{ \label{phase3} Phase 3 dataflow}
-\end{center}
-\end{figure}
+
+\paragraph{Phase 3 : exposure analysis}
 
 The Phase 3 system operates on the combined Phase 2 results from an
@@ -1881,4 +1936,11 @@
 \end{itemize}
 
+\begin{figure}
+\begin{center}
+\resizebox{8cm}{!}{\includegraphics{pics/phase3}}
+\caption{ \label{phase3} Phase 3 dataflow}
+\end{center}
+\end{figure}
+
 In the Phase 2 analysis, the astrometric solutions were determined
 independently for each chip.  These solutions are limited by the
@@ -1914,7 +1976,17 @@
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subsubsection{Phase 4 : image combination}
+
+\paragraph{Phase 4 : image combination}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Phase 4 Concept}
+
+Phase 4 processing within the \PS{} image processing pipeline is
+the final stage of processing for a science image.  It operates on
+each sky cell that has overlapping imaging data from the exposure(s)
+being processed, and produces the main output image data products of
+the pipeline --- the difference images and a deep static sky image ---
+along with the associated catalogs of static and variable sources.
 
 \begin{figure}
@@ -1924,15 +1996,4 @@
 \end{center}
 \end{figure}
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Phase 4 Concept}
-
-Phase 4 processing within the \PS{} image processing pipeline is
-the final stage of processing for a science image.  It operates on
-each sky cell that has overlapping imaging data from the exposure(s)
-being processed, and produces the main output image data products of
-the pipeline --- the difference images and a deep static sky image ---
-along with the associated catalogs of static and variable sources.
 
 Prior to Phase 4, the Phase 3 process produces the following products:
@@ -1951,7 +2012,7 @@
 These modules are each explained below.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Combine Images}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Combine Images}
 
 The first module for Phase 4 is to combine the images from each
@@ -1995,7 +2056,7 @@
 \end{enumerate}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Identify Sources}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Identify Sources}
 
 This module identifies sources in the combined sky cell image.  The
@@ -2008,7 +2069,7 @@
 the IPP Object Database.
  
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Transient Identification}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Transient Identification}
 
 This module identifies variable/moving sources.  The inputs are:
@@ -2057,7 +2118,7 @@
 \end{enumerate}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Add to Static Sky}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Add to Static Sky}
 
 This module adds the combined sky cell image into the static sky, so
@@ -2090,7 +2151,7 @@
 \end{enumerate}
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\paragraph{Notes}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Notes}
 
 \begin{itemize}
@@ -2108,17 +2169,29 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsection{Basic detrend image creation}
+\paragraph{Calibration Image Processing}
+
+The IPP Calibration Image Pipelines perform the tasks needed to
+generate high-quality calibration images from the input image
+dataset.  These operations may be performed on whatever timescales are
+appropriate and necessary to maintain the quality and relevance of the
+calibration images.  There are four distinct types of calibration
+image pipelines:  the basic detrend creation pipeline, the photometric
+correction image creation pipeline, the fringe pattern generation
+pipeline, and the sky foreground pattern generation pipeline.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Cal 1: Basic detrend image creation}
 
 The basic detrend image creation pipeline collects the appropriate
-input detrend images (bias, dark, flat, etc?) and generates a master
-image by combining the input images in some optimal way
+input detrend images (bias, dark, dome flat, etc) and generates a
+master image by combining the input images in some optimal way
 \tbd{median/sigma-clipping/etc}.  The master image is used to
 determine input image residuals so that poor input images can be
 iteratively rejected.
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subsubsection{Fringe pattern and sky foreground model creation}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Cal 2: Fringe pattern and sky foreground model creation}
 
 The fringe model creation and sky foreground model creation pipelines
@@ -2129,10 +2202,9 @@
 structure: both require processing of the input images, both determine
 a set of principal components as a function of specific input
-parameters.  
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-\subsubsection{Photometric flat correction image creation}
+parameters.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{Cal 3: Photometric flat correction image creation}
 
 The photometric flat-field correction uses images which have been
@@ -2146,4 +2218,68 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
+\paragraph{Calibration Test Processing}
+
+The calibration test processing tests observations to determine if the
+calibrations need updating.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{CalTest 1: Detrend frame testing}
+
+A newly-acquired master detrend frame, having been combined (using Cal
+1 or Cal 2) are simply differenced from the old detrend frames.  If
+there exist significant residuals, the newly-acquired detrend frame
+is adopted as the detrend frame of choice.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{CalTest 2: Photometric flat correction testing}
+
+Newly-acquired photometry of many objects (initially, this may be
+standard star fields, but once the PS1 catalog is available, it should
+be possible to use all photometry acquired over a given time period)
+are compared with previously-acquired photometry.  If there exist
+significant residuals, a new photometric flat correction should be
+produced from the newly-acquired photometry.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\paragraph{Reference Catalog Processing}
+
+The IPP reference catalog pipelines use the data in the IPP Metadata
+Database and the IPP Object Database to determined improved
+astrometric and photometric calibration references.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{AstroRef: Astrometric Reference Catalog creation}
+
+This processing stage shall use many observations over a given time
+period to fit a consistent global astrometric solution, resulting in a
+high quality and internally-consistent astrometric catalog that may be
+published.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subparagraph{PhotoRef: Photometric Reference Catalog creation}
+
+This processing stage shall use many observations over a given time
+period to fit a consistent global photometric solution, resulting in a
+high quality and internally-consistent photometric catalog that may be
+published.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\subsection{Reference Catalogs}
+
+The IPP will employ reference catalogs in order to calibrate the
+photometry and astrometry.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
 \subsubsection{Astrometric Reference Catalog}
 
@@ -2162,5 +2298,5 @@
 sufficient.
 
-For PS4, the PS1 catalogue shall be used.
+For PS4, the PS1 catalog shall be used.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -2170,4 +2306,6 @@
 \subsection{Modules}
 
+\tbd{What goes here?  There will be modules?}
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -2176,4 +2314,6 @@
 \subsection{\PS{} Library}
 
+See PSDC-430-007 for the design of the \PS{} Library, PSLib.
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -2181,4 +2321,6 @@
 
 \subsection{Internal Interfaces}
+
+\tbd{To be updated and expanded.}
 
 Internal interfaces consist of queries to the IMD or IPS, insertion of
@@ -2208,11 +2350,13 @@
 \subsection{External Interfaces}
 
+\tbd{This whole section to be updated.}
+
 This subsection describes the interfaces between the IPP and other
 \PS{} systems and the external clients.  The interfaces are
-illustrated in Figure \tbd{NN}.  Incoming data is received by either
-the IPS (pixels), the IMD (metadata), or the IOD (objects).  Requests
-for data by external clients are also made to these three databases.
-Requests for data made by the IPP are generated by the IPP Scheduler
-or the science processing pipelines.
+illustrated in Figure~\ref{fig:functionalities}.  Incoming data is
+received by either the IPS (pixels), the IMD (metadata), or the IOD
+(objects).  Requests for data by external clients are also made to
+these three databases.  Requests for data made by the IPP are
+generated by the IPP Scheduler or the science processing pipelines.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%