Index: /trunk/doc/Makefile =================================================================== --- /trunk/doc/Makefile (revision 6032) +++ /trunk/doc/Makefile (revision 6033) @@ -1,5 +1,5 @@ # Makefile for all IPP main engineering documents -DIR = pslib modules design hardware pantasks psphot misc +DIR = pslib modules design hardware ipptools pantasks psphot misc all: Index: /trunk/doc/dvo/.cvsignore =================================================================== --- /trunk/doc/dvo/.cvsignore (revision 6033) +++ /trunk/doc/dvo/.cvsignore (revision 6033) @@ -0,0 +1,1 @@ +*.log *.dvi *.aux *.toc *.log *.out *.lof *.tbr *.tbd *.pdf Index: /trunk/doc/dvo/Makefile =================================================================== --- /trunk/doc/dvo/Makefile (revision 6033) +++ /trunk/doc/dvo/Makefile (revision 6033) @@ -0,0 +1,29 @@ + +PDFLATEX = env TEXINPUTS=.:LaTeX:$(TEXINPUTS): pdflatex +PSLATEX = env TEXINPUTS=.:LaTeX:$(TEXINPUTS): latex + +help: + @echo "USAGE: make (target)" + @echo " targets: dvo all" + +dvo: dvo.pdf +all : dvo + +%.pdf: %.tex + $(PSLATEX) $*.tex + $(PSLATEX) $*.tex + dvips -z -t letter -o $*.ps $*.dvi + ps2pdf $*.ps $*.pdf + thumbpdf --modes=dvips $*.pdf + $(PSLATEX) $*.tex + dvips -z -t letter -o $*.ps $*.dvi + ps2pdf $*.ps $*.pdf + @rm -f $*.ps $*.dvi $*.aux $*.log $*.tbr $*.tbd $*.toc $*.tpm $*.lof body.tmp head.tmp + +clean : + $(RM) *.log *.dvi *.aux *.toc *.tbd *.tbr *.tpm *.lof *.out *~ core body.tmp head.tmp + +dist : clean + $(RM) *.pdf + +empty: clean Index: /trunk/doc/dvo/dvo.tex =================================================================== --- /trunk/doc/dvo/dvo.tex (revision 6033) +++ /trunk/doc/dvo/dvo.tex (revision 6033) @@ -0,0 +1,1297 @@ +\documentclass[panstarrs,spec]{panstarrs} + +\title{DVO : the Desktop Virtual Observatory} +\subtitle{Astronomical Object Databasing in the IPP} +\author{Eugene Magnier} +\audience{IPP} +\shorttitle{PanTasks for IPP} +\group{Pan-STARRS IPP} +\project{Pan-STARRS IPP} +\organization{Institute for Astronomy} +\version{DR} +\docnumber{PSDC-xxx-xxx} + +\begin{document} +\maketitle + +\tableofcontents +\pagebreak +\pagenumbering{arabic} + +\tbd{substantial discussion of the photcodes and the photometry + transformation process} + +\section{Overview} + +DVO, the Desktop Virtual Observatory, is a software system which +stores data related to astronomical objects derived from various +sources, and provides mechanisms to related multiple detections +together as astronomical objects. DVO deals with two related +concepts: {\em objects} and {\em detections}. The {\em objects} are +descriptions of astronomical objects while the {\em detections} are +the specific measurements of those objects, typically measured from +astronomical images. A collection of {\em detections} may be used to +derive average quantities which describe a particular {\em object}. A +third class of measurement to be considered are those supplied by +external references. Such measurements may be treated as {\em +detections}, with the caveat that access to the raw measurements and +metadata are usually unavailable: the reported measurements and errors +must be accepted as they are reported. + +DVO stores the collections of detections which were derived from +specific images. It provides a mechanism to determine the image from +which a specific detection was derived, and in conjunction with the +Image Server locate the corresponding data file. DVO also makes it +possible to extract all detections derived from a specific image and +to determine quantities such as the pixel coordinates of the detection +on the image. + +DVO also has the capability to associate multiple detections of a +specific object. Several major classes of objects will be present, +each of which must be handled correctly. DVO distinguished the +following types of objects. + +{\bf Stars, compact galaxies, and QSOs} will have nearly fixed +locations relative to other distant stars, with only small deviations +for individual measurements. The association between multiple +detections of such objects is made on the basis of their coincident +positions. DVO determines the average position of the object and the +deviations of the individual detections from that average on the basis +of the ensemble of individual detection. + +{\bf Solar System Objects} do not have a fixed location. Detections +of such objects are linked by their orbits, and depend on both the +position and the time of the image. DVO does not attempt to make this +link; this is the role of the MOPS system. However, it has the +ability to accept identifications made externally with specified +detections and to return the identifier of the moving object +associated with the specific detections. These associations also +include descriptive information such as the offset of the detection +from the predicted location of the detection based on the orbit. This +functionality is required to allow DVO to ignore known moving object +detections from other types of queries. + +{\bf High-proper-motion objects} in the general vicinity of the solar +system fall in between these first two classes of objects. Their +proper motion and parallax response is significant enough ($>0.2$ +arcsec in 1 year) that they are not well-described by an average +location and a collection of offsets. These objects are better +described by a distance and a proper motion vector. DVO provides the +association between the specific detections and an average object +which includes finite parallax and proper motion. + +{\bf Orphaned detections} are not associated with a specific +astronomical object of any of the above classes. Most of these will +be spurious (not representing real objects), some will be from solar +system objects for which orbits are not yet determined, some will be +from faint stars near the detection limits, and some will be from +short-term transients which have only been detected once. DVO +maintains these detections until they have been associated with one of +the objects above. DVO provides mechanisms by which individual +detections may be migrated back and forth between the orphan state and +association with an astronomical object. + +DVO stores the information about the detection, the related objects, +and the images which provided the measurements. For every detection, +DVO provides the mechanisms to link the detection back to the image +which supplied it. DVO also provides the capability to determine the +images containing a specific location but for which no detection was +made. The minimum set of information which must be carried for these +non-detections is the image and the associated object or orphan. + +DVO also stores the relationships between various +photometric systems and the evolution of that relationship. It +provides mechanisms to convert between the measured instrumental +magnitude of a detection with a specific filter, detector, and +telescope, and at a particular time and the implied magnitude in the +average Pan-STARRS photometry system, given a determined set of +calibrations. It also provides the capability to convert magnitudes +in one system to the magnitudes in another system; an example of such +a conversion is between the average Pan-STARRS filter systems and the +various reference systems appropriate for those filters. + +\section{DVO Database Tables} + +\begin{figure} +\resizebox{4.5in}{!}{\includegraphics{pics/dvo.01.ps}} +\caption{\label{fig:DVOtables} \small Data types managed by DVO} +\end{figure} + +Figure~\ref{fig:DVOtables} illustrates the data managed by DVO, and +Table~\ref{tab:DVOtables} provides a complete listing. The contents +of these tables are outlined in Appendix~\ref{sec:DVOTableContents}. +Below, the use of these tables by DVO software is discussed below. +Several of the tables are not just simple tables in the database but +are instead table groups divided into many subtables, each of which +represents a portion of the sky (a {\tt region}). These subtables may +also be distributed across different computers to distribute the +processing load. + +\subsection{Sky Regions Table} + +The {\tt Regions} table is used to subdivide the tables of images, +objects, and detections, etc, as discussed above. DVO +divides the sky into a hierarchy of regions (portions of the sky) each +of which is in turn subdivided into smaller portions. Since nearly +all interactions with DVO performed by the IPP are limited +in spatial coverage, subdividing the tables allows a specific +interaction to search only a small subset of the data. The table of +images is the smallest of the three; the table of detections is likely +to be the largest. As a result, the {\tt Images} table group will be +subdivided at a shallow hierarchical level, while the {\tt Objects} +and {\tt Detections} are subdivided on deeper (more finely sampled) +levels. The {\tt Regions} table defines the boundaries of the sky +regions and specifies if the region corresponds to an {\tt Images} +table, an {\tt Objects} table, and/or a {\tt Detections} table. It +also specifies which regions in the next level of the hierarchy are +contained by the region, and which parent region it belongs to. In +addition to improving the spatial access to the image, object, and +detection data, the {\tt Regions} table allows for multiple computers +to serve the database tables. The region file specifies the machine +which stores the specific table. Figure~\ref{fig:APDBRegions} +illustrates schematically the subdivision of the sky and the +association between different levels of the hierarchy with different +subtables. + +\begin{figure} +\begin{center} +\resizebox{6in}{!}{\includegraphics{pics/dvo.02.ps}} +\caption{DVO Regions and Image / Object tables} +\label{fig:DVOskyregions} +\end{center} +\end{figure} + +\subsection{Images Table Group} + +The {\tt Images} table group lists all of the images which provided +the data in DVO. These tables are subdivided by region on +the sky. In general, the images listed in this table correspond to +the Chips. This group of tables includes sufficient astrometric +parameters to represent the coordinates of the detections to a +sufficient accuracy. Parallel to the Images table is the Mosaic +table. This table is very similar to the Images table, but defines +the Mosaic which corresponds to a group of Images. The parameters +include the astrometric information needed to define the camera +distortion. + +\subsection{Image Overlaps Table Group} + +The specific subtable of {\tt Images} which contains a given image is +the one which contains the center pixel of that image. An additional +table group, {\tt Image Overlaps} (with the same subtable organization +as the {\tt Images} subtables), lists images which overlap that +specific subtable. Thus, given a particular coordinate, in order to +find that images which overlap that coordinate, it is necessary to +search the images in the {\tt Images} subtable which includes that +coordinate, and all images in the {\tt ImageOverlaps} subtable for +that coordinate. + +\begin{table}[hb] +\begin{center} +\caption{DVO Database Tables\label{tab:DVOtables}} +\begin{tabular}{ll} +\hline +\hline +{\bf Table Name} & {\bf Description} \\ +\hline +Images & The images that have objects in the DB. \\ +Image Overlaps & Image regions which are touched by specific images. \\ +Objects & The objects --- average properties of multiple detections of the same object. \\ +Average Magnitudes & Average photometry in multiple filters \\ +Solar System Objects & Identification of solar system objects \\ +Matched Detections & Detections of sources in an image identified with an Object. \\ +Orphaned Detections & Detections of sources in an image not identified with an Object. \\ +Non-detections & Non-detections of objects in an image. \\ +SkyRegions & spatial distribution of tables \\ +Filters & Filters understood by the system. \\ +Photcodes & Transformations between different photometric systems \\ +Zero Points & History of Zero-point \& Airmass terms \\ +Distortion Models & History of Optical Distortion terms \\ +Database Hosts & computers used to store the tables \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\subsection{Objects Table Group} + +The {\tt Objects} table group (also divided by region) stores the +average parameters for each astronomical object. Certain details of +this table have not yet been specified. In particular, objects with +significant parallax and/or proper motion may potentially be stored in +a distinct table. Solar system object identifications, to the extent +average properties are maintained in DVO, will certainly +be stored in a separate table. + +\subsection{Average Magnitudes Table Group} + +A related table, also divided into the same regions, is the {\tt +Average Magnitudes} table. In this table, there are multiple rows per +object, one for each of the primary filters of interest for which +photometric averaging is performed. This organization makes the +number of primary (averaged) filters a configurable value. + +\subsection{Matched Detections Table Group} + +The {\tt Matched Detections} table stores all of the measurements of +astronomical objects on specific images. This table includes all +detections associated with the average {\tt Objects}. As discussed +below, bright objects (above a configuration-specified signal-to-noise +level) are defined object even if only one detection has been found at +that position. Faint orphaned objects are not added to this list or +the list of objects. The different types of detections (P2, +P4$\Delta$, P4$\Sigma$) are distinguished by their photometry codes. +(This is only valid if DVO does not store different +quantities for these types of detections.) + +\subsection{Orphaned Detections Table Group} + +The {\tt Orphaned Detections} table stores the detections which have +not been correlated with an existing object. This table is only +populated for objects below a configuration-specified signal-to-noise +limit (e.g., 5$\sigma$). Bright orphaned detections are assigned an +object and added to the {\tt Matched Detections} table. + +\subsection{Non-detections Table Group} + +The {\tt Non-detections} table stores information about detection +failures for each object. If an image is added to the database which +overlaps an object but the object is not detected, an entry is made in +this table. In practice, this table may store only the most recent +non-detection and the total number, or a similar reduced set of +non-detection statistics. + +\subsection{Other Reference Tables} + +The {\tt Filters} table identifies all of the physical filters +(specific pieces of glass) known to the system. A related table, {\tt +Photcodes}, defines relationships between photometry systems. A +photometry system may consist of a detector, telescope, and specific +filter, or it may be a derived photometry system. The {\tt Database +Machines} table identifies all of the computers available to DVO. + +\section{Database Table I/O} + +\begin{figure} +\resizebox{4.5in}{!}{\includegraphics{pics/dvo.03.ps}} +\caption{\label{fig:DVOformats} \small DVO Table I/O } +\end{figure} + +DVO allows for a flexible representation of its data on disk. Data +may be written to disk in one four possible mode: RAW, FITS MEF, FITS +SPLIT, and MYSQL. These modes define the overall organization of the +data on disk. In the RAW mode, the data is written to disk in a +pseudo-FITS table format which consists of a simple FITS header +describing the layout followed by the binary data in a block. This +storage mode is maintained for historical reasons. There are also two +types of FITS modes in which the data tables are written as valid FITS +Binary Tables. In the SPLIT format, every data table is written as a +separate file, while in the MEF format, the object and detection +tables are bundled together into a single FITS file with multiple +table extensions. The MEF format has the advantage of minimize the +proliferation of files, while the SPLIT format is required to make use +of the fastest read/write capabilities of DVO. DVO makes use of these +raw data formats as a throughput risk mitigation strategy. As +discussed below, this strategy has proven very successful. + +There are also multiple formats in which the data may be stored. The +different formats define which specific database table columns are +stored and with what numerical format and precision. +Figure~\ref{fig:DVOformat} illustrates the conversion process which +DVO performs when loading in the data. When DVO loads data from a +file-based table (FITS or RAW), it first loads from the disk file into +a data structure representing the external format in use. The +external structure is then converted into the internal format. The +internal structure is always specified to be the superset of all +external data formats. This capability allows DVO to maintain +backwards compatibility with data tables written with early versions. +As DVO is extended and new elements are added to the tables, it is +only necessary to define the methods to convert the new internal table +into the external table. In addition, DVO makes use of autocoded +table manipulation and I/O APIs which are generated for each data +structure based on a descriptive table. This makes it easy to add new +data types and input/output methods without significant re-coding. + +\section{addstar : Insert Image \& Detection Set} + +\begin{figure} +\resizebox{4.5in}{!}{\includegraphics{pics/dvo.04.ps}} +\caption{\label{catalog} \small a figure } +\end{figure} + +\tbd{fill out discussion of the addstar client/server implementation} + +One of the most basic operations needed by DVO is to insert a +collection of detections derived from a specific image, and add the +definition of that image to the database. This operation is critical +in terms of the processing throughput. After the detections have been +assigned to the appropriate regions, they are matched against all +objects in the {\tt Objects} table. Matches are performed only on the +basis of positional coincidence, using a matching radius which may +depend on the image astrometry errors, or may be a fixed distance. +Any matched detections are added to the {\tt Matched Detections} +table. Any unmatched detections brighter than the Faint Detection +cut-off are specified as a new {\tt Object} and also added to the {\tt +Matched Detections} table. Any faint unmatched detections are added +to the {\tt Orphaned Detections} table. This division is important +because it allows the automatic association of new detections with +existing bright objects while limiting the I/O volume required to make +the detections. In general, there will be many fewer {\tt Objects} +than {\tt Detections}, and there will be fewer bright orphans than +faint orphans. + +\subsection{addstar -refs : Insert Reference Objects} + +This operation is very similar to the previous one. A collection of +reference objects are added to the database as a collection of +detections. The reference photometry should in general be given its +own photometry code. The reference data is different from the image +detection set because the associated image information is not +included. Thus, no corresponding images are added to the database. + +\section{relphot : Relative Photometry Analysis} + +This operation uses the overlaps of images and multiple observations +of the same objects to determine the relative photometry zero-points +for a collection of images. This is a task that wil be run much more +infrequently than the object insertion tasks. + +\section{relphot} + +\begin{verbatim} + * load data + o images: match photcode and time range, reset flags + o measure: select subset matching restritions on: photcode, time, dM, Minst, Mag, dophot == 1 + * iterate to find Mcal, image.flags: + * write out modified Mcal values to image table + * write out modified Mcal values to measure table + * write out modified Mrel values to average table +\end{verbatim} + +relphot has two primary purposes: + +\begin{verbatim} + * calculate Mcal for images / measures + * calculate Mrel for stars +\end{verbatim} + +relphot can also be used to determine the mosaic grid used to generate photometrically corrected flats (-grid option). + +\subsection{data exclusion} + +relphot uses only a subset of the photometry data to calculate Mcal +and Mrel. In the first stage, calculation of the Mcal values, relphot +loads the photometry data from each relevant catalog and creates an +internal subset catalog with the function bcatalog, excluding some of +the irrelevant data. In addition, it uses flags to mark some of the +data as invalid for the processing. bcatalog exclusions + +\begin{verbatim} + * measure.photcode not equivalent to requested photcode + * measure.dophot != 1 + * measure.Mcat > MAG_LIM + * measure.dM > SIGMA_LIM + * measure.Minst out of range (ImagMin - ImagMax) [optional] + * measure.t out of range (TSTART, TSTOP) +\end{verbatim} + +flagged data +flagged image data + +images can be flagged by setting bits of image.code stars can be +flagged by setting bits of average.code measures can be flagged by +setting bits of measure.flag + +\begin{verbatim} +image.code +ID_IMAGE_NOCAL : ignore, irrelevant +ID_IMAGE_POOR : image measured bad +ID_IMAGE_SKIP : externally known bad dMcal > VALUE +FLAG_IMAGE_SCATTER clean_images fabs(Mcal) > VALUE +FLAG_IMAGE_ZEROPT clean_images dMcal > VALUE +FLAG_IMAGE_SCATTER clean_mosaics fabs(Mcal) > VALUE +FLAG_IMAGE_ZEROPT clean_mosaics mark_images does not seem to do +anything useful? +average.code Ngood < MEAS_TOOFEW setMrel Ngood < +MEAS_TOOFEW clean_measures ChiSq > STAR_CHISQ clean_stars dM > +STAR_SCATTER clean_stars average.code (STAR_BAD) is not saved by +relphot: it is set by clean_stars, clean_measures, and setMrel, but +not setMrelOutput. STAR_BAD should only be internal since it depends +on the photcode, but is not associated with a specific photcode in the +data. Just in case, it is reset to 0 in setMrelFinal. measure.flag X,Y +out of range setExclusions 3 sigma clipping clean_measures +\end{verbatim} + +setting Mrel final value + +setMrelFinal is used to set the final average.Mrel values. We do this +in 4 stages. In each stage, we set the Mrel values for stars which +have not already been set, based on the current exclusion settings. At +successive stages, we relax the exclusions, allowing the more spurious +objects to have a valid Mrel value to be set. In this loop, we +actually run setMrelOutput twice: once to get the approximate Mrel +value, then we flag the outlier measurements with +\code{clean_measure}, then we redetermine the Mrel values on this +basis, and mark the stars for exclusion from the next iteration. + +\begin{verbatim} + exclude on + photcode 0 1 2 3 + time range 0 1 2 3 + MEAS_POOR 0 1 2 3 + MEAS_TOOFEW 0 1 2 3 + dophot == 10 0 1 2 + inst mag 0 1 2 + dophot != 1,2 0 1 + ID_IMAGE_POOR 0 1 + ID_IMAGE_SKIP 0 1 + dophot != 1 0 + measure.dM 0 +\end{verbatim} + +for all relphot runs, Mrel is re-calculated, and measures are marked at least if they are outliers in mag or ccd area. setMrel.output needs to do a few things differently from setMrel: + +\begin{verbatim} + * set measure.Mcal (skipped in setMrel.basic) + * set average.Mrel if N < TOO_FEW (not STAR_BAD) (optional!) + * use MAX (stats.error, stats.sigma) (optionally) + * allow STAR_BAD? +\end{verbatim} + +\section{uniphot : Zero Point Analysis} + +This operation uses the time history of relative photometry zero +points for images and the spatial overlap information to determine a +best set of image zero points which have a specific time scale for the +atmospheric stability. + +\section{global astrometry analysis} + +This operation uses the reference and image detections to determine an +optical distortion model for the camera and static astrometry model +components. The astrometry model includes: (1) field distortion +introduced by the telescope optics, which is a smoothly-varying +function of the field position relative to the center of the telescope +boresite coordinates. (2) focal plane geometry, which includes the +chip positions and rotations in the focal relative to the boresite, +along with chip-dependent plate-scale modifications needed to +represent tilts or warps of the individual detectors relative to the +ideal flat focal plane. . + +\begin{table} +\begin{center} +\caption{DBO Detection Classes \& Object Parameters\label{tab:APdetections}} +\begin{tabular}{lrrrr} +\hline +\hline +Object Parameter & P2 & P4S & P4D & SS \\ +\hline +PSF x,y, covar, $\alpha,\delta$ & + & + & + & + \\ +PSF mag, $\sigma_{\rm mag}$ & + & + & + & + \\ +star/gal sep & + & + & + & + \\ +$\sigma_x$, $\sigma_y$, $\theta$ & + & + & + & + \\ +local sky data & + & + & + & + \\ +Petrosian R, M, $R_{50}$, $R_{90}$ & - & + & - & + \\ +S\'ersic R, M, AB, $\phi$, $\nu$ & - & + & - & + \\ +W.L. $\gamma_1$, $\gamma_2$, pol. terms & - & - & - & + \\ +exp. spaced aps., Poisson noise, variance & - & - & - & + \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\section{DVO shell} + +\subsection{User Commands} + +\begin{verbatim} + gcat -- get catalog at location + gimages -- get images at location + gstar -- get star statistics + extract -- extract average vectors from catalogs + mextract -- extract measurement vectors from catalogs + imstats -- plot image statistics + imextract -- extract image vectors from database + lcat -- list catalogs in display region + cmatch -- match two catalogs +\end{verbatim} + +There are a variety of other commands which directly refer to the +photometry database. Some of these functions extract data of various +types from the database, others perform more complex plotting +operations. The commands listed above are those which simply extract +data from the database. The first three list information relevant to a +specific RA, DEC location on the sky: gcat (RA) (DEC) lists the +catalog at the specified location and places the name in the variable +\$CATNAME, gimages (RA) (DEC) lists all images which overlap the +specified location, gstars (RA) (DEC) (RADIUS) lists data about the +stars within a specified radius of the specified location (all numbers +above are given in decimal degrees). Similarly, lcat lists the +catalogs in the region. Imstats lists statistics about each image + +The next three commands extract a specific piece of information from +the photometry database and places it in a vector. First, extract will +extract average values for each star and place it in a vector. Next, +mextract will extract measurement values for each star and place it in +a vector: as a result a single star may have multiple entries in the +measurement vectors. Finally, imextract will extract image statistics +into vectors (not yet implemented). + +DVO provides several ways to access the photometry information stored +in the database. Several simple commands allow the user to extract 1 +dimensional information directly from one of the primary database +tables. The commands are: + +\begin{verbatim} + * imextract + * avextract + * mextract + * imsearch + * detsearch +\end{verbatim} + +imextract + +This command allows the user to extract one of the columns from the +image table, applying filtering as desired: + +\begin{verbatim} + dvo: imextract + USAGE: imextract (value) [-region] [-time start range] [-photcode photcode] + dvo: imextract help + value may be one of the following: + ra dec airmass Mcal dMcal Xm photcode time fwhm exptime nstar ncal sky flag +\end{verbatim} + +The extracted data is saved in a vector with the same name used to +select the column. The vector name will have the same case as the +choice given, but the column selection is case-insensitive (since +there are no ambiguities in the database columns names by case). + +avextract + +This command allows the user to extract data from one of the Average +table columns: + +\begin{verbatim} + dvo: avextract + USAGE: avextract (from) (value) [options] + from: cpt name or 'all' + value: average.parameter or photcode + dvo: avextract all help + value may be one of the following: + ra dec dmag Nmeas Nmiss Xm Xp Nphot Ncode flag type +\end{verbatim} + +This command takes as the first argument the name of one of the +database regions. Alternatively, all regions currently displayed may +be selection with the word 'all'. The second option specifies which +column to select from the Average table. In addition to the basic data +columns (ra, dec, etc), the magnitude-related average values (mag, +dmag, Xm, Nphot, Ncode) are coupled to a photcode, which is thus +required for these selections. The value 'mag' may also be subsituted +with a primary or secondary photcode. Eg: + +\begin{verbatim} + avextract all ra : select ra for all objects in displayed region + avextract all g : select g magnitudes + avextract all mag -photcode r : select r magnitudes + avextract all Xm -photcode r : select chisq values for r average mags +\end{verbatim} + +mextract + +This command allows the user to extract data from one of the Measure +table columns: + +\begin{verbatim} + dvo: mextract + USAGE: mextract (from) (value) [options] + from: cpt name or 'all' + value: measure.parameter or photcode + dvo: mextract all help + value may be one of the following: + ra dR dec dD mag dmag Mrel Mcal photcode time fwhm dophot xccd yccd xmosaic ymosaic flags +\end{verbatim} + +This command takes as the first argument the name of one of the +database regions. Alternatively, all regions currently displayed may +be selection with the word 'all'. The second option specifies which +column to select from the Measure table. In addition to the basic data +columns (ra, dec, etc), the magnitude-related average values (mag, +dmag) are coupled to a photcode, which is thus required for these +selections. The value 'mag' may also be subsituted with a primary or +secondary photcode. Eg: + +\begin{verbatim} + mextract all ra : select ra for all objects in displayed region + mextract all g : select g magnitudes + mextract all mag -photcode r : select r magnitudes + mextract all Xm -photcode r : select chisq values for r average mags +\end{verbatim} + +option filtering + +The following extraction options allow the user to restrict the +selections: + +-time (start) (range) + +select data for images within the given time range. The start date is +given in the format YYYY/MM/DD,hh:mm:ss (any of these element may be +dropped, in which case they default to 00 [for hh,mm,ss] or 01 [for +MM,DD]). The date may also be written as a number of days followed by +j (JD) or J (MJD). The two special date names "now" and "today" are +also valid. The range is written as a number followed by a unit, with +valid units of s (seconds), m (minutes), h (hours), d (days). Times +are in UT. For example: + +\begin{verbatim} + -time 2001/1/1 30d : select the range starting at midnight on 2001/1/1 + and ending 30 days (86400*30 seconds) later. + + -time now -3h : select the time range starting three hours ago and + ending now. +\end{verbatim} + +-region + +restrict the selection to the currently display portion of the +sky. This filter expects a portion of the sky to be plotted, and will +only select data for images in the part of the sky. The algorithm for +selecting the displayed region may not be perfect, so images near the +boundaries may be unexpected included or excluded (this depends on the +exact overlap details). Multiple queries with the same region will +result in the same subset of images selected. + +-photcode (photcode) + +This restricts the selection to the given photcode. Images may only +have photcodes of 'Dependent' type (eg, CFH12K.R.00). This filter +allows the selection of images by exact match (selected photcode is +Dependent type) or by equivalence match (selected photcode is Primary +or Secondary type). Thus, selecting images based on CFH12K.R.00 will +ignore images with photcode CFH12K.R.10, while selecting images based +on photcode 'r' will return all images with CFH12K.R.*, since all are +equivalent to 'r'. (NOTE: these details depend on the layout of the +photcode table). fill this out with all other restrictions provided by +photometry.c + +\begin{verbatim} + catalog -- plot catalog stars + cgrid -- plot sky coordinate grid + cplot -- plot vectors in sky coordinates + czplot -- plot scaled vectors in sky coordinates + images -- plot image boxes + imdense -- image density plot + lcurve -- plot lightcurve for a star + pcat -- plot catalog boundaries + region -- define sky region for plot + resid -- plot residuals + simage -- plot stars in an image + \end{verbatim} + + +There are two types of database plotting functions: those that display +or refer to the spatial charateristics of the data and those that +refer to other types of charatersitics, such as the time domain. The +graphics window 0 is reserved for all plots of objects on the sky. The +command region defines the current sky coordinates for plots in +graphic window 0. The command pcat plots the outline of all photometry +database files which are within the currently defined region (and by +default, only those with data). images plots the outline of the images +in the image database, while imdense shows the number of images at a +location by randomly spacing dots within the boundary of the +images. The command cgrid draws a grid in celestial coordinates on the +for the current region. + +The most complex, but also one of the most useful command is catalog, +which plots the positions of stars in the photometry database (and +others) on the sky. There are many options to this command. One set +allows the user to plot stars from the photometry database (the +default), from the HST GSC, or from an ASCII text file with RA, DEC, +and Mag in specified columns. If the ASCII file has a fixed number of +bytes per line, the data can be more quickly loaded. The size of the +points may be scaled by the star magnitude, by the number of +observations of the star, or by the number of missing datapoints for +the star. In addition, points may be plotted only if they land in +specified magnitude ranges, or with specified numbers of measurements, +or missed measurements. Also, objects may be plotted only if they have +a specified Average.code, so that only asteroids or only perfect stars +may be plotted. The plotted vectors may be saved, if desired, and the +source catalog epoch may be specified as different from J2000 (only +valid for ASCII data). + +Several other commands relate to non-spatial charateristics of images +and stars. lcurve will plot a light curve for all stars within some +radius of a point. resid plots the photometry residuals for a +particular region file. Some Examples + +\begin{figure} +%\resizebox{4.5in}{!}{\includegraphics{pics/fullsky}} +\caption{\label{allsky} \small Map of the entire sky, and images added to database. } +\end{figure} + +Fig.~\ref{allsky} shows a map of the entire sky, and the location of +the images currently in the database. This picture was made with the +following commands: (output is not shown) + +\begin{verbatim} + dvo: region 0 0 90 gls + dvo: cgrid + dvo: style -lw 2 -c red + dvo: images + dvo: ps +\end{verbatim} + +In this example, on the graphics window, the image boxes are shown in +red. The user now has the possiblitiy of using the cursor command to +narrow in on a specific region, and so forth. + +\begin{figure} +%\resizebox{4.5in}{!}{\includegraphics{pics/polar}} +\caption{\label{polar} \small Map of +the sky in polar project, and images added to database. } +\end{figure} + +Fig.~\ref{allsky} shows a map of the entire sky, and the location of +the images currently in the database from a polar project. This +picture was made with the following commands: (output is not shown) + +\begin{verbatim} + dvo: region 0 0 90 zea + dvo: cgrid + dvo: style -lw 2 -c red + dvo: images + dvo: ps +\end{verbatim} + +In this example, on the graphics window, the image boxes are shown in +red. The user now has the possiblitiy of using the cursor command to +narrow in on a specific region, and so forth. + +\begin{figure} +%\resizebox{4.5in}{!}{\includegraphics{pics/catalog}} +\caption{\label{catalog} \small Comparison between HST GSC and +photometry database astrometry. } +\end{figure} + +Fig.~\ref{catalog} shows an example comparison of the photometry +database star positions and the HST Guide Star Catalog star +positions. The crosses are all objects in the photometry database, +while the boxes are only the stars identified as USNO stars. The +circles are the stars from the HST GSC. The size of both points is a +function of brightness. This plot was made with the following commands +(starting from the previous image): + +\begin{verbatim} + dvo: cursor (typed 1 on region of interest) + 1 137.097858 22.698305 + q 137.097858 22.698305 + dvo: region $R1 $D1 0.2 TAN + dvo: cgrid + dvo: box + dvo: style -pt 0 + dvo: gcat $R1 $D1 + 0 n2230/1951.cpt * + dvo: style -pt 2; cat -all -m 12 18 + dvo: style -pt 1; cat -all -m 12 18 -ID $USNO + dvo: style -pt 7; cat -all -m 12 18 -g +\end{verbatim} + +\section{other user tools} + +\subsection{delstar} + +\subsection{getstar} + +\subsection{imphotset} + +imphotset allows you to set certain phot.image table entries. here are +the options: + +imphotset [-photcode code] [-name foo] [-trange (start) (stop)] -flag +and value + +Here is a complete list of relphot configuration variable names, a +quick description, and reasonable values to start with: + +\begin{verbatim} + --- configuration variables used by relphot --- + MAG_LIM : float + ignore measurements fainter than this absolute magnitude + + SIGMA_LIM : float + ignore measurements with magnitude error larger than this value + + STAR_SCATTER : float + mark stars as bad if their scatter is larger than this value + + IMAGE_SCATTER : float + mark images as bad if their scatter is larger than this value + + IMAGE_OFFSET : float + mark images as bad if the absolute value of thie zero point offset + is larger than this number + + STAR_CHISQ : float + mark stars as variable if their reduced chisq are larger than this value + + STAR_TOOFEW : int + mark stars as bad if the have fewer than this number of valid measurements + + IMAGE_TOOFEW : int + mark images as bad if the have fewer than this number of valid measurements + + IMAGE_GOOD_FRACTION : float + mark images as bad if the have fewer than this fraction of valid measurements + + IMAGE_CATALOG : string + name of the image catalog file + + IMAGE_CATALOG_TEMPLATE : string + name of the template file to create the image catalog file + + CATALOG_TEMPLATE : string + name of the template file to create the catalog file + + GSCFILE : string + name of the GSC region table + + CATDIR : string + directory where the database is stored + + PHOTCODE_FILE : string + file containing photometry code information + + ZERO_PT : float + default zero point for random data + + RELPHOT_GRID_X : int + scale of mosaic correction grid + + RELPHOT_GRID_Y : int + scale of mosaic correction grid + + RELPHOT_GRID_BINNING : int + deprecated + + CAMERA_CONFIG : string + name of the file containing descriptive information about the camera + + --- sample ConfigFile entries with typical values --- + + MAG_LIM 24.0 + SIGMA_LIM 0.05 + STAR_SCATTER 0.05 + IMAGE_SCATTER 0.05 + IMAGE_OFFSET 0.2 + STAR_CHISQ 10.0 + STAR_TOOFEW 3 + IMAGE_TOOFEW 10 + IMAGE_GOOD_FRACTION + IMAGE_CATALOG $CATDIR/Images.dat + IMAGE_CATALOG_TEMPLATE $REFSDIR/elixir/template.cat + CATALOG_TEMPLATE $REFSDIR/elixir/template.cat + GSCFILE $REFSDIR/gsc/GSCregions.tbl + CATDIR $CATDIR + PHOTCODE_FILE $CONFDIR/camera/$CAMERA.photcode + ZERO_PT 25.0 + RELPHOT_GRID_X 4 + RELPHOT_GRID_Y 8 + RELPHOT_GRID_BINNING 512 + CAMERA_CONFIG $CONFDIR/camera/$CAMERA.config +\end{verbatim} + + +\section{Performance} + +DVO design partly driven by the need to make the +detection-object associations quickly and to processes the incoming +detections at a sufficiently high rate to meet the throughput +requirements. For each upload of the object detections from a +complete FPA, DVO must match roughly $1.4 \times 10^{6}$ +detections from an FPA with roughly $6.4 \times 10^{6}$ objects, +including orphaned bright detections. This corresponds to roughly 640 +MB, if each object uses 100 bytes for its descriptive informations +(more than is currently specified in the Object table). With a +throughput of 100 MB/s for reads from a RAID, DVO can +perform the data read in a fraction of a second if the data is +distributed across 10 computers. + +\appendix +\pagebreak + +\section{DVO Tables} +\label{sec:DVOTableContents} + +\begin{table}[bh] +\begin{center} +\caption{Images\label{tab:images}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +\code{NAME} & \code{char[32]} & name of original image \\ +\code{TZERO} & \code{e_time} & readout time (row 0) \\ +\code{COORDS} & \code{Coords} & astrometry \\ +\code{NSTAR} & \code{unsigned int} & number of stars on image \\ +\code{SECZ} & \code{float} & airmass, mag \\ +\code{NX} & \code{short} & image width \\ +\code{NY} & \code{short} & image height \\ +\code{APMIFIT} & \code{float} & aperture correction \\ +\code{DAPMIFIT} & \code{float} & apmifit error \\ +\code{MCAL} & \code{float} & calibration mag \\ +\code{DMCAL} & \code{float} & error on Mcal \\ +\code{XM} & \code{short} & image chisq \\ +\code{SOURCE} & \code{short} & photcode \\ +\code{EXPTIME} & \code{float} & exposure time (seconds) \\ +\code{ST} & \code{float} & sidereal time of exposure \\ +\code{LAT} & \code{float} & observatory latitude (degrees) \\ +\code{DETECTION_LIMIT} & \code{unsigned char} & detection limit (10*mag) \\ +\code{SATURATION_LIMIT} & \code{unsigned char} & saturation limit (10*mag) \\ +\code{CERROR} & \code{unsigned char} & astrometric error (50*arcsec) \\ +\code{FWHM_X} & \code{unsigned char} & PSF x width, (25*arcsec) \\ +\code{FWHM_Y} & \code{unsigned char} & PSF y width, (25*arcsec) \\ +\code{TRATE} & \code{unsigned char} & scan rate, (100 usec/pixel) \\ +\code{CODE} & \code{char} & image quality flag \\ +\code{CCDNUM} & \code{unsigned char} & CCD ID number \\ +\code{ORDER} & \code{short} & Mrel polynomial order \\ +\code{MREL_POLY} & \code{short[14]} & Mrel polynomial \\ +\code{DUMMY} & \code{char[18]} & expansion \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Objects\label{tab:Objects}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +\code{RA} & \code{double} & RA, \\ +\code{DEC} & \code{double} & DEC, \\ +\code{MAG} & \code{float} & primary mag, \\ +\code{MAG_ERR} & \code{float} & error on primary mag, \\ +{\it \code{V_RAf}} & \code{float} & proper-motion (arcsec/year), \\ +{\it \code{V_DEC}} \code{float} & proper-motion (arcsec/year) \\ +{\it \code{PAR}} \code{float} & parallax (arcseconds) \\ +{\it \code{D_V_RA}} \code{float} & proper-motion error (arcsec/year) \\ +{\it \code{D_V_DEC}} \code{float} & proper-motion error (arcsec/year) \\ +{\it \code{D_PAR}} \code{float} & parallax error (arcseconds) \\ +\code{SIGMA_POS} & \code{short} & position scatter, \\ +\code{CHISQ_MAG} & \code{short} & chisq for primary mag, \\ +\code{CHISQ_GAL} & \code{short} & chisq for galaxy mags, \\ +\code{NMEAS} & \code{unsigned short} & number of measures \\ +\code{NMISS} & \code{unsigned short} & number of missings \\ +\code{CODE} & \code{unsigned short} & ID code (star, ghost, etc) \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{SecFilt Magnitudes\label{tab:SecFilt} - NOTE: corresponding + photcodes defined externally for the table sequence, Average Object + association defined by sequence } +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +\code{MAG} & \code{float} & other mags, mags \\ +\code{MAG_ERR} & \code{float} & scatter on mag mags \\ +\code{MAG_CHI} & \code{short} & chisq on mag [100*log(value)] \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Matched Detections\label{tab:Detections}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +\code{D_RA} & \code{float} & RA offset, arcsec \\ +\code{D_DEC} & \code{float} & DEC offset, arcsec \\ +\code{MAG} & \code{float} & catalog mag, mag \\ +\code{MCAL} & \code{float} & image cal mag, mag \\ +\code{MGAL} & \code{float} & galaxy mag, mag \\ +\code{DM} & \code{float} & mag error, mag \\ +\code{AIRMASS} & \code{float} & (airmass - 1), airmass \\ +\code{DT} & \code{float} & exposure time, 2.5*log(exptime) \\ +\code{FWX} & \code{short} & object fwhm major axis, 1/100 of arcsec \\ +\code{FWY} & \code{short} & object fwhm minor axis, 1/100 of arcsec \\ +\code{THETA} & \code{unsigned char} & angle wrt ccd X dir, (0xff/360) deg \\ +\code{DOPHOT} & \code{char} & dophot type \\ +\code{SOURCE} & \code{unsigned short} & photcode \\ +\code{FLAGS} & \code{unsigned short} & flags for various uses \\ +\code{T} & \code{unsigned int} & time in seconds (UNIX) \\ +\code{AVEREF} & \code{unsigned int} & reference to average entry \\ +\code{OBJECT ID} & & \\ +\code{SKY} & & \\ +\code{D_SKY} & & \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Orphaned Detections\label{tab:Orphans}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +$\alpha$ & & \\ +$\delta$ & & \\ +$\sigma_{\alpha}$ & & \\ +$\sigma_{\delta}$ & & \\ +$M_{\rm inst}$ & & \\ +$M_{\rm cal}$ & & \\ +$\sigma_{\rm mag}$& & \\ +photcode & & \\ +type & & \\ +flags & & \\ +time/date & & \\ +airmass & & \\ +$\sigma_{x}$ & & \\ +$\sigma_{y}$ & & \\ +$\theta$ & & \\ +exptime & & \\ +sky & & \\ +$\sigma_{\rm sky}$& & \\ +etc & & \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Non-detections\label{tab:NonDetects}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +object ID & & \\ +$N_{\rm non-det}$ & & \\ +last time/date & & \\ +last mag & & \\ +faintest time/date & & \\ +faintest mag & & \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Regions\label{tab:Regions}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +\code{R_MIN} & \code{float} & \\ +\code{R_MAX} & \code{float} & \\ +\code{D_MIN} & \code{float} & \\ +\code{D_MAX} & \code{float} & \\ +\code{CHILD_S} & \code{int} & sequence number in full table of first child \\ +\code{CHILD_E} & \code{int} & sequence number in full table of last child + 1 \\ +\code{PARENT} & \code{int} & sequence number in full table of parent \\ +\code{INDEX} & \code{int} & sequence number in full table of this entry \\ +\code{DEPTH} & \code{char} & depth of this entry \\ +\code{CHILD} & \code{char} & does this entry have children? \\ +\code{TABLE} & \code{char} & does this entry have a table? \\ +\code{NAME} & \code{char[21]} & name / filename \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Image Overlaps\label{tab:ImageOverlaps}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +Image ID & & \\ +Region Table & & \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Filters\label{tab:Filters}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +Filter ID & & \\ +Filter name & & \\ +Photcode & & \\ +$\lambda_0$ & & \\ +$\delta_\lambda$ & & \\ +$\epsilon$ & & \\ +transmission curve& & \\ +time/date & & \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Photcodes\label{tab:Photcodes}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +\code{CODE} & \code{unsigned short} & code number (stored in Measure.source) \\ +\code{NAME} & \code{char[32]} & name for filter combination \\ +\code{TYPE} & \code{char} & PRI/SEC/DEP/REF \\ +\code{C_LAM} & \code{short} & primary phot calibration terms (millimags) \\ +\code{C_LAM_ERR} & \code{short} & primary phot calibration terms (millimags) \\ +\code{X_ERR} & \code{short} & primary phot calibration terms (millimags) \\ +\code{K} & \code{float} & secondary phot calibration terms (millimags) \\ +\code{C1} & \code{int} & color is average.M[c1] - average.M[c2] \\ +\code{C2} & \code{int} & color is average.M[c1] - average.M[c2] \\ +\code{EQUIV} & \code{int} & this dependent filter is equivalent to equiv PRI/SEC \\ +\code{NC} & \code{int} & number of color terms \\ +\code{X} & \code{float[4]} & \code{color terms X[0]*mc + X[1]*mc^2 + X[2]*mc^3} \\ +Telescope & & \\ +Camera & & \\ +Detector & & \\ +Filter & & \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Zero Point History\label{tab:Zpts}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +\code{ZP_OBS} & \code{float} & measured zero point, mag \\ +\code{ZP_REF} & \code{float} & nominal zero point, mag \\ +\code{ZP_ERR} & \code{float} & error on zero point, mag \\ +\code{C_AIRMASS} & \code{float} & airmass coeff, mag per airmass \\ +\code{C_COLOR} & \code{float} & color coeff, mag per mag \\ +\code{START_TIME} & \code{e_time} & start time of measurement, seconds since 1 Jan 1970 UT \\ +\code{STOP_TIME} & \code{e_time} & stop time of measurement, seconds since 1 Jan 1970 UT \\ +\code{C1_CODE} & \code{short} & code 1 for color, photcode \\ +\code{C2_CODE} & \code{short} & code 2 for color, photcode \\ +\code{PHOTCODE} & \code{short} & photcode, photcode \\ +\code{LABEL} & \code{char[64]} & data label \\ +\code{REFCODE} & \code{rawshort} & photcode, photcode \\ +\code{N_TIME} & \code{int} & number of times \\ +\code{N_MEAS} & \code{int} & number of measurements \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Distortion History\label{tab:Distortions}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +Camera & & \\ +Telescope & & \\ +distortion terms & & \\ +time/date & & \\ +residuals / error & & \\ +N stars & & \\ +N images & & \\ +astrom ref set & & \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Database Hosts\label{tab:APDBHosts}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +machine name & & \\ +machine ID & & \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\begin{table}[bh] +\begin{center} +\caption{Solar System Objects\label{tab:SSObjs}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column Name} & {\bf Datatype } & {\bf Description} \\ +\hline +SSO ID & & \\ +$N_{\rm det}$ & & \\ +\hline +\end{tabular} +\end{center} +\end{table} + +\end{document} + +\begin{figure} +\begin{center} +\resizebox{4.5in}{!}{\includegraphics{pics/APDB}} +\caption{DVO components} +\label{fig:Components} +\end{center} +\end{figure} + +DVO provides interfaces to extract lists of objects and +detections based on various query parameters. It provides the +capability to extract all detections associated with a specific +object, all non-detections of that object, all non-detections of an +orphan, and summary statistics from these collections. It will also +return all objects or detections within specified spatial regions +including regions bounded by great circles (RA,DEC; GLAT,GLON; +ELAT,ELON) and regions described by a location and a search radius. +It will also return the image parameters associated with a specific +detection including image coordinates of the detection, exposure time, +time and date of the detection, etc. + +As shown in Figure~\ref{fig:Components}, DVO consists of the following +components: + +\begin{itemize} +\item AP Database database tables +\item AP Database database engine +\item AP Database servers +\item AP Database client APIs +\end{itemize} + Index: /trunk/doc/pantasks/.cvsignore =================================================================== --- /trunk/doc/pantasks/.cvsignore (revision 6033) +++ /trunk/doc/pantasks/.cvsignore (revision 6033) @@ -0,0 +1,1 @@ +*.log *.dvi *.aux *.toc *.log *.out *.lof *.tbr *.tbd *.pdf Index: /trunk/doc/pantasks/Makefile =================================================================== --- /trunk/doc/pantasks/Makefile (revision 6033) +++ /trunk/doc/pantasks/Makefile (revision 6033) @@ -0,0 +1,29 @@ + +PDFLATEX = env TEXINPUTS=.:LaTeX:$(TEXINPUTS): pdflatex +PSLATEX = env TEXINPUTS=.:LaTeX:$(TEXINPUTS): latex + +help: + @echo "USAGE: make (target)" + @echo " targets: pantasks all" + +pantasks: pantasks.pdf +all : pantasks + +%.pdf: %.tex + $(PSLATEX) $*.tex + $(PSLATEX) $*.tex + dvips -z -t letter -o $*.ps $*.dvi + ps2pdf $*.ps $*.pdf + thumbpdf --modes=dvips $*.pdf + $(PSLATEX) $*.tex + dvips -z -t letter -o $*.ps $*.dvi + ps2pdf $*.ps $*.pdf + @rm -f $*.ps $*.dvi $*.aux $*.log $*.tbr $*.tbd $*.toc $*.tpm $*.lof body.tmp head.tmp + +clean : + $(RM) *.log *.dvi *.aux *.toc *.tbd *.tbr *.tpm *.lof *.out *~ core body.tmp head.tmp + +dist : clean + $(RM) *.pdf + +empty: clean Index: /trunk/doc/pantasks/pantasks.tex =================================================================== --- /trunk/doc/pantasks/pantasks.tex (revision 6033) +++ /trunk/doc/pantasks/pantasks.tex (revision 6033) @@ -0,0 +1,910 @@ +\documentclass[panstarrs,spec]{panstarrs} + +\title{PanTasks} +\subtitle{the IPP Scheduler and Controller system} +\author{Eugene Magnier} +\audience{IPP} +\shorttitle{PanTasks} +\group{Pan-STARRS IPP} +\project{Pan-STARRS IPP} +\organization{Institute for Astronomy} +\version{DR} +\docnumber{PSDC-xxx-xxx} + +\begin{document} +\maketitle + +\tableofcontents +\pagebreak +\pagenumbering{arabic} + +\section{Overview} + +This document discusses the design of PanTasks. PanTasks is the IPP +tool which manages the sequencing of data analysis steps and, with the +related tool `PControl', distributes the data processing across a +cluster of computers. + +The purpose of PanTasks is to manage the automatic construction and +execution of inter-related (often repetative) operations. PanTasks +uses a set of rules to define UNIX commands, and their corresponding +command-line arguments, to be performed on some regular, repeated +basis. The utility of PanTasks is that it can easily define an +analysis system which is completely state-based, as opposed to an +event-driven system. + +The two basic units of PanTasks operation are the 'task' and the +'job'. A 'job' is simply a command the user would execute on a +command line; it consists of a command along with optional command +line arguments. A 'task' is a generic description of a type of job in +which the details of any specific piece of data are omitted. The task +defines the UNIX command which corresponds to the job, and it provides +rules for determining the identity of data for the job. The task also +defines tests which are used to decide if the job may be executed. +Finally, it defines a polling frequency in which PanTasks should +attempt to construct a new job for the task. + +For example, we may want to regularly copy files from one location to +another. Perhaps the name of the next available file is available in +a database table, and can be retrieved with the command 'nextFile'. +Perhaps we wish to check for new files every 60 seconds. Thus, the +task is to copy files, while a specific job of this task might be of +the form \code{cp newfile newpath}. With PanTasks, we would define a +copy task somewhat like the following: + +\begin{verbatim} + task copyfile + periods -exec 60.0 + + task.exec + $file = `nextFile` + if ($file == "none") + break + end + command cp $file $newpath + end + + task.exit 0 + queuepush copied $file + end + + task.exit 1 + queuepush failure $file + end + end +\end{verbatim} + +In this simple example, the task is attempted every 60 seconds. If +there is no new file (output of 'nextFile' is 'none'), then no job +results from this task. If there is a new file, the copy command is +performed. This example is deceptively simple because one could +easily imagine writing a stand-along program which performs the same +thing. The important advantages of using PanTasks for this type of +operation are: +\begin{itemize} +\item the output from one job can be used to spawn a number of other tasks. +\item the success or failure state of each job can be used to spawn + other tasks. +\item the details of the job and data test are kept separated from the + rules which connect tasks together. +\item the relationships between tasks are kept together in a single + location. +\end{itemize} + +In addition to these organizational advantages, PanTasks also provides +a direct connection to the tool for monitoring parallel jobs, +pcontrol. Thus the jobs spawned by PanTasks can be defined to run in +the background locally or on any of the computers in the parallel +processing cluster. + + +\section{PanTasks : the Scheduler} + +The purpose of PanTasks is to manage the automatic construction and +execution of inter-related (often repetative) operations. PanTasks uses +a set of rules to define UNIX commands, and their corresponding +command-line arguments, to be performed on some regular, repeated +basis. The utility of PanTasks is that it can easily define an analysis +system which is completely state-based, as opposed to an event-driven +system. + +Consider, for example, a telescope which obtains a collection of +images over the course of a night. Every minute or two, it takes an +image and writes the image to some disk. An event-driven analysis +system would involve having the telescope initiate a process at the +end of the exposure. This process would perform an analysis, write +some output, then send trigger another process. This type of operation +works very well for a simple set up with reliable hardware. Such a +system becomes more difficult to maintain when hardware failures occur +or when multiple systems need to interact with each other. When +failures occur, the triggering information (the events) is easily +lost, thus some mechanisms are needed to detect these failures and +either re-send the trigger or send an alternative failure-mode +trigger. Or, if two systems need to interact, one or the other system +must block for results from the first. Stopping and restarting such an +analysis system is very delicate since the appropriate triggers must +be set up some how, eg by noticing which images have not succeeded and +restarting them at the appropriate stage. All of these types of +methods of handling complexity and failures are essentially +state-based rules. PanTasks allows the easy definition of a totally +state-based analysis system. + +In a state-based system, some mechanism examines the state of the +system and decides which actions to perform based on the current +state. In the illustration above, the mechanism could examine the +images available (either by examining the disk or by examining the +state of a data table) and decide to perform an operation based on +what images are available. This makes it very easy to handle +complexity and errors. If an analysis fails, the state either is not +successfully updated or the error state is recorded, both situations +being easy to detect and easy to handle. Restarting the system simply +involves starting the state-monitoring mechanism. Combining results +from multiple input sources simply involves watching for the multiple +inputs to be available. PanTasks provides a mechanism to define state +monitors, and to define the actions which are performed when those +states occur. PanTasks action consist of initiating UNIX commands, where +the arguments of those commands may depend on the results of the state +tests. + +\subsection{Tasks vs Jobs} + +The primary function of PanTasks is to repeatedly perform tasks, and +execute jobs on the basis of those tasks. A task consists of a set of +rules which describe system state tests to perform on a regular time +scale. Based on the results of those state tests, the task will then +choose whether or not to construct a job. The task also defines +actions to perform upon the completion of a job, based upon the output +and exit status of the job. A task thus defines the repeat period. It +may optionally define valid or invalid time ranges (eg, Mon-Fri or +10:00-17:00, etc). The task may also specify that the job be run +locally (ie, in the background on the same computer as PanTasks) or +remotely by the parallel process controller (pcontrol). A job may even +be restricted to a specific computer managed by pcontrol. An example +of a simple tasks is given below. + +\begin{verbatim} + task datalist + command ls /data/foo + periods -exec 5.0 + periods -timeout 50.0 + periods -poll 1.0 + + task.exit 0 + queueprint stdout + queuedelete stdout + end + + task.exit 1 + queuepush failure "task failed" + end + end + \end{verbatim} + + +This task does not perform any system state tests; it is simply +constructs a new job every 5.0 seconds. The job in this case is always +the same: ls /data/foo . When the job finished, if the job exit status +is 0 (normal UNIX success status), the resulting output is printed to +the screen. If the job returns an exit status of 1 (a failure), the +failure queue receives a single entry. Although they are not defined +in this case, it is also possible to specify the action to be taken if +the job crashes (does not exit normally) or if it times out (runs +beyond the specified timeout period). A slightly more complex task +which performs a state test and constructs a command based on that +test is shown below + +\begin{verbatim} + task datalist + periods -exec 5.0 + periods -timeout 50.0 + periods -poll 1.0 + + task.exec + $file = `next.file` + if ($file == "none") + break + end + command cp /data/foo/$file /data/bar + end + + task.exit 0 + queueprint stdout + queuedelete stdout + queuepush copied $file + end + + task.exit 1 + queuepush failure $file + end + end +\end{verbatim} + + The task.exec macro is executed by PanTasks every 5.0 seconds. This + macro executes a (hypothetical user-defined) UNIX command (next.file) + which examines the system state, return either a filename or the word + "none". If the result of this test is "none", the task does nothing: + no job is constructed. Otherwise, a job is constructed using the name + of the file returned by the state test. Successful jobs have the + filename added to the 'copied' queue, while failed jobs add the + filename to the 'failure' queue. + +\subsection{Parallel vs Local Job Processing} + +Jobs which are generated by PanTasks tasks may either be run locally +(forked in the background on the same machine as PanTasks) or run on the +IPP parallel process controller, pcontrol. The default is for the job +to be run locally. If a job should be run on the parallel controller, +this can be specified by including the command host (hostname) in the +definition of a task. If the value of (hostname) is 'anyhost', then +pcontrol may select any of its host computers to run the job according +to its own rules. If the value of (hostname) is one of the computers +managed by pcontrol, then that machine will be selected for the job, +if it is available. This amounts to a preference to use that machine, +but pcontrol is allowed to substitute a different machine if it +chooses. If the host command is given the option -required, then +pcontrol is forced to use the named host, even if the machine is down, +unknown, or otherwise unavailable. If the machine is not available, +pcontrol will simply hold onto the job until the machine is available +or the job is deleted. Note that PanTasks may delete jobs from pcontrol +if they remain pending for too long (see period -timeout). + +It is possible to interact directly with the parallel processor to +examine the current status, halt the parallel processor, etc. Commands +to the parallel processor are defined under the controller +command. The following controller commands are available: + +\begin{itemize} + +\item controller host (command) (hostname): Manage the parallel + controller collection of hosts. This command can be used to add a + new host, the delete one of the existing hosts, to turn a host on or + off, and to check the status of a host + + \begin{itemize} + \item controller host add (hostname): add a new host. + + \item controller host delete (hostname): delete a host. + + \item controller host on (hostname): tell pcontrol that the host is on. + + \item controller host off (hostname): tell pcontrol that the host is off. + + \item controller host retry (hostname): tell pcontrol to retry the host connection. + + \item controller host check (hostname): check the current status of a host. + \end{itemize} + + \item controller exit: stop controller execution. + + \item controller status: report controller current status. + + \item controller check: check job or host status. + + \item controller output: print accumulated messages from the controller. +\end{itemize} + +It is also possible to specify a host for a task which has not been +identified to the controller. If such a host is required, the +controller will simply keep the associated jobs in the pending state +until such a machine exists. See the pcontrol documentation for +further discussion of the controller manipuation of jobs and hosts. + +\subsection{Task Restrictions} + +Tasks may have restrictions on when they create jobs and how +frequently they create jobs. The task command trange is used to +specify a valid or invalid time range for a task. A valid time range +limits the task evaluation to that time period. An invalid time range +excludes task evaluation from the time period. Any number of time +range restrictions may be defined, and the union of all restrictions +will define if a job may be created. By default, the time range is an +inclusive time range: the task is evaluated only if the current time +falls within the specified time range. Alternatively, if the -exclude +flag is given, the time range is exclusive, in which case the task is +not evaluated if the current time falls within this range. + +The time range may be given as a range of absolute dates as follows: + +\begin{verbatim} + trange YYYY/MM/DD,HH:MM:SS YYYY/MM/DD,HH:MM:SS +\end{verbatim} + +where the two dates specify the start and end of the time range. In +either of these date representations, the least-significant elements +of the date and time may be dropped, defaulting to 00 (in the case of +hours, minutes, and seconds) or 01 (in the case of day and +months). Rather than specifying an end date, it is also valid to +specify a time interval from the starting date. The time interval is +specified as a number followed by a unit indicated by a single letter: +d (days), h (hours), m (minutes), s (seconds). + +The time range may also be specified as a repeated period of time, +either as a time of day or a day and time of week. In the first case, +the time range is specified as follows: + + +\begin{verbatim} + trange HH:MM:SS HH:MM:SS + \end{verbatim} + + +where again the least-significant elements may be dropped and default +to 00. This type of restriction defines a time range which is valid +every day. The alternative is to specify a time range within the week, +in the following form: + +\begin{verbatim} + trange DAY@HH:MM:SS DAY@HH:MM:SS +\end{verbatim} + +where the value of DAY may take on any of the three letter day-of-week +names (Sun, Mon, Tue, etc). This restriction specifies a start and end +time within a week which is evaluated for each week. + +Below are several examples of valid time range restrictions + +\begin{verbatim} + trange 2005/01/01 2005/12/31 (only run during 2005!) + trange 18:00 00:00 (only run from 6pm until midnight) + trange 00:00 06:00 (only run from midnight until 6am) + trange Mon@08:00 Fri@17:00 (only run between Mon morning and Fri afternoon) + trange -exclude 12:00 13:00 (skip 1 hour from noon) +\end{verbatim} + +Note that the current definition of trange does not include time zone +information. This means that all times are relative to UT. This should +be addressed by adding a timezone environment variable to PanTasks and +by allowing the trange to define a timezone offset. + +It is also possible to restrict the total number of jobs which are +spawned for a given task. This is done with the nmax command, which is +given as part of the task definition. Once a task has constructed nmax +jobs, it stops task evaluation. It is possible to redefine the value +of nmax at any time by redefining the task. Any time the task is +redefined, the new values for any task concept will override the +existing values for the task concept. + +\subsection{Inter-Task and Inter-Job Communications} + +There are several ways in which the results of jobs may be used to +influence other jobs. These include: + +\begin{itemize} +\item external communications +\item job exit status +\item job stdout parsing +\end{itemize} + +It is always possible for the interprocess communication to be +performed externally: all jobs may simply write results to an external +data source which is queried as part of the task evaluation. PanTasks +may interact with UNIX programs using Opihi system interaction +functions. These interaction methods include: the backticks for +setting Opihi variables: + +\begin{verbatim} + $variable = `UNIX Command` +\end{verbatim} + +The exec command (which executes a UNIX command) and the backticks +both receive the UNIX command exit status, setting the variable +\code{$STATUS}. It is also possible to set a variable list to the +output of a UNIX command: + +\begin{verbatim} + list var -x "UNIX Command" +\end{verbatim} + +In this last case, the values \code{$var:0 - $var:N-1} are set to the +value of the stdout lines from the UNIX command, and the value +\code{$var:n} is set to the number of output lines. + +Fine-grained control over the job exit status is available with the +task.exit macro command. This allows a task to define an exit macro +which is performed for different exit status conditions. The argument +to the task.exit command is the exit status value which triggers the +macro. This may consist of any valid numeric exit status value +(0-255). It may also have the value crash, in which case the macro is +executed if the program exited as a result of a signal (ie, +segmentation fault, etc). Finally, if may have the value default, in +which case, the macro is run if no other macro describes the exit +status. + +Jobs may transmit their results back to PanTasks for further evaluation +through the standard output and standard error streams. Whenever a job +exits, the complete stdout and stderr streams from the job are pushed +onto the PanTasks queues stdout and stderr. The job exit macros may then +parse these queues, moving the results into other PanTasks / Opihi data +containers (queues, variables, vectors, whatever is appropriate). Note +that currently, the output data is simply pushed onto these output +queues. It is currently the responsibility of the PanTasks programmer to +use or dispose of the data in these queues. This may change in the +future: the queues may be flushed for each job completion. + +\subsection{Running the scheduler} + +Once a set of tasks has been defined, the scheduler can be +started. The scheduler will run in the background, at regular +intervals examining the collection of tasks and jobs. In these +periods, the scheduler attempts to construct new jobs and checks on +the status of jobs which may have finished, either locally or on the +controller. To start the scheduler, give the command run. To stop the +scheduler, given the command stop. The current status of the +scheduler, controller, and any jobs which have been spawned are listed +with the status command. + +It is also possible to kill or delete individual jobs by hand with the +commands kill (jobID) or delete (jobID). Other features + +\subsection{PanTasks Command Summary} + +\begin{verbatim} + controller -- controller commands + task -- define a schedulable task + host -- define host machine for a task + nmax -- define maximum number of jobs for a task + trange -- define valid/invalid time periods for a task + task.exit -- define exit macros for a task + task.exec -- define pre-exec macro for a task + command -- define executed command for a task + periods -- define time scales for a task + run -- run the scheduler + stop -- stop the scheduler + pulse -- set the scheduler update period + status -- get system status + kill -- kill job + delete -- delete job + verbose -- set/toggle verbose mode +\end{verbatim} + +\begin{figure} +\begin{center} +\includegraphics[scale=0.85]{pics/pantasks.01.ps} +\caption{\label{queues} PanTasks queues and MDDB tables} +\end{center} +\end{figure} + +\begin{figure} +\begin{center} +\includegraphics[scale=0.85]{pics/pantasks.02.ps} +\caption{\label{queues} PanTasks queues and MDDB tables} +\end{center} +\end{figure} + +\begin{figure} +\begin{center} +\includegraphics[scale=0.85]{pics/pantasks.03.ps} +\caption{\label{queues} PanTasks queues and MDDB tables} +\end{center} +\end{figure} + +\newpage +\section{pcontrol : the PanTasks parallel controller} + +The IPP uses a group of computers to store and process images and to +manipulate collections of detections. These computers perform any of a +large number of analysis stages or other processing tasks without +significant interprocess communication. It is necessary to have a +mechanism which initiates computing tasks on the different computers, +which monitors the tasks as they are executed, which handles the +output and the errors from these tasks, and which reacts to the +failure of any of the computing nodes. The system responsible for the +tasks in the IPP is pcontrol. + +\subsection{Host States} + +pcontrol maintains a table of available processing computers (hosts) +and tracks their status. Hosts managed by pcontrol are allowed to be +in one of several states: off, down, idle, busy, and done. These +states have the following meanings: + +If the host is off, it is known to pcontrol, but pcontrol does not +have an active connection to the machine. Hosts which are off are not +available for jobs, and pcontrol does not attempt to initiate a +connection to them. + +When pcontrol is told to consider a machine on, the machine is moved +from the off state to the down state. Pcontrol attempts to initiate a +connection to the host. Connections are made by running a remote +client on the host, using the specified connection method. The +connection method may be ssh, rsh, or an equivalent remote shell +connection. The choice is specified by the COMMAND Opihi variable. The +remote connection starts a dedicated remote client which must accept +the pcontrol client commands and respond appropriately. The provided +remote client is called pclient, though in principal other equivalent +programs could be used by setting the Opihi variable SHELL (this +feature more generally allows a user to specify a path to the remote +client, if it is not in the user's path). A pcontrol user may force a +host to transition to the off state with the command host off +(hostname). ( Note that this command will set only one of the +connections to the named host to off. If multiple connections to a +machine have been defined, multiple off commands must be sent). + +If the remote connection is successful, the connected host is moved by +pcontrol from the down state to the idle state. If the connection is +unsuccessful, pcontrol will try again after a certain period of +time. If the connection continues to be unsuccessful, the retry period +is doubled for each successiver connection attempt. If the user wants +to force pcontrol to retry the connection to a machine (if, for +example, the timeout is now very long, but the user knows the +machine's ethernet cable has been re-inserted...), this can be +achieved with the command host retry (hostname). A host which is down +is in the limbo state between off and idle. + +Once pcontrol has made a successful connection to the host, the host +is in the idle state. At this point, it is ready to accept jobs from +pcontrol for execution. Pcontrol repeatedly queries the hosts to check +that they are still alive. If a host is discovered to be unresponsive, +and particularly if the remote pipe connection has closed, then the +machine is moved back to the down state. + +Hosts which are idle may accept a job from pcontrol. A job simply +consists of a bare UNIX command, without redirection of standard input +or standard output. The host will initiate the job, and pcontrol will +place the host into the busy state. The remote client, pclient, runs +the job in the background and will continue to accept input from +pcontrol. pcontrol will continue to check the status of the host, and +now also the status of the specific job. As before, if the connection +breaks, pcontrol will migrate the host to the down state. Any job +already initiated on a host which goes down will be returned for later +processing, so the job will not be lost. + +When the job exits, pclient tells pcontrol that the job is completed, +and specifies the exit status. At this point, pcontrol will move the +host from busy to done state. It will stay in this state until +pcontrol can determine the ending conditions and reset the remote +client. pcontrol requests the standard error and standard output from +the job from pclient. pcontrol stores this data with its information +about the completed job, and send a reset command to the remote +client. Once these cleanup tasks are successfully completed, pcontrol +will move the host to the idle state, ready for further jobs. + +Each physical computer may have multiple processors. pcontrol treats +each processor independently. It is up to the system configuration if +each computer needs to reserve one of its CPUs to manage background +tasks or if pcontrol should attempt to send one task per CPU and let +the operating system handle the I/O load. some of this behavior will +probably be eventually more intelligent. For example, the commands +which turn a host on or off should be able to do the same operation to +all host connections for the same machine name. + +A machine may be completely removed from pcontrol's host tables with +the command host delete (hostname). + +\subsection{Jobs} + +The pcontrol accepts new jobs with the command job ..., in which the +ellipsis represents the command and arguments of a valid UNIX +command. The commands are run under sh, and are executed in the user's +home directory. (If it is desired, we can easily add a command to tell +pclient to perform cd). Users should be wary of the conditions under +which the remote jobs are run. If the nodes in question all +cross-mount the same home directories, multiple jobs which interact +with the same named file may produce unexpected results. The +controller cannot enforce good behavior on the part of the remote +jobs; it is the responsibility of the user to ensure that conflicts do +not arise by, eg, always using unique output file names. + +Other issues may arise from the fact that pcontrol may be choosing any +of the hosts to run the job. Typical failures arise if the user does +not realize that specific jobs do not behave the same on all machines, +or if a necessary resource (eg, some input data file) is only +available or accessible from some of the hosts. It is the +responsibility of the task to wait for network lags (ie, NFS delays). + +pcontrol gives each task a unique internal identifier (Job ID) +equivalent to the process ID used in UNIX. When a job is submitted to +pcontrol, the command echoes back the Job ID. This ID may be used by +other pcontrol commands to obtain information about or interact with +the job. + +A job may specify a specific host for the task execution. The host +specified for a job may be required, or desired. In the first case, +pcontrol, will only run the job on the specified host, waiting until +it is available before attempting the job. In the second case, +pcontrol will attempt to send the job to the specified host, but if +the host is unavailable (how long? what conditions?), pcontrol will +allow the job to be sent to an alternative host. pcontrol attempts to +honor the requests for required and desired hosts, giving priority +first to required-host jobs, then to the desired-host jobs, and +finally to all other jobs. To specify a host for a job, the following +commands are used: + +\begin{verbatim} + job -host (command and arguments...) + job +host (command and arguments...) +\end{verbatim} + + +The first case specifies a desired host, while the second specifies a +required host. It is also possible to specify the special host name +anyhost, which is equivalent to not specifying a host at all. + +Job priority / urgency levels are not implemented at this time. + +I/O vs CPU tasks are not currently distinguished by pcontrol + +pcontrol stores the stdout and stderr for each completed job. To +retrieve these data from these streams, the user issues the commands +stdout (JobID) and stderr (JobID). The result is a single line +specifying the number of bytes to expect, followed by a dump of the +buffers, followed by the prompt. It is the user's responsibility to +relieve pcontrol of this data load by deleting jobs once they are no +longer needed. Job deletion is performed with the command delete +(JobID). + +Jobs are moved between the following states by pcontrol: + +\begin{itemize} +\item pending: the job has not yet been executed. +\item busy: the job is currently being executed. +\item done: the job has completed, but the stdout/stderr has not been processed by pcontrol. +\item exit: the job has completed with a valid exit status +\item crash: the job has completed with a crash status (exit on signal). +\end{itemize} + +\subsection{Miscellaneous Commands} + +It is possible to check the status of a single host or job with the +user command check. + +pcontrol continuously examines the stack of jobs, adjusting their +state as needed and extracting their output when it is ready. These +checks are performed in the background, with pcontrol ready to accept +further commands from the user in the foreground. These checks are +performed after every keystroke, and also after an inactivity +timeout. The interrupt interval defaults to 1 second, but may be +adjusted with the pulse command, which takes as an argument, the +number of microseconds for the timeout. + +the pcontrol system status may be examined with the command +status. This provides a dump of the job stacks and the host stacks. + +It is possible to list the jobs currently in a specific stack, +corresponding to the list of jobs with a given state. This is done +with the command jobstack (stackname). The valid stack names are +pending, busy, exit, crash, and done. The result is a list of all jobs +on the specified stack. This is useful to determine quickly which jobs +have exited or crashed. + +A specific job may be killed with the command kill (JobID). This +command is only valid for a job in the busy state. Any job in the +pending, exit, or crash state may be deleted with the delete (JobID) +command. This is necessary to free the memory associated with the job +and its output streams. + +The command verbose (mode) turns the verbosity of the pcontrol +operations on or off. + +The pcontrol and the IPP Image Server have related needs for +information from the combined storage-and-processing nodes regarding +which nodes are available. It is not yet clear if this information is +best stored in a single location (either pcontrol or IPP Image +Server), which provides the information to other systems on demand, or +if both systems should maintain the information. Also, it may be +necessary to distinguish nodes which are available for processing from +those that are available to serve data as part of the IPP Image +Server. + +\subsection{Command Summary} + +\begin{verbatim} + check -- get job or host status + delete -- delete job + host -- add / delete / modify host + job -- add job + jobstack -- list jobs for a single stack + kill -- kill job + pulse -- set system pulse + status -- get system status + stderr -- get stderr buffer for job + stdout -- get stdout buffer for job + verbose -- set the verbose mode for job +\end{verbatim} + +\begin{figure} +\begin{center} +\includegraphics[scale=0.85]{pics/pantasks.04.ps} +\caption{\label{queues} PanTasks queues and MDDB tables} +\end{center} +\end{figure} + +\begin{figure} +\begin{center} +\includegraphics[scale=0.85]{pics/pantasks.05.ps} +\caption{\label{queues} PanTasks queues and MDDB tables} +\end{center} +\end{figure} + +\begin{figure} +\begin{center} +\includegraphics[scale=0.85]{pics/pantasks.06.ps} +\caption{\label{queues} PanTasks queues and MDDB tables} +\end{center} +\end{figure} + +\newpage +\section{pclient} + +pclient is the remote process monitor for pcontrol, the parallel +process controller. + +The program pclient is used to support the remote jobs which are run +on the remote hosts by pcontrol. The concept of pclient is to act as a +buffer between the job running on the remote host and pcontrol. The +pcontrol design uses (by default) ssh connections initiated by +pcontrol to the remote hosts. These connections execute the remote +program of pclient. The use of a remote login process lets the UNIX +system take care of the user authentication issues. In this case, the +recommended practice is to set up ssh to allow the connection to the +remote host without additional authentication using the appropriate +authorized keys (see this article on ssh issues). + +It is convenient to keep a continuous connection to the remote +hosts. This avoids incurring the overhead of authentication for each +command which is executed, while keeping a high-quality user +authentication process in place. + +pclient acts as a buffer between pcontrol and the remote background +process, allowing the continuous connection to remain viable without +samping pcontrol with output from the jobs. + +\section{Command Summary} + +pclient has a very limited command set, as follows: + +\begin{verbatim} + job : start the job (UNIX command) in the background. + check : return the current job status + status : return the current job status (?) + stdout : dump the stdout stream accumulated from the job + back to the calling program. + stderr : dump the stderr stream accumulated from the job + back to the calling program. + reset : kill (if needed) the job and reset to accept + another job. +\end{verbatim} + +\begin{figure} +\begin{center} +\includegraphics[scale=0.85]{pics/pantasks.07.ps} +\caption{\label{queues} PanTasks queues and MDDB tables} +\end{center} +\end{figure} + +\appendix +\pagebreak + +\section{Example : Pcopy.pro} + +Below is an example script for psched which demonstrates the +scheduling system. This parallel-copying script implements the +Pan-STARRS image copying system, which requests images from the summit +and copies them to the appropriate computer. The first task in the +script queries an external system for new image names with the +function new.images. In the case of Pan-STARRS, this would be a +request from OTIS, the observatory controlling system. The second task +initiates the individual image copies, with separate CCDs being copied +to separate computers. This script uses the concept of having specific +machines assigned to specific CCDs (as Pan-STARRS intends to +operate). The association is determined by calling the external +function chip.host, providing the identifier of the chip in +question. This returns an appropriate host. The copy.image function +copies the file and also sends a message to the summit system to +inform it that the image has been successfully copied. + +\begin{verbatim} +verbose on + queueinit newImages + exec echo 0 > new.last + exec cp -f raw.list new.list + + controller host add po01 + controller host add po02 + controller host add po03 + controller host add po04 + + # identify the images ready for copy + # new entries are added to queue newImages + # need to compare the new list with the ones already being processed + task new.images + command new.images + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 5 + + # success + task.exit 0 + local i j Nstdout Nimages + # compare output with new.image queue + # keep only new entries + queuesize stdout -var Nstdout + for i 0 $Nstdout + queuepop stdout -var line + queuepush newImages -uniq -key 0 "$line" + end + end + + # locked list + task.exit 1 + echo "new.images: exec failure" + $new.image.failure ++ + end + + # default exit status + task.exit default + echo "new.images: unknown exit status: $EXIT" + $new.image.failure ++ + end + + # operation times out? + task.exit timeout + echo "new.images: timeout" + $new.image.failure ++ + end + end + + # copy new images, sending job to desired host + task copy.images + periods -poll 0.2 + periods -exec 1 + periods -timeout 5 + + task.exec + queuesize newImages -var N + if ($N == 0) break + # if ($network == 0) break + # if ($filesystem == 1) break + + queuepop newImages -var line + list tmp -split $line + $filename = $tmp:0 + $chip = $tmp:1 + $state = $tmp:2 + if ($state == new) + # copy this image + queuepush newImages -replace -key 0 "$filename $chip run" + else + # ignore this image + queuepush newImages -replace -key 0 "$filename $chip $state" + break + end + # echo $chip + $host = `chip.host $chip` + # echo $host + host $host + # echo "starting copy for $filename on $host..." + command copy.image $filename $chip + end + + # can I have access to argc,argv? + + # success + task.exit 0 + echo "done copy..." + queuepop stdout -var line + list tmp -split $line + $filename = $tmp:0 + $chip = $tmp:1 + exec mark.image $filename + queuepush newImages -replace -key 0 "$filename $chip copy" + end + + # default exit status + task.exit default + echo "new.images: unknown exit status: $EXIT" + $new.image.failure ++ + end + + # operation times out? + task.exit timeout + echo "new.images: timeout" + $new.image.failure ++ + end + end + +\end{verbatim} + +\end{document} Index: /trunk/doc/psphot/Makefile =================================================================== --- /trunk/doc/psphot/Makefile (revision 6032) +++ /trunk/doc/psphot/Makefile (revision 6033) @@ -5,5 +5,5 @@ help: @echo "USAGE: make (target)" - @echo " targets: psphot add" + @echo " targets: psphot all" psphot: psphot.pdf