Index: /branches/czw_branch/20100519/PS-IPP-Config/lib/PS/IPP/Config.pm =================================================================== --- /branches/czw_branch/20100519/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 28163) +++ /branches/czw_branch/20100519/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 28164) @@ -56,4 +56,5 @@ our $PS_TABLE_ID_STACK = 3; our $PS_TABLE_ID_DIFF = 4; +our $PS_TABLE_ID_STATICSKY = 5; our $parser = PS::IPP::Metadata::Config->new; # Metadata parser Index: /branches/czw_branch/20100519/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /branches/czw_branch/20100519/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 28163) +++ /branches/czw_branch/20100519/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 28164) @@ -1116,4 +1116,9 @@ $astrom_file = $run->{cam_path_base} . ".smf"; my $astrom_file_resolved = $ipprc->file_resolve($astrom_file); + if (!$astrom_file_resolved) { + print STDERR "cannot resolve astrometry file: $astrom_file\n"; + setErrorCodes($rowList, $PSTAMP_NOT_AVAILABLE); + next; + } $command .= " -astrom $astrom_file_resolved"; } else { Index: /branches/czw_branch/20100519/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- /branches/czw_branch/20100519/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 28163) +++ /branches/czw_branch/20100519/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 28164) @@ -17,4 +17,5 @@ our @EXPORT_OK = qw( read_request_file + get_error_string $PSTAMP_CENTER_IN_PIXELS $PSTAMP_RANGE_IN_PIXELS @@ -82,8 +83,41 @@ our $PSTAMP_NO_OVERLAP = 28; + use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Config qw( :standard ); + +my @errorStrings = qw( +PSTAMP_SUCCESS +PS_EXIT_UNKNOWN_ERROR +PS_EXIT_SYS_ERROR +PS_EXIT_CONFIG_ERROR +PS_EXIT_PROG_ERROR +PS_EXIT_DATA_ERROR +PS_EXIT_TIMEOUT_ERROR +undefined +undefined +undefined +PSTAMP_SYSTEM_ERROR +PSTAMP_NOT_IMPLEMENTED +PSTAMP_UNKNOWN_ERROR +undefined +undefined +undefined +undefined +undefined +undefined +undefined +PSTAMP_DUP_REQUEST +PSTAMP_INVALID_REQUEST +PSTAMP_UNKNOWN_PRODUCT +PSTAMP_NO_IMAGE_MATCH +PSTAMP_NOT_DESTREAKED +PSTAMP_NOT_AVAILABLE +PSTAMP_GONE +PSTAMP_NO_JOBS_QUEUED +PSTAMP_NO_OVERLAP +); sub read_request_file { @@ -205,3 +239,13 @@ } +sub get_error_string { + my $error_code = shift; + my $error_string; + if ($error_code >= 0) { + $error_string = $errorStrings[$error_code]; + } + $error_string = "unknown" if !$error_string; + return $error_string; +} + 1; Index: /branches/czw_branch/20100519/dbconfig/cam.md =================================================================== --- /branches/czw_branch/20100519/dbconfig/cam.md (revision 28163) +++ /branches/czw_branch/20100519/dbconfig/cam.md (revision 28164) @@ -15,9 +15,14 @@ magicked S64 0 software_ver STR 16 - maskfrac_npix S32 0 - maskfrac_static F32 0.0 - maskfrac_dynamic F32 0.0 - maskfrac_magic F32 0.0 - maskfrac_advisory F32 0.0 + maskfrac_ref_npix S32 0 + maskfrac_ref_static F32 0.0 + maskfrac_ref_dynamic F32 0.0 + maskfrac_ref_magic F32 0.0 + maskfrac_ref_advisory F32 0.0 + maskfrac_max_npix S32 0 + maskfrac_max_static F32 0.0 + maskfrac_max_dynamic F32 0.0 + maskfrac_max_magic F32 0.0 + maskfrac_max_advisory F32 0.0 note STR 255 END @@ -98,9 +103,14 @@ fault S16 0 # Key NOT NULL software_ver STR 16 - maskfrac_npix S32 0 - maskfrac_static F32 0.0 - maskfrac_dynamic F32 0.0 - maskfrac_magic F32 0.0 - maskfrac_advisory F32 0.0 + maskfrac_ref_npix S32 0 + maskfrac_ref_static F32 0.0 + maskfrac_ref_dynamic F32 0.0 + maskfrac_ref_magic F32 0.0 + maskfrac_ref_advisory F32 0.0 + maskfrac_max_npix S32 0 + maskfrac_max_static F32 0.0 + maskfrac_max_dynamic F32 0.0 + maskfrac_max_magic F32 0.0 + maskfrac_max_advisory F32 0.0 quality S16 0 END Index: /branches/czw_branch/20100519/dbconfig/changes.txt =================================================================== --- /branches/czw_branch/20100519/dbconfig/changes.txt (revision 28163) +++ /branches/czw_branch/20100519/dbconfig/changes.txt (revision 28164) @@ -1541,51 +1541,5 @@ ALTER TABLE distRun ADD COLUMN alternate TINYINT DEFAULT 0 AFTER no_magic; --- Tables for static sky analysis - --- A static sky analysis set -CREATE TABLE staticskyRun ( - ss_id BIGINT AUTO_INCREMENT, -- unique identifier - state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) - workdir VARCHAR(255) NOT NULL, -- working directory - label VARCHAR(64), -- processing label - data_group VARCHAR(64), -- group for data - dist_group VARCHAR(64), -- group for distribution - reduction VARCHAR(64), -- reduction class (for altering recipe) - note VARCHAR(255), -- note - registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered - PRIMARY KEY(ss_id), - KEY(state), - KEY(label), - KEY(data_group), - KEY(dist_group) -) ENGINE=innodb DEFAULT CHARSET=latin1; - --- Inputs for static sky analysis -CREATE TABLE staticskyInput ( - ss_id BIGINT, -- static sky identifier - stack_id BIGINT, -- stack identifier - PRIMARY KEY(ss_id,stack_id), - KEY(stack_id), - FOREIGN KEY(ss_id) REFERENCES staticskyRun(ss_id), - FOREIGN KEY(stack_id) REFERENCES stackSumSkyfile(stack_id) -) ENGINE=innodb DEFAULT CHARSET=latin1; - --- Result of static sky analysis -CREATE TABLE staticskyResult ( - ss_id BIGINT, -- static sky identifier - path_base VARCHAR(255) NOT NULL, -- root name for outputs - dtime_phot FLOAT, -- elapsed time for photometry - dtime_script FLOAT, -- elapsed time for script - sources INT, -- number of sources - hostname VARCHAR(64) NOT NULL, -- host that executed script - good_frac FLOAT, -- good fraction of skycell - quality SMALLINT NOT NULL, -- bad quality flag - fault SMALLINT NOT NULL, -- fault code - PRIMARY KEY(ss_id), - KEY(good_frac), - KEY(fault), - KEY(quality), - FOREIGN KEY(ss_id) REFERENCES staticskyRun(ss_id) -) ENGINE=innodb DEFAULT CHARSET=latin1; +-- staticsky tables were re-configured and the entry moved below ALTER TABLE pstampDependent ADD COLUMN component VARCHAR(64) after stage_id; @@ -1680,2 +1634,128 @@ ALTER TABLE diffSkyfile ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic; +ALTER TABLE chipRun CHANGE maskfrac_npix maskfrac_npix FLOAT; +ALTER TABLE camRun CHANGE maskfrac_npix maskfrac_npix FLOAT; +ALTER TABLE warpRun CHANGE maskfrac_npix maskfrac_npix FLOAT; +ALTER TABLE diffRun CHANGE maskfrac_npix maskfrac_npix FLOAT; + +ALTER TABLE camRun CHANGE maskfrac_npix maskfrac_ref_npix FLOAT; +ALTER TABLE camRun CHANGE maskfrac_static maskfrac_ref_static FLOAT; +ALTER TABLE camRun CHANGE maskfrac_dynamic maskfrac_ref_dynamic FLOAT; +ALTER TABLE camRun CHANGE maskfrac_magic maskfrac_ref_magic FLOAT; +ALTER TABLE camRun CHANGE maskfrac_advisory maskfrac_ref_advisory FLOAT; + +ALTER TABLE camProcessedExp CHANGE maskfrac_npix maskfrac_ref_npix INT; +ALTER TABLE camProcessedExp CHANGE maskfrac_static maskfrac_ref_static FLOAT; +ALTER TABLE camProcessedExp CHANGE maskfrac_dynamic maskfrac_ref_dynamic FLOAT; +ALTER TABLE camProcessedExp CHANGE maskfrac_magic maskfrac_ref_magic FLOAT; +ALTER TABLE camProcessedExp CHANGE maskfrac_advisory maskfrac_ref_advisory FLOAT; + +ALTER TABLE camRun ADD COLUMN maskfrac_max_npix FLOAT after maskfrac_ref_advisory; +ALTER TABLE camRun ADD COLUMN maskfrac_max_static FLOAT after maskfrac_max_npix; +ALTER TABLE camRun ADD COLUMN maskfrac_max_dynamic FLOAT after maskfrac_max_static; +ALTER TABLE camRun ADD COLUMN maskfrac_max_magic FLOAT after maskfrac_max_dynamic; +ALTER TABLE camRun ADD COLUMN maskfrac_max_advisory FLOAT after maskfrac_max_magic; + +ALTER TABLE camProcessedExp ADD COLUMN maskfrac_max_npix INT after maskfrac_ref_advisory; +ALTER TABLE camProcessedExp ADD COLUMN maskfrac_max_static FLOAT after maskfrac_max_npix; +ALTER TABLE camProcessedExp ADD COLUMN maskfrac_max_dynamic FLOAT after maskfrac_max_static; +ALTER TABLE camProcessedExp ADD COLUMN maskfrac_max_magic FLOAT after maskfrac_max_dynamic; +ALTER TABLE camProcessedExp ADD COLUMN maskfrac_max_advisory FLOAT after maskfrac_max_magic; + +-- these tables have not yet been used, and the definitions changed somewhat + +-- A static sky analysis set +CREATE TABLE staticskyRun ( + sky_id BIGINT AUTO_INCREMENT, -- unique identifier + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + note VARCHAR(255), -- note + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(sky_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Inputs for static sky analysis +CREATE TABLE staticskyInput ( + sky_id BIGINT, -- static sky identifier + stack_id BIGINT, -- stack identifier + PRIMARY KEY(sky_id,stack_id), + KEY(stack_id), + FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id), + FOREIGN KEY(stack_id) REFERENCES stackSumSkyfile(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Result of static sky analysis +CREATE TABLE staticskyResult ( + sky_id BIGINT, -- static sky identifier + path_base VARCHAR(255) NOT NULL, -- root name for outputs + dtime_phot FLOAT, -- elapsed time for photometry + dtime_script FLOAT, -- elapsed time for script + sources INT, -- number of sources + num_inputs INT, -- number inputs used + hostname VARCHAR(64) NOT NULL, -- host that executed script + good_frac FLOAT, -- good fraction of skycell + quality SMALLINT NOT NULL, -- bad quality flag + fault SMALLINT NOT NULL, -- fault code + PRIMARY KEY(sky_id), + KEY(good_frac), + KEY(fault), + KEY(quality), + FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +-- Version 1.1.69 +CREATE TABLE warpSummary ( + warp_id BIGINT, + tangent_plane VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(warp_id), + KEY(tangent_plane), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffSummary ( + diff_id BIGINT, + tangent_plane VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(diff_id), + KEY(tangent_plane), + FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackSummary ( + stack_id BIGINT, + tangent_plane VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(stack_id), + KEY(tangent_plane), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociation ( + sass_id BIGINT AUTO_INCREMENT, + data_group VARCHAR(64) NOT NULL, + tangent_plane VARCHAR(64) NOT NULL, + tess_id VARCHAR(64) NOT NULL, + filter VARCHAR(64) NOT NULL, + PRIMARY KEY(sass_id), + KEY(data_group), + KEY(tangent_plane), + KEY(tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociationMap ( + sass_id BIGINT, + stack_id BIGINT, + PRIMARY KEY(sass_id, stack_id), + FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/czw_branch/20100519/dbconfig/chip.md =================================================================== --- /branches/czw_branch/20100519/dbconfig/chip.md (revision 28163) +++ /branches/czw_branch/20100519/dbconfig/chip.md (revision 28164) @@ -15,5 +15,5 @@ magicked S64 0 software_ver STR 16 - maskfrac_npix S32 0 + maskfrac_npix F32 0.0 maskfrac_static F32 0.0 maskfrac_dynamic F32 0.0 Index: /branches/czw_branch/20100519/dbconfig/diff.md =================================================================== --- /branches/czw_branch/20100519/dbconfig/diff.md (revision 28163) +++ /branches/czw_branch/20100519/dbconfig/diff.md (revision 28164) @@ -16,5 +16,5 @@ magicked S64 0 software_ver STR 16 - maskfrac_npix S32 0 + maskfrac_npix F32 0.0 maskfrac_static F32 0.0 maskfrac_dynamic F32 0.0 @@ -75,2 +75,8 @@ maskfrac_advisory F32 0.0 END + +diffSummary METADATA + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + tangent_plane STR 32 # Primary Key + path_base STR 255 +END Index: /branches/czw_branch/20100519/dbconfig/ipp.m4 =================================================================== --- /branches/czw_branch/20100519/dbconfig/ipp.m4 (revision 28163) +++ /branches/czw_branch/20100519/dbconfig/ipp.m4 (revision 28164) @@ -20,4 +20,5 @@ include(diff.md) include(stack.md) +include(staticsky.md) include(det.md) include(label.md) Index: /branches/czw_branch/20100519/dbconfig/stack.md =================================================================== --- /branches/czw_branch/20100519/dbconfig/stack.md (revision 28163) +++ /branches/czw_branch/20100519/dbconfig/stack.md (revision 28164) @@ -53,2 +53,21 @@ quality S16 0 END + +stackSummary METADATA + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) + tangent_plane STR 32 # Primary Key + path_base STR 255 +END + +stackAssociation METADATA + sass_id S64 0 # Primary Key AUTO_INCREMENT + data_group STR 64 # Key + tangent_plane STR 64 # Key + tess_id STR 64 # Key + filter STR 64 +END + +stackAssociationMap METADATA + sass_id S64 0 # Primary Key fkey(sass_id) ref stackAssociation(sass_id) + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) +END Index: /branches/czw_branch/20100519/dbconfig/staticsky.md =================================================================== --- /branches/czw_branch/20100519/dbconfig/staticsky.md (revision 28163) +++ /branches/czw_branch/20100519/dbconfig/staticsky.md (revision 28164) @@ -1,3 +1,4 @@ # Tables for static sky analysis +# XXX add tess_id and skycell_id here? staticskyRun METADATA @@ -24,4 +25,5 @@ dtime_script F32 0.0 sources S32 0 + num_inputs S32 0 hostname STR 64 good_frac F32 0.0 # Key Index: /branches/czw_branch/20100519/dbconfig/warp.md =================================================================== --- /branches/czw_branch/20100519/dbconfig/warp.md (revision 28163) +++ /branches/czw_branch/20100519/dbconfig/warp.md (revision 28164) @@ -26,5 +26,5 @@ magicked S64 0 software_ver STR 16 - maskfrac_npix S32 0 + maskfrac_npix F32 0.0 maskfrac_static F32 0.0 maskfrac_dynamic F32 0.0 @@ -79,2 +79,8 @@ label STR 64 # Primary Key END + +warpSummary METADATA + warp_id S64 0 # Primary Key fkey(warp_id) ref warpRun(warp_id) + tangent_plane STR 32 # Primary Key + path_base STR 255 +END Index: /branches/czw_branch/20100519/doc/stamps/stampsICD.tex =================================================================== --- /branches/czw_branch/20100519/doc/stamps/stampsICD.tex (revision 28163) +++ /branches/czw_branch/20100519/doc/stamps/stampsICD.tex (revision 28164) @@ -9,6 +9,6 @@ \project{Pan-STARRS IPP} \organization{Institute for Astronomy} -\version{DR} -\docnumber{PSDC-940-xxx} +\version{1.0} +\docnumber{PSDC-940-011-00} \newcommand\ugriz{$u^\prime g^\prime r^\prime i^\prime z^\prime$} @@ -49,5 +49,5 @@ Each row in a request table contains a postage stamp request specification. Each request specification causes zero or more postage stamp jobs to be processed. -An image is added to the output file set for each successful job. +Images are added to the output file set for each successful job. The request specification give the parameters that the PSS uses to determine the images of interest @@ -125,5 +125,5 @@ \subsubsection{Image Types} -All requests need IMG\_TYPE. This parameter selects the stage in the processing pipeline +All requests need IMG\_TYPE. This parameter selects the stage of the processing pipeline for the images of interest. The allowed values for IMG\_TYPE are raw, chip, warp, diff, and stack. @@ -139,5 +139,5 @@ IMG\_TYPE that contain the center. The set of images of interest may be limited by specifying certain criteria as described -in Section \ref{ImageCuts}. NOTE: this request type is not yet implemented. +in Section \ref{ImageCuts}. \item{\textbf{byexp}} The image of interest is the image resulting from a particular exposure @@ -149,11 +149,11 @@ \item{\textbf{bydiff}} The image with given IMG\_TYPE corresponding to the difference image -whose IMAGEID is equal to ID. This type is useful for requesting stamps based on difference detections. +whose diff\_skyfile\_id is equal to ID. This type is useful for requesting stamps based on difference detections. \item{\textbf{byskycell}} Images are selected from the skycell with skycell\_id equal to the value for COMPONENT from the tessellation identified by TESS\_ID. This request type may be -used only with IMG\_TYPE warp, stack, and diff. +used with IMG\_TYPE warp, stack, and diff. The set of images may be limited by filter and date as described in Section \ref{ImageCuts}. -NOTE: this request type is not yet implemented. + \end{itemize} @@ -231,7 +231,7 @@ the stamps are created the cells are mosaicked into a single image.) -Any fractional parts in pixel coordinate values are ignored. (XXX: is that true?) - -For JOB\_TYPE stamp the value of \code{OPTION\_MASK} is used to select various other options. +\subsection{Request Options} + +For \code{JOB_TYPE} stamp the value of \code{OPTION_MASK} is used to select various other options. It is a bitwise OR of the following values. @@ -239,17 +239,19 @@ \begin{center} \caption{}\label{optionmask} -\begin{tabular}{ll} -\hline -\hline -{\bf OPTION\_MASK value} & {\bf Description } \\ -\hline -1 & create postage stamp of the image pixels \\ -2 & create postage stamp of the mask pixels \\ -4 & create postage stamp of the variance pixels \\ -8 & return the sources file (.cmf) for the images' run \\ -16 & return the psf file for the images' run \\ -32 & return the background model file for the images chipRun (applies only for chip) \\ -1024 & make stamps from inverse difference images (applies only for diff) \\ -2048 & make stamps from unconvolved stack images (applies only for stack) \\ +\begin{tabular}{lll} +\hline +\hline +{\bf C Macro name} & {\bf Value} & {\bf Description } \\ +\hline +PSTAMP\_SELECT\_IMAGE & 1 & create postage stamp of the image pixels \\ +PSTAMP\_SELECT\_MASK & 2 & create postage stamp of the mask pixels \\ +PSTAMP\_SELECT\_VARIANCE & 4 & create postage stamp of the variance pixels \\ +PSTAMP\_SELECT\_CMF & 8 & return the sources file (.cmf) for the images' run \\ +PSTAMP\_SELECT\_PSF & 16 & return the psf file for the images' run \\ +PSTAMP\_SELECT\_BACKMDL & 32 & return the background model file for the images chipRun (applies only for chip) \\ +PSTAMP\_SELECT\_INVERSE & 1024 & make stamps from inverse difference images (applies only for diff) \\ +PSTAMP\_SELECT\_UNCONV & 2048 & make stamps from unconvolved stack images (applies only for stack) \\ +PSTAMP\_USE\_IMFILE\_ID & 16384 & for byid requests select images using image id instead of stage id \\ +PSTAMP\_NO\_WAIT\_FOR\_UPDATE & 32768 & if images are not available because they have been cleaned don't wait \\ \hline \end{tabular} @@ -264,6 +266,37 @@ For requests with IMG\_TYPE diff or if \code{REQ_TYPE} is bydiff and the corresponding difference run is a 'bothways' diff the images are selected based on the inverse difference -images if \code{(OPTION_MASK & 1024)} is non-zero. Otherwise from the images are selected based -on the 'positive' images. +images if \code{(OPTION_MASK & PSTAMP_SELECT_INVERSE)} is non-zero. +Otherwise from the images are selected based on the 'positive' images. + +For \code{REQ_TYPE} byid normally the \code{ID} value is the "Stage ID" or "Run ID" of the requested image. +If the bit \code{PSTAMP_USE_IMFILE_ID} is included in the \code{OPTION_MASK} for a byid request, +the \code{ID} for the request is the "Image ID" instead of the "Stage ID". +Table ~\ref{byid_ids} gives the IPP database column that corresponds to these identefiers for the various +stages. The "Image ID" is useful because it may be used to select component of a run. For example +the chip\_imfile\_id values are recorded in the DVO image table so it is easy to create a postage +stamp request from a DVO measurement. XXX: point to an example. + +\begin{table}[h] +\begin{center} +\caption{}\label{byid_ids} +\begin{tabular}{lll} +\hline +\hline +{\bf IMG\_TYPE/stage} & {\bf stage ID} & {\bf Image ID } \\ +\hline +raw & exp\_id & raw\_imfile\_id \\ +chip & chip\_id & chip\_imfile\_id \\ +warp & warp\_id & warp\_skyfile\_id \\ +diff & diff\_id & diff\_skyfile\_id \\ +stack & stack\_id & stack\_id \\ +\hline +\end{tabular} +\end{center} +\end{table} +Due to limited space on the IPP cluster, images files are deleted some time after they are created. Data in +this state is referred to as "cleaned". By default the postage stamp server will automatically regenerate +the pixels for requested images, if possible. This is referred to as the update process. +If the user does not wish to wait for the updates to complete he should include +\code{PSTAMP_NO_WAIT_FOR_UPDATE} in the \code{OPTION_MASK}. @@ -306,4 +339,5 @@ \code{ROWNUM} & U32 & identifier for this row in table\\ \code{ERROR_CODE} & U32 & Error code (see text) \\ +\code{ERROR_STR} & U32 & Name corresponding to \code{ERROR_CODE}\\ \code{IMG_NAME} & string & Name of image in Data Store file set \\ \code{JOB_ID} & S64 & Internal postage stamp server job id that produced this image \\ @@ -383,10 +417,10 @@ PSTAMP\_INVALID\_REQUEST & 21 & Error in a request specification. See parse\_error.txt \\ PSTAMP\_UNKNOWN\_PRODUCT & 22 & Unknown product in request specification \\ -PSTAMP\_NO\_IMAGE\_MATCH & 23 & No images matched the request \\ +PSTAMP\_NO\_IMAGE\_MATCH & 23 & No images matched the request \\ PSTAMP\_NOT\_DESTREAKED & 24 & Image matched, but not yet de-streaked \\ PSTAMP\_NOT\_AVAILABLE & 25 & Image not available (temporary) \\ -PSTAMP\_GONE & 26 & Image is no longer available \\ -PSTAMP\_NO\_JOBS\_QUEUED & 27 & Request specification yielded no jobs. See parse\_error.txt \\ -PSTAMP\_NO\_OVERLAP & 28 & Center not contained in any image of interest \\ +PSTAMP\_GONE & 26 & Image is no longer available \\ +PSTAMP\_NO\_JOBS\_QUEUED & 27 & Request specification yielded no jobs. See parse\_error.txt \\ +PSTAMP\_NO\_OVERLAP & 28 & Center coordinates are not contained in any image of interest \\ \hline Index: /branches/czw_branch/20100519/ippScripts/Build.PL =================================================================== --- /branches/czw_branch/20100519/ippScripts/Build.PL (revision 28163) +++ /branches/czw_branch/20100519/ippScripts/Build.PL (revision 28164) @@ -55,4 +55,5 @@ scripts/diff_skycell.pl scripts/stack_skycell.pl + scripts/staticsky.pl scripts/magic_tree.pl scripts/magic_process.pl Index: /branches/czw_branch/20100519/ippScripts/scripts/staticsky.pl =================================================================== --- /branches/czw_branch/20100519/ippScripts/scripts/staticsky.pl (revision 28163) +++ /branches/czw_branch/20100519/ippScripts/scripts/staticsky.pl (revision 28164) @@ -21,4 +21,6 @@ use PS::IPP::Metadata::List qw( parse_md_list ); use Data::Dumper; +use File::Temp qw( tempfile ); +use File::Basename; use PS::IPP::Config 1.01 qw( :standard ); @@ -29,5 +31,5 @@ my $missing_tools; my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); -my $ppSub = can_run('ppSub') or (warn "Can't find ppSub" and $missing_tools = 1); +my $psphotStack = can_run('psphotStack') or (warn "Can't find psphotStack" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); if ($missing_tools) { @@ -36,7 +38,8 @@ } -my ($sky_id, $dbname, $threads, $outroot, $reduction, $inverse, $run_state, $verbose, $no_update, $no_op, $redirect, $save_temps); +my ($sky_id, $camera, $dbname, $threads, $outroot, $reduction, $inverse, $run_state, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( 'sky_id=s' => \$sky_id, # Diff identifier + 'camera|c=s' => \$camera, # Camera name 'dbname|d=s' => \$dbname, # Database name 'threads=s' => \$threads, # Number of threads to use @@ -53,19 +56,19 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --sky_id --outroot", + -msg => "Required options: --sky_id --outroot --camera (--run_state)", -exitval => 3, ) unless defined $sky_id and - defined $outroot; - -my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $sky_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration - -# XXX camera is not known here; cannot use filerules... -# my $logDest = $ipprc->filename("LOG.EXP", $outroot); -my $logDest = "$outroot.log"; -$logDest .= ".update" if $run_state eq "update"; + defined $outroot and + defined $camera; +# and defined $run_state; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $sky_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot); +# $logDest .= ".update" if $run_state eq "update"; $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $sky_id, $PS_EXIT_SYS_ERROR ) if $redirect; -my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF); +# my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_STATICSKY); my $outbase = basename($outroot); @@ -91,45 +94,29 @@ } -# XXX do we NEED to define the camera? -# &my_die("Unable to identify camera", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless defined $camera; -# $ipprc->define_camera($camera); - # generate the input print $listFile "INPUT MULTI\n"; +my $nInputs = @$files; foreach my $file (@$files) { print $listFile "INPUT METADATA\n"; - $path_base = $file->{path_base}; - my $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $file->{path_base} ); # Mask name - my $maskCnv = $ipprc->filename("PPSTACK.OUTPUT.MASK", $file->{path_base} ); # Mask name - my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name - my $psfCnv = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $file->{path_base} ); # PSF name - - my $imageRaw = $ipprc->filename("PPSTACK.UNCONV", $file->{path_base} ); # Mask name - my $maskRaw = $ipprc->filename("PPSTACK.UNCONV.MASK", $file->{path_base} ); # Mask name - my $weightRaw = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $file->{path_base} ); # Weight name - - my $sources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $file->{path_base}); # Sources name - - &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image ); - &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask ); - &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $weight ); - &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) if ($convolve and not $ipprc->file_exists( $psf )); - &my_die("Sources $sources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $sources ); - - print $listFile " RAW:IMAGE STR " . $imageRaw . "\n"; - print $listFile " RAW:MASK STR " . $maskRaw . "\n"; - print $listFile " RAW:VARIANCE STR " . $weightRaw . "\n"; - - print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; - print $listFile " CNV:MASK STR " . $maskCnv . "\n"; - print $listFile " CNV:VARIANCE STR " . $weightCnv . "\n"; - print $listFile " CNV:PSF STR " . $psfCnv . "\n"; - - print $listFile " SOURCES STR " . $sources . "\n"; - - print $listFile "END\n\n"; - + # XXX if we take the input from 'warp', we will need to make different selections here + my $path_base = $file->{path_base}; + print "input: $path_base\n"; + + my $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $path_base ); # Image name + my $maskCnv = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base ); # Mask name + my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name + + my $imageRaw = $ipprc->filename("PPSTACK.UNCONV", $path_base ); # Image name + my $maskRaw = $ipprc->filename("PPSTACK.UNCONV.MASK", $path_base ); # Mask name + my $weightRaw = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $path_base ); # Weight name + + my $sources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $path_base ); # Sources name + + # XXX is this the correct PSF file? + my $psfCnv = $ipprc->filename("PPSTACK.TARGET.PSF", $path_base ); # PSF name + + # XXX we could make some different choices if some inputs do not exist... &my_die("Couldn't find input: $imageRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageRaw"); &my_die("Couldn't find input: $maskRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskRaw"); @@ -140,4 +127,17 @@ &my_die("Couldn't find input: $psfCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); &my_die("Couldn't find input: $sources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources"); + + print $listFile " RAW:IMAGE STR " . $imageRaw . "\n"; + print $listFile " RAW:MASK STR " . $maskRaw . "\n"; + print $listFile " RAW:VARIANCE STR " . $weightRaw . "\n"; + + print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; + print $listFile " CNV:MASK STR " . $maskCnv . "\n"; + print $listFile " CNV:VARIANCE STR " . $weightCnv . "\n"; + print $listFile " CNV:PSF STR " . $psfCnv . "\n"; + + print $listFile " SOURCES STR " . $sources . "\n"; + + print $listFile "END\n\n"; } @@ -149,30 +149,6 @@ } -print "reduction: $reduction\n"; +print "reduction: $reduction\n"; print "recipe_psphot: $recipe_psphot\n"; - -# Get the output filenames -my $outputName = $ipprc->filename("PPSUB.OUTPUT", $outroot); -my $outputMask = $ipprc->filename("PPSUB.OUTPUT.MASK", $outroot); -my $outputVariance = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $outroot); -my $outputSources = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $outroot); -my $jpeg1Name = $ipprc->filename("PPSUB.OUTPUT.JPEG1", $outroot); -my $jpeg2Name = $ipprc->filename("PPSUB.OUTPUT.JPEG2", $outroot); -my $configuration = $ipprc->filename("PPSUB.CONFIG", $outroot); -my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); -my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); - -if ($run_state eq 'update') { - $traceDest .= '.update'; - $outputStats .= '.update'; -} - -my ($inverseName, $inverseMask, $inverseVariance, $inverseSources); -if ($inverse) { - $inverseName = $ipprc->filename("PPSUB.INVERSE", $outroot); - $inverseMask = $ipprc->filename("PPSUB.INVERSE.MASK", $outroot); - $inverseVariance = $ipprc->filename("PPSUB.INVERSE.VARIANCE", $outroot); - $inverseSources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $outroot); -} my $cmdflags; @@ -180,37 +156,13 @@ # Perform subtraction { - my $command = "$ppSub $outroot"; - $command .= " -inimage $input"; - $command .= " -refimage $template"; - $command .= " -inmask $inputMask"; - $command .= " -refmask $templateMask"; - $command .= " -invariance $inputVariance"; - $command .= " -refvariance $templateVariance"; - $command .= " -insources $inputSources"; - $command .= " -refsources $templateSources"; - $command .= " -stats $outputStats"; + my $command = "$psphotStack $outroot"; + $command .= " -input $listName"; $command .= " -threads $threads" if defined $threads; - if ($run_state eq "new") { - $command .= " -dumpconfig $configuration"; - } else { - my $configurationReal = $ipprc->file_resolve($configuration) or &my_die("Couldn't resolve configuration file: $configuration", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR); - $command .= " -ipprc $configurationReal"; - } - $command .= " -save-inconv" if defined $saveInConv; - $command .= " -save-refconv" if defined $saveRefConv; - $command .= " -recipe PPSUB $recipe_ppSub"; $command .= " -recipe PSPHOT $recipe_psphot"; - $command .= " -recipe PPSTATS WARPSTATS"; - $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; - $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; - $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF"; - if ($run_state eq "new") { - $command .= " -photometry"; - } - $command .= " -inverse" if $inverse; - $command .= " -tracedest $traceDest -log $logDest"; - $command .= " -dbname $dbname" if defined $dbname; - $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id; - $command .= " -source_id $source_id" if defined $source_id; + # $command .= " -dumpconfig $configuration"; + # $command .= " -tracedest $traceDest -log $logDest"; + # $command .= " -dbname $dbname" if defined $dbname; + # $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id; + # $command .= " -source_id $source_id" if defined $source_id; unless ($no_op) { @@ -219,41 +171,57 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppSub: $error_code", $sky_id, $skycell_id, $error_code); - } - - my $outputStatsReal = $ipprc->file_resolve($outputStats); - &my_die("Couldn't find expected output file: $outputStats", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); + &my_die("Unable to perform ppSub: $error_code", $sky_id, $error_code); + } + + # my $outputStatsReal = $ipprc->file_resolve($outputStats); + # &my_die("Couldn't find expected output file: $outputStats", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); # measure chip stats - $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppStatsFromMetadata: $error_code", $sky_id, $skycell_id, $error_code); - } - foreach my $line (@$stdout_buf) { - $cmdflags .= " $line"; - } - chomp $cmdflags; + # $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL"; + # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + # run(command => $command, verbose => $verbose); + # unless ($success) { + # $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + # &my_die("Unable to perform ppStatsFromMetadata: $error_code", $sky_id, $error_code); + # } + # foreach my $line (@$stdout_buf) { + # $cmdflags .= " $line"; + # } + # chomp $cmdflags; my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag if (!$quality) { - &my_die("Couldn't find expected output file: $outputName", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); - &my_die("Couldn't find expected output file: $outputMask", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); - &my_die("Couldn't find expected output file: $outputVariance", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance); - &my_die("Couldn't find expected output file: $outputSources", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); - &my_die("Couldn't find expected output file: $jpeg1Name", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg1Name); - &my_die("Couldn't find expected output file: $jpeg2Name", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg2Name); - if ($inverse) { - &my_die("Couldn't find expected output file: $inverseName", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseName); - &my_die("Couldn't find expected output file: $inverseMask", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseMask); - &my_die("Couldn't find expected output file: $inverseVariance", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseVariance); - &my_die("Couldn't find expected output file: $inverseSources", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseSources); - if ($run_state eq 'new') { - &my_die("Couldn't find expected output file: $configuration", $sky_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration); - } - } + + # Get the output filenames + # we have one set of output files per input file set + for (my $i = 0; $i < @$files; $i++) { + my $outputName = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot); + my $outputMask = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot); + my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot); + my $outputSources = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot); + + &my_die("Couldn't find expected output file: $outputName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + &my_die("Couldn't find expected output file: $outputMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance); + &my_die("Couldn't find expected output file: $outputSources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + } + + #my $configuration = $ipprc->filename("PPSUB.CONFIG", $outroot); + #my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); + #my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); + + # if ($run_state eq 'update') { + # $traceDest .= '.update'; + # $outputStats .= '.update'; + # } + + my $chisqName = $ipprc->filename("PSPHOT.CHISQ.IMAGE", $outroot); + my $chisqMask = $ipprc->filename("PSPHOT.CHISQ.MASK", $outroot); + my $chisqVariance = $ipprc->filename("PSPHOT.CHISQ.VARIANCE", $outroot); + + &my_die("Couldn't find expected output file: $chisqName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqName); + &my_die("Couldn't find expected output file: $chisqMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqMask); + &my_die("Couldn't find expected output file: $chisqVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqVariance); } } else { @@ -266,14 +234,14 @@ # Add the subtraction result { - my $command = "$difftool -sky_id $sky_id -skycell_id $skycell_id"; - $command .= " -magicked $magicked" if $magicked; - if ($run_state eq 'new') { - $command .= " -adddiffskyfile -path_base $outroot"; - $command .= " $cmdflags"; - $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $command .= " -hostname $host" if defined $host; - } else { - $command .= " -tofullskyfile"; - } + my $command = "$staticskytool -sky_id $sky_id"; + # if ($run_state eq 'new') { + $command .= " -addresult -path_base $outroot"; + $command .= " -num_inputs $nInputs"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + # } else { + # $command .= " -tofullskyfile"; + # } $command .= " -dbname $dbname" if defined $dbname; @@ -285,5 +253,5 @@ "Unable to perform difftool -adddiffskyfile" : "Unable to perform difftool -tofullskyfile"; - &my_die("$err_message: $error_code", $sky_id, $skycell_id, $error_code); + &my_die("$err_message: $error_code", $sky_id, $error_code); } } @@ -300,6 +268,6 @@ warn($msg); - if (defined $sky_id and defined $skycell_id and not $no_update) { - my $command = "$difftool -sky_id $sky_id -skycell_id $skycell_id -fault $exit_code"; + if (defined $sky_id and not $no_update) { + my $command = "$staticskytool -sky_id $sky_id -fault $exit_code"; if ($run_state eq 'new') { $command .= " -adddiffskyfile"; Index: /branches/czw_branch/20100519/ippToPsps/config/detection/map.xml =================================================================== --- /branches/czw_branch/20100519/ippToPsps/config/detection/map.xml (revision 28163) +++ /branches/czw_branch/20100519/ippToPsps/config/detection/map.xml (revision 28164) @@ -100,4 +100,5 @@ + Index: /branches/czw_branch/20100519/ippToPsps/config/detection/tables.xml =================================================================== --- /branches/czw_branch/20100519/ippToPsps/config/detection/tables.xml (revision 28163) +++ /branches/czw_branch/20100519/ippToPsps/config/detection/tables.xml (revision 28164) @@ -83,5 +83,5 @@ - + @@ -149,5 +149,5 @@ - + Index: /branches/czw_branch/20100519/ippToPsps/config/init/data.xml =================================================================== --- /branches/czw_branch/20100519/ippToPsps/config/init/data.xml (revision 28163) +++ /branches/czw_branch/20100519/ippToPsps/config/init/data.xml (revision 28164) @@ -30,33 +30,33 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Index: /branches/czw_branch/20100519/ippToPsps/config/init/tables.xml =================================================================== --- /branches/czw_branch/20100519/ippToPsps/config/init/tables.xml (revision 28163) +++ /branches/czw_branch/20100519/ippToPsps/config/init/tables.xml (revision 28164) @@ -57,4 +57,5 @@ + Index: /branches/czw_branch/20100519/ippToPsps/scripts/ippToPsps_run.pl =================================================================== --- /branches/czw_branch/20100519/ippToPsps/scripts/ippToPsps_run.pl (revision 28163) +++ /branches/czw_branch/20100519/ippToPsps/scripts/ippToPsps_run.pl (revision 28164) @@ -87,22 +87,11 @@ my ($distGroup) = @_; - if ($distGroup eq "MD01") {return $distGroup;} - if ($distGroup eq "MD02") {return $distGroup;} - if ($distGroup eq "MD03") {return $distGroup;} - if ($distGroup eq "MD04") {return $distGroup;} - if ($distGroup eq "MD05") {return $distGroup;} - if ($distGroup eq "MD06") {return $distGroup;} - if ($distGroup eq "MD07") {return $distGroup;} - if ($distGroup eq "MD08") {return $distGroup;} - if ($distGroup eq "MD09") {return $distGroup;} - if ($distGroup eq "MD10") {return $distGroup;} - if ($distGroup eq "M31") {return $distGroup;} - if ($distGroup eq "sts") {return "STS";} - if ($distGroup eq "SweetSpot") {return "SS";} - if ($distGroup eq "3PI") {return $distGroup;} - if ($distGroup eq "ThreePi") {return "3PI";} - if ($distGroup eq "SAS") {return "3PI";} - - print "* Do not understand distribution group: '$distGroup'\n"; + if ($distGroup =~ m/^MD([0-1][0-9])$/i) {return "MD$1";} + if ($distGroup =~ m/^M31$/i) {return "M31";} + if ($distGroup =~ m/^sts$/i) {return "STS";} + if ($distGroup =~ m/^SweetSpot$/i) {return "SSS";} + if ($distGroup =~ m/^(3PI)|(ThreePi)|(SAS)$/i) {return "3PI";} + + print "* Unknown distribution group: '$distGroup'\n"; return undef; } @@ -114,4 +103,8 @@ ####################################################################################### sub process { + my $lastExpId = getLastExpId(); + + print "*******************************************************************************\n*\n"; + print "* Last successfully processed exposure ID was $lastExpId\n"; # Set up the database @@ -137,4 +130,5 @@ AND chipRun.exp_id = rawExp.exp_id AND camRun.label LIKE '%$label%' + AND rawExp.exp_id > $lastExpId ORDER BY rawExp.exp_id ASC SQL @@ -153,20 +147,11 @@ my $batchId = 0; my $batchesPerJob = 1; - - # get a new jobId - my $jobId = getJobId(); - $jobId++; - if ($jobId < 0) { print "* Cannot find sensible jobId ($jobId)\n"; - return 0; - } - - print "*******************************************************************************\n*\n"; - my $anyBatches = 0; - $query->execute; open (LOGFILE, ">> $logFile"); + + my $jobId = -1; # loop round exposures @@ -174,5 +159,7 @@ my ($expId, $distGroup) = @row; - print "* Exposure ID = $expId with dist group = '$distGroup'\n"; + $jobId = getNewJobId(); + + print "* Exposure ID = $expId with dist group = '$distGroup' as job ID $jobId\n"; my $surveyType = getSurveyTypeFromDistGroup($distGroup); @@ -192,5 +179,4 @@ mkdir($jobOutputPath, 0777); mkdir($batchOutputPath, 0777); - setJobId($jobId); # run IppToPsps program @@ -222,5 +208,4 @@ print LOGFILE "$jobId $expId\n"; - $jobId++; $batchId = 0; $anyBatches = 0; @@ -228,7 +213,11 @@ else { - print "* No batches to publish\n" + print "* No batches to publish\n"; + print LOGFILE "$jobId $expId ERROR\n"; } } + else { + print LOGFILE "$jobId $expId ERROR\n"; + } if ($batchType eq "init" ) {print "* Wrote initialisation batch\n"; last;} @@ -246,8 +235,20 @@ ####################################################################################### # +# Gets the last processed exposure ID (ini file for now - will be Db eventually TODO) +# +######################################################################################## +sub getLastExpId { + + my $lastExpId = `tail $logFile -n 1 | awk '{print \$2}'`; + chomp($lastExpId); + return $lastExpId; +} + +####################################################################################### +# # Gets the current jobId (ini file for now - will be Db eventually TODO) # ######################################################################################## -sub getJobId { +sub getNewJobId { my $jobId = -1; @@ -260,16 +261,5 @@ close (MYFILE); - if ($verbose) { print "* Current jobId = $jobId\n"; } - - return $jobId; -} - -####################################################################################### -# -# Sets the current jobId (in file for now - will be Db eventually TODO) -# -######################################################################################## -sub setJobId { - my ($jobId) = @_; + $jobId++; open (MYFILE, "> $jobIdFilePath"); @@ -277,10 +267,10 @@ close (MYFILE); - if ($verbose) { print "* New jobId = $jobId\n"; } -} - -####################################################################################### -# -#run IppToPsps + return $jobId; +} + +####################################################################################### +# +# run IppToPsps # ####################################################################################### Index: /branches/czw_branch/20100519/ippToPsps/src/ippToPspsBatchDetection.c =================================================================== --- /branches/czw_branch/20100519/ippToPsps/src/ippToPspsBatchDetection.c (revision 28163) +++ /branches/czw_branch/20100519/ippToPsps/src/ippToPspsBatchDetection.c (revision 28164) @@ -124,4 +124,5 @@ float instMagErr[MAXDETECT]; float peakMag[MAXDETECT]; + uint64_t flags[MAXDETECT]; long objID[MAXDETECT]; long detectID[MAXDETECT]; @@ -284,4 +285,7 @@ ippObjID[s] = (uint64_t)dvoDetections[d].ave.catID*1000000000 + (uint64_t)dvoDetections[d].ave.objID; ippDetectID[s] = dvoDetections[d].meas.detID; + + // the 'photFlags' are the same as those in the 'FLAGS' column of the smf + flags[s] = ((uint64_t)dvoDetections[d].meas.dbFlags << 32) | (uint64_t)dvoDetections[d].meas.photFlags; imageID[s] = pspsImageId; obsTimes[s] = obsTime; @@ -330,4 +334,5 @@ fits_write_col(this->fitsOut, TFLOAT, DETECTION_PEAKADU, 1, 1, nDet, peakFlux, &status); fits_write_col(this->fitsOut, TSTRING, DETECTION_ASSOCDATE, 1, 1, nDet, assocDate, &status); + fits_write_col(this->fitsOut, TLONGLONG, DETECTION_INFOFLAG, 1, 1, nDet, flags, &status); if (numOfDuplicates||numInvalidFlux) fits_delete_rowlist(this->fitsOut, removeList, numOfDuplicates+numInvalidFlux, &status); Index: /branches/czw_branch/20100519/ippTools/notes.txt =================================================================== --- /branches/czw_branch/20100519/ippTools/notes.txt (revision 28163) +++ /branches/czw_branch/20100519/ippTools/notes.txt (revision 28164) @@ -1,2 +1,14 @@ + +2010.05.24 EAM + + staticskytool: + + definebyquery : choose input stacks that meet a list of + restrictions. Require a specific list of filters? + Require a minimum number of inputs? + + eg: sst definebyquery -filterlist g,r,i,z,y [-label md04%] [-ra_min x] [-dec_min x] ... + + can we do the filter selects in the database? 2008.07.19 EAM Index: /branches/czw_branch/20100519/ippTools/share/Makefile.am =================================================================== --- /branches/czw_branch/20100519/ippTools/share/Makefile.am (revision 28163) +++ /branches/czw_branch/20100519/ippTools/share/Makefile.am (revision 28164) @@ -289,4 +289,9 @@ stacktool_sumskyfile.sql \ stacktool_tosum.sql \ + staticskytool_definebyquery_select.sql \ + staticskytool_definebyquery_inputs.sql \ + staticskytool_inputs.sql \ + staticskytool_todo.sql \ + staticskytool_result.sql \ warptool_change_skyfile_data_state.sql \ warptool_change_run_state.sql \ Index: /branches/czw_branch/20100519/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/czw_branch/20100519/ippTools/share/pxadmin_create_tables.sql (revision 28163) +++ /branches/czw_branch/20100519/ippTools/share/pxadmin_create_tables.sql (revision 28164) @@ -277,5 +277,5 @@ magicked BIGINT, software_ver VARCHAR(16), - maskfrac_npix INT, + maskfrac_npix FLOAT, maskfrac_static FLOAT, maskfrac_dynamic FLOAT, @@ -400,9 +400,14 @@ magicked BIGINT, software_ver VARCHAR(16), - maskfrac_npix INT, - maskfrac_static FLOAT, - maskfrac_dynamic FLOAT, - maskfrac_magic FLOAT, - maskfrac_advisory FLOAT, + maskfrac_ref_npix FLOAT, + maskfrac_ref_static FLOAT, + maskfrac_ref_dynamic FLOAT, + maskfrac_ref_magic FLOAT, + maskfrac_ref_advisory FLOAT, + maskfrac_max_npix FLOAT, + maskfrac_max_static FLOAT, + maskfrac_max_dynamic FLOAT, + maskfrac_max_magic FLOAT, + maskfrac_max_advisory FLOAT, note VARCHAR(255), PRIMARY KEY(cam_id), @@ -483,9 +488,14 @@ fault SMALLINT NOT NULL, software_ver VARCHAR(16), - maskfrac_npix INT, - maskfrac_static FLOAT, - maskfrac_dynamic FLOAT, - maskfrac_magic FLOAT, - maskfrac_advisory FLOAT, + maskfrac_ref_npix INT, + maskfrac_ref_static FLOAT, + maskfrac_ref_dynamic FLOAT, + maskfrac_ref_magic FLOAT, + maskfrac_ref_advisory FLOAT, + maskfrac_max_npix INT, + maskfrac_max_static FLOAT, + maskfrac_max_dynamic FLOAT, + maskfrac_max_magic FLOAT, + maskfrac_max_advisory FLOAT, PRIMARY KEY(cam_id), KEY(fault), @@ -892,5 +902,5 @@ magicked BIGINT, software_ver VARCHAR(16), - maskfrac_npix INT, + maskfrac_npix FLOAT, maskfrac_static FLOAT, maskfrac_dynamic FLOAT, @@ -969,4 +979,13 @@ label VARCHAR(64), PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE warpSummary ( + warp_id BIGINT, + tangent_plane VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(warp_id), + KEY(tangent_plane), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1041,4 +1060,33 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE stackSummary ( + stack_id BIGINT, + tangent_plane VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(stack_id), + KEY(tangent_plane), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociation ( + sass_id BIGINT AUTO_INCREMENT, + data_group VARCHAR(64) NOT NULL, + tangent_plane VARCHAR(64) NOT NULL, + tess_id VARCHAR(64) NOT NULL, + filter VARCHAR(64) NOT NULL, + PRIMARY KEY(sass_id), + KEY(data_group), + KEY(tangent_plane), + KEY(tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociationMap ( + sass_id BIGINT, + stack_id BIGINT, + PRIMARY KEY(sass_id, stack_id), + FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE diffRun ( diff_id BIGINT AUTO_INCREMENT, @@ -1056,5 +1104,5 @@ magicked BIGINT, software_ver VARCHAR(16), - maskfrac_npix INT, + maskfrac_npix FLOAT, maskfrac_static FLOAT, maskfrac_dynamic FLOAT, @@ -1137,4 +1185,13 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE diffSummary ( + diff_id BIGINT, + tangent_plane VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(diff_id), + KEY(tangent_plane), + FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE magicRun ( magic_id BIGINT AUTO_INCREMENT, @@ -1607,5 +1664,5 @@ -- A static sky analysis set CREATE TABLE staticskyRun ( - ss_id BIGINT AUTO_INCREMENT, -- unique identifier + sky_id BIGINT AUTO_INCREMENT, -- unique identifier state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) workdir VARCHAR(255) NOT NULL, -- working directory @@ -1616,5 +1673,5 @@ note VARCHAR(255), -- note registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered - PRIMARY KEY(ss_id), + PRIMARY KEY(sky_id), KEY(state), KEY(label), @@ -1625,9 +1682,9 @@ -- Inputs for static sky analysis CREATE TABLE staticskyInput ( - ss_id BIGINT, -- static sky identifier + sky_id BIGINT, -- static sky identifier stack_id BIGINT, -- stack identifier - PRIMARY KEY(ss_id,stack_id), + PRIMARY KEY(sky_id,stack_id), KEY(stack_id), - FOREIGN KEY(ss_id) REFERENCES staticskyRun(ss_id), + FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id), FOREIGN KEY(stack_id) REFERENCES stackSumSkyfile(stack_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1635,18 +1692,19 @@ -- Result of static sky analysis CREATE TABLE staticskyResult ( - ss_id BIGINT, -- static sky identifier + sky_id BIGINT, -- static sky identifier path_base VARCHAR(255) NOT NULL, -- root name for outputs dtime_phot FLOAT, -- elapsed time for photometry dtime_script FLOAT, -- elapsed time for script sources INT, -- number of sources + num_inputs INT, -- number of input images hostname VARCHAR(64) NOT NULL, -- host that executed script good_frac FLOAT, -- good fraction of skycell quality SMALLINT NOT NULL, -- bad quality flag fault SMALLINT NOT NULL, -- fault code - PRIMARY KEY(ss_id), + PRIMARY KEY(sky_id), KEY(good_frac), KEY(fault), KEY(quality), - FOREIGN KEY(ss_id) REFERENCES staticskyRun(ss_id) + FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/czw_branch/20100519/ippTools/share/staticskytool_definebyquery.sql =================================================================== --- /branches/czw_branch/20100519/ippTools/share/staticskytool_definebyquery.sql (revision 28163) +++ /branches/czw_branch/20100519/ippTools/share/staticskytool_definebyquery.sql (revision 28164) @@ -1,14 +1,22 @@ -SELECT - tess_id, - skycell_id, - COUNT(*) AS num, - COUNT(DISTINCT filter) AS num_filter -FROM stackRun -JOIN stackSumSkyfile USING(stack_id) -WHERE stackRun.state = 'full' - AND stackSumSkyfile.fault = 0 - AND stackSumSkyfile.quality = 0 --- WHERE hook %s -GROUP BY - tess_id, - skycell_id +SELECT + tess_id, + skycell_id, + num_filter +FROM ( + SELECT + tess_id, + skycell_id, + COUNT(DISTINCT filter) AS num_filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + WHERE stackRun.state = 'full' + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + -- WHERE hook 1 %s + -- restrict by stackSumSkyfile.good_frac, stackRun.skycell_id, stackRun.label + GROUP BY + tess_id, + skycell_id + ) AS TMP +WHERE num_filter == %d + Index: /branches/czw_branch/20100519/ippTools/share/staticskytool_definebyquery_inputs.sql =================================================================== --- /branches/czw_branch/20100519/ippTools/share/staticskytool_definebyquery_inputs.sql (revision 28164) +++ /branches/czw_branch/20100519/ippTools/share/staticskytool_definebyquery_inputs.sql (revision 28164) @@ -0,0 +1,14 @@ + SELECT + MAX(stack_id) as stack_id, + tess_id, + skycell_id, + filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + WHERE stackRun.state = 'full' + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + AND stackRun.tess_id = '%s' + AND stackRun.skycell_id = '%s' + -- WHERE hook %s + GROUP BY stackRun.filter Index: /branches/czw_branch/20100519/ippTools/share/staticskytool_definebyquery_select.sql =================================================================== --- /branches/czw_branch/20100519/ippTools/share/staticskytool_definebyquery_select.sql (revision 28164) +++ /branches/czw_branch/20100519/ippTools/share/staticskytool_definebyquery_select.sql (revision 28164) @@ -0,0 +1,22 @@ +SELECT + tess_id, + skycell_id, + num_filter +FROM ( + SELECT + tess_id, + skycell_id, + COUNT(DISTINCT filter) AS num_filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + WHERE stackRun.state = 'full' + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + -- WHERE hook 1 %s + -- restrict by stackSumSkyfile.good_frac, stackRun.skycell_id, stackRun.label + GROUP BY + tess_id, + skycell_id + ) AS TMP +WHERE num_filter = %d + Index: /branches/czw_branch/20100519/ippTools/share/staticskytool_inputs.sql =================================================================== --- /branches/czw_branch/20100519/ippTools/share/staticskytool_inputs.sql (revision 28164) +++ /branches/czw_branch/20100519/ippTools/share/staticskytool_inputs.sql (revision 28164) @@ -0,0 +1,7 @@ +SELECT + staticskyRun.*, + staticskyInput.stack_id, + stackSumSkyfile.path_base +FROM staticskyRun +JOIN staticskyInput USING(sky_id) +JOIN stackSumSkyfile USING (stack_id) Index: /branches/czw_branch/20100519/ippTools/share/staticskytool_result.sql =================================================================== --- /branches/czw_branch/20100519/ippTools/share/staticskytool_result.sql (revision 28164) +++ /branches/czw_branch/20100519/ippTools/share/staticskytool_result.sql (revision 28164) @@ -0,0 +1,11 @@ +SELECT + staticskyResult.*, + staticskyRun.state, + staticskyRun.tess_id, + staticskyRun.skycell_id, + staticskyRun.filter, + staticskyRun.workdir, + staticskyRun.label, +FROM staticskyRun +JOIN staticskyResult +USING(stack_id) Index: /branches/czw_branch/20100519/ippTools/share/staticskytool_todo.sql =================================================================== --- /branches/czw_branch/20100519/ippTools/share/staticskytool_todo.sql (revision 28164) +++ /branches/czw_branch/20100519/ippTools/share/staticskytool_todo.sql (revision 28164) @@ -0,0 +1,15 @@ +SELECT + staticskyRun.sky_id, + staticskyRun.workdir, + staticskyRun.reduction, + staticskyRun.label, + staticskyRun.state +FROM staticskyRun +JOIN staticskyInput USING (sky_id) +JOIN stackRun USING (stack_id) +LEFT JOIN staticskyResult USING(sky_id) +WHERE + (staticskyRun.state = 'new' AND staticskyResult.sky_id IS NULL) + -- WHERE hook %s +GROUP BY sky_id +HAVING SUM(IF(stackRun.state = 'full', 1, 0)) = COUNT(staticskyInput.sky_id) Index: /branches/czw_branch/20100519/ippTools/share/warptool_towarped.sql =================================================================== --- /branches/czw_branch/20100519/ippTools/share/warptool_towarped.sql (revision 28163) +++ /branches/czw_branch/20100519/ippTools/share/warptool_towarped.sql (revision 28164) @@ -12,7 +12,8 @@ workdir, magicked, - path_base + path_base, + priority FROM ( - SELECT DISTINCT + (SELECT DISTINCT warpSkyCellMap.warp_id, warpImfile.warp_skyfile_id, @@ -27,5 +28,6 @@ warpRun.workdir, chipRun.magicked, - CAST(NULL AS CHAR(255)) AS path_base + CAST(NULL AS CHAR(255)) AS path_base, + IFNULL(Label.priority, 10000) AS priority FROM warpRun JOIN warpSkyCellMap @@ -50,4 +52,5 @@ LEFT JOIN warpMask ON warpRun.label = warpMask.label + LEFT JOIN Label ON warpRun.label = Label.label WHERE warpRun.state = 'new' @@ -60,8 +63,10 @@ AND warpMask.label IS NULL AND warpSkyCellMap.fault = 0 + AND (Label.active OR Label.active IS NULL) -- where hook 1 %s -- limit hook 1 %s + ) UNION - SELECT + (SELECT warpSkyCellMap.warp_id, warpImfile.warp_skyfile_id, @@ -76,5 +81,6 @@ warpRun.workdir, MIN(chipProcessedImfile.magicked) AS magicked, - warpSkyfile.path_base + warpSkyfile.path_base, + IFNULL(Label.priority, 10000) AS priority FROM warpRun JOIN warpImfile USING(warp_id) @@ -86,4 +92,5 @@ JOIN rawExp USING(exp_id) LEFT JOIN chipProcessedImfile USING(chip_id, class_id) + LEFT JOIN Label ON warpRun.label = Label.label WHERE warpRun.state = 'update' AND warpSkyfile.data_state = 'update' @@ -94,8 +101,9 @@ -- this blocks processing until all the chip inputs have been destreaked AND (warpSkyfile.magicked = 0 OR chipProcessedImfile.magicked >= 0) + AND (Label.active OR Label.active IS NULL) -- where hook 2 %s GROUP BY warp_id, skycell_id HAVING COUNT(warpSkyCellMap.class_id) = COUNT(chipProcessedImfile.class_id) -- limit hook 2 %s + ) ) as towarped - Index: /branches/czw_branch/20100519/ippTools/src/Makefile.am =================================================================== --- /branches/czw_branch/20100519/ippTools/src/Makefile.am (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/Makefile.am (revision 28164) @@ -10,4 +10,5 @@ flatcorr \ dqstatstool \ + labeltool \ magictool \ magicdstool \ @@ -21,4 +22,5 @@ regtool \ stacktool \ + staticskytool \ warptool \ receivetool \ @@ -55,4 +57,5 @@ faketool.h \ dqstatstool.h \ + labeltool.h \ magictool.h \ magicdstool.h \ @@ -64,4 +67,5 @@ regtool.h \ stacktool.h \ + staticskytool.h \ warptool.h \ pubtool.h @@ -153,4 +157,10 @@ dqstatstoolConfig.c +labeltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +labeltool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +labeltool_SOURCES = \ + labeltool.c \ + labeltoolConfig.c + magictool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) magictool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la @@ -188,4 +198,10 @@ stacktool.c \ stacktoolConfig.c + +staticskytool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +staticskytool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +staticskytool_SOURCES = \ + staticskytool.c \ + staticskytoolConfig.c pxadmin_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) Index: /branches/czw_branch/20100519/ippTools/src/camtool.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/camtool.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/camtool.c (revision 28164) @@ -490,12 +490,18 @@ PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); - PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); - PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); - PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); - PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); - PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); - - psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", - ver_pslib,ver_psmodules,ver_psphot,ver_psastro,ver_ppstats,ver_ppimage,ver_streaks); + PXOPT_LOOKUP_S32(maskfrac_ref_npix, config->args, "-maskfrac_ref_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_ref_static, config->args, "-maskfrac_ref_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_ref_dynamic, config->args, "-maskfrac_ref_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_ref_magic, config->args, "-maskfrac_ref_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_ref_advisory, config->args, "-maskfrac_ref_advisory", false, false); + + PXOPT_LOOKUP_S32(maskfrac_max_npix, config->args, "-maskfrac_max_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_max_static, config->args, "-maskfrac_max_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_max_dynamic, config->args, "-maskfrac_max_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_max_magic, config->args, "-maskfrac_max_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_max_advisory, config->args, "-maskfrac_max_advisory", false, false); + +/* psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", */ +/* ver_pslib,ver_psmodules,ver_psphot,ver_psastro,ver_ppstats,ver_ppimage,ver_streaks); */ psString software_ver = NULL; if ((ver_pslib)&&(ver_psmodules)) { @@ -632,9 +638,14 @@ fault, software_ver, - maskfrac_npix, - maskfrac_static, - maskfrac_dynamic, - maskfrac_magic, - maskfrac_advisory, + maskfrac_ref_npix, + maskfrac_ref_static, + maskfrac_ref_dynamic, + maskfrac_ref_magic, + maskfrac_ref_advisory, + maskfrac_max_npix, + maskfrac_max_static, + maskfrac_max_dynamic, + maskfrac_max_magic, + maskfrac_max_advisory, quality ); @@ -674,7 +685,10 @@ } // Set chipRun.maskfrac* to the appropriate values. - if (maskfrac_npix) { - if (!pxSetRunMaskfrac(config, "camRun", "cam_id",cam_id, maskfrac_npix, maskfrac_static, - maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + if (maskfrac_ref_npix) { + if (!pxCamSetRunMaskfrac(config, "camRun", "cam_id",cam_id, + (float) maskfrac_ref_npix, maskfrac_ref_static, + maskfrac_ref_dynamic, maskfrac_ref_magic, maskfrac_ref_advisory, + (float) maskfrac_max_npix, maskfrac_max_static, + maskfrac_max_dynamic, maskfrac_max_magic, maskfrac_max_advisory)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /branches/czw_branch/20100519/ippTools/src/camtoolConfig.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/camtoolConfig.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/camtoolConfig.c (revision 28164) @@ -182,9 +182,15 @@ psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); - psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); - psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); - psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); - psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); - psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_ref_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_ref_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_ref_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_ref_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_ref_advisory", 0, "define advisory mask fraction", NAN); + + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_max_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_max_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_max_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_max_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_max_advisory", 0, "define advisory mask fraction", NAN); Index: /branches/czw_branch/20100519/ippTools/src/chiptool.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/chiptool.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/chiptool.c (revision 28164) @@ -141,5 +141,5 @@ ) { - // start a transaction so we don't end up with an exp without any associted + // start a transaction so we don't end up with an exp without any associated // imfiles if (!psDBTransaction(config->dbh)) { @@ -1398,5 +1398,5 @@ psString software_ver = NULL; - psS32 maskfrac_npix = 0; + psS64 maskfrac_npix = 0; psF32 maskfrac_static = 0; psF32 maskfrac_dynamic = 0; @@ -1417,10 +1417,10 @@ } psMetadataAddStr(row, PS_LIST_TAIL, "software_ver", 0, "Software version", software_ver); - psMetadataAddS32(row, PS_LIST_TAIL, "maskfrac_npix", 0, "Number of pixels used for mask stats", maskfrac_npix); + psMetadataAddS32(row, PS_LIST_TAIL, "maskfrac_npix", 0, "Number of pixels used for mask stats", (psS32) maskfrac_npix); psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_static", 0, "Fraction of static masked pixels", maskfrac_static); psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_dynamic", 0, "Fraction of dynamic masked pixels", maskfrac_dynamic); psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_magic", 0, "Fraction of magic masked pixels", maskfrac_magic); psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_advisory", 0, "Fraction of advisory pixels", maskfrac_advisory); - psWarning("ADVANCE %ld %s %d %f %f %f %f\n",chip_id,software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory); +/* psWarning("ADVANCE %ld %s %d %f %f %f %f\n",chip_id,software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory); */ chipRunRow *chipRun = chipRunObjectFromMetadata(row); if (!psDBTransaction(config->dbh)) { @@ -1430,23 +1430,27 @@ // Set chipRun.software_ver to the appropriate value - if (!pxSetRunSoftware(config, "chipRun", "chip_id", chip_id, software_ver)) { - psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, - chip_id); - psFree(output); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (software_ver) { + if (!pxSetRunSoftware(config, "chipRun", "chip_id", chip_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); } - return(false); } // Set chipRun.maskfrac* to the appropriate values. - if (!pxSetRunMaskfrac(config, "chipRun", "chip_id",chip_id, maskfrac_npix, maskfrac_static, - maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { - psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, - chip_id); - psFree(output); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (maskfrac_npix) { + if (!pxSetRunMaskfrac(config, "chipRun", "chip_id",chip_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); } - return(false); } // set chipRun.state to 'stop' and update the magicked state Index: /branches/czw_branch/20100519/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/chiptoolConfig.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/chiptoolConfig.c (revision 28164) @@ -89,7 +89,7 @@ psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group (LIKE comparison)", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); @@ -101,5 +101,5 @@ psMetadata *setimfiletoupdateArgs = psMetadataAlloc(); psMetadataAddS64(setimfiletoupdateArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID (required)", 0); - psMetadataAddStr(setimfiletoupdateArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id (required)", NULL); + psMetadataAddStr(setimfiletoupdateArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", NULL); psMetadataAddStr(setimfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0,"new value for label", NULL); Index: /branches/czw_branch/20100519/ippTools/src/difftool.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/difftool.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/difftool.c (revision 28164) @@ -44,4 +44,6 @@ static bool definewarpwarpMode(pxConfig *config); static bool definestackstackMode(pxConfig *config); +static bool tosummaryMode(pxConfig *config); +static bool addsummaryMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupskyfileMode(pxConfig *config); @@ -93,4 +95,6 @@ MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); MODECASE(DIFFTOOL_MODE_DEFINESTACKSTACK, definestackstackMode); + MODECASE(DIFFTOOL_MODE_TOSUMMARY, tosummaryMode); + MODECASE(DIFFTOOL_MODE_ADDSUMMARY, addsummaryMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); @@ -203,6 +207,7 @@ PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-data_group","data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-data_group","data_group","LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group","dist_group","LIKE"); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); if (!psListLength(where->list)) { @@ -716,5 +721,5 @@ psString software_ver = NULL; - psS32 maskfrac_npix = 0; + psS64 maskfrac_npix = 0; psF32 maskfrac_static = 0; psF32 maskfrac_dynamic = 0; @@ -2249,4 +2254,135 @@ } +static bool tosummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "diffSkyfile.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "diffRun.state", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "diffSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("difftool_tosummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "diffSkyfile")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + if (where2) { + if (psListLength(where->list)) { + psStringAppend(&query, " %s", where2); + } else { + psStringAppend(&query, " WHERE 1 %s", where2); + } + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return(true); +} +static bool addsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(tangent_plane, config->args, "-tangent_plane", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + psString query = pxDataGet("difftool_addsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, diff_id, tangent_plane, path_base)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psS64 numUpdated = psDBAffectedRows(config->dbh); + + if (numUpdated != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); + psFree(query); + return false; + } + + psFree(query); + + // Print anything here? + + return(true); +} + + static bool pendingcleanuprunMode(pxConfig *config) { Index: /branches/czw_branch/20100519/ippTools/src/difftool.h =================================================================== --- /branches/czw_branch/20100519/ippTools/src/difftool.h (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/difftool.h (revision 28164) @@ -32,4 +32,6 @@ DIFFTOOL_MODE_ADDDIFFSKYFILE, DIFFTOOL_MODE_ADVANCE, + DIFFTOOL_MODE_TOSUMMARY, + DIFFTOOL_MODE_ADDSUMMARY, DIFFTOOL_MODE_DIFFSKYFILE, DIFFTOOL_MODE_REVERTDIFFSKYFILE, Index: /branches/czw_branch/20100519/ippTools/src/difftoolConfig.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/difftoolConfig.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/difftoolConfig.c (revision 28164) @@ -64,11 +64,12 @@ psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define by label instead of diff ID", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "define by data_group instead of diff ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define by label instead of diff ID (LIKE comparison)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "define by data_group instead of diff ID (LIKE comparison)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "define by dist_group instead of diff ID (LIKE comparison)", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); - psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_diff_mode", 0, "specify type of difference (WW=1,WS=2,SW=3,SS=4)", 0); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_diff_mode", 0, "specify type of difference (WW=1,WS=2,SW=3,SS=4)", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); @@ -322,4 +323,29 @@ psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -tosummary + psMetadata *tosummaryArgs = psMetadataAlloc(); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(tosummaryArgs , PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", NULL); + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-template", 0, "apply exposure args to template of bothways diff", false); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(tosummaryArgs , PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); + psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL); + + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(tosummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsummary + psMetadata *addsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-tangent_plane", 0, "set tangent plane", NULL); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL); + // -pendingcleanupskyfile psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc(); @@ -379,5 +405,4 @@ psMetadata *importrunArgs = psMetadataAlloc(); psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - @@ -414,4 +439,6 @@ PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full", DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); + PXOPT_ADD_MODE("-tosummary", "show runs that can be summarized", DIFFTOOL_MODE_TOSUMMARY, tosummaryArgs); + PXOPT_ADD_MODE("-addsummary", "add entry to the summary table", DIFFTOOL_MODE_ADDSUMMARY, addsummaryArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /branches/czw_branch/20100519/ippTools/src/labeltool.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/labeltool.c (revision 28164) +++ /branches/czw_branch/20100519/ippTools/src/labeltool.c (revision 28164) @@ -0,0 +1,290 @@ +/* + * magictool.c + * + * Copyright (C) 2006-2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "labeltool.h" + +static bool definelabelMode(pxConfig *config); +static bool updatelabelMode(pxConfig *config); +static bool deletelabelMode(pxConfig *config); +static bool listlabelMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = labeltoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(LABELTOOL_MODE_DEFINELABEL, definelabelMode); + MODECASE(LABELTOOL_MODE_UPDATELABEL, updatelabelMode); + MODECASE(LABELTOOL_MODE_DELETELABEL, deletelabelMode); + MODECASE(LABELTOOL_MODE_LISTLABEL, listlabelMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool definelabelMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + // XXX: perhaps we should have a default priority? + PXOPT_LOOKUP_S32(priority, config->args, "-set_priority", true, false); + + // optional + PXOPT_LOOKUP_BOOL(inactive, config->args, "-set_inactive", false); + PXOPT_LOOKUP_STR(comment, config->args, "-set_comment", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + LabelRow *row = LabelRowAlloc( + label, + priority, + inactive ? false : true, + comment + ); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to allocate Label object"); + return false; + } + if (!LabelInsertObject(config->dbh, row)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + return false; + } + + if (!LabelPrintObject(stdout, row, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(row); + return false; + } + + psFree(row); + + return true; +} + + +static bool updatelabelMode(pxConfig *config) +{ + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + + // optional (at least one is required) + PXOPT_LOOKUP_S32(priority, config->args, "-set_priority", false, false); + PXOPT_LOOKUP_BOOL(inactive, config->args, "-set_inactive", false); + PXOPT_LOOKUP_BOOL(active, config->args, "-set_active", false); + PXOPT_LOOKUP_STR(comment, config->args, "-set_comment", false, false); + + if (! (priority || active || inactive || comment) ) { + psError(PS_ERR_UNKNOWN, true, "at least one -set option is required\n"); + return false; + } + if (active && inactive) { + psError(PS_ERR_UNKNOWN, true, "only one of -active and -inactive may be supplied"); + return false; + } + + psString query = psStringCopy("UPDATE Label SET"); + char sep = ' '; + + if (priority) { + psStringAppend(&query, "%c priority = %d", sep, priority); + sep = ','; + } + if (active) { + psStringAppend(&query, "%c active = %d", sep, 1); + sep = ','; + } + if (inactive) { + psStringAppend(&query, "%c active = %d", sep, 0); + sep = ','; + } + if (comment) { + psStringAppend(&query, "%c comment = '%s'", sep, comment); + sep = ','; + } + + psStringAppend(&query, " WHERE label = '%s'", label); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool deletelabelMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + psString query = NULL; + + psStringAppend(&query, "DELETE FROM Label WHERE label = '%s'", label); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool listlabelMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + + PXOPT_LOOKUP_BOOL(active, config->args, "-active", false); + PXOPT_LOOKUP_BOOL(inactive, config->args, "-inactive", false); + if (active && inactive) { + psError(PS_ERR_UNKNOWN, true, "only one of -active and inactive may be supplied"); + return false; + } + PXOPT_LOOKUP_BOOL(lowtohigh, config->args, "-lowtohigh", false); + PXOPT_LOOKUP_BOOL(hightolow, config->args, "-hightolow", false); + if (lowtohigh && hightolow) { + psError(PS_ERR_UNKNOWN, true, "only one of -lowtohigh and -hightolow may be supplied"); + return false; + } + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = psStringCopy("SELECT * FROM Label\n"); + + char *sep = " WHERE "; + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + sep = " AND "; + } + psFree(where); + + if (active) { + psStringAppend(&query, "%s active\n", sep); + sep = " AND "; + } else if (inactive) { + psStringAppend(&query, "%s NOT active\n", sep); + sep = " AND "; + } + + if (lowtohigh || hightolow) { + char *order = lowtohigh ? "" : "DESC"; + psStringAppend(&query, "\nORDER BY priority %s\n", order); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("labeltool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "Label", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /branches/czw_branch/20100519/ippTools/src/labeltool.h =================================================================== --- /branches/czw_branch/20100519/ippTools/src/labeltool.h (revision 28164) +++ /branches/czw_branch/20100519/ippTools/src/labeltool.h (revision 28164) @@ -0,0 +1,35 @@ +/* + * labeltool.h + * + * Copyright (C) 2010 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef LABELTOOL_H +#define LABELTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + LABELTOOL_MODE_NONE = 0x0, + LABELTOOL_MODE_DEFINELABEL, + LABELTOOL_MODE_UPDATELABEL, + LABELTOOL_MODE_DELETELABEL, + LABELTOOL_MODE_LISTLABEL, +} labeltoolMode; + +pxConfig *labeltoolConfig(pxConfig *config, int argc, char **argv); + +#endif // LABELTOOL_H Index: /branches/czw_branch/20100519/ippTools/src/labeltoolConfig.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/labeltoolConfig.c (revision 28164) +++ /branches/czw_branch/20100519/ippTools/src/labeltoolConfig.c (revision 28164) @@ -0,0 +1,113 @@ +/* + * labeltoolConfig.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "labeltool.h" + +pxConfig *labeltoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definelabel + psMetadata *definelabelArgs = psMetadataAlloc(); + psMetadataAddStr(definelabelArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); + psMetadataAddS64(definelabelArgs, PS_LIST_TAIL, "-set_priority", 0, "define priority (required)", 0); + psMetadataAddBool(definelabelArgs, PS_LIST_TAIL, "-set_inactive", 0, "set label inactive", false); + psMetadataAddStr(definelabelArgs, PS_LIST_TAIL, "-set_comment", 0, "define label comment", NULL); + psMetadataAddBool(definelabelArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updatelabel + psMetadata *updatelabelArgs = psMetadataAlloc(); + psMetadataAddStr(updatelabelArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison) (required)", NULL); + psMetadataAddS64(updatelabelArgs, PS_LIST_TAIL, "-set_priority", 0, "define new priority", 0); + psMetadataAddBool(updatelabelArgs, PS_LIST_TAIL, "-set_active", 0, "set label active", false); + psMetadataAddBool(updatelabelArgs, PS_LIST_TAIL, "-set_inactive", 0, "set label inactive", false); + psMetadataAddStr(updatelabelArgs, PS_LIST_TAIL, "-set_comment", 0, "define new label comment", NULL); + + // -deletelabel + psMetadata *deletelabelArgs = psMetadataAlloc(); + psMetadataAddStr(deletelabelArgs, PS_LIST_TAIL, "-label", 0, "label to delete (required)", NULL); + + // -listlabel + psMetadata *listlabelArgs = psMetadataAlloc(); + psMetadataAddStr(listlabelArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison)", NULL); + psMetadataAddBool(listlabelArgs, PS_LIST_TAIL, "-hightolow", 0, "order by priority high to low", false); + psMetadataAddBool(listlabelArgs, PS_LIST_TAIL, "-lowtohigh", 0, "order by priority low to high", false); + psMetadataAddBool(listlabelArgs, PS_LIST_TAIL, "-active", 0, "list active labels", false); + psMetadataAddBool(listlabelArgs, PS_LIST_TAIL, "-inactive", 0, "list inactive labels", false); + psMetadataAddU64(listlabelArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listlabelArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definelabel", "", LABELTOOL_MODE_DEFINELABEL, definelabelArgs); + PXOPT_ADD_MODE("-updatelabel", "", LABELTOOL_MODE_UPDATELABEL, updatelabelArgs); + PXOPT_ADD_MODE("-deletelabel", "", LABELTOOL_MODE_DELETELABEL, deletelabelArgs); + PXOPT_ADD_MODE("-listlabel", "", LABELTOOL_MODE_LISTLABEL, listlabelArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/czw_branch/20100519/ippTools/src/pxtools.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/pxtools.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/pxtools.c (revision 28164) @@ -49,108 +49,133 @@ psString pxMergeCodeVersions(psString version1, psString version2) { - psString out = NULL; - - bool mod1 = false; - bool mod2 = false; - - psS32 num1; - psS32 num2; - psS32 numO; - - if (!version1) { - psStringAppend(&out,version2); + psString out = NULL; + + bool mod1 = false; + bool mod2 = false; + + psS32 num1; + psS32 num2; + psS32 numO; + + if (!version1) { + psStringAppend(&out, "%s", version2); + return(out); + } + if (!version2) { + psStringAppend(&out, "%s", version1); + return(out); + } + + if (strchr(version1,'M')) { + psStringSubstitute(&version1,"M",""); + mod1 = true; + } + if (strchr(version2,'M')) { + psStringSubstitute(&version2,"M",""); + mod2 = true; + } + + num1 = strtol(version1,NULL,10); + num2 = strtol(version2,NULL,10); + + if (num1 >= num2) { + numO = num1; + } + else { + numO = num2; + } + + psStringAppend(&out,"%" PRId32,numO); + if (mod1 || mod2) { + psStringAppend(&out,"M"); + } return(out); - } - if (!version2) { - psStringAppend(&out,version1); - return(out); - } - - if (strchr(version1,'M')) { - psStringSubstitute(&version1,"M",""); - mod1 = true; - } - if (strchr(version2,'M')) { - psStringSubstitute(&version2,"M",""); - mod2 = true; - } - - num1 = strtol(version1,NULL,10); - num2 = strtol(version2,NULL,10); - - if (num1 >= num2) { - numO = num1; - } - else { - numO = num2; - } - - psStringAppend(&out,"%" PRId32,numO); - if (mod1 || mod2) { - psStringAppend(&out,"M"); - } - return(out); } bool pxCoalesceRunStatus(pxConfig *config, const psString dbQFile, psS64 stage_id, psString *software_ver, - psS32 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, + psS64 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, psF32 *maskfrac_magic, psF32 *maskfrac_advisory) { - psString query = pxDataGet(dbQFile); + psString query = pxDataGet(dbQFile); /* psString text_id = NULL; */ /* psStringAppend(&text_id," %" PRId64,stage_id); */ /* psStringSubstitute(&query,text_id,"@STAGE_ID@"); */ /* psFree(text_id); */ - if (!p_psDBRunQueryF(config->dbh, query, stage_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (!p_psDBRunQueryF(config->dbh, query, stage_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } psFree(query); - return(false); - } - psFree(query); - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return(false); - } - - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *row = output->data[i]; - - psS32 this_npix = psMetadataLookupS32(NULL, row, "maskfrac_npix"); - psF32 this_static = psMetadataLookupF32(NULL, row, "maskfrac_static"); - psF32 this_dynamic = psMetadataLookupF32(NULL, row, "maskfrac_dynamic"); - psF32 this_magic = psMetadataLookupF32(NULL, row, "maskfrac_magic"); - psF32 this_advisory = psMetadataLookupF32(NULL, row, "maskfrac_advisory"); - psString this_version = psMetadataLookupStr(NULL, row, "software_ver"); - - *software_ver = pxMergeCodeVersions(*software_ver,this_version); - - *maskfrac_static = ((*maskfrac_static * *maskfrac_npix) + (this_npix * this_static)) / (this_npix + *maskfrac_npix); - *maskfrac_dynamic = ((*maskfrac_dynamic * *maskfrac_npix) + (this_npix * this_dynamic)) / (this_npix + *maskfrac_npix); - *maskfrac_magic = ((*maskfrac_magic * *maskfrac_npix) + (this_npix * this_magic)) / (this_npix + *maskfrac_npix); - *maskfrac_advisory = ((*maskfrac_advisory * *maskfrac_npix) + (this_npix * this_advisory)) / (this_npix + *maskfrac_npix); - *maskfrac_npix += this_npix; - } - psFree(output); - return(true); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS32 this_npix = psMetadataLookupS32(NULL, row, "maskfrac_npix"); + psF32 this_static = psMetadataLookupF32(NULL, row, "maskfrac_static"); + psF32 this_dynamic = psMetadataLookupF32(NULL, row, "maskfrac_dynamic"); + psF32 this_magic = psMetadataLookupF32(NULL, row, "maskfrac_magic"); + psF32 this_advisory = psMetadataLookupF32(NULL, row, "maskfrac_advisory"); + psString this_version = psMetadataLookupStr(NULL, row, "software_ver"); + + *software_ver = pxMergeCodeVersions(*software_ver,this_version); +/* printf("%ld : %d %f %f %f %f <-> %ld %f %f %f %f\n",i,this_npix,this_static,this_dynamic,this_magic,this_advisory, */ +/* *maskfrac_npix,*maskfrac_static,*maskfrac_dynamic,*maskfrac_magic,*maskfrac_advisory); */ + if (this_npix > 0) { + *maskfrac_static = ((*maskfrac_static * *maskfrac_npix) + (this_npix * this_static)) / (this_npix + *maskfrac_npix); + *maskfrac_dynamic = ((*maskfrac_dynamic * *maskfrac_npix) + (this_npix * this_dynamic)) / (this_npix + *maskfrac_npix); + *maskfrac_magic = ((*maskfrac_magic * *maskfrac_npix) + (this_npix * this_magic)) / (this_npix + *maskfrac_npix); + *maskfrac_advisory = ((*maskfrac_advisory * *maskfrac_npix) + (this_npix * this_advisory)) / (this_npix + *maskfrac_npix); + *maskfrac_npix += this_npix; + } + } + psFree(output); + return(true); } bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, psString software_ver) { - char *query = "UPDATE %s SET software_ver = '%s' WHERE %s = %" PRId64; - printf(query,tableName,software_ver,stage_id_name,stage_id); - if (!p_psDBRunQueryF(config->dbh,query,tableName,software_ver,stage_id_name,stage_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to set software version for %s %" PRId64,stage_id_name,stage_id); - return(false); - } - - return(true); -} + char *query = "UPDATE %s SET software_ver = '%s' WHERE %s = %" PRId64; +/* printf(query,tableName,software_ver,stage_id_name,stage_id); */ + if (!p_psDBRunQueryF(config->dbh,query,tableName,software_ver,stage_id_name,stage_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to set software version for %s %" PRId64,stage_id_name,stage_id); + return(false); + } + + return(true); +} + bool pxSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, - psS32 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, + psS64 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, psF32 maskfrac_magic, psF32 maskfrac_advisory) { - char *query = "UPDATE %s SET maskfrac_npix = %d, maskfrac_static = %f, maskfrac_dynamic = %f, maskfrac_magic = %f, maskfrac_advisory = %f WHERE %s = %" PRId64; - if (!p_psDBRunQueryF(config->dbh,query,tableName,maskfrac_npix,maskfrac_static, - maskfrac_dynamic, maskfrac_magic,maskfrac_advisory,stage_id_name,stage_id)) { + char *query = "UPDATE %s SET maskfrac_npix = %f, maskfrac_static = %f, maskfrac_dynamic = %f, maskfrac_magic = %f, maskfrac_advisory = %f WHERE %s = %" PRId64; + if (!p_psDBRunQueryF(config->dbh,query,tableName,(float) maskfrac_npix,maskfrac_static, + maskfrac_dynamic, maskfrac_magic,maskfrac_advisory,stage_id_name,stage_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to set maskfrac stats for %s %" PRId64,stage_id_name,stage_id); + return(false); + } + + + + return(true); +} + +bool pxCamSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psS64 maskfrac_ref_npix, psF32 maskfrac_ref_static, psF32 maskfrac_ref_dynamic, + psF32 maskfrac_ref_magic, psF32 maskfrac_ref_advisory, + psS64 maskfrac_max_npix, psF32 maskfrac_max_static, psF32 maskfrac_max_dynamic, + psF32 maskfrac_max_magic, psF32 maskfrac_max_advisory) { + char *query = "UPDATE %s SET maskfrac_ref_npix = %f, maskfrac_ref_static = %f, maskfrac_ref_dynamic = %f, maskfrac_ref_magic = %f, maskfrac_ref_advisory = %f, maskfrac_max_npix = %f, maskfrac_max_static = %f, maskfrac_max_dynamic = %f, maskfrac_max_magic = %f, maskfrac_max_advisory = %f WHERE %s = %" PRId64; + if (!p_psDBRunQueryF(config->dbh,query,tableName,(float) maskfrac_ref_npix,maskfrac_ref_static, + maskfrac_ref_dynamic, maskfrac_ref_magic,maskfrac_ref_advisory, + (float) maskfrac_max_npix,maskfrac_max_static, + maskfrac_max_dynamic, maskfrac_max_magic,maskfrac_max_advisory, + stage_id_name,stage_id)) { psError(PS_ERR_UNKNOWN, false, "failed to set maskfrac stats for %s %" PRId64,stage_id_name,stage_id); @@ -268,11 +293,11 @@ char *comma = ","; -# define addColumn(_tab, _val) \ - do { \ - if (_val) { \ - psStringAppend(pQuery, "%s %s.%s = '%s'", separator, _tab, #_val, _val); \ - separator = comma; \ - } \ - } while (0) +# define addColumn(_tab, _val) \ + do { \ + if (_val) { \ + psStringAppend(pQuery, "%s %s.%s = '%s'", separator, _tab, #_val, _val); \ + separator = comma; \ + } \ + } while (0) addColumn(runTable, state); @@ -411,5 +436,5 @@ if (import_version && strcmp(import_version, schema_version)) { psError(PS_ERR_UNKNOWN, true, "input file schema_version: %s does not match data base: %s", - import_version, schema_version); + import_version, schema_version); return false; } else if (!import_version) { Index: /branches/czw_branch/20100519/ippTools/src/pxtools.h =================================================================== --- /branches/czw_branch/20100519/ippTools/src/pxtools.h (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/pxtools.h (revision 28164) @@ -55,11 +55,16 @@ psString pxMergeCodeVersions(psString version1, psString version2); bool pxCoalesceRunStatus(pxConfig *config, const psString dbQFile, psS64 stage_id, psString *software_ver, - psS32 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, + psS64 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, psF32 *maskfrac_magic, psF32 *maskfrac_advisory); bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, psString software_ver); bool pxSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, - psS32 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, + psS64 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, psF32 maskfrac_magic, psF32 maskfrac_advisory); +bool pxCamSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, + psS64 maskfrac_ref_npix, psF32 maskfrac_ref_static, psF32 maskfrac_ref_dynamic, + psF32 maskfrac_ref_magic, psF32 maskfrac_ref_advisory, + psS64 maskfrac_max_npix, psF32 maskfrac_max_static, psF32 maskfrac_max_dynamic, + psF32 maskfrac_max_magic, psF32 maskfrac_max_advisory); bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows); Index: /branches/czw_branch/20100519/ippTools/src/stacktool.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/stacktool.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/stacktool.c (revision 28164) @@ -40,4 +40,6 @@ static bool sumskyfileMode(pxConfig *config); static bool revertsumskyfileMode(pxConfig *config); +static bool tosummaryMode(pxConfig *config); +static bool addsummaryMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupskyfileMode(pxConfig *config); @@ -76,4 +78,6 @@ MODECASE(STACKTOOL_MODE_SUMSKYFILE, sumskyfileMode); MODECASE(STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileMode); + MODECASE(STACKTOOL_MODE_TOSUMMARY, tosummaryMode); + MODECASE(STACKTOOL_MODE_ADDSUMMARY, addsummaryMode); MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); @@ -1125,4 +1129,117 @@ } #endif +static bool tosummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackSumSkyfile.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "stackRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "stackRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("stacktool_tosummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return(true); +} +static bool addsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + PXOPT_LOOKUP_STR(tangent_plane, config->args, "-tangent_plane", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + psString query = pxDataGet("stacktool_addsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, stack_id, tangent_plane, path_base)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psS64 numUpdated = psDBAffectedRows(config->dbh); + + if (numUpdated != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); + psFree(query); + return false; + } + + psFree(query); + + // Print anything here? + + return(true); +} static bool pendingcleanuprunMode(pxConfig *config) Index: /branches/czw_branch/20100519/ippTools/src/stacktool.h =================================================================== --- /branches/czw_branch/20100519/ippTools/src/stacktool.h (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/stacktool.h (revision 28164) @@ -34,4 +34,6 @@ STACKTOOL_MODE_SUMSKYFILE, STACKTOOL_MODE_REVERTSUMSKYFILE, + STACKTOOL_MODE_TOSUMMARY, + STACKTOOL_MODE_ADDSUMMARY, STACKTOOL_MODE_PENDINGCLEANUPRUN, STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, Index: /branches/czw_branch/20100519/ippTools/src/stacktoolConfig.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/stacktoolConfig.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/stacktoolConfig.c (revision 28164) @@ -204,4 +204,24 @@ psMetadataAddBool(revertsumskyfileArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", 0); + // -tosummary + psMetadata *tosummaryArgs = psMetadataAlloc(); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by stackRun dist_group (LIKE comparison)", NULL); + + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(tosummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsummary + psMetadata *addsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-stack_id", 0, "set stack ID", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-tangent_plane", 0, "set tangent plane", NULL); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL); + // -pendingcleanuprun psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); @@ -257,4 +277,6 @@ PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", STACKTOOL_MODE_DONECLEANUP, donecleanupArgs); PXOPT_ADD_MODE("-updatesumskyfile", "update fault code for sumskyfile", STACKTOOL_MODE_UPDATESUMSKYFILE, updatesumskyfileArgs); + PXOPT_ADD_MODE("-tosummary", "show runs that can be summarized", STACKTOOL_MODE_TOSUMMARY, tosummaryArgs); + PXOPT_ADD_MODE("-addsummary", "add entry to the summary table", STACKTOOL_MODE_ADDSUMMARY, addsummaryArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", STACKTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", STACKTOOL_MODE_IMPORTRUN, importrunArgs); Index: /branches/czw_branch/20100519/ippTools/src/staticskytool.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/staticskytool.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/staticskytool.c (revision 28164) @@ -1,4 +1,4 @@ /* - * stacktool.c + * staticskytool.c * * Copyright (C) 2007-2010 Joshua Hoblitt, Paul Price @@ -29,23 +29,16 @@ #include "pxtools.h" -#include "stacktool.h" +#include "staticskytool.h" static bool definebyqueryMode(pxConfig *config); -static bool definerunMode(pxConfig *config); static bool updaterunMode(pxConfig *config); -static bool addinputskyfileMode(pxConfig *config); -static bool inputskyfileMode(pxConfig *config); -static bool tosumMode(pxConfig *config); -static bool addsumskyfileMode(pxConfig *config); -static bool sumskyfileMode(pxConfig *config); -static bool revertsumskyfileMode(pxConfig *config); -static bool pendingcleanuprunMode(pxConfig *config); -static bool pendingcleanupskyfileMode(pxConfig *config); -static bool donecleanupMode(pxConfig *config); -static bool updatesumskyfileMode(pxConfig *config); +static bool inputsMode(pxConfig *config); +static bool todoMode(pxConfig *config); +static bool addresultMode(pxConfig *config); +static bool resultMode(pxConfig *config); +static bool revertMode(pxConfig *config); +static bool updateresult(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); - -static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state); # define MODECASE(caseName, func) \ @@ -60,5 +53,5 @@ psLibInit(NULL); - pxConfig *config = stacktoolConfig(NULL, argc, argv); + pxConfig *config = staticskytoolConfig(NULL, argc, argv); if (!config) { psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); @@ -67,14 +60,14 @@ switch (config->mode) { - MODECASE(STACKTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); - MODECASE(STACKTOOL_MODE_UPDATERUN, updaterunMode); - MODECASE(STACKTOOL_MODE_INPUTS, inputsMode); - MODECASE(STACKTOOL_MODE_TODO, todoMode); - MODECASE(STACKTOOL_MODE_ADDRESULT, addresultMode); - MODECASE(STACKTOOL_MODE_RESULT, resultMode); - MODECASE(STACKTOOL_MODE_REVERT, revertMode); - MODECASE(STACKTOOL_MODE_UPDATERESULT, updateresultMode); - MODECASE(STACKTOOL_MODE_EXPORTRUN, exportrunMode); - MODECASE(STACKTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(STATICSKYTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(STATICSKYTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(STATICSKYTOOL_MODE_INPUTS, inputsMode); + MODECASE(STATICSKYTOOL_MODE_TODO, todoMode); + MODECASE(STATICSKYTOOL_MODE_ADDRESULT, addresultMode); + MODECASE(STATICSKYTOOL_MODE_RESULT, resultMode); + MODECASE(STATICSKYTOOL_MODE_REVERT, revertMode); + MODECASE(STATICSKYTOOL_MODE_UPDATERESULT, updateresult); + MODECASE(STATICSKYTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(STATICSKYTOOL_MODE_IMPORTRUN, importrunMode); default: psAbort("invalid option (this should not happen)"); @@ -114,86 +107,57 @@ PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); - psMetadata *where = psMetadataAlloc(); - psMetadata *having = psMetadataAlloc(); // HAVING clause - - PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); - PXOPT_COPY_STR(config->args, where, "-select_skycell_id", "stackRun.skycell_id", "=="); - pxAddLabelSearchArgs(config, where, "-select_label", "stackRun.label", "LIKE"); - pxAddLabelSearchArgs(config, where, "-select_filter", "stackRun.filter", "LIKE"); - - // these are used to build the HAVING restriction - PXOPT_COPY_S32(config->args, having, "-min_num", "num", ">="); - PXOPT_COPY_S32(config->args, having, "-min_filter", "num_filter", ">="); + psMetadata *whereMD = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); + pxAddLabelSearchArgs(config, whereMD, "-select_label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); + + // find the number of requested filters: + psMetadataItem *filters = psMetadataLookup(config->args, "-select_filter"); + psAssert (filters, "-select_filter must exist"); + psAssert (filters->type == PS_DATA_METADATA_MULTI, "-select_filter should be a multi container"); + psAssert (filters->data.list->n, "-select_filter should at least have a place-holder"); + int num_filter = filters->data.list->n; + if (num_filter < 2) { + psError(PXTOOLS_ERR_CONFIG, false, "invalid request: only 1 filter selected"); + psFree(whereMD); + return false; + } PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); - - if (!psListLength(where->list)) { - psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); - psFree(where); - psFree(having); - return false; - } psString select = pxDataGet("staticskytool_definebyquery_select.sql"); if (!select) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - psFree(where); - psFree(having); - return false; - } - - psString where1 = psStringCopy(""); - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&where1, "\nAND %s", whereClause); - psFree(whereClause); - } - - psString where2 = psStringCopy(""); - if (label) { - psStringAppend(&where2, "\nWHERE stackRun.label = '%s'", label); - } - - // Restriction on aggregated quantities using HAVING - { - psString havingClause = NULL; // HAVING string - if (psListLength(having->list)) { - havingClause = psDBGenerateWhereConditionSQL(having, NULL); - } - - if (min_new > 0) { - if (havingClause) { - psStringAppend(&havingClause, " AND"); - } - psStringAppend(&havingClause, - " (num_warp - num_stack >= %d OR (num_warp >= %d AND num_stack IS NULL))", - min_new, min_new); - } - if (isfinite(min_frac)) { - if (havingClause) { - psStringAppend(&havingClause, " AND"); - } - // Avoiding division by zero - psStringAppend(&havingClause, " (num_warp >= %f * num_stack OR num_stack IS NULL)", - (double)min_frac); - } - if (havingClause) { - psStringAppend(&select, " HAVING %s", havingClause); - psFree(havingClause); - } - } - psFree(having); - - if (!p_psDBRunQueryF(config->dbh, select, where1, where2)) { + psFree(whereMD); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + // this 'where' is used for both staticskytool_definebyquery_select.sql and staticskytool_definebyquery_inputs.sql + psString where = NULL; + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&where, "\nAND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + if (!p_psDBRunQueryF(config->dbh, select, where, num_filter)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(select); - psFree(where1); - psFree(where2); return false; } psFree(select); - psFree(where1); - psFree(where2); + + // we now have a list of (tess_id, skycell_id) that (potentially) meet out needs + // we now need to loop over all of these and for each pair, select the best set of + // inputs psArray *output = p_psDBFetchResult(config->dbh); @@ -212,5 +176,5 @@ } if (!psArrayLength(output)) { - psWarning("stacktool: no rows found"); + psWarning("staticskytool: no rows found"); psFree(output); psFree(where); @@ -219,5 +183,5 @@ if (pretend) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "stackSkycells", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "staticskyInput", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -230,11 +194,6 @@ } - psString insert = NULL; // Insertion query - if (randomLimit > 0) { - insert = pxDataGet("stacktool_definebyquery_insert_random_part1.sql"); - } else { - insert = pxDataGet("stacktool_definebyquery_insert.sql"); - } - if (!insert) { + psString inputsSQL = pxDataGet("staticskytool_definebyquery_inputs.sql"); + if (!inputsSQL) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); @@ -242,29 +201,4 @@ } - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&insert, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - if (randomLimit > 0) { - psString part2 = pxDataGet("stacktool_definebyquery_insert_random_part2.sql"); - if (!part2) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - psFree(insert); - return false; - } - psStringAppend(&insert, "%s", part2); - psFree(part2); - } - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(where); - return false; - } - - psArray *list = psArrayAllocEmpty(16); // List of runs, to print for (long i = 0; i < output->n; i++) { psMetadata *row = output->data[i]; // Row from select @@ -276,6 +210,5 @@ psError(PS_ERR_UNKNOWN, false, "failed to lookup skycell_id"); psFree(output); - psFree(insert); - psFree(list); + psFree(inputsSQL); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -288,6 +221,5 @@ psError(PS_ERR_UNKNOWN, false, "failed to lookup tess_id"); psFree(output); - psFree(insert); - psFree(list); + psFree(inputsSQL); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -296,139 +228,101 @@ } - psString filter = psMetadataLookupStr(&status, row, "filter"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to lookup filter"); + // query for the inputs + if (!p_psDBRunQueryF(config->dbh, inputsSQL, tess_id, skycell_id, where)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(output); - psFree(insert); - psFree(list); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - // create a new stackRun for this stack - stackRunRow *run = stackRunRowAlloc( - 0, // ID - "new", // state - workdir, - label, - data_group ? data_group : label, - dist_group, - reduction, - dvodb, - registered, - skycell_id, - tess_id, - filter, - note); - - if (!stackRunInsertObject(config->dbh, run)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(inputsSQL); + return false; + } + + psArray *inputs = p_psDBFetchResult(config->dbh); + if (!inputs) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(inputs); psFree(output); - psFree(run); - psFree(insert); - psFree(list); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - // figure out the new stack_id - psS64 stack_id = psDBLastInsertID(config->dbh); - run->stack_id = stack_id; - - psArrayAdd(list, list->n, run); - psFree(run); - - // Create a suitable insertion query for this run - psString thisInsert = psStringCopy(insert); - psString idString = NULL; - psStringAppend(&idString, "%" PRId64, stack_id); - psStringSubstitute(&thisInsert, idString, "@STACK_ID@"); - psFree(idString); - - // replace @FILTER@, @SKYCELL_ID@, @RANDOM_LIMIT@ - psStringSubstitute(&thisInsert, filter, "@FILTER@"); - psStringSubstitute(&thisInsert, skycell_id, "@SKYCELL_ID@"); - - if (randomLimit > 0) { - psString limString = NULL; - psStringAppend(&limString, "%d", randomLimit); - psStringSubstitute(&thisInsert, limString, "@RANDOM_LIMIT@"); - psFree(limString); - } - - // XXX this insert uses a select to generate the list of warp_ids for the stack, - // we have applied a set of criteria above (WHERE) to select the relevant warps - // this insert needs to use exactly the same restrictions (race condition is probably not critical) - // the insert below seems to only restrict matches to the skycell, tess, and filter - if (!p_psDBRunQuery(config->dbh, thisInsert)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(thisInsert); - psFree(insert); + psFree(inputsSQL); + return false; + } + if (!psArrayLength(inputs)) { + psWarning("staticskytool ERROR: no rows found for known tess_id, skycell_id?"); + continue; + } + + // XXX if we are unable to guarantee that all selected inputs have all and only the + // requested filters, see the code at the bottom of this file (ifdef'ed out for now) + + // insert a new staticsky entry and find its new id + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(inputs); psFree(output); - psFree(list); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(thisInsert); - -# if (0) - { - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - psFree(where); - return false; - } - if (!psArrayLength(output)) { - psWarning("stacktool (definebyquery, insert): no rows found"); + psFree(inputsSQL); + return false; + } + + // create a chipRun + if (!staticskyRunInsert(config->dbh, + 0x0, // sky_id + "new", // state + workdir, + label, + data_group, + dist_group, + reduction, + registered, + note + ) + ) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(inputs); psFree(output); - psFree(where); - return true; - } - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "stackSkycells", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - psFree(where); - return false; - } - psFree(output); - psFree(where); - return true; - } -# endif - - } + psFree(inputsSQL); + return false; + } + + psS64 sky_id = psDBLastInsertID(config->dbh); + + // loop over the possible inputs and record only the valid ones + for (int j = 0; j < inputs->n; j++) { + psMetadata *inputRow = inputs->data[j]; // Row from select + + // pull out the skycell_id, tess_id, filter + psS64 stack_id = psMetadataLookupS64(&status, inputRow, "stack_id"); + psAssert(status, "failed to find stack_id?"); + + // add a staticskyInput entry + if (!staticskyInputInsert(config->dbh, sky_id, stack_id)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(inputs); + psFree(output); + psFree(inputsSQL); + return false; + } + } + psFree (inputs); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(inputsSQL); + return false; + } + } + psFree(inputsSQL); psFree(output); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(list); - return false; - } - - if (!stackRunPrintObjects(stdout, list, !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(list); - return false; - } - psFree(list); return true; } @@ -438,13 +332,8 @@ PS_ASSERT_PTR_NON_NULL(config, false); -#ifdef notdef - PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); -#endif psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); if (!psListLength(where->list)) { psFree(where); @@ -453,29 +342,12 @@ } - psString query = psStringCopy("UPDATE stackRun"); + psString query = psStringCopy("UPDATE staticskyRun"); // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true); - + bool result = pxUpdateRun(config, where, &query, "staticskyRun", "sky_id", "staticskyResult", true); psFree(query); psFree(where); return result; - -#ifdef notdef - // Hack-y work around to make stacktool more like the other tools, without breaking other stuff (hopefully). - - if ((state)&&(stack_id)) { - // set detRun.state to state - return setstackRunState(config, stack_id, state); - } - - if ((state)&&(label)) { - return setstackRunStateByLabel(config, label, state); - } - - psError(PS_ERR_UNKNOWN, false, "Required options not found."); - return false; -#endif } @@ -485,15 +357,14 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // XXX require at least a stack id (add better search options) - PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + // XXX require at least a sky id (add better search options) + PXOPT_LOOKUP_S64(sky_id, config->args, "-sky_id", true, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = pxDataGet("stacktool_inputskyfile.sql"); + psString query = pxDataGet("staticskytool_inputs.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -502,6 +373,6 @@ if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "stackInputSkyfile"); - psStringAppend(&query, " AND %s", whereClause); + psString whereClause = psDBGenerateWhereConditionSQL(where, "staticskyInput"); + psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); } @@ -537,5 +408,5 @@ } if (!psArrayLength(output)) { - psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psTrace("staticskytool", PS_LOG_INFO, "no rows found"); psFree(output); return true; @@ -544,5 +415,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "stackInputSkyfile", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "staticskyInput", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -561,13 +432,12 @@ PS_ASSERT_PTR_NON_NULL(config, false); - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-sky_id", "sky_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "staticskyRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = pxDataGet("stacktool_tosum.sql"); + psString query = pxDataGet("staticskytool_todo.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -575,10 +445,15 @@ } - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString where = NULL; + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&where, "\n AND %s", whereClause); + psFree(whereClause); + psFree(whereMD); // treat limit == 0 as "no limit" @@ -589,5 +464,6 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { + // the where clause is required and matches the WHERE hook format string + if (!p_psDBRunQueryF(config->dbh, query, where)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -611,5 +487,5 @@ } if (!psArrayLength(output)) { - psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psTrace("staticskytool", PS_LOG_INFO, "no rows found"); psFree(output); return true; @@ -618,5 +494,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "staticskyResult", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -626,9 +502,7 @@ psFree(output); - return true; } - static bool addresultMode(pxConfig *config) { @@ -636,31 +510,14 @@ // required - PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); + PXOPT_LOOKUP_S64(sky_id, config->args, "-sky_id", true, false); // optional - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); - PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); - PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); - PXOPT_LOOKUP_F32(dtime_stack, config->args, "-dtime_stack", false, false); - PXOPT_LOOKUP_F32(dtime_match_mean, config->args, "-dtime_match_mean", false, false); - PXOPT_LOOKUP_F32(dtime_match_stdev, config->args, "-dtime_match_stdev", false, false); - PXOPT_LOOKUP_F32(dtime_initial, config->args, "-dtime_initial", false, false); - PXOPT_LOOKUP_F32(dtime_reject, config->args, "-dtime_reject", false, false); - PXOPT_LOOKUP_F32(dtime_final, config->args, "-dtime_final", false, false); PXOPT_LOOKUP_F32(dtime_phot, config->args, "-dtime_phot", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); - PXOPT_LOOKUP_F32(match_mean, config->args, "-match_mean", false, false); - PXOPT_LOOKUP_F32(match_stdev, config->args, "-match_stdev", false, false); - PXOPT_LOOKUP_F32(match_rms, config->args, "-match_rms", false, false); - PXOPT_LOOKUP_F32(stamps_mean, config->args, "-stamps_mean", false, false); - PXOPT_LOOKUP_F32(stamps_stdev, config->args, "-stamps_stdev", false, false); - PXOPT_LOOKUP_S32(stamps_min, config->args, "-stamps_min", false, false); - PXOPT_LOOKUP_S32(reject_images, config->args, "-reject_images", false, false); - PXOPT_LOOKUP_F32(reject_pix_mean, config->args, "-reject_pix_mean", false, false); - PXOPT_LOOKUP_F32(reject_pix_stdev, config->args, "-reject_pix_stdev", false, false); PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false); PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); + PXOPT_LOOKUP_S32(num_inputs, config->args, "-num_inputs", false, false); // default values @@ -668,4 +525,5 @@ PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + // XXX not sure we need a transaction here... if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -673,31 +531,11 @@ } - // XXX need to validate the stack_id here - // XXX instead of validiting it here we should just use forgein key - // constrants - if (!stackSumSkyfileInsert(config->dbh, - stack_id, - uri, + if (!staticskyResultInsert(config->dbh, + sky_id, path_base, - bg, - bg_stdev, - dtime_stack, - dtime_match_mean, - dtime_match_stdev, - dtime_initial, - dtime_reject, - dtime_final, dtime_phot, dtime_script, - match_mean, - match_stdev, - match_rms, - stamps_mean, - stamps_stdev, - stamps_min, - reject_images, - reject_pix_mean, - reject_pix_stdev, sources, + num_inputs, hostname, good_frac, @@ -712,14 +550,4 @@ } - if (fault == 0) { - if (!setstackRunState(config, stack_id, "full")) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "failed to change stackRun's state"); - return false; - } - } - // point of no return if (!psDBCommit(config->dbh)) { @@ -731,5 +559,5 @@ } - +// XXX what is this used by? what filters are needed? static bool resultMode(pxConfig *config) { @@ -737,16 +565,8 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); - PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE"); - PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); - -// The following three selectors are incompatible with the sql so omit them -// PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); -// PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); -// PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "staticskyRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "staticskyRun.data_group", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); @@ -755,5 +575,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = pxDataGet("stacktool_sumskyfile.sql"); + psString query = pxDataGet("staticskytool_result.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -801,5 +621,5 @@ } if (!psArrayLength(output)) { - psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psTrace("staticskytool", PS_LOG_INFO, "no rows found"); psFree(output); return true; @@ -807,5 +627,5 @@ if (psArrayLength(output)) { - if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "staticskyResult", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -819,9 +639,9 @@ } - static bool revertMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); +# if (0) psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); @@ -836,5 +656,5 @@ // Delete product - psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); + psString delete = pxDataGet("staticskytool_revertsumskyfile_delete.sql"); if (!delete) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -857,16 +677,15 @@ int numRows = psDBAffectedRows(config->dbh); // Number of row affected - psLogMsg("stacktool", PS_LOG_INFO, "Deleted %d rows", numRows); + psLogMsg("staticskytool", PS_LOG_INFO, "Deleted %d rows", numRows); psFree(where); +# endif return true; } - -static bool updateresultState(pxConfig *config, psS64 stack_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(state, false); - +# if (0) +static bool updateresultState(pxConfig *config) +{ // check that state is a valid string value if (!pxIsValidState(state)) { @@ -881,12 +700,13 @@ return false; } - return true; } - -static bool updateresultMode(pxConfig *config) +# endif + +static bool updateresult(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); +# if (0) PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); @@ -901,4 +721,5 @@ psFree (where); +# endif return true; } @@ -906,4 +727,5 @@ bool exportrunMode(pxConfig *config) { +# if (0) typedef struct ExportTable { char tableName[80]; @@ -934,7 +756,7 @@ ExportTable tables [] = { - {"stackRun", "stacktool_export_run.sql"}, - {"stackInputSkyfile", "stacktool_export_input_skyfile.sql"}, - {"stackSumSkyfile", "stacktool_export_sum_skyfile.sql"}, + {"stackRun", "staticskytool_export_run.sql"}, + {"stackInputSkyfile", "staticskytool_export_input_skyfile.sql"}, + {"stackSumSkyfile", "staticskytool_export_sum_skyfile.sql"}, }; @@ -999,4 +821,5 @@ fclose (f); +# endif return true; } @@ -1004,4 +827,5 @@ bool importrunMode(pxConfig *config) { +# if (0) unsigned int nFail; @@ -1072,4 +896,39 @@ } +# endif return true; } + +# if (0) + // now we need to loop over all requested filters and check that each is found + // XXX is this needed? haven't we required that we only match the requested filters + // and that N unique filters are found? + + psVector *found = psVectorAlloc(inputs->n, PS_TYPE_U8); + psVectorInit (found, 0); + + psMetadataItem *filter = NULL; + psListIterator *iter = psListIteratorAlloc (filters->data.list, PS_LIST_HEAD, false); + while ((filter = psListGetAndIncrement(iter))) { + + bool foundOne = false; + + psAssert (filter->type == PS_DATA_STR, "filter is not a string?"); + for (int j = 0; !foundOne && (j < inputs->n); j++) { + if (found->data.U8[j]) continue; + psMetadata *inputRow = inputs->data[j]; // Row from select + + // pull out the skycell_id, tess_id, filter + psString inFilter = psMetadataLookupStr(&status, row, "filter"); + psAssert (status); + + found->data.U8[j] = true; + foundOne = true; + } + if (!foundOne) { + // this required filter was not found in the inputs, skip the entry + skip(); + } + } +# endif + Index: /branches/czw_branch/20100519/ippTools/src/staticskytoolConfig.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/staticskytoolConfig.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/staticskytoolConfig.c (revision 28164) @@ -52,5 +52,5 @@ psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple required)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); @@ -89,12 +89,13 @@ // -addresult psMetadata *addresultArgs = psMetadataAlloc(); - psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-sky_id", 0, "define stack ID (required)", 0); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-sky_id", 0, "define sky ID (required)", 0); psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_phot", 0, "define photometry time", NAN); psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); psMetadataAddS32(addresultArgs, PS_LIST_TAIL, "-sources", 0, "number of sources", 0); + psMetadataAddS32(addresultArgs, PS_LIST_TAIL, "-num_inputs", 0, "number of inputs", 0); psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-good_frac", 0, "define fraction of good pixels", NAN); - psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0) + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); @@ -102,6 +103,6 @@ psMetadata *resultArgs= psMetadataAlloc(); psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); - psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); - psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); + // psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); + // psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); @@ -109,4 +110,5 @@ psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(resultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(resultArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false); // -revert @@ -143,5 +145,5 @@ PXOPT_ADD_MODE("-inputs", "Get inputs", STATICSKYTOOL_MODE_INPUTS, inputsArgs); PXOPT_ADD_MODE("-todo", "Get runs to do", STATICSKYTOOL_MODE_TODO, todoArgs); - PXOPT_ADD_MODE("-addresult", "Add result of run", STATICSKYTOOL_MODE_ADDRESULT, addresultsArgs); + PXOPT_ADD_MODE("-addresult", "Add result of run", STATICSKYTOOL_MODE_ADDRESULT, addresultArgs); PXOPT_ADD_MODE("-result", "Get result of run", STATICSKYTOOL_MODE_RESULT, resultArgs); PXOPT_ADD_MODE("-revert", "Revert failed run", STATICSKYTOOL_MODE_REVERT, revertArgs); Index: /branches/czw_branch/20100519/ippTools/src/warptool.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/warptool.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/warptool.c (revision 28164) @@ -48,4 +48,6 @@ static bool maskedMode(pxConfig *config); static bool unblockMode(pxConfig *config); +static bool tosummaryMode(pxConfig *config); +static bool addsummaryMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupwarpMode(pxConfig *config); @@ -101,4 +103,6 @@ MODECASE(WARPTOOL_MODE_MASKED, maskedMode); MODECASE(WARPTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(WARPTOOL_MODE_TOSUMMARY, tosummaryMode); + MODECASE(WARPTOOL_MODE_ADDSUMMARY, addsummaryMode); MODECASE(WARPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupwarpMode); @@ -408,7 +412,7 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "warpRun.reduction", "=="); PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); - PXOPT_COPY_STR(config->args, where, "-data_group","warpRun.data_group","=="); - PXOPT_COPY_STR(config->args, where, "-dist_group","warpRun.dist_group","=="); - pxAddLabelSearchArgs(config, where, "-label", "warpRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group","warpRun.data_group","LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group","warpRun.dist_group","LIKE"); + pxAddLabelSearchArgs(config, where, "-label", "warpRun.label", "LIKE"); PXOPT_COPY_TIME(config->args, where, "-registered_begin", "warpRun.registered", ">="); @@ -940,15 +944,12 @@ // treat limit == 0 as "no limit" - psString limitString = psStringCopy("\n"); + psString limitString = psStringCopy("\nORDER BY priority DESC, warp_id"); if (limit) { - // We apply the limit to both sides of the UNION - // to avoid slow queries and to the query itself - // to satisfy the user's requested limit + // We apply the limit to both sides of the UNION to avoid slow queries + // and to the query itself to satisfy the user's requested limit psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit)); psStringAppend(&query, " %s", limitString); } -#define TWO_WHERES -#ifdef TWO_WHERES if (!p_psDBRunQueryF(config->dbh, query, whereStr, limitString, whereStr, limitString)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -956,11 +957,4 @@ return false; } -#else - if (!p_psDBRunQueryF(config->dbh, query, whereStr)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } -#endif psFree(limitString); psFree(whereStr); @@ -1163,5 +1157,5 @@ psString software_ver = NULL; - psS32 maskfrac_npix = 0; + psS64 maskfrac_npix = 0; psF32 maskfrac_static = 0; psF32 maskfrac_dynamic = 0; @@ -1181,25 +1175,29 @@ return(false); } - // Set warpRun.software_ver to the appropriate value - if (!pxSetRunSoftware(config, "warpRun", "warp_id", warp_id, software_ver)) { - psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, - warp_id); - psFree(output); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + // Set warpRun.software_ver to the appropriate value if ( + if (software_ver) { + if (!pxSetRunSoftware(config, "warpRun", "warp_id", warp_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, + warp_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); } - return(false); } // Set warpRun.maskfrac* to the appropriate values. - if (!pxSetRunMaskfrac(config, "warpRun", "warp_id", warp_id, maskfrac_npix, maskfrac_static, - maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { - psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, - warp_id); - psFree(output); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (maskfrac_npix) { + if (!pxSetRunMaskfrac(config, "warpRun", "warp_id", warp_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, + warp_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); } - return(false); - } + } @@ -1554,4 +1552,133 @@ } +static bool tosummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_tosummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "warpSkyfile")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + if (where2) { + psStringAppend(&query, " %s", where2); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return(true); +} +static bool addsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(tangent_plane, config->args, "-tangent_plane", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + psString query = pxDataGet("warptool_addsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, warp_id, tangent_plane, path_base)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psS64 numUpdated = psDBAffectedRows(config->dbh); + + if (numUpdated != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); + psFree(query); + return false; + } + + psFree(query); + + // Print anything here? + + return(true); +} + static bool pendingcleanuprunMode(pxConfig *config) { Index: /branches/czw_branch/20100519/ippTools/src/warptool.h =================================================================== --- /branches/czw_branch/20100519/ippTools/src/warptool.h (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/warptool.h (revision 28164) @@ -44,4 +44,6 @@ WARPTOOL_MODE_MASKED, WARPTOOL_MODE_UNBLOCK, + WARPTOOL_MODE_TOSUMMARY, + WARPTOOL_MODE_ADDSUMMARY, WARPTOOL_MODE_PENDINGCLEANUPRUN, WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, Index: /branches/czw_branch/20100519/ippTools/src/warptoolConfig.c =================================================================== --- /branches/czw_branch/20100519/ippTools/src/warptoolConfig.c (revision 28163) +++ /branches/czw_branch/20100519/ippTools/src/warptoolConfig.c (revision 28164) @@ -136,7 +136,7 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by warpRun reduction class", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by warpRun state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by warpRun data_group", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by warpRun dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label (LIKE comparison)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by warpRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by warpRun dist_group (LIKE comparison)", NULL); psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-registered_begin", 0, "search for runs by registration time (>=)", NULL); psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-registered_end", 0, "search for runs by registration time (<)", NULL); @@ -321,4 +321,32 @@ psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + // -tosummary + psMetadata *tosummaryArgs = psMetadataAlloc(); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); + psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-filter", 0, "search for exposures by filter", NULL); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-magicked", 0, "search by magic id", 0); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by warpRun dist_group", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for runs that have not been destreaked", false); + + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(tosummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsummary + psMetadata *addsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-warp_id", 0, "set warp ID", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-tangent_plane", 0, "set tangent plane", NULL); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL); + // -pendingcleanuprun psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); @@ -430,4 +458,7 @@ PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs); PXOPT_ADD_MODE("-setskyfiletoupdate", "set cleaned skyfile to be updated", WARPTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateArgs); + PXOPT_ADD_MODE("-tosummary", "show runs that can be summarized", WARPTOOL_MODE_TOSUMMARY, tosummaryArgs); + PXOPT_ADD_MODE("-addsummary", "add entry to the summary table", WARPTOOL_MODE_ADDSUMMARY, addsummaryArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", WARPTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", WARPTOOL_MODE_IMPORTRUN, importrunArgs); Index: /branches/czw_branch/20100519/ippconfig/gpc1/camera.config =================================================================== --- /branches/czw_branch/20100519/ippconfig/gpc1/camera.config (revision 28163) +++ /branches/czw_branch/20100519/ippconfig/gpc1/camera.config (revision 28164) @@ -153,4 +153,9 @@ BURNTOOL.STATE.GOOD.UPDATE S16 13 # for upddate processing accept earlier version -FOV F32 15000 # Field of view of unvignetted region in FPA pixels. -NPIX_INTERCHIP S32 0 +FOV_REF F32 20930 # Field of view of unvignetted region in FPA pixels. +FOV_MAX F32 22720 +#NPIX_REF S32 1357799527 +#NPIX_MAX S32 1482790357 +NPIX_REF S32 1333206539 +NPIX_MAX S32 1455690816 +NPIX_INTERCHIP S32 103794483 Index: /branches/czw_branch/20100519/ippconfig/gpc1/psphot.config =================================================================== --- /branches/czw_branch/20100519/ippconfig/gpc1/psphot.config (revision 28163) +++ /branches/czw_branch/20100519/ippconfig/gpc1/psphot.config (revision 28164) @@ -36,4 +36,5 @@ BACKGROUND.XBIN S32 32 # size of background superpixels BACKGROUND.YBIN S32 32 # size of background superpixels + OUTPUT.FORMAT STR PS1_DV1 END Index: /branches/czw_branch/20100519/ippconfig/recipes/filerules-mef.mdc =================================================================== --- /branches/czw_branch/20100519/ippconfig/recipes/filerules-mef.mdc (revision 28163) +++ /branches/czw_branch/20100519/ippconfig/recipes/filerules-mef.mdc (revision 28164) @@ -307,4 +307,6 @@ PPSKYCELL.JPEG1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg JPEG NONE CHIP TRUE NONE PPSKYCELL.JPEG2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg JPEG NONE CHIP TRUE NONE +PPSKYCELL.BIN1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.fits IMAGE NONE FPA TRUE NONE +PPSKYCELL.BIN2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.fits IMAGE NONE FPA TRUE NONE LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE Index: /branches/czw_branch/20100519/ippconfig/recipes/filerules-simple.mdc =================================================================== --- /branches/czw_branch/20100519/ippconfig/recipes/filerules-simple.mdc (revision 28163) +++ /branches/czw_branch/20100519/ippconfig/recipes/filerules-simple.mdc (revision 28164) @@ -272,4 +272,6 @@ PPSKYCELL.JPEG1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg JPEG NONE CHIP TRUE NONE PPSKYCELL.JPEG2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg JPEG NONE CHIP TRUE NONE +PPSKYCELL.BIN1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.fits IMAGE NONE FPA TRUE NONE +PPSKYCELL.BIN2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.fits IMAGE NONE FPA TRUE NONE LOG.IMFILE OUTPUT {OUTPUT}.imfile.log TEXT NONE FPA TRUE NONE Index: /branches/czw_branch/20100519/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/czw_branch/20100519/ippconfig/recipes/filerules-split.mdc (revision 28163) +++ /branches/czw_branch/20100519/ippconfig/recipes/filerules-split.mdc (revision 28164) @@ -292,4 +292,6 @@ PPSKYCELL.JPEG1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg JPEG NONE CHIP TRUE NONE PPSKYCELL.JPEG2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg JPEG NONE CHIP TRUE NONE +PPSKYCELL.BIN1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.fits IMAGE NONE FPA TRUE NONE +PPSKYCELL.BIN2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.fits IMAGE NONE FPA TRUE NONE LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE Index: /branches/czw_branch/20100519/ippconfig/recipes/ppSkycell.config =================================================================== --- /branches/czw_branch/20100519/ippconfig/recipes/ppSkycell.config (revision 28163) +++ /branches/czw_branch/20100519/ippconfig/recipes/ppSkycell.config (revision 28164) @@ -4,3 +4,3 @@ BIN1 S32 4 # Binning factor for output 1 BIN2 S32 4 # Binning factor for output 2 - +MAKEFITS BOOL T # Make fits image as well? Index: /branches/czw_branch/20100519/ippconfig/recipes/ppStats.config =================================================================== --- /branches/czw_branch/20100519/ippconfig/recipes/ppStats.config (revision 28163) +++ /branches/czw_branch/20100519/ippconfig/recipes/ppStats.config (revision 28164) @@ -130,9 +130,15 @@ ANALYSIS STR PSASTRO.HEADER - ANALYSIS STR MASKFRAC_STATIC - ANALYSIS STR MASKFRAC_MAGIC - ANALYSIS STR MASKFRAC_ADVISORY - ANALYSIS STR MASKFRAC_DYNAMIC - ANALYSIS STR MASKFRAC_NPIX + ANALYSIS STR MASKFRAC_REF_STATIC + ANALYSIS STR MASKFRAC_REF_MAGIC + ANALYSIS STR MASKFRAC_REF_ADVISORY + ANALYSIS STR MASKFRAC_REF_DYNAMIC + ANALYSIS STR MASKFRAC_REF_NPIX + + ANALYSIS STR MASKFRAC_MAX_STATIC + ANALYSIS STR MASKFRAC_MAX_MAGIC + ANALYSIS STR MASKFRAC_MAX_ADVISORY + ANALYSIS STR MASKFRAC_MAX_DYNAMIC + ANALYSIS STR MASKFRAC_MAX_NPIX END Index: /branches/czw_branch/20100519/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- /branches/czw_branch/20100519/ippconfig/recipes/ppStatsFromMetadata.config (revision 28163) +++ /branches/czw_branch/20100519/ippconfig/recipes/ppStatsFromMetadata.config (revision 28164) @@ -282,9 +282,15 @@ # Mask stats values - ENTRY VAL MASKFRAC_NPIX S32 CONSTANT -maskfrac_npix - ENTRY VAL MASKFRAC_STATIC F32 CONSTANT -maskfrac_static - ENTRY VAL MASKFRAC_DYNAMIC F32 CONSTANT -maskfrac_dynamic - ENTRY VAL MASKFRAC_MAGIC F32 CONSTANT -maskfrac_magic - ENTRY VAL MASKFRAC_ADVISORY F32 CONSTANT -maskfrac_advisory + ENTRY VAL MASKFRAC_REF_NPIX S32 CONSTANT -maskfrac_ref_npix + ENTRY VAL MASKFRAC_REF_STATIC F32 CONSTANT -maskfrac_ref_static + ENTRY VAL MASKFRAC_REF_DYNAMIC F32 CONSTANT -maskfrac_ref_dynamic + ENTRY VAL MASKFRAC_REF_MAGIC F32 CONSTANT -maskfrac_ref_magic + ENTRY VAL MASKFRAC_REF_ADVISORY F32 CONSTANT -maskfrac_ref_advisory + + ENTRY VAL MASKFRAC_MAX_NPIX S32 CONSTANT -maskfrac_max_npix + ENTRY VAL MASKFRAC_MAX_STATIC F32 CONSTANT -maskfrac_max_static + ENTRY VAL MASKFRAC_MAX_DYNAMIC F32 CONSTANT -maskfrac_max_dynamic + ENTRY VAL MASKFRAC_MAX_MAGIC F32 CONSTANT -maskfrac_max_magic + ENTRY VAL MASKFRAC_MAX_ADVISORY F32 CONSTANT -maskfrac_max_advisory END Index: /branches/czw_branch/20100519/ippconfig/recipes/psphot.config =================================================================== --- /branches/czw_branch/20100519/ippconfig/recipes/psphot.config (revision 28163) +++ /branches/czw_branch/20100519/ippconfig/recipes/psphot.config (revision 28164) @@ -342,5 +342,4 @@ # Recipe overrides for DIFF DIFF METADATA - OUTPUT.FORMAT STR PS1_DV1 END Index: /branches/czw_branch/20100519/magic/Makefile =================================================================== --- /branches/czw_branch/20100519/magic/Makefile (revision 28163) +++ /branches/czw_branch/20100519/magic/Makefile (revision 28164) @@ -1,6 +1,16 @@ + +MAGIC_DIR = ~/magic_tarballs + +# necessary support for libQt (only used by magic) +QTDIR = qt-everywhere-opensource-src-4.6.1 +QTSRC = $(MAGIC_DIR)/qt-everywhere-opensource-src-4.6.1.tar.gz +QTDEP := $(shell test -f $(QTSRC) && echo $(QTSRC)) + +PSCFG = ../psconfig/psconfig.csh +PSPREFIX := $(shell csh $(PSCFG) $(PSVERSION) --prefix) default: all -all: ssa-core-cpp magic verify +all: qt-everywhere ssa-core-cpp magic verify install: magic.install verify.install @@ -12,8 +22,37 @@ update: # get the source code (replace this with SVN interactions) - tar xvzf ~/magic.tgz - tar xvzf ~/ssa-core-cpp.tgz + tar xvzf $(MAGIC_DIR)/magic.tgz + tar xvzf $(MAGIC_DIR)/ssa-core-cpp.tgz + tar xvzf $(QTSRC) cp Makefile.magic magic/Makefile.magic cp magic/VerifyStreaks/main.cpp verify/VerifyStreaks.cpp + +qt-everywhere: $(QTDIR)/lib/libQtCore.so + +QTOPT = -fast \ + -opensource \ + -nomake tools \ + -nomake examples \ + -nomake demos \ + -nomake docs \ + -nomake translations \ + -no-xmlpatterns \ + -no-multimedia \ + -no-audio-backend \ + -no-phonon \ + -no-phonon-backend \ + -no-svg \ + -no-webkit \ + -no-javascript-jit \ + -no-script \ + -no-scripttools + +# the target (QTDEP) is blank unless QTSRC exists +$(QTDIR)/lib/libQtCore.so: $(QTDEP) + echo $(PSCFG) + echo $(PSPREFIX) + if [ -d $(QTDIR) ]; then (cd $(QTDIR) && echo "yes" | ./configure $(QTOPT) -prefix $(PSPREFIX)); fi + if [ -d $(QTDIR) ]; then (cd $(QTDIR) && make); fi + if [ -d $(QTDIR) ]; then (cd $(QTDIR) && make install); fi ssa-core-cpp: ssa-core-cpp/build/libSSA.a Index: /branches/czw_branch/20100519/ppSkycell/src/ppSkycell.h =================================================================== --- /branches/czw_branch/20100519/ppSkycell/src/ppSkycell.h (revision 28163) +++ /branches/czw_branch/20100519/ppSkycell/src/ppSkycell.h (revision 28164) @@ -16,4 +16,5 @@ int bin1, bin2; // Binning factors pmConfig *config; // Configuration + bool doFits; // hold whether to do fits as well. } ppSkycellData; Index: /branches/czw_branch/20100519/ppSkycell/src/ppSkycellCamera.c =================================================================== --- /branches/czw_branch/20100519/ppSkycell/src/ppSkycellCamera.c (revision 28163) +++ /branches/czw_branch/20100519/ppSkycell/src/ppSkycellCamera.c (revision 28164) @@ -103,4 +103,12 @@ return false; } + if (!pmFPAfileDefineOutput(data->config, NULL, "PPSKYCELL.BIN1")) { + psError(psErrorCodeLast(), false, "Unable to define output."); + return false; + } + if (!pmFPAfileDefineOutput(data->config, NULL, "PPSKYCELL.BIN2")) { + psError(psErrorCodeLast(), false, "Unable to define output."); + return false; + } // Now the camera has been determined, we can read the recipe @@ -116,4 +124,6 @@ data->bin2 = psMetadataLookupS32(NULL, recipe, "BIN2"); + data->doFits = psMetadataLookupBool(NULL, recipe, "MAKEFITS"); + if (data->bin1 <= 0 || data->bin2 <= 0) { psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find legitimate values for BIN1 and BIN2"); Index: /branches/czw_branch/20100519/ppSkycell/src/ppSkycellLoop.c =================================================================== --- /branches/czw_branch/20100519/ppSkycell/src/ppSkycellLoop.c (revision 28163) +++ /branches/czw_branch/20100519/ppSkycell/src/ppSkycellLoop.c (revision 28164) @@ -284,9 +284,15 @@ pmFPAfileActivate(data->config->files, true, "PPSKYCELL.JPEG1"); pmFPAfileActivate(data->config->files, true, "PPSKYCELL.JPEG2"); + if (data->doFits) { + pmFPAfileActivate(data->config->files, true, "PPSKYCELL.BIN1"); + pmFPAfileActivate(data->config->files, true, "PPSKYCELL.BIN2"); + } + pmFPAview *view = filesIterateDown(data->config); // View to readout + pmCell *cell1 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.JPEG1"); // Rebinned cell 1 pmCell *cell2 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.JPEG2"); // Rebinned cell 2 - psFree(view); + pmReadout *ro1 = pmReadoutAlloc(cell1), *ro2 = pmReadoutAlloc(cell2); // Binned readouts @@ -298,5 +304,5 @@ ro1->data_exists = cell1->data_exists = cell1->parent->data_exists = true; ro2->data_exists = cell2->data_exists = cell2->parent->data_exists = true; - + pmFPAfile *file1 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.JPEG1", 0); file1->save = true; @@ -305,4 +311,40 @@ file2->save = true; file2->fileIndex = i; + + if (data->doFits) { + pmCell *Fcell1 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.BIN1"); // Rebinned cell 1 + pmCell *Fcell2 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.BIN2"); // Rebinned cell 2 + // This is a hack to get a functioning header created so the fits images can be written out. + psMetadataAddS32(Fcell1->concepts,PS_LIST_TAIL,"CELL.XPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell1->concepts,PS_LIST_TAIL,"CELL.YPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell1->concepts,PS_LIST_TAIL,"CELL.READDIR", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell2->concepts,PS_LIST_TAIL,"CELL.XPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell2->concepts,PS_LIST_TAIL,"CELL.YPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell2->concepts,PS_LIST_TAIL,"CELL.READDIR", PS_META_REPLACE,"",1); + + psMetadataAddS32(Fcell1->parent->concepts,PS_LIST_TAIL,"CHIP.XPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell1->parent->concepts,PS_LIST_TAIL,"CHIP.YPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell2->parent->concepts,PS_LIST_TAIL,"CHIP.XPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell2->parent->concepts,PS_LIST_TAIL,"CHIP.YPARITY", PS_META_REPLACE,"",1); + + pmReadout *Fro1 = pmReadoutAlloc(Fcell1), *Fro2 = pmReadoutAlloc(Fcell2); // Binned readouts + + Fro1->image = image1; + Fro2->image = image2; + Fro1->mask = mask1; + Fro2->mask = mask2; + + Fro1->data_exists = Fcell1->data_exists = Fcell1->parent->data_exists = true; + Fro2->data_exists = Fcell2->data_exists = Fcell2->parent->data_exists = true; + + pmFPAfile *fits1 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.BIN1", 0); + fits1->save = true; + fits1->fileIndex = i; + pmFPAfile *fits2 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.BIN2", 0); + fits2->save = true; + fits2->fileIndex = i; + } + + psFree(view); #if 0 { Index: /branches/czw_branch/20100519/ppStack/src/ppStackCombineFinal.c =================================================================== --- /branches/czw_branch/20100519/ppStack/src/ppStackCombineFinal.c (revision 28163) +++ /branches/czw_branch/20100519/ppStack/src/ppStackCombineFinal.c (revision 28164) @@ -138,24 +138,4 @@ } - // Set exposure time correctly - { - float exptime = 0.0; // Summed exposure time - for (int i = 0; i < options->num; i++) { - if (options->inputMask->data.U8[i]) { - continue; - } - exptime += options->exposures->data.F32[i]; - } - - { - psMetadataItem *item = psMetadataLookup(outRO->parent->concepts, "CELL.EXPOSURE"); - item->data.F32 = exptime; - } - { - psMetadataItem *item = psMetadataLookup(outRO->parent->parent->parent->concepts, "FPA.EXPOSURE"); - item->data.F32 = exptime; - } - } - // Put version information into the header pmHDU *hdu = pmHDUFromCell(outRO->parent); Index: /branches/czw_branch/20100519/ppStack/src/ppStackSources.c =================================================================== --- /branches/czw_branch/20100519/ppStack/src/ppStackSources.c (revision 28163) +++ /branches/czw_branch/20100519/ppStack/src/ppStackSources.c (revision 28164) @@ -77,8 +77,8 @@ PS_ASSERT_VECTOR_SIZE(inputMask, sourceLists->n, false); -#if defined(TESTING) && 0 +#if defined(TESTING) && 1 { // Deliberately induce a major transparency difference - psArray *sources = sourceLists->data[1]; // Sources to correct + psArray *sources = sourceLists->data[0]; // Sources to correct for (int i = 0; i < sources->n; i++) { pmSource *source = sources->data[i]; // Source of interest @@ -283,10 +283,10 @@ } psArray *sources = sourceLists->data[i]; // Sources of interest - float magCorr = zp->data.F32[i] + trans->data.F32[i] - 2.5*log10(options->sumExposure); + float magCorr = zp->data.F32[i] - trans->data.F32[i] - 2.5*log10(options->sumExposure); if (zpExpNum == numGoodImages) { // Using measured zero points, so attempt to set target zero point magCorr -= zpTarget; } - options->norm->data.F32[i] = -magCorr; + options->norm->data.F32[i] = magCorr; psLogMsg("ppStack", PS_LOG_INFO, "Applying scale correction to image %d: %f mag (%f)\n", Index: /branches/czw_branch/20100519/ppSub/src/ppSubMakePSF.c =================================================================== --- /branches/czw_branch/20100519/ppSub/src/ppSubMakePSF.c (revision 28163) +++ /branches/czw_branch/20100519/ppSub/src/ppSubMakePSF.c (revision 28164) @@ -136,5 +136,5 @@ // Copy the PSF-related data - psMetadataIterator *iter = psMetadataIteratorAlloc(inputRO->analysis, PS_LIST_HEAD, "^PSF\\.CLUMP.*"); + psMetadataIterator *iter = psMetadataIteratorAlloc(inputRO->analysis, PS_LIST_HEAD, "^PSF.*"); psMetadataItem *item; // Item from iteration while ((item = psMetadataGetAndIncrement(iter))) { Index: /branches/czw_branch/20100519/psModules/src/detrend/pmMaskStats.c =================================================================== --- /branches/czw_branch/20100519/psModules/src/detrend/pmMaskStats.c (revision 28163) +++ /branches/czw_branch/20100519/psModules/src/detrend/pmMaskStats.c (revision 28164) @@ -8,4 +8,5 @@ #include #include + #include "pmHDU.h" #include "pmFPA.h" @@ -16,11 +17,11 @@ -# define ESCAPE { \ - psError(PS_ERR_UNKNOWN, false, "I/O failure in pmMaskStats"); \ - psFree (view); \ - return false; \ - } +#define ESCAPE { \ + psError(psErrorCodeLast(), false, "I/O failure in pmMaskStats"); \ + psFree (view); \ + return false; \ + } - + @@ -30,9 +31,9 @@ bool status; - - psU16 staticMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.STATIC"); - psU16 magicMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.MAGIC"); - psU16 dynamicMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.DYNAMIC"); - psU16 advisoryMaskVal = psMetadataLookupU32(&status, config->recipes, "MASKSTAT.ADVISORY"); + + psImageMaskType staticMaskVal = psMetadataLookupImageMask(&status, config->recipes, "MASKSTAT.STATIC"); + psImageMaskType magicMaskVal = psMetadataLookupImageMask(&status, config->recipes, "MASKSTAT.MAGIC"); + psImageMaskType dynamicMaskVal = psMetadataLookupImageMask(&status, config->recipes, "MASKSTAT.DYNAMIC"); + psImageMaskType advisoryMaskVal = psMetadataLookupImageMask(&status, config->recipes, "MASKSTAT.ADVISORY"); psS32 Npix_valid = 0; @@ -54,24 +55,24 @@ if (!cell->process || !cell->file_exists) {continue; } while ((readout = pmFPAviewNextReadout(view, fpa, 1)) != NULL) { - if (!readout->data_exists) {continue; } + if (!readout->data_exists) {continue; } - psImage *mask = readout->mask; - if (!pmSingleImageMaskStats(mask,&Npix_valid,&Npix_static,&Npix_magic, - &Npix_dynamic,&Npix_advisory, - staticMaskVal,magicMaskVal, - dynamicMaskVal,advisoryMaskVal)) { - psError(PS_ERR_UNKNOWN, false, "Unable to calculate masks for readout."); - return(false); - } - psMetadataAddS32(readout->analysis, PS_LIST_TAIL,"MASKFRAC_NPIX", 0, - "Number of valid pixels", Npix_valid); - psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_STATIC", 0, - "Fraction of pixels statically masked", (float) Npix_static / Npix_valid); - psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_DYNAMIC", 0, - "Fraction of pixels dynamically masked", (float) Npix_dynamic / Npix_valid); - psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_MAGIC", 0, - "Fraction of pixels magically masked", (float) Npix_magic / Npix_valid); - psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_ADVISORY", 0, - "Fraction of pixels masked as an advisory", (float) Npix_advisory / Npix_valid); + psImage *mask = readout->mask; + if (!pmSingleImageMaskStats(mask,&Npix_valid,&Npix_static,&Npix_magic, + &Npix_dynamic,&Npix_advisory, + staticMaskVal,magicMaskVal, + dynamicMaskVal,advisoryMaskVal)) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate masks for readout."); + return(false); + } + psMetadataAddS32(readout->analysis, PS_LIST_TAIL,"MASKFRAC_NPIX", 0, + "Number of valid pixels", Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_STATIC", 0, + "Fraction of pixels statically masked", (float) Npix_static / Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_DYNAMIC", 0, + "Fraction of pixels dynamically masked", (float) Npix_dynamic / Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_MAGIC", 0, + "Fraction of pixels magically masked", (float) Npix_magic / Npix_valid); + psMetadataAddF32(readout->analysis,PS_LIST_TAIL, "MASKFRAC_ADVISORY", 0, + "Fraction of pixels masked as an advisory", (float) Npix_advisory / Npix_valid); } } @@ -83,8 +84,8 @@ bool pmSingleImageMaskStats(psImage *mask, - psS32 *Npix_valid, psS32 *Npix_static, psS32 *Npix_magic, - psS32 *Npix_dynamic, psS32 *Npix_advisory, - psU16 staticMaskVal, psU16 magicMaskVal, - psU16 dynamicMaskVal, psU16 advisoryMaskVal) { + psS32 *Npix_valid, psS32 *Npix_static, psS32 *Npix_magic, + psS32 *Npix_dynamic, psS32 *Npix_advisory, + psImageMaskType staticMaskVal, psImageMaskType magicMaskVal, + psImageMaskType dynamicMaskVal, psImageMaskType advisoryMaskVal) { PS_ASSERT_IMAGE_NON_NULL(mask, false); *Npix_valid = 0; @@ -95,23 +96,22 @@ psImageMaskType **maskData = mask->data.PS_TYPE_IMAGE_MASK_DATA; - psWarning("HERE: %ld %d %d\n",(long) maskData, mask->numRows,mask->numCols); - for (int i = 0; i < mask->numRows - 1; i++) { - for (int j = 0; j < mask->numCols - 1; j++) { + for (int i = 0; i < mask->numRows; i++) { + for (int j = 0; j < mask->numCols; j++) { *Npix_valid += 1; if (maskData[i][j] & staticMaskVal) { - *Npix_static += 1; - continue; + *Npix_static += 1; + continue; } if (maskData[i][j] & dynamicMaskVal) { - *Npix_dynamic += 1; - continue; + *Npix_dynamic += 1; + continue; } if (maskData[i][j] & magicMaskVal) { - *Npix_magic += 1; - continue; + *Npix_magic += 1; + continue; } if (maskData[i][j] & advisoryMaskVal) { - *Npix_advisory += 1; - continue; + *Npix_advisory += 1; + continue; } } Index: /branches/czw_branch/20100519/psModules/src/detrend/pmMaskStats.h =================================================================== --- /branches/czw_branch/20100519/psModules/src/detrend/pmMaskStats.h (revision 28163) +++ /branches/czw_branch/20100519/psModules/src/detrend/pmMaskStats.h (revision 28164) @@ -4,8 +4,8 @@ bool pmFPAMaskStats(pmFPA *fpa, pmConfig *config); bool pmSingleImageMaskStats(psImage *mask, - psS32 *Npix_valid, psS32 *Npix_static, psS32 *Npix_magic, - psS32 *Npix_dynamic, psS32 *Npix_advisory, - psU16 staticMaskVal, psU16 magicMaskVal, - psU16 dynamicMaskVal, psU16 advisoryMaskVal); + psS32 *Npix_valid, psS32 *Npix_static, psS32 *Npix_magic, + psS32 *Npix_dynamic, psS32 *Npix_advisory, + psImageMaskType staticMaskVal, psImageMaskType magicMaskVal, + psImageMaskType dynamicMaskVal, psImageMaskType advisoryMaskVal); Index: /branches/czw_branch/20100519/psastro/src/psastroMaskUpdates.c =================================================================== --- /branches/czw_branch/20100519/psastro/src/psastroMaskUpdates.c (revision 28163) +++ /branches/czw_branch/20100519/psastro/src/psastroMaskUpdates.c (revision 28164) @@ -30,5 +30,11 @@ int colEnd(psU16 corner_list,int i,int x_0,int y_0,int R,int max) { if ((corner_list == 0x04)||(corner_list == 0x05)) { - return((int) fabs(x_0 - sqrt(pow(R,2) - pow(y_0 - i,2)))); + int v = (int) fabs(x_0 - sqrt(pow(R,2) - pow(y_0 - i,2))); + if (v > max) { + return(max); + } + else { + return(v); + } } else { @@ -54,5 +60,11 @@ (corner_list == 0x03)||(corner_list == 0x0b)|| (corner_list == 0x07)) { - return((int) fabs(y_0 - sqrt(pow(R,2) - pow(x_0 - j,2)))); + int v = (int) fabs(y_0 - sqrt(pow(R,2) - pow(x_0 - j,2))); + if (v > max) { + return(max); + } + else { + return(v); + } } else { @@ -62,5 +74,7 @@ } -/** +/* #define MASK_DEBUG 1 */ + +/* * create a mask or mask regions based on the collection of reference stars that * are in the vicinity of each chip */ @@ -112,7 +126,6 @@ return false; } - psLogMsg ("psastro", PS_LOG_INFO, "generating a bright-star mask"); - + bool REFSTAR_MASK_BLEED = psMetadataLookupBool (&status, recipe, "REFSTAR_MASK_BLEED"); @@ -137,9 +150,15 @@ psU16 advisoryMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.ADVISORY"); - psS32 Npix_valid = 0; - psS32 Npix_static = 0; - psS32 Npix_magic = 0; - psS32 Npix_dynamic = 0; - psS32 Npix_advisory = 0; + psS32 Npix_ref_valid = 0; + psS32 Npix_ref_static = 0; + psS32 Npix_ref_magic = 0; + psS32 Npix_ref_dynamic = 0; + psS32 Npix_ref_advisory = 0; + + psS32 Npix_max_valid = 0; + psS32 Npix_max_static = 0; + psS32 Npix_max_magic = 0; + psS32 Npix_max_dynamic = 0; + psS32 Npix_max_advisory = 0; psU16 corner_list = 0x00; @@ -177,10 +196,8 @@ // Get camera specific mask stat options - psF32 FOV = psMetadataLookupF32(&status, fpaMask->camera, "FOV"); // Almost certainly doesn't exist. Placeholder for now. - psS32 CONSTANT_BLANK = psMetadataLookupS32(&status, fpaMask->camera, "NPIX_INTERCHIP"); // Need to add this as well. - - Npix_valid += CONSTANT_BLANK; - Npix_static += CONSTANT_BLANK; - + psF32 FOV_REF = psMetadataLookupF32(&status, fpaMask->camera, "FOV_REF"); + psF32 FOV_MAX = psMetadataLookupF32(&status, fpaMask->camera, "FOV_MAX"); + psS32 NPIX_REF = psMetadataLookupS32(&status, fpaMask->camera, "NPIX_REF"); + psS32 NPIX_MAX = psMetadataLookupS32(&status, fpaMask->camera, "NPIX_MAX"); // select the reference mask fpa :: we use this to determine cell boundaries @@ -209,5 +226,8 @@ // open/load files as needed if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE; - +#ifdef MASK_DEBUG + psImage *masktest = psImageAlloc(10500,10500,PS_TYPE_U16); + psImageMaskType **maskIData = masktest->data.PS_TYPE_IMAGE_MASK_DATA; +#endif // this loop selects the matched stars for all chips while ((chip = pmFPAviewNextChip (view, fpa, 1)) != NULL) { @@ -435,15 +455,10 @@ while ((cell = pmFPAviewNextCell(view, fpa, 1)) != NULL) { if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE; - psWarning("In CELL: FOV: %f CONSTANT: %d Mask bits: %d %d %d %d\n", - FOV,CONSTANT_BLANK,staticMaskVal,magicMaskVal,dynamicMaskVal,advisoryMaskVal); - if (!cell->process || !cell->file_exists) {continue; } while ((readout = pmFPAviewNextReadout(view, fpa, 1)) != NULL) { pmReadout *readoutMask = pmFPAviewThisReadout (view, outMask->fpa); if (!pmFPAfileIOChecks (config, viewMask, PM_FPA_BEFORE)) ESCAPE; - psWarning("In READOUT: FOV: %f CONSTANT: %d Mask bits: %d %d %d %d\n", - FOV,CONSTANT_BLANK,staticMaskVal,magicMaskVal,dynamicMaskVal,advisoryMaskVal); - if (!readoutMask->data_exists) {continue; } + psPlane coordFPA; psPlane coordCell; @@ -453,84 +468,119 @@ psImageMaskType **maskData = mask->data.PS_TYPE_IMAGE_MASK_DATA; // Dance coordinates around - // Calculate which corners fall within the field of view. + // Calculate which corners fall within the field of view. If this chip is fully contained, we can + // do a simple scan instead of checking it falls within the FOV. // 0x04 0x08 // 0x01 0x02 + corner_list = 0; - coordFPA.x = 0.0; - coordFPA.y = 0.0; - psPlaneTransformApply(&coordCell,chip->fromFPA,&coordFPA); - if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV,2)) { + coordCell.x = 0.0; + coordCell.y = 0.0; + psPlaneTransformApply(&coordFPA,chip->toFPA,&coordCell); + if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV_REF,2)) { corner_list = corner_list | 0x01; } - - coordCell.x = mask->numCols - 1; - psPlaneTransformApply(&coordCell,chip->fromFPA,&coordFPA); - if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV,2)) { + coordCell.x = (1.0 * mask->numCols - 1.0); + psPlaneTransformApply(&coordFPA,chip->toFPA,&coordCell); + if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV_REF,2)) { corner_list = corner_list | 0x02; } - - coordCell.y = mask->numRows - 1; - psPlaneTransformApply(&coordCell,chip->fromFPA,&coordFPA); - if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV,2)) { + coordCell.y = 1.0 * (mask->numRows - 1); + psPlaneTransformApply(&coordFPA,chip->toFPA,&coordCell); + if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV_REF,2)) { corner_list = corner_list | 0x08; } - - coordCell.x = 0; - psPlaneTransformApply(&coordCell,chip->fromFPA,&coordFPA); - if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV,2)) { + coordCell.x = 0.0; + psPlaneTransformApply(&coordFPA,chip->toFPA,&coordCell); + if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV_REF,2)) { corner_list = corner_list | 0x04; } - - int x_0 = coordFPA.x; - int y_0 = coordFPA.y; - - + // Scan over the valid regions of the image and count masked pixels - if ((corner_list == 0x04)||(corner_list == 0x08)|| - (corner_list == 0x05)||(corner_list == 0x0a)) { - for (int i = 0; i < mask->numRows - 1; i++) { - for (int j = colStart(corner_list,i,x_0,y_0,FOV,mask->numCols - 1); - j < colEnd(corner_list,i,x_0,y_0,FOV,mask->numCols - 1); j++) { - Npix_valid++; - if (maskData[i][j] & staticMaskVal) { - Npix_static++; - continue; - } - if (maskData[i][j] & dynamicMaskVal) { - Npix_dynamic++; - continue; - } - if (maskData[i][j] & magicMaskVal) { - Npix_magic++; - continue; - } - if (maskData[i][j] & advisoryMaskVal) { - Npix_advisory++; - continue; - } + for (int i = 0; i < mask->numRows - 1; i++) { + for (int j = 0; j < mask->numCols - 1; j++) { + coordCell.x = j; + coordCell.y = i; + coordFPA.x = 0.0; + coordFPA.y = 0.0; + int region = 0; + + if (corner_list == 0x0f) { + Npix_ref_valid++; + Npix_max_valid++; + region = 1; +/* #ifdef MASK_DEBUG */ +/* psPlaneTransformApply(&coordFPA,chip->toFPA,&coordCell); */ +/* maskIData[(int) (coordFPA.y/4 + 5250)][(int) (coordFPA.x/4 + 5250)] = */ +/* maskIData[(int) (coordFPA.y/4 + 5250)][(int) (coordFPA.x/4 + 5250)] + 0x01; */ +/* #endif */ } - } - } - else { - for (int j = 0; j < mask->numCols - 1; j++) { - for (int i = rowStart(corner_list,j,x_0,y_0,FOV,mask->numRows - 1); - i < rowEnd(corner_list,j,x_0,y_0,FOV,mask->numRows - 1); i++) { - Npix_valid++; - if (maskData[i][j] & staticMaskVal) { - Npix_static++; - continue; - } - if (maskData[i][j] & dynamicMaskVal) { - Npix_dynamic++; - continue; - } - if (maskData[i][j] & magicMaskVal) { - Npix_magic++; - continue; - } - if (maskData[i][j] & advisoryMaskVal) { - Npix_advisory++; - continue; - } + if (!region) { + psPlaneTransformApply(&coordFPA,chip->toFPA,&coordCell); + if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV_REF,2)) { + Npix_ref_valid++; + Npix_max_valid++; + region = 1; +/* #ifdef MASK_DEBUG */ +/* maskIData[(int) (coordFPA.y/4 + 5250)][(int) (coordFPA.x/4 + 5250)] = */ +/* maskIData[(int) (coordFPA.y/4 + 5250)][(int) (coordFPA.x/4 + 5250)] + 0x01; */ +/* #endif */ + } + else if (pow(coordFPA.x,2) + pow(coordFPA.y,2) <= pow(FOV_MAX,2)) { + Npix_max_valid++; + region = 2; +/* #ifdef MASK_DEBUG */ +/* maskIData[(int) (coordFPA.y/4 + 5250)][(int) (coordFPA.x/4 + 5250)] = */ +/* maskIData[(int) (coordFPA.y/4 + 5250)][(int) (coordFPA.x/4 + 5250)] + 0x01; */ +/* #endif */ + } + } + if (!region) { + continue; + } + + if (maskData[i][j] & staticMaskVal) { +#ifdef MASK_DEBUG + psPlaneTransformApply(&coordFPA,chip->toFPA,&coordCell); + maskIData[(int) (coordFPA.y/4 + 5250)][(int) (coordFPA.x/4 + 5250)] = + maskIData[(int) (coordFPA.y/4 + 5250)][(int) (coordFPA.x/4 + 5250)] + 1; +#endif + if (region == 1) { + Npix_ref_static++; + Npix_max_static++; + } + if (region == 2) { + Npix_max_static++; + } + continue; + } + if (maskData[i][j] & dynamicMaskVal) { + if (region == 1) { + Npix_ref_dynamic++; + Npix_max_dynamic++; + } + if (region == 2) { + Npix_max_dynamic++; + } + continue; + } + if (maskData[i][j] & magicMaskVal) { + if (region == 1) { + Npix_ref_magic++; + Npix_max_magic++; + } + if (region == 2) { + Npix_max_magic++; + } + continue; + } + if (maskData[i][j] & advisoryMaskVal) { + if (region == 1) { + Npix_ref_advisory++; + Npix_max_advisory++; + } + if (region == 2) { + Npix_max_advisory++; + } + continue; } } @@ -555,4 +605,11 @@ if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE; +#ifdef MASK_DEBUG + psFits *maskFits = psFitsOpen("/data/ipp007.0/watersc1/mask.test.fits","w"); + psFitsWriteImage(maskFits,NULL,masktest,1,"mask"); + + psFree(maskFits); + psFree(masktest); +#endif if (COUNT_GHOSTS) { // save nGhosts to update header. @@ -566,14 +623,27 @@ } - psMetadataAddS32(stats,PS_LIST_TAIL, "MASKFRAC_NPIX", 0, - "Number of valid pixels", Npix_valid); - psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_STATIC", 0, - "Fraction of pixels statically masked", (float) Npix_static / Npix_valid); - psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_DYNAMIC", 0, - "Fraction of pixels dynamically masked", (float) Npix_dynamic / Npix_valid); - psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_MAGIC", 0, - "Fraction of pixels magically masked", (float) Npix_magic / Npix_valid); - psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_ADVISORY", 0, - "Fraction of pixels masked as an advisory", (float) Npix_advisory / Npix_valid); + Npix_ref_static += (NPIX_REF - Npix_ref_valid); + Npix_max_static += (NPIX_MAX - Npix_max_valid); + psMetadataAddS32(stats,PS_LIST_TAIL, "MASKFRAC_REF_NPIX", 0, + "Number of valid pixels", Npix_ref_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_REF_STATIC", 0, + "Fraction of pixels statically masked", (float) Npix_ref_static / NPIX_REF); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_REF_DYNAMIC", 0, + "Fraction of pixels dynamically masked", (float) Npix_ref_dynamic / NPIX_REF); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_REF_MAGIC", 0, + "Fraction of pixels magically masked", (float) Npix_ref_magic / NPIX_REF); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_REF_ADVISORY", 0, + "Fraction of pixels masked as an advisory", (float) Npix_ref_advisory / NPIX_REF); + + psMetadataAddS32(stats,PS_LIST_TAIL, "MASKFRAC_MAX_NPIX", 0, + "Number of valid pixels", Npix_max_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_MAX_STATIC", 0, + "Fraction of pixels statically masked", (float) Npix_max_static / NPIX_MAX); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_MAX_DYNAMIC", 0, + "Fraction of pixels dynamically masked", (float) Npix_max_dynamic / NPIX_MAX); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_MAX_MAGIC", 0, + "Fraction of pixels magically masked", (float) Npix_max_magic / NPIX_MAX); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_MAX_ADVISORY", 0, + "Fraction of pixels masked as an advisory", (float) Npix_max_advisory / NPIX_MAX); // deactivate all files Index: /branches/czw_branch/20100519/psconfig/psbuild =================================================================== --- /branches/czw_branch/20100519/psconfig/psbuild (revision 28163) +++ /branches/czw_branch/20100519/psconfig/psbuild (revision 28164) @@ -10,4 +10,6 @@ $profile = 0; $developer = 0; +$magic = 0; +$operations = 0; $start = ""; $stop = ""; @@ -59,4 +61,8 @@ shift; next; } + if ($ARGV[0] eq "-magic") { + $magic = 1; + shift; next; + } if ($ARGV[0] eq "-optimize") { $optimize = 1; @@ -69,4 +75,11 @@ if ($ARGV[0] eq "-dev") { $developer = 1; + shift; next; + } + # basic settings for the operations installation + if ($ARGV[0] eq "-ops") { + $developer = 1; + $optimize = 1; + $magic = 1; shift; next; } @@ -167,4 +180,18 @@ sub build_distribution { + + # the operations system needs to first update the magic software + if ($magic) { + $homedir = `pwd`; chomp $homedir; + chdir "../magic"; + + if (! -d magic || ! -d ssa-core-cpp || ! -d qt-everywhere-opensource-src-4.6.1) { + $status = vsystem ("make update"); + if ($status) { die "failed to untar magic directories"; } + } else { + print "magic directories exist; run make update manually if needed\n"; + } + chdir $homedir; + } # use psconfig.csh to set needed build aliases Index: /branches/czw_branch/20100519/psphot/src/psphotMagnitudes.c =================================================================== --- /branches/czw_branch/20100519/psphot/src/psphotMagnitudes.c (revision 28163) +++ /branches/czw_branch/20100519/psphot/src/psphotMagnitudes.c (revision 28164) @@ -87,5 +87,5 @@ bool IGNORE_GROWTH = psMetadataLookupBool (&status, recipe, "IGNORE_GROWTH"); bool INTERPOLATE_AP = psMetadataLookupBool (&status, recipe, "INTERPOLATE_AP"); - bool DIFF_STATS = psMetadataLookupBool (&status, recipe, "INTERPOLATE_AP"); + bool DIFF_STATS = psMetadataLookupBool (&status, recipe, "DIFF_STATS"); pmSourcePhotometryMode photMode = PM_SOURCE_PHOT_APCORR | PM_SOURCE_PHOT_WEIGHT; Index: /branches/czw_branch/20100519/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /branches/czw_branch/20100519/pstamp/scripts/pstamp_checkdependent.pl (revision 28163) +++ /branches/czw_branch/20100519/pstamp/scripts/pstamp_checkdependent.pl (revision 28164) @@ -80,4 +80,10 @@ my $whole_run = ($component eq 'all'); +# XXX: whole_run was a concept that isn't practical. Having one dependent for +# a whole run makes finding errors too hard. We always have a dependent for each +# component. +# XXX: remove the unneeded code +my_die("component = 'all' not supported", $PS_EXIT_PROG_ERROR) if $whole_run;; + if ($whole_run) { if ($stage eq "chip") { @@ -135,30 +141,27 @@ print "skipping $command\n"; } -} elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update') and - ($whole_run or ($it->{data_state} eq 'cleaned'))) { - - # Update has not been queued for this component yet. - - # XXX: if $whole_run this is going to repeatedly call -setimfiletoupdate. This will be a no-op - # but is wasteful. - # Maybe we should deal with having three states for pstampDependent.state. 'new', 'run', 'full' - # need to queue the updates +} elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update')) { + # For warp and diff stages we need to call the 'queue_update' subroutines even if the + # data_state is update in order to check the earlier stages in the pipeline + # For example if warpSkyfile is in update state but the chip run that it depends on hasn't + # been updated we need to go and queue it. + + if ($it->{fault}) { + my_die("Component faulted on update dep_id: $dep_id", + $PS_EXIT_SYS_ERROR); + } + if ($stage eq 'chip') { - # queue_update_chip takes an array so that queue_update_warp can pass it set of chips + # check_states_chip takes an array so that check_states_warp can pass it set of chips my $chips = [$it]; - $status = queue_update_chip($it->{chip_id}, $whole_run, $chips, $rlabel, $need_magic); + $status = check_states_chip($it->{chip_id}, $whole_run, $chips, $rlabel, $need_magic); } elsif ($stage eq 'warp') { - $status = queue_update_warp($it, $whole_run, $rlabel, $need_magic); + $status = check_states_warp($it, $whole_run, $rlabel, $need_magic); } elsif ($stage eq 'diff') { - $status = queue_update_diff($it, $whole_run, $rlabel, $need_magic); + $status = check_states_diff($it, $whole_run, $rlabel, $need_magic); } else { my_die("Unexpected stage found $stage", $PS_EXIT_PROG_ERROR); } } else { - # update for this component has been queued by hasn't completed yet or the state is an error state - # XXX: in this state we aren't checking the dependents anymore. If something goes wrong this - # will never complete. Consider calling the 'queue_update' functions again just to check. They are relatively cheap - - print "${stage}Run $stage_id state is $it->{state} $component data_state is $it->{data_state}\n"; @@ -170,13 +173,16 @@ my $job_fault = 0; - # temporary hack to deal with data with old burntool tables. This check is now done by pstampparse.pl - # so runs in this state should not get queued +if (0) { if ($stage eq 'chip') { + # XXX: There is no need to check this anymore. All magicked chipRuns have abs(burntool_state) >= 13 + # If something changes that causes an error, we will figure that out from the chip failure my $burntool_state = $it->{burntool_state}; - if ($burntool_state and (abs($burntool_state) < 14)) { + # XXX: get the value of 13 from the ppImage recipe + if ($burntool_state and (abs($burntool_state) < 13)) { print STDERR "chip $it->{chip_id} $it->{class_id} has burntool_state $burntool_state. Not avaiable.\n"; $job_fault = $PSTAMP_NOT_AVAILABLE; } } +} if ($state eq 'error_cleaned') { $job_fault = $PSTAMP_NOT_AVAILABLE; @@ -191,5 +197,5 @@ if (!$job_fault and ($stage eq 'chip')) { # chip processing is done, start destreaking. - $job_fault = queue_update_magicDSRun($stage, $stage_id, $rlabel, $need_magic, $it->{raw_magicked}, $it->{dsRun_state}); + $job_fault = check_states_magicDSRun($stage, $stage_id, $rlabel, $need_magic, $it->{raw_magicked}, $it->{dsRun_state}); } if ($job_fault) { @@ -201,5 +207,5 @@ -sub queue_update_chip { +sub check_states_chip { my $chip_id = shift; my $whole_run = shift; # if true queue entire run for update @@ -216,6 +222,8 @@ $raw_all_magicked &= ($chip->{raw_magicked} > 0); - if ($chip->{state} =~ /error/) { + if (($chip->{state} =~ /error/) or ($chip->{state} =~ /purged/) or ($chip->{state} =~ /scrubbed/)) { + print "chipRun state is $chip->{chip_id} is in state $chip->{state} cannot update\n"; faultJobs('stop', undef, undef, $PSTAMP_GONE); + return; } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) { my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id -class_id $chip->{class_id}"; @@ -237,4 +245,10 @@ $dsRun_state = $run->{dsRun_state}; $raw_all_magicked = ($run->{raw_magicked} > 0); + my $state = $run->{state}; + if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/)) { + print "chipRun state is $run->{chip_id} is in state $state cannot update\n"; + faultJobs('stop', undef, undef, $PSTAMP_GONE); + return; + } # providing no -class_id arguments changes all imfiles with data_state = 'cleaned' to 'update' @@ -253,13 +267,13 @@ } - my $status = queue_update_magicDSRun('chip', $chip_id, $rlabel, $need_magic, $raw_all_magicked, $dsRun_state); + my $status = check_states_magicDSRun('chip', $chip_id, $rlabel, $need_magic, $raw_all_magicked, $dsRun_state); return $status; } -sub queue_update_warp { +sub check_states_warp { # check status of input chips. If state is not updatable set error code for job - # if chipProcessedImfile.state is cleaned call queue_update_chip + # if chipProcessedImfile.state is cleaned call check_states_chip # need to code warptool -setskyfiletoupdate @@ -301,5 +315,5 @@ } - if ($chips_ready) { + if ($chips_ready and $skycell->{data_state} ne 'update') { # the reason we defer setting the warp to update is so that we can handle error conditions at previous # stages more easily. @@ -317,5 +331,5 @@ } } elsif (scalar @chipsToUpdate > 0) { - return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic); + return check_states_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic); } } else { @@ -354,5 +368,5 @@ (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) { my $data = [$chipRun]; - return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic); + return check_states_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic); } } @@ -361,5 +375,5 @@ return 0; } -sub queue_update_diff { +sub check_states_diff { my $metadata = shift; my $whole_run = shift; # if true queue entire run for update @@ -392,5 +406,5 @@ if ($warp->{data_state} ne 'full') { - return queue_update_warp($warp, 0, $rlabel, $need_magic); + return check_states_warp($warp, 0, $rlabel, $need_magic); } # warps are ready fall through and queue the diff update @@ -404,5 +418,5 @@ if ($warp1->{data_state} ne 'full') { $warps_ready = 0; - $status = queue_update_warp($warp1, 0, $rlabel, $need_magic); + $status = check_states_warp($warp1, 0, $rlabel, $need_magic); if ($status) { return $status; @@ -415,5 +429,5 @@ if ($warp2->{data_state} ne 'full') { $warps_ready = 0; - $status = queue_update_warp($warp2, 0, $rlabel, $need_magic); + $status = check_states_warp($warp2, 0, $rlabel, $need_magic); } @@ -467,5 +481,5 @@ if ($warp->{data_state} ne 'full') { - return queue_update_warp($warp, 0, $rlabel, $need_magic); + return check_states_warp($warp, 0, $rlabel, $need_magic); } # warps are ready fall through and queue the diff update @@ -474,15 +488,17 @@ } - my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; + if ($skycell->{data_state} ne 'update') { + my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } } } else { @@ -599,5 +615,5 @@ } -sub queue_update_magicDSRun { +sub check_states_magicDSRun { my $stage = shift; my $stage_id = shift; @@ -608,7 +624,7 @@ # XXX: this code assumes that destreaking is handled at the chip stage - my_die ("queue_update_magicDSRun only works for stage chip", $PS_EXIT_PROG_ERROR) if $stage ne 'chip'; - - # if called from queue_update_warp dsRun_state is unknown. Go find it. + my_die ("check_states_magicDSRun only works for stage chip", $PS_EXIT_PROG_ERROR) if $stage ne 'chip'; + + # if called from check_states_warp dsRun_state is unknown. Go find it. if (!$dsRun_state) { my $command = "$chiptool -listrun -chip_id $stage_id"; Index: /branches/czw_branch/20100519/pstamp/scripts/pstamp_finish.pl =================================================================== --- /branches/czw_branch/20100519/pstamp/scripts/pstamp_finish.pl (revision 28163) +++ /branches/czw_branch/20100519/pstamp/scripts/pstamp_finish.pl (revision 28164) @@ -176,4 +176,6 @@ my $exp_id = $job->{exp_id}; + my $error_string = get_error_string($fault); + if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) { # this request had a duplicate request name yet the parser didn't give @@ -220,5 +222,5 @@ # add line to the table definition file - print $tdf "$rownum|$fault|$img_name|$job_id|"; + print $tdf "$rownum|$fault|$error_string|$img_name|$job_id|"; # ra_deg and dec_deg are the coordinates of center of the stamp @@ -241,5 +243,5 @@ } else { print STDERR "no reglist file for job $job_id\n" if $verbose; - print $tdf "$rownum|$fault|0|$job_id|"; + print $tdf "$rownum|$fault|$error_string|0|$job_id|"; print $tdf "0|0|"; # center of (non-existent) stamp print $tdf "$exp_info|"; Index: /branches/czw_branch/20100519/pstamp/scripts/pstamp_request_file =================================================================== --- /branches/czw_branch/20100519/pstamp/scripts/pstamp_request_file (revision 28163) +++ /branches/czw_branch/20100519/pstamp/scripts/pstamp_request_file (revision 28164) @@ -29,7 +29,11 @@ printhelp($0) if $help; + pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --input", - -exitval => 3) unless defined $input; + +unless (defined $input) { + print STDERR "Required options: --input\n"; + printhelp($0); +} # The header kewords Index: /branches/czw_branch/20100519/pstamp/scripts/pstamp_results_file.pl =================================================================== --- /branches/czw_branch/20100519/pstamp/scripts/pstamp_results_file.pl (revision 28163) +++ /branches/czw_branch/20100519/pstamp/scripts/pstamp_results_file.pl (revision 28164) @@ -54,4 +54,6 @@ # error code from processing this image { name => 'ERROR_CODE',type => 'V', writetype => TULONG }, + # error string correspoding to ERROR_CODE + { name => 'ERROR_STR',type => '24A', writetype => TSTRING }, # name of the postage stamp image { name => 'IMG_NAME', type => '64A', writetype => TSTRING }, Index: /branches/czw_branch/20100519/pstamp/scripts/pstampparse.pl =================================================================== --- /branches/czw_branch/20100519/pstamp/scripts/pstampparse.pl (revision 28163) +++ /branches/czw_branch/20100519/pstamp/scripts/pstampparse.pl (revision 28164) @@ -811,5 +811,5 @@ # chipRun's can be in full state if destreaking is necessary if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned') and - ($stage eq 'chip' and $state eq 'full')) { + ($stage ne 'chip' and $state eq 'full')) { my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR); } Index: /branches/czw_branch/20100519/pstamp/scripts/pstamprequest =================================================================== --- /branches/czw_branch/20100519/pstamp/scripts/pstamprequest (revision 28164) +++ /branches/czw_branch/20100519/pstamp/scripts/pstamprequest (revision 28164) @@ -0,0 +1,374 @@ +#!/bin/env perl +### +### pstamprequest +### +### Program to make a postage stamp request table for a set of coordinates +### + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions ) ; # :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Config qw( :standard ); +use PS::IPP::PStamp::RequestFile qw( :standard ); +use PS::IPP::PStamp::Job qw( :standard ); +use File::Temp qw(tempfile); +use File::Basename qw(basename); +use IPC::Cmd 0.36 qw( can_run run ); +use Carp; +use POSIX; + +my $verbose; +my $save_temps; + +# list file columns +# RA DEC FILTER MJD_MIN MJD_MAX + +my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base); + +my ($image, $mask, $variance, $cmf, $psf, $backmdl, $inverse); +my ($unconvolved, $use_imfile_id, $no_wait); + +my $default_size = 100; +my $job_type = 'stamp'; +my $req_type = 'bycoord'; +my $stage = 'chip'; +my $option_mask; +my $width = $default_size; +my $height = $default_size; +my $project = 'gpc1'; +my $coord_mask; +my $pixcenter; +my $arcseconds; + +my $id; +my $tess_id = 'null'; +my $component = 'null'; +my $data_group = 'null'; +my $filter; +my $mjd_min = 0; +my $mjd_max = 0; +my $comment; + +my $missing_tools; +my $pstamp_request_file = can_run('pstamp_request_file') or (warn "Can't find required program pstamp_request_file" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); +} + +GetOptions( + 'list=s' => \$list, # list of coordinates if undef ra and dec or x and y are required + 'ra=s' => \$ra, # + 'dec=s' => \$dec, + 'x=s' => \$x, + 'y=s' => \$y, + 'width=i' => \$width, + 'height=i' => \$height, + 'pixcenter' => \$pixcenter, + 'arcseconds' => \$arcseconds, + 'coord_mask=i' => \$coord_mask, + 'job_type=s' => \$job_type, + 'output=s' => \$output, + 'req_name=s' => \$req_name, + 'req_name_base=s' => \$req_name_base, + 'stage=s' => \$stage, + 'project=s' => \$project, + 'req_type=s' => \$req_type, + 'id=s' => \$id, + 'tess_id=s' => \$tess_id, + 'component=s' => \$component, + 'data_group=s' => \$data_group, + 'filter=s' => \$filter, + 'mjd_min=s' => \$mjd_min, + 'mjd_max=s' => \$mjd_max, + 'comment=s' => \$comment, + + 'option_mask=i' => \$option_mask, + 'image' => \$image, + 'mask' => \$mask, + 'variance' => \$variance, + 'cmf' => \$cmf, + 'psf' => \$psf, + 'backmdl' => \$backmdl, + 'inverse' => \$inverse, + 'unconvolved' => \$unconvolved, + 'use_imfile_id' => \$use_imfile_id, + 'do_not_wait' => \$no_wait, + + 'verbose' => \$verbose, + 'save-temps' => \$save_temps, +) or pod2usage(2); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( -msg => "Required options: --req_name | --req_name_base --stage ") + unless (defined $req_name or defined $req_name_base) + and defined $stage; + +pod2usage( -msg => "Invalid job_type: $job_type", -exitval =>1 ) + unless ($job_type eq 'stamp' or $job_type eq 'get_image'); + +if ($job_type eq 'stamp') { + if (defined $list) { + pod2usage( -msg => "--ra --dec --x --y are not used with --list", -exitval =>1 ) + if defined $x or defined $y; + } elsif (!$pixcenter) { + pod2usage( -msg => "Required options for stamp requests: --ra and --dec or --list", -exitval =>1 ) + unless (defined $ra and defined $dec); + # to simplify code we just use $x and $y from here + $x = $ra; + $y = $dec; + } else { + pod2usage( -msg => "Required options for stamp requests: --x and --y or --list", -exitval =>1 ) + unless (defined $x and defined $y); + } +} else { + pod2usage( -msg => "req_type must be byid or byexp for get_image requests", + -exitval =>1 ) unless $req_type eq 'byid' or $req_type eq 'byexp'; + $x = 0; + $y = 0; +} + +if ($req_type eq 'byid') { + pod2usage( -msg => "ID required for get_image requests", -exitval =>1 ) + if !defined $id; + die("ID must be number for byid requests") if !validID($id); +} elsif ($req_type eq 'byexp') { + pod2usage( -msg => "ID (exp_name) required for byexp requests", -exitval =>1 ) + if !defined $id; +} elsif ($req_type eq 'bydiff') { + pod2usage( -msg => "ID (exp_name) required for bydiff requests", -exitval =>1 ) + if !defined $id; +} elsif ($req_type eq 'byskycell') { + pod2usage( -msg => "tess_id and skycell_id required for byskycell requests", -exitval =>1 ) + if !(defined $tess_id and defined $component); +} elsif ($req_type ne 'bycoord') { + pod2usage( -msg => "$req_type is not a valid req_type", -exitval =>1 ); +} + +$id = 0 if !$id; + +unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack') { + die "$stage is not a valid value for stage\n"; +} + +checkFilter($filter, 'null', $filter) if $filter; +checkMJD($mjd_min, 0, "") if $mjd_min; +checkMJD($mjd_max, 0, "" ) if $mjd_min; + +# user supplied option mask takes precedence +if (!defined $option_mask) { + $option_mask = 0; + if ($job_type eq 'stamp') { + $option_mask |= $PSTAMP_SELECT_IMAGE if $image; + $option_mask |= $PSTAMP_SELECT_MASK if $mask; + $option_mask |= $PSTAMP_SELECT_VARIANCE if $variance; + $option_mask = $PSTAMP_SELECT_IMAGE if $option_mask == 0; + + # if no image was requested make a stamp of the image + + $option_mask |= $PSTAMP_SELECT_CMF if $cmf; + $option_mask |= $PSTAMP_SELECT_PSF if $psf; + $option_mask |= $PSTAMP_SELECT_BACKMDL if $backmdl; + $option_mask |= $PSTAMP_SELECT_UNCONV if $unconvolved; + $option_mask |= $PSTAMP_USE_IMFILE_ID if $use_imfile_id; + $option_mask |= $PSTAMP_SELECT_INVERSE if $inverse; + } + $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_wait; +} + + +# user supplied coord_mask takes precedence +if (!defined $coord_mask) { + $coord_mask = 0; + if ($pixcenter) { + $coord_mask |= $PSTAMP_CENTER_IN_PIXELS; + } + if (!$arcseconds) { + $coord_mask |= $PSTAMP_RANGE_IN_PIXELS; + } +} + + +# if req_name is not supplied use the current date and time to build one off of the base +if (!$req_name) { + my $datestr = strftime "%Y%m%dT%H%M%S", gmtime; + $req_name .= $req_name_base . $datestr; +} + +# ok ready to go + +my $rows; +if ($list) { + $rows = readList($list); +} else { + $rows = []; + push @$rows, buildRow("", $comment, $x, $y, $filter, $mjd_min, $mjd_max); +} + +my ($tdf, $table_def_name) = tempfile ("/tmp/tabledef.XXXX", UNLINK => !$save_temps); +print $tdf "$req_name 1\n"; +my $rownum = 0; +foreach my $row (@$rows) { + $rownum++; + my $line = "$rownum $row->{ra}\t$row->{dec}\t$width $height" + . " $coord_mask $job_type $option_mask $project $req_type" + . " $stage $id $tess_id $component $data_group" + . " $row->{filter} $row->{mjd_min} $row->{mjd_max}"; + + if ($row->{comment} and $row->{comment} ne '') { + $line .= " |$row->{comment}" ; + } else { + $line .= " |$comment" if $comment; + } + + print "$line\n" if $verbose; + print $tdf "$line\n"; +} +close $tdf; + +{ + my $command = "$pstamp_request_file --input $table_def_name --req_name $req_name"; + $command .= " --output $output" if $output; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + print STDERR @$stderr_buf; + exit $error_code >> 8; + } +} + +exit 0; + +sub readList { + my $file = shift; + open IN, "<$file" or die "failed to open $file for input\n"; + + my @rows; + + my $linenumber = 0; + foreach my $line () { + $linenumber++; + chomp $line; + next if !$line; + next if ($line =~ /^#/); + + my ($spec, $comment) = split /\|/, $line; + $comment = '' if !defined $comment; + + my @vals = split " ", $spec; + push @rows, buildRow("at line number $linenumber", $comment, @vals); + } + close IN; + + return \@rows; +} + +sub buildRow { + my $linenumber = shift; + my $comment = shift; + my @vals = @_; + + my $row = {}; + $row->{ra} = checkRA($vals[0], $linenumber); + $row->{dec} = checkDEC($vals[1], $linenumber); + $row->{filter} = checkFilter($vals[2], $filter, $linenumber); + $row->{mjd_min} = checkMJD($vals[3], $mjd_min, $linenumber);; + $row->{mjd_max} = checkMJD($vals[4], $mjd_max, $linenumber);; + $row->{comment} = $comment; + + return $row; +} + +sub validNumber { + my $val = shift; + + return 0 if !defined $val; + + return ($val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/); +} + +sub validID +{ + my $val = shift; + + return 0 if !$val; + + return ! ($val =~ /\D/); +} + +sub checkCoord { + my $c = shift; + my $linenumber = shift; + + my $result; + if ($c =~ /\:/) { + # sexagesmial format not valid for pixel coordinates + die "invalid coordinate value $c found $linenumber\n" if $pixcenter; + + my ($d, $m, $s) = split '\:', $c; + die "invalid coordinate value $c found $linenumber\n" + unless validNumber($d) and validNumber($m) and validNumber($s); + my $sign; + if ($c =~ /^-/) { # checking $d < 0 doesn't work for $d = -0 + $sign = -1; + $d = -$d; + } else { + $sign = 1; + } + $result = $sign * ($d + ($m + $s/60.) /60.); + } else { + die "$c is not valid coordinate value $linenumber\n" + unless validNumber($c); + $result = $c; + } + return $result; +} + +sub checkRA { + my $ra = shift; + my $linenumber = shift; + my $checked = checkCoord($ra, $linenumber); + + if ($ra =~ /\:/) { + return $checked * 360. / 24.; + } else { + return $checked; + } +} + +sub checkDEC { + return checkCoord(@_); +} + +sub checkFilter { + my $f = shift; + my $default = shift; + my $linenumber = shift; + + $default = 'null' if !defined $default; + + $f = $default if (!defined$f or $f eq 'null'); + + return $f; +} + +sub checkMJD { + my $mjd = shift; + my $default = shift; + my $linenumber = shift;; + + return $default if (!$mjd) ; + + die "invalid mjd value '$mjd' found $linenumber\n" if !validNumber($mjd); + + return $mjd; +} + +sub checkStage { + my $stage = shift; + +} Index: /branches/czw_branch/20100519/tools/errors.pl =================================================================== --- /branches/czw_branch/20100519/tools/errors.pl (revision 28163) +++ /branches/czw_branch/20100519/tools/errors.pl (revision 28164) @@ -106,4 +106,5 @@ my %variance; my %error_reading; +my %all_rejected; my %unknown; my %bad; @@ -205,4 +206,9 @@ if ($line =~ /Error reading .*, (\S+)/) { $error_reading{$name} = $1; + $found = 1; + last; + } + if ($line =~ /All images rejected/) { + $all_rejected{$name} = $log; $found = 1; last; @@ -228,4 +234,5 @@ print_failure(\%variance, "'Variance renormalisation' errors"); print_failure(\%error_reading, "'Error reading' errors"); +print_failure(\%all_rejected, "'All images rejected' errors"); print_failure(\%unknown, "Unknown errors"); print_failure(\%bad, "Bad logs");