Index: /trunk/dbconfig/changes.txt =================================================================== --- /trunk/dbconfig/changes.txt (revision 28374) +++ /trunk/dbconfig/changes.txt (revision 28375) @@ -1634,4 +1634,5 @@ ALTER TABLE diffSkyfile ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic; + ALTER TABLE chipRun CHANGE maskfrac_npix maskfrac_npix FLOAT; ALTER TABLE camRun CHANGE maskfrac_npix maskfrac_npix FLOAT; @@ -1730,5 +1731,4 @@ -- this adds the new tables - CREATE TABLE minidvodbRun ( minidvodb_id BIGINT AUTO_INCREMENT, @@ -1741,6 +1741,4 @@ PRIMARY KEY(minidvodb_id), KEY(minidvodb_name), INDEX(minidvodb_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; - - CREATE TABLE minidvodbProcessed ( @@ -1791,2 +1789,52 @@ FOREIGN KEY(diff_phot_id) REFERENCES diffPhotRun(diff_phot_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; + + +-- Version 1.1.69 warp/stack/diff summary and stack associations. + +CREATE TABLE warpSummary ( + warp_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(warp_id), + KEY(projection_cell), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE diffSummary ( + diff_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(diff_id), + KEY(projection_cell), + FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociation ( + sass_id BIGINT AUTO_INCREMENT, + data_group VARCHAR(64) NOT NULL, + projection_cell VARCHAR(64) NOT NULL, + tess_id VARCHAR(64) NOT NULL, + filter VARCHAR(64) NOT NULL, + PRIMARY KEY(sass_id), + KEY(data_group), + KEY(projection_cell), + KEY(tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociationMap ( + sass_id BIGINT, + stack_id BIGINT, + PRIMARY KEY(sass_id, stack_id), + FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackSummary ( + sass_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(sass_id), + KEY(projection_cell), + FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /trunk/dbconfig/diff.md =================================================================== --- /trunk/dbconfig/diff.md (revision 28374) +++ /trunk/dbconfig/diff.md (revision 28375) @@ -75,2 +75,8 @@ maskfrac_advisory F32 0.0 END + +diffSummary METADATA + diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) + projection_cell STR 32 # Primary Key + path_base STR 255 +END Index: /trunk/dbconfig/stack.md =================================================================== --- /trunk/dbconfig/stack.md (revision 28374) +++ /trunk/dbconfig/stack.md (revision 28375) @@ -53,2 +53,21 @@ quality S16 0 END + +stackSummary METADATA + sass_id S64 0 # Primary Key fkey(sass_id) ref stackAssociation(sass_id) + projection_cell STR 32 # Primary Key + path_base STR 255 +END + +stackAssociation METADATA + sass_id S64 0 # Primary Key AUTO_INCREMENT + data_group STR 64 # Key + projection_cell STR 64 # Key + tess_id STR 64 # Key + filter STR 64 +END + +stackAssociationMap METADATA + sass_id S64 0 # Primary Key fkey(sass_id) ref stackAssociation(sass_id) + stack_id S64 0 # Primary Key fkey(stack_id) ref stackRun(stack_id) +END Index: /trunk/dbconfig/warp.md =================================================================== --- /trunk/dbconfig/warp.md (revision 28374) +++ /trunk/dbconfig/warp.md (revision 28375) @@ -79,2 +79,8 @@ label STR 64 # Primary Key END + +warpSummary METADATA + warp_id S64 0 # Primary Key fkey(warp_id) ref warpRun(warp_id) + projection_cell STR 32 # Primary Key + path_base STR 255 +END Index: /trunk/ippScripts/Build.PL =================================================================== --- /trunk/ippScripts/Build.PL (revision 28374) +++ /trunk/ippScripts/Build.PL (revision 28375) @@ -109,4 +109,5 @@ scripts/bundle_detrends.pl scripts/ipp_cluster_load_monitor.pl + scripts/skycell_jpeg.pl scripts/diffphot.pl )], Index: /trunk/ippScripts/MANIFEST =================================================================== --- /trunk/ippScripts/MANIFEST (revision 28374) +++ /trunk/ippScripts/MANIFEST (revision 28375) @@ -42,3 +42,4 @@ scripts/bundle_detrends.pl scripts/ipp_cluster_load_monitor.pl +scripts/skycell_jpeg.pl t/00_distribution.t Index: /trunk/ippScripts/scripts/skycell_jpeg.pl =================================================================== --- /trunk/ippScripts/scripts/skycell_jpeg.pl (revision 28375) +++ /trunk/ippScripts/scripts/skycell_jpeg.pl (revision 28375) @@ -0,0 +1,360 @@ +#! /usr/bin/env perl + +use warnings; +use strict; +use Carp; +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use File::Temp qw( tempfile ); + +my $db; + +my ($stage, $stage_id, $outroot, $camera, $dbname, $logfile, $verbose, $save_temps); +my $missing_tools; +my ($help,$masks); +my ($tempFile,$tempName,$maskFile,$maskName); +my $ppSkycell = can_run('ppSkycell') or (warn "Can't find ppSkycell" and $missing_tools = 1); +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +GetOptions( + 'help|h' => \$help, + 'stage=s' => \$stage, + 'stage_id=s' => \$stage_id, + 'outroot=s' => \$outroot, + 'dbname=s' => \$dbname, + 'camera=s' => \$camera, + 'logfile=s' => \$logfile, + 'verbose' => \$verbose, + 'save_temps' => \$save_temps, + 'masks' => \$masks, +# 'no-op' => \$no_op, +# 'no-update' => \$no_update, + ) or pod2usage ( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV; +pod2usage( + -msg => "Required options: --stage --stage_id --path_base\nHelpful options: --camera --dbname", + -exitval => 3, + ) unless + defined $stage and defined $stage_id and defined $outroot; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up", $stage_id, $PS_EXIT_CONFIG_ERROR); # this is used for PATH, NEB filename conversions + +my %stages = ('warp' => 1, 'stack' => 1, 'diff' => 1); + +unless (exists($stages{$stage})) { + my_die("Unknown stage '$stage'",$stage_id,$PS_EXIT_UNKNOWN_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +if ($stage eq 'warp') { + my $imfiles; + my $command = "$warptool -warped -warp_id $stage_id"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform warptool: $error_code", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + # no results found; + } + + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata config doc", $stage_id, $error_code); + + my %tangents = (); + + foreach my $imfile (@$imfiles) { + my $skycell_id = $imfile->{skycell_id}; + my $path_base = $imfile->{path_base}; + my $quality = $imfile->{quality}; + my $state = $imfile->{data_state}; + my $projection_cell = $skycell_id; + if ($quality == 8007 or $state ne 'full') { + next; + } + + $projection_cell =~ s/^(.*)\..*$/$1/; + + unless (exists($tangents{$projection_cell})) { + # Make a temp file and fill, but be sure to save + ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{FILE} = $tempFile; + $tangents{$projection_cell}{NAME} = $tempName; + if ($masks) { + my ($maskFile, $maskName) = tempfile("/tmp/skycell.$projection_cell.masks.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{MFILE} = $maskFile; + $tangents{$projection_cell}{MNAME} = $maskName; + } + } + print "$skycell_id $projection_cell\n"; + my $file = $ipprc->filename("PSWARP.OUTPUT", $path_base, $skycell_id); + print "$file $state $quality\n"; + my $f_fh = $tangents{$projection_cell}{FILE}; + print $f_fh "$file\n"; + if ($masks) { + my $mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base, $skycell_id); + print "$mask\n"; + my $m_fh = $tangents{$projection_cell}{MFILE}; + print $m_fh "$mask\n"; + } + } + foreach my $projection_cell (keys %tangents) { + $command = "$ppSkycell -images $tangents{$projection_cell}{NAME}"; + if ($masks) { + $command .= " -masks $tangents{$projection_cell}{MNAME} "; + } + $command .= " ${outroot}.${projection_cell} "; + print "$command\n"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code); + } + # Update database: + $command = "$warptool -addsummary -warp_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update warp summary: $error_code", $stage_id, $error_code); + } + } + if (scalar (keys %tangents) == 0) { + my $projection_cell = 'NULL'; + $command = "$warptool -addsummary -warp_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update diff summary: $error_code", $stage_id, $error_code); + } + } +} #end warp stage +if ($stage eq 'diff') { + my $imfiles; + my $command = "$difftool -diffskyfile -diff_id $stage_id"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform difftool: $error_code", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + # no results found; + } + + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata config doc", $stage_id, $error_code); + + my %tangents = (); + + foreach my $imfile (@$imfiles) { + my $skycell_id = $imfile->{skycell_id}; + my $path_base = $imfile->{path_base}; + my $quality = $imfile->{quality}; + my $state = $imfile->{data_state}; + my $projection_cell = $skycell_id; + if ($quality == 8007 or $quality = 1010 or $state ne 'full') { + next; + } + + $projection_cell =~ s/^(.*)\..*$/$1/; + + unless (exists($tangents{$projection_cell})) { + # Make a temp file and fill, but be sure to save + ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{FILE} = $tempFile; + $tangents{$projection_cell}{NAME} = $tempName; + if ($masks) { + my ($maskFile, $maskName) = tempfile("/tmp/skycell.$projection_cell.masks.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{MFILE} = $maskFile; + $tangents{$projection_cell}{MNAME} = $maskName; + } + } + print "$skycell_id $projection_cell\n"; + my $file = $ipprc->filename("PPSUB.OUTPUT", $path_base, $skycell_id); + print "$file $state $quality\n"; + my $f_fh = $tangents{$projection_cell}{FILE}; + print $f_fh "$file\n"; + if ($masks) { + my $mask = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base, $skycell_id); + print "$mask\n"; + my $m_fh = $tangents{$projection_cell}{MFILE}; + print $m_fh "$mask\n"; + } + } + foreach my $projection_cell (keys %tangents) { + $command = "$ppSkycell -images $tangents{$projection_cell}{NAME}"; + if ($masks) { + $command .= " -masks $tangents{$projection_cell}{MNAME} "; + } + $command .= " ${outroot}.${projection_cell} "; + print "$command\n"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code); + } + # Update database: + $command = "$difftool -addsummary -diff_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update diff summary: $error_code", $stage_id, $error_code); + } + } + if (scalar (keys %tangents) == 0) { + my $projection_cell = 'NULL'; + $command = "$difftool -addsummary -diff_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update diff summary: $error_code", $stage_id, $error_code); + } + } +} #end diff stage +if ($stage eq 'stack') { + my $imfiles; + my $command = "$stacktool -sassskyfile -sass_id $stage_id"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform stacktool: $error_code", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + # no results found; + } + + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata config doc", $stage_id, $error_code); + + my %tangents = (); + + foreach my $imfile (@$imfiles) { + my $skycell_id = $imfile->{skycell_id}; + my $path_base = $imfile->{path_base}; + my $quality = $imfile->{quality}; + my $state = $imfile->{state}; + my $fault = $imfile->{fault}; + my $projection_cell = $skycell_id; + if ($quality == 8007 or $state ne 'full') { + next; + } + + $projection_cell =~ s/^(.*)\..*$/$1/; + + unless (exists($tangents{$projection_cell})) { + # Make a temp file and fill, but be sure to save + ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{FILE} = $tempFile; + $tangents{$projection_cell}{NAME} = $tempName; + if ($masks) { + my ($maskFile, $maskName) = tempfile("/tmp/skycell.$projection_cell.masks.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{MFILE} = $maskFile; + $tangents{$projection_cell}{MNAME} = $maskName; + } + } + print "$skycell_id $projection_cell\n"; + my $file = $ipprc->filename("PPSTACK.OUTPUT", $path_base, $skycell_id); + print "$file $state $quality\n"; + my $f_fh = $tangents{$projection_cell}{FILE}; + print $f_fh "$file\n"; + if ($masks) { + my $mask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base, $skycell_id); + print "$mask\n"; + my $m_fh = $tangents{$projection_cell}{MFILE}; + print $m_fh "$mask\n"; + } + } + foreach my $projection_cell (keys %tangents) { + $command = "$ppSkycell -images $tangents{$projection_cell}{NAME}"; + if ($masks) { + $command .= " -masks $tangents{$projection_cell}{MNAME} "; + } + $command .= " ${outroot}.${projection_cell} "; + print "$command\n"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code); + } + # Update database: + $command = "$stacktool -addsummary -sass_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update stack summary: $error_code", $stage_id, $error_code); + } + } + if (scalar (keys %tangents) == 0) { + my $projection_cell = 'NULL'; + $command = "$stacktool -addsummary -sass_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } +# print "$command\n"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update stack summary: $error_code", $stage_id, $error_code); + } + } + +} #end stack stage + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $stage = shift; # stage name + my $stage_id = shift; # identifier + my $exit_code = shift; # Exit code + # outputImage and path_base are globals + + carp($msg); + exit $exit_code; +} Index: /trunk/ippTasks/diff.pro =================================================================== --- /trunk/ippTasks/diff.pro (revision 28374) +++ /trunk/ippTasks/diff.pro (revision 28375) @@ -11,4 +11,5 @@ book init diffSkyfile #book init diffCleanup +book init diffPendingSummary ### Database lists @@ -16,4 +17,5 @@ $diffAdvance_DB = 0 $diff_revert_DB = 0 +$diffSummary_DB = 0 #$diffCleanup_DB = 0 @@ -27,4 +29,5 @@ macro diff.reset book init diffSkyfile + book init diffPendingSummary # book init diffCleanup end @@ -44,4 +47,10 @@ active false end + task diff.summary.load + active true + end + task diff.summary.run + active true + end end @@ -58,4 +67,10 @@ end task diff.revert + active false + end + task diff.summary.load + active false + end + task diff.summary.run active false end @@ -193,5 +208,4 @@ # host anyhost # $WORKDIR = $WORKDIR_TEMPLATE - if (($DIFF_MODE == 1)||("$DIFF_MODE" == "NULL")) $DIFF_TAG = "" @@ -477,2 +491,150 @@ end end + +### Load tasks for doing the diffs +### Tasks are loaded into diffPendingSkyCell. +task diff.summary.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1200 + npending 1 + + stdout NULL + stderr $LOGDIR/diff.summary.log + + task.exec + if ($LABEL:n == 0) break + $run = difftool -tosummary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffSummary_DB + $run = $run -dbname $DB:$diffSummary_DB + $diffSummary_DB ++ + if ($diffSummary_DB >= $DB:n) set diffSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + # XXX change tess_id to tess_dir when db is updated + ipptool2book stdout diffPendingSummary -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffPendingSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffPendingSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the diff overlaps +### Tasks are taken from diffPendingSkyCell. +task diff.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages diffPendingSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in diffPendingSkyCell (pantaskState == INIT) + book getpage diffPendingSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffPendingSummary $pageName pantaskState RUN + book getword diffPendingSummary $pageName diff_id -var DIFF_ID + book getword diffPendingSummary $pageName camera -var CAMERA + book getword diffPendingSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword diffPendingSummary $pageName dbname -var DBNAME + book getword diffPendingSummary $pageName tess_id -var TESS_DIR + book getword diffPendingSummary $pageName diff_mode -var DIFF_MODE + book getword diffPendingSummary $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + host anyhost + strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR +# set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + +# if ("$PATH_BASE" == "NULL") + + if (($DIFF_MODE == 1)||("$DIFF_MODE" == "NULL")) + $DIFF_TAG = "" + end + if ($DIFF_MODE == 2) + $DIFF_TAG = "WS." + end + if ($DIFF_MODE == 3) + $DIFF_TAG = "SW." + end + if ($DIFF_MODE == 4) + $DIFF_TAG = "SS." + end + + basename $TESS_DIR -var TESS_ID + + ## generate outroot specific to this exposure + sprintf outroot "%s/%s/%s.%sdif.%s.summary" $WORKDIR $TESS_ID $TESS_ID $DIFF_TAG $DIFF_ID + + + stdout $LOGDIR/diff.summary.log + stderr $LOGDIR/diff.summary.log + + $run = skycell_jpeg.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit diffPendingSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword diffPendingSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffPendingSummary $options:0 pantaskState TIMEOUT + end +end Index: /trunk/ippTasks/stack.pro =================================================================== --- /trunk/ippTasks/stack.pro (revision 28374) +++ /trunk/ippTasks/stack.pro (revision 28375) @@ -10,4 +10,5 @@ ### Initialise the books containing the tasks to do book init stackSumSkyfile +book init stackPendingSummary #book init stackCleanup @@ -16,4 +17,5 @@ $stack_revert_DB = 0 #$stackCleanup_DB = 0 +$stackSummary_DB = 0 ### Check status of stacking tasks @@ -26,4 +28,5 @@ macro stack.reset book init stackSumSkyfile + book init stackPendingSummary # book init stackCleanup end @@ -40,4 +43,10 @@ active false end + task stack.summary.load + active true + end + task stack.summary.run + active true + end end @@ -51,4 +60,10 @@ end task stack.revert + active false + end + task stack.summary.load + active false + end + task stack.summary.run active false end @@ -349,4 +364,7 @@ # end + + + task stack.revert host local @@ -400,2 +418,132 @@ end +### Load tasks for doing the stack summary +### Tasks are loaded into stackPendingSummary +task stack.summary.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1200 + trange 07:00:00 08:00:00 -nmax 1 + npending 1 + + stdout NULL + stderr $LOGDIR/stack.summary.load + + task.exec + if ($LABEL:n == 0) break + $run = stacktool -tosummary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$stackSummary_DB + $run = $run -dbname $DB:$stackSummary_DB + $stackSummary_DB ++ + if ($stackSummary_DB >= $DB:n) set stackSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackPendingSummary -key sass_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackPendingSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackPendingSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for doing the stack summary +task stack.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages stackPendingSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in stackPendingSummary (pantaskState == INIT) + book getpage stackPendingSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackPendingSummary $pageName pantaskState RUN + book getword stackPendingSummary $pageName sass_id -var SASS_ID + book getword stackPendingSummary $pageName camera -var CAMERA + book getword stackPendingSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword stackPendingSummary $pageName dbname -var DBNAME + book getword stackPendingSummary $pageName tess_id -var TESS_DIR + book getword stackPendingSummary $pageName state -var RUN_STATE + book getword stackPendingSummary $pageName projection_cell -var projection_cell + + # set the host and workdir + host anyhost + strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR + + basename $TESS_DIR -var TESS_ID + + ## generate outroot specific to this association + sprintf outroot "%s/%s/%s.stk.%s.summary" $WORKDIR $TESS_ID $TESS_ID $SASS_ID + + stdout $LOGDIR/stack.summary.log + stderr $LOGDIR/stack.summary.log + + $run = skycell_jpeg.pl --stage stack --stage_id $SASS_ID --camera $CAMERA --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit stackPendingSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword stackPendingSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackPendingSummary $options:0 pantaskState TIMEOUT + end +end + Index: /trunk/ippTasks/warp.pro =================================================================== --- /trunk/ippTasks/warp.pro (revision 28374) +++ /trunk/ippTasks/warp.pro (revision 28375) @@ -13,4 +13,5 @@ book init warpPendingSkyCell book init warpPendingCleanup +book init warpPendingSummary ### Database lists @@ -19,4 +20,5 @@ $warp_revert_overlap_DB = 0 $warp_revert_warped_DB = 0 +$warpSummary_DB = 0 ### Check status of warping tasks @@ -30,4 +32,5 @@ book init warpInputExp book init warpPendingSkyCell + book init warpPendingSummary end @@ -55,4 +58,10 @@ active true end + task warp.summary.load + active true + end + task warp.summary.run + active true + end end @@ -78,4 +87,10 @@ end task warp.revert.warped + active false + end + task warp.summary.load + active false + end + task warp.summary.run active false end @@ -525,2 +540,136 @@ end end + +### Load tasks for doing the warps +### Tasks are loaded into warpPendingSkyCell. +task warp.summary.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1200 + npending 1 + + stdout NULL + stderr $LOGDIR/warp.summary.log + + task.exec + if ($LABEL:n == 0) break + $run = warptool -tosummary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warpSummary_DB + $run = $run -dbname $DB:$warpSummary_DB + $warpSummary_DB ++ + if ($warpSummary_DB >= $DB:n) set warpSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + # XXX change tess_id to tess_dir when db is updated + ipptool2book stdout warpPendingSummary -key warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpPendingSkyCell. +task warp.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages warpPendingSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in warpPendingSkyCell (pantaskState == INIT) + book getpage warpPendingSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingSummary $pageName pantaskState RUN + book getword warpPendingSummary $pageName warp_id -var WARP_ID + book getword warpPendingSummary $pageName camera -var CAMERA + book getword warpPendingSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword warpPendingSummary $pageName dbname -var DBNAME + book getword warpPendingSummary $pageName tess_id -var TESS_DIR + book getword warpPendingSummary $pageName exp_tag -var EXP_TAG + book getword warpPendingSummary $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + host anyhost + strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR +# set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + +# if ("$PATH_BASE" == "NULL") + ## generate outroot specific to this exposure + basename $TESS_DIR -var TESS_ID + + sprintf outroot "%s/%s/%s.wrp.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID $TESS_ID + + + stdout $LOGDIR/warp.summary.log + stderr $LOGDIR/warp.summary.log + + $run = skycell_jpeg.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit warpPendingSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword warpPendingSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingSummary $options:0 pantaskState TIMEOUT + end +end Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 28374) +++ /trunk/ippTools/share/Makefile.am (revision 28375) @@ -126,4 +126,6 @@ difftool_skyfile.sql \ difftool_todiffskyfile.sql \ + difftool_tosummary.sql \ + difftool_addsummary.sql \ disttool_definebyquery_camera.sql \ disttool_definebyquery_chip.sql \ @@ -278,4 +280,5 @@ regtool_updatebyquery.sql \ regtool_updatebyqueryimfile.sql \ + stacktool_associationdefine_select.sql \ stacktool_definebyquery_insert.sql \ stacktool_definebyquery_insert_random_part1.sql \ @@ -293,5 +296,8 @@ stacktool_revertsumskyfile_delete.sql \ stacktool_sumskyfile.sql \ + stacktool_sassskyfile.sql \ stacktool_tosum.sql \ + stacktool_tosummary.sql \ + stacktool_addsummary.sql \ staticskytool_definebyquery_select.sql \ staticskytool_definebyquery_inputs.sql \ @@ -326,4 +332,6 @@ warptool_updateskyfile.sql \ warptool_warped.sql \ + warptool_tosummary.sql \ + warptool_addsummary.sql \ diffphottool_definerun.sql \ diffphottool_input.sql \ Index: /trunk/ippTools/share/difftool_addsummary.sql =================================================================== --- /trunk/ippTools/share/difftool_addsummary.sql (revision 28375) +++ /trunk/ippTools/share/difftool_addsummary.sql (revision 28375) @@ -0,0 +1,5 @@ +INSERT INTO diffSummary (diff_id,projection_cell,path_base) VALUES ( + %lld, -- warp_id + '%s', -- projection_cell + '%s' -- path_base + ) Index: /trunk/ippTools/share/difftool_tosummary.sql =================================================================== --- /trunk/ippTools/share/difftool_tosummary.sql (revision 28375) +++ /trunk/ippTools/share/difftool_tosummary.sql (revision 28375) @@ -0,0 +1,5 @@ +SELECT DISTINCT diff_id,"GPC1" as camera,diffRun.workdir,diffRun.tess_id,diffRun.diff_mode,diffRun.state + FROM diffRun + LEFT JOIN diffSummary USING(diff_id) +WHERE diffRun.state = 'full' AND + diffSummary.projection_cell IS NULL Index: /trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_create_tables.sql (revision 28374) +++ /trunk/ippTools/share/pxadmin_create_tables.sql (revision 28375) @@ -981,4 +981,13 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE warpSummary ( + warp_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(warp_id), + KEY(projection_cell), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE stackRun ( stack_id BIGINT AUTO_INCREMENT, @@ -1049,4 +1058,33 @@ KEY(quality), FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociation ( + sass_id BIGINT AUTO_INCREMENT, + data_group VARCHAR(64) NOT NULL, + projection_cell VARCHAR(64) NOT NULL, + tess_id VARCHAR(64) NOT NULL, + filter VARCHAR(64) NOT NULL, + PRIMARY KEY(sass_id), + KEY(data_group), + KEY(projection_cell), + KEY(tess_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackAssociationMap ( + sass_id BIGINT, + stack_id BIGINT, + PRIMARY KEY(sass_id, stack_id), + FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE stackSummary ( + sass_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(sass_id), + KEY(projection_cell), + FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1147,4 +1185,13 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE diffSummary ( + diff_id BIGINT, + projection_cell VARCHAR(64) NOT NULL, + path_base VARCHAR(255) NOT NULL, + PRIMARY KEY(diff_id), + KEY(projection_cell), + FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE magicRun ( magic_id BIGINT AUTO_INCREMENT, Index: /trunk/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_drop_tables.sql (revision 28374) +++ /trunk/ippTools/share/pxadmin_drop_tables.sql (revision 28375) @@ -41,10 +41,15 @@ DROP TABLE IF EXISTS warpSkyfile; DROP TABLE IF EXISTS warpMask; +DROP TABLE IF EXISTS warpSummary; DROP TABLE IF EXISTS diffRun; DROP TABLE IF EXISTS diffInputSkyfile; DROP TABLE IF EXISTS diffSkyfile; +DROP TABLE IF EXISTS diffSummary; DROP TABLE IF EXISTS stackRun; DROP TABLE IF EXISTS stackInputSkyfile; DROP TABLE IF EXISTS stackSumSkyfile; +DROP TABLE IF EXISTS stackSummary; +DROP TABLE IF EXISTS stackAssociation; +DROP TABLE IF EXISTS stackAssociationMap; DROP TABLE IF EXISTS magicRun; DROP TABLE IF EXISTS magicInputSkyfile; Index: /trunk/ippTools/share/stacktool_addsummary.sql =================================================================== --- /trunk/ippTools/share/stacktool_addsummary.sql (revision 28375) +++ /trunk/ippTools/share/stacktool_addsummary.sql (revision 28375) @@ -0,0 +1,5 @@ +INSERT INTO stackSummary (sass_id,projection_cell,path_base) VALUES ( + %lld, -- warp_id + '%s', -- projection_cell + '%s' -- path_base +) Index: /trunk/ippTools/share/stacktool_associationdefine_select.sql =================================================================== --- /trunk/ippTools/share/stacktool_associationdefine_select.sql (revision 28375) +++ /trunk/ippTools/share/stacktool_associationdefine_select.sql (revision 28375) @@ -0,0 +1,19 @@ +SELECT + sass_id, + data_group, + projection_cell, + tess_id, + filter + FROM stackAssociation +RIGHT JOIN ( + SELECT + data_group, + tess_id, + filter, + CASE WHEN LOCATE('.',skycell_id,9) > 0 THEN + SUBSTRING_INDEX(skycell_id,'.',2) ELSE + SUBSTRING_INDEX(skycell_id,'.',1) END + AS projection_cell + FROM stackRun + WHERE stackRun.stack_id = @STACK_ID@ + ) AS RUN USING (data_group,tess_id,filter,projection_cell) LIMIT 1 Index: /trunk/ippTools/share/stacktool_sassskyfile.sql =================================================================== --- /trunk/ippTools/share/stacktool_sassskyfile.sql (revision 28375) +++ /trunk/ippTools/share/stacktool_sassskyfile.sql (revision 28375) @@ -0,0 +1,23 @@ +SELECT + stackAssociation.sass_id, + stackAssociation.projection_cell, + stackSumSkyfile.*, + stackRun.state, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + stackRun.workdir, + stackRun.label, + (SELECT rawExp.camera FROM + stackInputSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun ON warpRun.fake_id = fakeRun.fake_id + JOIN camRun ON camRun.cam_id = fakeRun.cam_id + JOIN chipRun ON camRun.chip_id = chipRun.chip_id + JOIN rawExp ON chipRun.exp_id = rawExp.exp_id + where stack_id = stackRun.stack_id limit 1 + ) as camera +FROM stackRun +JOIN stackSumSkyfile USING(stack_id) +JOIN stackAssociationMap USING(stack_id) +JOIN stackAssociation USING(sass_id) Index: /trunk/ippTools/share/stacktool_tosummary.sql =================================================================== --- /trunk/ippTools/share/stacktool_tosummary.sql (revision 28375) +++ /trunk/ippTools/share/stacktool_tosummary.sql (revision 28375) @@ -0,0 +1,13 @@ +SELECT DISTINCT sass_id,rawExp.camera,stackRun.workdir,stackRun.tess_id,stackRun.state + FROM stackRun + JOIN stackInputSkyfile ON stackRun.stack_id = stackInputSkyfile.stack_id + JOIN stackAssociationMap ON stackRun.stack_id = stackAssociationMap.stack_id + JOIN stackAssociation USING(sass_id) + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN stackSummary USING(sass_id) +WHERE stackRun.state = 'full' AND + stackSummary.projection_cell IS NULL Index: /trunk/ippTools/share/warptool_addsummary.sql =================================================================== --- /trunk/ippTools/share/warptool_addsummary.sql (revision 28375) +++ /trunk/ippTools/share/warptool_addsummary.sql (revision 28375) @@ -0,0 +1,5 @@ +INSERT INTO warpSummary (warp_id,projection_cell,path_base) VALUES( + %lld, -- warp_id + '%s', -- projection_cell + '%s' -- path_base +) Index: /trunk/ippTools/share/warptool_tosummary.sql =================================================================== --- /trunk/ippTools/share/warptool_tosummary.sql (revision 28375) +++ /trunk/ippTools/share/warptool_tosummary.sql (revision 28375) @@ -0,0 +1,7 @@ +SELECT DISTINCT warp_id,rawExp.camera,warpRun.workdir,warpRun.tess_id,rawExp.exp_tag,warpRun.state + FROM warpRun + JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) + LEFT JOIN warpSummary USING(warp_id) +WHERE warpRun.state = 'full' AND + warpSummary.projection_cell IS NULL Index: /trunk/ippTools/src/difftool.c =================================================================== --- /trunk/ippTools/src/difftool.c (revision 28374) +++ /trunk/ippTools/src/difftool.c (revision 28375) @@ -44,4 +44,6 @@ static bool definewarpwarpMode(pxConfig *config); static bool definestackstackMode(pxConfig *config); +static bool tosummaryMode(pxConfig *config); +static bool addsummaryMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupskyfileMode(pxConfig *config); @@ -93,4 +95,6 @@ MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); MODECASE(DIFFTOOL_MODE_DEFINESTACKSTACK, definestackstackMode); + MODECASE(DIFFTOOL_MODE_TOSUMMARY, tosummaryMode); + MODECASE(DIFFTOOL_MODE_ADDSUMMARY, addsummaryMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); @@ -2250,4 +2254,135 @@ } +static bool tosummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "diffSkyfile.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "diffRun.state", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "diffSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("difftool_tosummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "diffSkyfile")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + if (where2) { + if (psListLength(where->list)) { + psStringAppend(&query, " %s", where2); + } else { + psStringAppend(&query, " AND 1 %s", where2); + } + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return(true); +} +static bool addsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + psString query = pxDataGet("difftool_addsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, diff_id, projection_cell, path_base)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psS64 numUpdated = psDBAffectedRows(config->dbh); + + if (numUpdated != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); + psFree(query); + return false; + } + + psFree(query); + + // Print anything here? + + return(true); +} + + static bool pendingcleanuprunMode(pxConfig *config) { Index: /trunk/ippTools/src/difftool.h =================================================================== --- /trunk/ippTools/src/difftool.h (revision 28374) +++ /trunk/ippTools/src/difftool.h (revision 28375) @@ -32,4 +32,6 @@ DIFFTOOL_MODE_ADDDIFFSKYFILE, DIFFTOOL_MODE_ADVANCE, + DIFFTOOL_MODE_TOSUMMARY, + DIFFTOOL_MODE_ADDSUMMARY, DIFFTOOL_MODE_DIFFSKYFILE, DIFFTOOL_MODE_REVERTDIFFSKYFILE, Index: /trunk/ippTools/src/difftoolConfig.c =================================================================== --- /trunk/ippTools/src/difftoolConfig.c (revision 28374) +++ /trunk/ippTools/src/difftoolConfig.c (revision 28375) @@ -323,4 +323,33 @@ psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -tosummary + psMetadata *tosummaryArgs = psMetadataAlloc(); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(tosummaryArgs , PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + + psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-magicked", 0, "search by magic id", 0); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL); + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for runs that have not been destreaked", false); + + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(tosummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsummary + psMetadata *addsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-projection_cell", 0, "set projection cell", NULL); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL); + // -pendingcleanupskyfile psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc(); @@ -380,5 +409,4 @@ psMetadata *importrunArgs = psMetadataAlloc(); psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - @@ -415,4 +443,6 @@ PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full", DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); + PXOPT_ADD_MODE("-tosummary", "show runs that can be summarized", DIFFTOOL_MODE_TOSUMMARY, tosummaryArgs); + PXOPT_ADD_MODE("-addsummary", "add entry to the summary table", DIFFTOOL_MODE_ADDSUMMARY, addsummaryArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /trunk/ippTools/src/stacktool.c =================================================================== --- /trunk/ippTools/src/stacktool.c (revision 28374) +++ /trunk/ippTools/src/stacktool.c (revision 28375) @@ -39,5 +39,8 @@ static bool addsumskyfileMode(pxConfig *config); static bool sumskyfileMode(pxConfig *config); +static bool sassskyfileMode(pxConfig *config); static bool revertsumskyfileMode(pxConfig *config); +static bool tosummaryMode(pxConfig *config); +static bool addsummaryMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupskyfileMode(pxConfig *config); @@ -75,5 +78,8 @@ MODECASE(STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileMode); MODECASE(STACKTOOL_MODE_SUMSKYFILE, sumskyfileMode); + MODECASE(STACKTOOL_MODE_SASSSKYFILE, sassskyfileMode); MODECASE(STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileMode); + MODECASE(STACKTOOL_MODE_TOSUMMARY, tosummaryMode); + MODECASE(STACKTOOL_MODE_ADDSUMMARY, addsummaryMode); MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); @@ -102,4 +108,84 @@ exit(exit_status); } +//stackAssociationRow *association = pxStackAssociationDefine(data_group,tess_id,filter,skycell_id); +stackAssociationRow *pxStackAssociationDefine(pxConfig *config, psS64 stack_id) { + psString select = pxDataGet("stacktool_associationdefine_select.sql"); + if (!select) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString idString = NULL; + psStringAppend(&idString, "%" PRId64, stack_id); + // Copy string to get around the issue with psStringSubstitute not believing + // that select is a psString. + psString rep = psStringCopy(select); + psFree(select); + select = rep; + psStringSubstitute(&select, idString, "@STACK_ID@"); + psFree(idString); + + if (!p_psDBRunQuery(config->dbh, select)) { + psError(PS_ERR_UNKNOWN,false, "database error"); + psFree(select); + return(NULL); + } + psFree(select); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return(NULL); + } + if (psArrayLength(output) != 1) { + psWarning("stacktool: incorrect number of rows found"); + psFree(output); + return(NULL); + } + psMetadata *outrow = psMetadataAlloc(); + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; + + printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,row,"sass_id"), + psMetadataLookupStr(NULL,row,"data_group"), + psMetadataLookupStr(NULL,row,"tess_id"), + psMetadataLookupStr(NULL,row,"filter"), + psMetadataLookupStr(NULL,row,"projection_cell")); + + if (psMetadataLookupS64(NULL,row,"sass_id") == PS_MAX_S64) { + psMetadataAddS64(outrow,PS_LIST_TAIL,"sass_id",PS_META_REPLACE,"",0); + } + else { + psMetadataAddS64(outrow,PS_LIST_TAIL,"sass_id",PS_META_REPLACE,"",psMetadataLookupS64(NULL,row,"sass_id")); + } + psMetadataAddStr(outrow,PS_LIST_TAIL,"data_group",PS_META_REPLACE,"",psMetadataLookupStr(NULL,row,"data_group")); + psMetadataAddStr(outrow,PS_LIST_TAIL,"tess_id",PS_META_REPLACE,"",psMetadataLookupStr(NULL,row,"tess_id")); + psMetadataAddStr(outrow,PS_LIST_TAIL,"filter",PS_META_REPLACE,"",psMetadataLookupStr(NULL,row,"filter")); + psMetadataAddStr(outrow,PS_LIST_TAIL,"projection_cell",PS_META_REPLACE,"", + psMetadataLookupStr(NULL,row,"projection_cell")); + } + printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,outrow,"sass_id"), + psMetadataLookupStr(NULL,outrow,"data_group"), + psMetadataLookupStr(NULL,outrow,"tess_id"), + psMetadataLookupStr(NULL,outrow,"filter"), + psMetadataLookupStr(NULL,outrow,"projection_cell")); + + + psFree(output); + stackAssociationRow *sassRow = stackAssociationObjectFromMetadata(outrow); + psFree(outrow); + return(sassRow); +} + + + + @@ -395,4 +481,48 @@ psFree(run); + //CZW Add an association entry here. + // Define the requested association, and insert it if it doesn't already exist + stackAssociationRow *association = pxStackAssociationDefine(config,stack_id); + psS64 sass_id; + if (!association->sass_id) { + psTrace("stacktool.association",2,"No required Association found. Adding."); + + if (!stackAssociationInsertObject(config->dbh,association)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(run); + psFree(insert); + psFree(list); + psFree(association); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + sass_id = psDBLastInsertID(config->dbh); + association->sass_id = sass_id; + } + // Insert the map entry for this row. + stackAssociationMapRow *maprow = stackAssociationMapRowAlloc(sass_id,stack_id); + if (!stackAssociationMapInsertObject(config->dbh,maprow)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(run); + psFree(insert); + psFree(list); + psFree(association); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + + // Create a suitable insertion query for this run psString thisInsert = psStringCopy(insert); @@ -552,4 +682,6 @@ run->stack_id = psDBLastInsertID(config->dbh); + //CZW Add an association entry here. + // insert the stackInputSkyfile rows psListIterator *iter = psListIteratorAlloc(warp_ids->data.list, 0, false); @@ -603,7 +735,8 @@ #endif psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "stackRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-sass_id", "stackAssociationMap.sass_id", "=="); if (!psListLength(where->list)) { psFree(where); @@ -612,4 +745,5 @@ } + //CZW join against stackAssociationMap psString query = psStringCopy("UPDATE stackRun"); @@ -1038,23 +1172,27 @@ } - -static bool revertsumskyfileMode(pxConfig *config) +static bool sassskyfileMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); - pxAddLabelSearchArgs(config, where, "-label", "stackRun.label", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); - - if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { - psFree(where); - psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); - return false; - } - - // Delete product - psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); - if (!delete) { + PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociation.sass_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackAssociation.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "stackAssociation.projection_cell", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "stackAssociation.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-data_group", "stackAssociation.data_group", "LIKE"); + +// The following three selectors are incompatible with the sql so omit them +// PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); +// PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); +// PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_sassskyfile.sql"); + if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; @@ -1063,88 +1201,11 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&delete, " AND %s", whereClause); + psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, delete)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(delete); - psFree(where); - return false; - } - psFree(delete); - - int numRows = psDBAffectedRows(config->dbh); // Number of row affected - psLogMsg("stacktool", PS_LOG_INFO, "Deleted %d rows", numRows); - - psFree(where); - - return true; -} - - -static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state); - return false; - } - - char *query = "UPDATE stackRun SET state = '%s' WHERE stack_id = %"PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, stack_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for stack_id %"PRId64, stack_id); - return false; - } - - return true; -} - -#ifdef notdef -static bool setstackRunStateByLabel(pxConfig *config, const char *label, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state); - return false; - } - - char *query = "UPDATE stackRun SET state = '%s' WHERE label = '%s'"; - if (!p_psDBRunQueryF(config->dbh, query, state, label)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for label %s", label); - return false; - } - - return true; -} -#endif - -static bool pendingcleanuprunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); - - psString query = pxDataGet("stacktool_pendingcleanuprun.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + psFree(where); @@ -1165,5 +1226,14 @@ psArray *output = p_psDBFetchResult(config->dbh); if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return false; } @@ -1174,31 +1244,233 @@ } + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool revertsumskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("stacktool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + + return true; +} + + +static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state); + return false; + } + + char *query = "UPDATE stackRun SET state = '%s' WHERE stack_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, stack_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for stack_id %"PRId64, stack_id); + return false; + } + + return true; +} + +#ifdef notdef +static bool setstackRunStateByLabel(pxConfig *config, const char *label, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state); + return false; + } + + char *query = "UPDATE stackRun SET state = '%s' WHERE label = '%s'"; + if (!p_psDBRunQueryF(config->dbh, query, state, label)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for label %s", label); + return false; + } + + return true; +} +#endif +static bool tosummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociationMap.sass_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackSumSkyfile.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "stackRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "stackRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("stacktool_tosummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupRun", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - -static bool pendingcleanupskyfileMode(pxConfig *config) + if (!ippdbPrintMetadatas(stdout, output, "stackRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return(true); +} +static bool addsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(sass_id, config->args, "-sass_id", true, false); + PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + psString query = pxDataGet("stacktool_addsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, sass_id, projection_cell, path_base)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psS64 numUpdated = psDBAffectedRows(config->dbh); + + if (numUpdated != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); + psFree(query); + return false; + } + + psFree(query); + + // Print anything here? + + return(true); +} + +static bool pendingcleanuprunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); - PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); psMetadata *where = psMetadataAlloc(); - if (stack_id) { - PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); - } + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociationMap.sass_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); - psString query = pxDataGet("stacktool_pendingcleanupskyfile.sql"); + psString query = pxDataGet("stacktool_pendingcleanuprun.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -1239,5 +1511,5 @@ // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupSkyfile", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupRun", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -1250,16 +1522,19 @@ } - -static bool donecleanupMode(pxConfig *config) +static bool pendingcleanupskyfileMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - - psString query = pxDataGet("stacktool_donecleanup.sql"); + if (stack_id) { + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + } + pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); + + psString query = pxDataGet("stacktool_pendingcleanupskyfile.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -1300,4 +1575,66 @@ // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupSkyfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-sass_id", "sass_id", "=="); + + psString query = pxDataGet("stacktool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true if (!ippdbPrintMetadatas(stdout, output, "stackDoneCleanup", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); @@ -1329,4 +1666,5 @@ } +//CZW I have not added sass information to the export/import run modes yet. bool exportrunMode(pxConfig *config) { Index: /trunk/ippTools/src/stacktool.h =================================================================== --- /trunk/ippTools/src/stacktool.h (revision 28374) +++ /trunk/ippTools/src/stacktool.h (revision 28375) @@ -33,5 +33,8 @@ STACKTOOL_MODE_ADDSUMSKYFILE, STACKTOOL_MODE_SUMSKYFILE, + STACKTOOL_MODE_SASSSKYFILE, STACKTOOL_MODE_REVERTSUMSKYFILE, + STACKTOOL_MODE_TOSUMMARY, + STACKTOOL_MODE_ADDSUMMARY, STACKTOOL_MODE_PENDINGCLEANUPRUN, STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, Index: /trunk/ippTools/src/stacktoolConfig.c =================================================================== --- /trunk/ippTools/src/stacktoolConfig.c (revision 28374) +++ /trunk/ippTools/src/stacktoolConfig.c (revision 28375) @@ -197,4 +197,16 @@ psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-all", 0, "enable search without arguments", false); + // -sassskyfile + psMetadata *sassskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(sassskyfileArgs, PS_LIST_TAIL, "-sass_id", 0, "search by stack association ID", 0); + psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); + psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-projection_cell", 0, "search by projection cell", 0); + + psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-data_group", 0, "search by stackAssociation.data_group (LIKE comparison)", NULL); + psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", NULL); + psMetadataAddU64(sassskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(sassskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(sassskyfileArgs, PS_LIST_TAIL, "-all", 0, "enable search without arguments", false); + // -revertsumskyfile psMetadata *revertsumskyfileArgs= psMetadataAlloc(); @@ -204,4 +216,25 @@ psMetadataAddBool(revertsumskyfileArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", 0); + // -tosummary + psMetadata *tosummaryArgs = psMetadataAlloc(); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-sass_id", 0, "search by stack association ID", 0); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by stackRun dist_group (LIKE comparison)", NULL); + + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(tosummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsummary + psMetadata *addsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-sass_id", 0, "set stack Association ID", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-projection_cell", 0, "set projection cell", NULL); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL); + // -pendingcleanuprun psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); @@ -252,4 +285,5 @@ PXOPT_ADD_MODE("-addsumskyfile", "", STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileArgs); PXOPT_ADD_MODE("-sumskyfile", "list results of stackRun", STACKTOOL_MODE_SUMSKYFILE, sumskyfileArgs); + PXOPT_ADD_MODE("-sassskyfile", "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE, sassskyfileArgs); PXOPT_ADD_MODE("-revertsumskyfile","", STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileArgs); PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); @@ -257,4 +291,6 @@ PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", STACKTOOL_MODE_DONECLEANUP, donecleanupArgs); PXOPT_ADD_MODE("-updatesumskyfile", "update fault code for sumskyfile", STACKTOOL_MODE_UPDATESUMSKYFILE, updatesumskyfileArgs); + PXOPT_ADD_MODE("-tosummary", "show runs that can be summarized", STACKTOOL_MODE_TOSUMMARY, tosummaryArgs); + PXOPT_ADD_MODE("-addsummary", "add entry to the summary table", STACKTOOL_MODE_ADDSUMMARY, addsummaryArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", STACKTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", STACKTOOL_MODE_IMPORTRUN, importrunArgs); Index: /trunk/ippTools/src/warptool.c =================================================================== --- /trunk/ippTools/src/warptool.c (revision 28374) +++ /trunk/ippTools/src/warptool.c (revision 28375) @@ -48,4 +48,6 @@ static bool maskedMode(pxConfig *config); static bool unblockMode(pxConfig *config); +static bool tosummaryMode(pxConfig *config); +static bool addsummaryMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupwarpMode(pxConfig *config); @@ -101,4 +103,6 @@ MODECASE(WARPTOOL_MODE_MASKED, maskedMode); MODECASE(WARPTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(WARPTOOL_MODE_TOSUMMARY, tosummaryMode); + MODECASE(WARPTOOL_MODE_ADDSUMMARY, addsummaryMode); MODECASE(WARPTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupwarpMode); @@ -1548,4 +1552,133 @@ } +static bool tosummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_tosummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "warpSkyfile")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + if (where2) { + psStringAppend(&query, " %s", where2); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return(true); +} +static bool addsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + psString query = pxDataGet("warptool_addsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, warp_id, projection_cell, path_base)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psS64 numUpdated = psDBAffectedRows(config->dbh); + + if (numUpdated != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); + psFree(query); + return false; + } + + psFree(query); + + // Print anything here? + + return(true); +} + static bool pendingcleanuprunMode(pxConfig *config) { Index: /trunk/ippTools/src/warptool.h =================================================================== --- /trunk/ippTools/src/warptool.h (revision 28374) +++ /trunk/ippTools/src/warptool.h (revision 28375) @@ -44,4 +44,6 @@ WARPTOOL_MODE_MASKED, WARPTOOL_MODE_UNBLOCK, + WARPTOOL_MODE_TOSUMMARY, + WARPTOOL_MODE_ADDSUMMARY, WARPTOOL_MODE_PENDINGCLEANUPRUN, WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, Index: /trunk/ippTools/src/warptoolConfig.c =================================================================== --- /trunk/ippTools/src/warptoolConfig.c (revision 28374) +++ /trunk/ippTools/src/warptoolConfig.c (revision 28375) @@ -321,4 +321,32 @@ psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + // -tosummary + psMetadata *tosummaryArgs = psMetadataAlloc(); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); + psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-filter", 0, "search for exposures by filter", NULL); + psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-magicked", 0, "search by magic id", 0); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL); + psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by warpRun dist_group", NULL); + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for runs that have not been destreaked", false); + + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(tosummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsummary + psMetadata *addsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-warp_id", 0, "set warp ID", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-projection_cell", 0, "set projection cell", NULL); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL); + // -pendingcleanuprun psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); @@ -430,4 +458,7 @@ PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs); PXOPT_ADD_MODE("-setskyfiletoupdate", "set cleaned skyfile to be updated", WARPTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateArgs); + PXOPT_ADD_MODE("-tosummary", "show runs that can be summarized", WARPTOOL_MODE_TOSUMMARY, tosummaryArgs); + PXOPT_ADD_MODE("-addsummary", "add entry to the summary table", WARPTOOL_MODE_ADDSUMMARY, addsummaryArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", WARPTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", WARPTOOL_MODE_IMPORTRUN, importrunArgs); Index: /trunk/ippconfig/recipes/filerules-mef.mdc =================================================================== --- /trunk/ippconfig/recipes/filerules-mef.mdc (revision 28374) +++ /trunk/ippconfig/recipes/filerules-mef.mdc (revision 28375) @@ -312,4 +312,6 @@ PPSKYCELL.JPEG1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg JPEG NONE CHIP TRUE NONE PPSKYCELL.JPEG2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg JPEG NONE CHIP TRUE NONE +PPSKYCELL.BIN1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.fits IMAGE NONE FPA TRUE NONE +PPSKYCELL.BIN2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.fits IMAGE NONE FPA TRUE NONE LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE Index: /trunk/ippconfig/recipes/filerules-simple.mdc =================================================================== --- /trunk/ippconfig/recipes/filerules-simple.mdc (revision 28374) +++ /trunk/ippconfig/recipes/filerules-simple.mdc (revision 28375) @@ -277,4 +277,6 @@ PPSKYCELL.JPEG1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg JPEG NONE CHIP TRUE NONE PPSKYCELL.JPEG2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg JPEG NONE CHIP TRUE NONE +PPSKYCELL.BIN1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.fits IMAGE NONE FPA TRUE NONE +PPSKYCELL.BIN2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.fits IMAGE NONE FPA TRUE NONE LOG.IMFILE OUTPUT {OUTPUT}.imfile.log TEXT NONE FPA TRUE NONE Index: /trunk/ippconfig/recipes/filerules-split.mdc =================================================================== --- /trunk/ippconfig/recipes/filerules-split.mdc (revision 28374) +++ /trunk/ippconfig/recipes/filerules-split.mdc (revision 28375) @@ -297,4 +297,6 @@ PPSKYCELL.JPEG1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg JPEG NONE CHIP TRUE NONE PPSKYCELL.JPEG2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg JPEG NONE CHIP TRUE NONE +PPSKYCELL.BIN1 OUTPUT {OUTPUT}.{FILE.INDEX}.b1.fits IMAGE NONE FPA TRUE NONE +PPSKYCELL.BIN2 OUTPUT {OUTPUT}.{FILE.INDEX}.b2.fits IMAGE NONE FPA TRUE NONE LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE Index: /trunk/ippconfig/recipes/ppSkycell.config =================================================================== --- /trunk/ippconfig/recipes/ppSkycell.config (revision 28374) +++ /trunk/ippconfig/recipes/ppSkycell.config (revision 28375) @@ -4,3 +4,3 @@ BIN1 S32 4 # Binning factor for output 1 BIN2 S32 4 # Binning factor for output 2 - +MAKEFITS BOOL T # Make fits image as well? Index: /trunk/ppImage/src/ppImageLoop.c =================================================================== --- /trunk/ppImage/src/ppImageLoop.c (revision 28374) +++ /trunk/ppImage/src/ppImageLoop.c (revision 28375) @@ -175,4 +175,5 @@ ESCAPE("Unable to measures pixel stats for image"); } + if (!ppImageMosaicChip(config, options, view, "PPIMAGE.CHIP", "PPIMAGE.OUTPUT")) { ESCAPE("Unable to mosaic chip"); @@ -181,4 +182,5 @@ // we perform photometry on the readouts of this chip in the output + psTimerStart(TIMER_PHOT); if (options->doPhotom) { Index: /trunk/ppImage/src/ppImagePhotom.c =================================================================== --- /trunk/ppImage/src/ppImagePhotom.c (revision 28374) +++ /trunk/ppImage/src/ppImagePhotom.c (revision 28375) @@ -11,7 +11,7 @@ pmCell *cell; pmReadout *readout; - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); psphotInit(); - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // find or define a pmFPAfile PSPHOT.INPUT pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.INPUT"); @@ -20,10 +20,10 @@ return false; } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // we make a new copy of the output chip to keep psphot from modifying the output image pmChip *oldChip = pmFPAviewThisChip (view, input->src); pmChip *newChip = pmFPAviewThisChip (view, input->fpa); pmChipCopy (newChip, oldChip); - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // iterate over the cells and readout for this chip while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { @@ -34,5 +34,5 @@ while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { if (! readout->data_exists) { continue; } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // run the actual photometry analysis if (!psphotReadout (config, view)) { Index: /trunk/ppSkycell/src/ppSkycell.h =================================================================== --- /trunk/ppSkycell/src/ppSkycell.h (revision 28374) +++ /trunk/ppSkycell/src/ppSkycell.h (revision 28375) @@ -16,4 +16,5 @@ int bin1, bin2; // Binning factors pmConfig *config; // Configuration + bool doFits; // hold whether to do fits as well. } ppSkycellData; Index: /trunk/ppSkycell/src/ppSkycellCamera.c =================================================================== --- /trunk/ppSkycell/src/ppSkycellCamera.c (revision 28374) +++ /trunk/ppSkycell/src/ppSkycellCamera.c (revision 28375) @@ -103,4 +103,12 @@ return false; } + if (!pmFPAfileDefineOutput(data->config, NULL, "PPSKYCELL.BIN1")) { + psError(psErrorCodeLast(), false, "Unable to define output."); + return false; + } + if (!pmFPAfileDefineOutput(data->config, NULL, "PPSKYCELL.BIN2")) { + psError(psErrorCodeLast(), false, "Unable to define output."); + return false; + } // Now the camera has been determined, we can read the recipe @@ -116,4 +124,6 @@ data->bin2 = psMetadataLookupS32(NULL, recipe, "BIN2"); + data->doFits = psMetadataLookupBool(NULL, recipe, "MAKEFITS"); + if (data->bin1 <= 0 || data->bin2 <= 0) { psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find legitimate values for BIN1 and BIN2"); Index: /trunk/ppSkycell/src/ppSkycellLoop.c =================================================================== --- /trunk/ppSkycell/src/ppSkycellLoop.c (revision 28374) +++ /trunk/ppSkycell/src/ppSkycellLoop.c (revision 28375) @@ -284,9 +284,15 @@ pmFPAfileActivate(data->config->files, true, "PPSKYCELL.JPEG1"); pmFPAfileActivate(data->config->files, true, "PPSKYCELL.JPEG2"); + if (data->doFits) { + pmFPAfileActivate(data->config->files, true, "PPSKYCELL.BIN1"); + pmFPAfileActivate(data->config->files, true, "PPSKYCELL.BIN2"); + } + pmFPAview *view = filesIterateDown(data->config); // View to readout + pmCell *cell1 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.JPEG1"); // Rebinned cell 1 pmCell *cell2 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.JPEG2"); // Rebinned cell 2 - psFree(view); + pmReadout *ro1 = pmReadoutAlloc(cell1), *ro2 = pmReadoutAlloc(cell2); // Binned readouts @@ -298,11 +304,48 @@ ro1->data_exists = cell1->data_exists = cell1->parent->data_exists = true; ro2->data_exists = cell2->data_exists = cell2->parent->data_exists = true; - + pmFPAfile *file1 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.JPEG1", 0); file1->save = true; - file1->index = i; + file1->fileIndex = i; pmFPAfile *file2 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.JPEG2", 0); file2->save = true; - file2->index = i; + file2->fileIndex = i; + + if (data->doFits) { + pmCell *Fcell1 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.BIN1"); // Rebinned cell 1 + pmCell *Fcell2 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.BIN2"); // Rebinned cell 2 + + // This is a hack to get a functioning header created so the fits images can be written out. + psMetadataAddS32(Fcell1->concepts,PS_LIST_TAIL,"CELL.XPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell1->concepts,PS_LIST_TAIL,"CELL.YPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell1->concepts,PS_LIST_TAIL,"CELL.READDIR", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell2->concepts,PS_LIST_TAIL,"CELL.XPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell2->concepts,PS_LIST_TAIL,"CELL.YPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell2->concepts,PS_LIST_TAIL,"CELL.READDIR", PS_META_REPLACE,"",1); + + psMetadataAddS32(Fcell1->parent->concepts,PS_LIST_TAIL,"CHIP.XPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell1->parent->concepts,PS_LIST_TAIL,"CHIP.YPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell2->parent->concepts,PS_LIST_TAIL,"CHIP.XPARITY", PS_META_REPLACE,"",1); + psMetadataAddS32(Fcell2->parent->concepts,PS_LIST_TAIL,"CHIP.YPARITY", PS_META_REPLACE,"",1); + + pmReadout *Fro1 = pmReadoutAlloc(Fcell1), *Fro2 = pmReadoutAlloc(Fcell2); // Binned readouts + + Fro1->image = image1; + Fro2->image = image2; + Fro1->mask = mask1; + Fro2->mask = mask2; + + Fro1->data_exists = Fcell1->data_exists = Fcell1->parent->data_exists = true; + Fro2->data_exists = Fcell2->data_exists = Fcell2->parent->data_exists = true; + + pmFPAfile *fits1 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.BIN1", 0); + fits1->save = true; + fits1->fileIndex = i; + pmFPAfile *fits2 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.BIN2", 0); + fits2->save = true; + fits2->fileIndex = i; + } + + psFree(view); #if 0 { Index: /trunk/psconfig/tagsets/ipp-2.9.dist =================================================================== --- /trunk/psconfig/tagsets/ipp-2.9.dist (revision 28374) +++ /trunk/psconfig/tagsets/ipp-2.9.dist (revision 28375) @@ -73,4 +73,5 @@ YYYYY ppTranslate ipp-2-9 -0 YYYYY ppViz ipp-2-9 -0 + YYYYY ppSkycell ipp-2-9 -0 YYYYY extsrc/gpcsw ipp-2-9 -0 Index: /trunk/psphot/src/psphotReadout.c =================================================================== --- /trunk/psphot/src/psphotReadout.c (revision 28374) +++ /trunk/psphot/src/psphotReadout.c (revision 28375) @@ -14,5 +14,5 @@ // by the multiple threads, not the total time used by all threads. psTimerStart ("psphotReadout"); - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); pmModelClassSetLimits(PM_MODEL_LIMITS_LAX); @@ -26,5 +26,5 @@ char *breakPt = psMetadataLookupStr (NULL, recipe, "BREAK_POINT"); psAssert (breakPt, "configuration error: set BREAK_POINT"); - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // set the photcode for this image if (!psphotAddPhotcode (config, view, "PSPHOT.INPUT")) { @@ -32,5 +32,5 @@ return false; } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // Generate the mask and weight images, including the user-defined analysis region of interest if (!psphotSetMaskAndVariance (config, view, "PSPHOT.INPUT")) { @@ -40,9 +40,10 @@ return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // generate a background model (median, smoothed image) if (!psphotModelBackground (config, view, "PSPHOT.INPUT")) { return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); if (!psphotSubtractBackground (config, view, "PSPHOT.INPUT")) { return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); @@ -51,5 +52,5 @@ return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // load the psf model, if suppled. FWHM_X,FWHM_Y,etc are determined and saved on // readout->analysis XXX this function currently only works with a single PSPHOT.INPUT @@ -58,5 +59,5 @@ return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // find the detections (by peak and/or footprint) in the image. if (!psphotFindDetections (config, view, "PSPHOT.INPUT", true)) { // pass 1 @@ -65,5 +66,5 @@ return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // construct sources and measure basic stats (saved on detections->newSources) if (!psphotSourceStats (config, view, "PSPHOT.INPUT", true)) { // pass 1 @@ -74,5 +75,5 @@ return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // find blended neighbors of very saturated stars (detections->newSources) if (!psphotDeblendSatstars (config, view, "PSPHOT.INPUT")) { @@ -80,5 +81,5 @@ return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // mark blended peaks PS_SOURCE_BLEND (detections->newSources) if (!psphotBasicDeblend (config, view, "PSPHOT.INPUT")) { @@ -86,5 +87,5 @@ return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // classify sources based on moments, brightness. if a PSF model has been loaded, the PSF // clump defined for it is used not measured (detections->newSources) @@ -93,4 +94,5 @@ return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // if we were not supplied a PSF model, determine the IQ stats here (detections->newSources) if (!psphotImageQuality (config, view, "PSPHOT.INPUT")) { // pass 1 @@ -101,5 +103,5 @@ return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // use bright stellar objects to measure PSF if we were supplied a PSF for any input file, // this step is skipped @@ -115,16 +117,16 @@ // XXX fix this in the new multi-input context // psphotLoadExtSources (config, view, "PSPHOT.INPUT"); // pass 1 - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // construct an initial model for each object, set the radius to fitRadius, set circular // fit mask (detections->newSources) psphotGuessModels (config, view, "PSPHOT.INPUT"); // pass 1 - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // merge the newly selected sources into the existing list // NOTE: merge OLD and NEW psphotMergeSources (config, view, "PSPHOT.INPUT"); - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // linear PSF fit to source peaks, subtract the models from the image (in PSF mask) psphotFitSourcesLinear (config, view, "PSPHOT.INPUT", false); // pass 1 (detections->allSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // identify CRs and extended sources (only unmeasured sources are measured) psphotSourceSize (config, view, "PSPHOT.INPUT", true); // pass 1 (detections->allSources) @@ -132,36 +134,36 @@ goto finish; } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // non-linear PSF and EXT fit to brighter sources // replace model flux, adjust mask as needed, fit, subtract the models (full stamp) psphotBlendFit (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // replace all sources psphotReplaceAllSources (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // linear fit to include all sources (subtract again) // NOTE : apply to ALL sources (extended + psf) psphotFitSourcesLinear (config, view, "PSPHOT.INPUT", true); // pass 2 (detections->allSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // if we only do one pass, skip to extended source analysis if (!strcasecmp (breakPt, "PASS1")) goto pass1finish; - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // NOTE: possibly re-measure background model here with objects subtracted / or masked // add noise for subtracted objects psphotAddNoise (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // find fainter sources // NOTE: finds new peaks and new footprints, OLD and FULL set are saved on detections psphotFindDetections (config, view, "PSPHOT.INPUT", false); // pass 2 (detections->peaks, detections->footprints) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // remove noise for subtracted objects (ie, return to normal noise level) // NOTE: this needs to operate only on the OLD sources psphotSubNoise (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // define new sources based on only the new peaks // NOTE: new sources are saved on detections->newSources psphotSourceStats (config, view, "PSPHOT.INPUT", false); // pass 2 (detections->newSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // set source type // NOTE: apply only to detections->newSources @@ -170,21 +172,21 @@ return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // create full input models, set the radius to fitRadius, set circular fit mask // NOTE: apply only to detections->newSources psphotGuessModels (config, view, "PSPHOT.INPUT"); // pass 2 (detections->newSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // replace all sources so fit below applies to all at once // NOTE: apply only to OLD sources (which have been subtracted) psphotReplaceAllSources (config, view, "PSPHOT.INPUT"); // pass 2 - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // merge the newly selected sources into the existing list // NOTE: merge OLD and NEW // XXX check on free of sources... psphotMergeSources (config, view, "PSPHOT.INPUT"); // (detections->newSources + detections->allSources -> detections->allSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // NOTE: apply to ALL sources psphotFitSourcesLinear (config, view, "PSPHOT.INPUT", true); // pass 3 (detections->allSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); pass1finish: @@ -192,13 +194,13 @@ // NOTE: applies only to NEW (unmeasured) sources psphotSourceSize (config, view, "PSPHOT.INPUT", false); // pass 2 (detections->allSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); psphotExtendedSourceAnalysis (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources) psphotExtendedSourceFits (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources) - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); finish: // plot positive sources // psphotSourcePlots (readout, sources, recipe); - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // measure aperture photometry corrections if (!psphotApResid (config, view, "PSPHOT.INPUT")) { // pass 1 (detections->allSources) @@ -206,8 +208,10 @@ return psphotReadoutCleanup (config, view, "PSPHOT.INPUT"); } - + printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast())); // calculate source magnitudes - psphotMagnitudes(config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources) - + if (!psphotMagnitudes(config, view, "PSPHOT.INPUT")) { // pass 1 (detections->allSources) + psErrorStackPrint(stderr, "Unable to do magnitudes."); + psErrorClear(); + } if (!psphotEfficiency(config, view, "PSPHOT.INPUT")) { // pass 1 psErrorStackPrint(stderr, "Unable to determine detection efficiencies from fake sources"); @@ -219,9 +223,19 @@ // replace background in residual image - psphotSkyReplace (config, view, "PSPHOT.INPUT"); // pass 1 - + if (!psphotSkyReplace (config, view, "PSPHOT.INPUT")) { // pass 1 + psErrorStackPrint(stderr, "Unable to replace sky"); + psErrorClear(); + +/* psLogMsg("psphot", 3, "failed on psphotSkyReplace"); */ +/* return(psphotReadoutCleanup(config, view, "PSPHOT.INPUT")); */ + } // drop the references to the image pixels held by each source - psphotSourceFreePixels (config, view, "PSPHOT.INPUT"); // pass 1 - + if (!psphotSourceFreePixels (config, view, "PSPHOT.INPUT")) { // pass 1 + psErrorStackPrint(stderr, "Unable to free source pixels"); + psErrorClear(); + +/* psLogMsg ("psphot", 3, "failed on psphotSourceFreePixels"); */ +/* return(psphotReadoutCleanup(config, view, "PSPHOT.INPUT")); */ + } // create the exported-metadata and free local data return psphotReadoutCleanup(config, view, "PSPHOT.INPUT");