Index: /trunk/dbconfig/changes.txt =================================================================== --- /trunk/dbconfig/changes.txt (revision 31434) +++ /trunk/dbconfig/changes.txt (revision 31435) @@ -2131,2 +2131,3 @@ UPDATE dbversion set schema_version = '1.1.70', updated= CURRENT_TIMESTAMP(); + Index: /trunk/dbconfig/lap.md =================================================================== --- /trunk/dbconfig/lap.md (revision 31435) +++ /trunk/dbconfig/lap.md (revision 31435) @@ -0,0 +1,31 @@ +lapSequence METADATA + seq_id S64 0 # Primary Key AUTO_INCREMENT + name STR 64 # Key + description STR 255 +END + +lapRun METADATA + lap_id S64 0 # Primary Key AUTO_INCREMENT + seq_id S64 0 # Key fkey (seq_id) ref lapSequence(seq_id) + tess_id STR 64 + projection_cell STR 64 # Key + filter STR 64 # Key + state STR 64 # Key + label STR 64 # Key + dist_group STR 64 + registered TAI NULL + fault S16 0 # Key + quick_sass_id S64 0 # fkey(quick_sass_id) ref stackAssociation(sass_id) + final_sass_id S64 0 # fkey(final_sass_id) ref stackAssociation(sass_id) +END + +lapExp METADATA + lap_id S64 0 # Primary Key fkey (lap_id) ref lapRun(lap_id) + exp_id S64 0 # Primary Key fkey (exp_id) ref rawExp(exp_id) + chip_id S64 0 # Key fkey (exp_id, chip_id) ref chipRun(exp_id, chip_id) + pair_id S64 0 # Key fkey (pair_id) ref chipRun(chip_id) + private BOOL f + pairwise BOOL f + active BOOL f + data_state STR 64 # Key +END Index: /trunk/ippScripts/Build.PL =================================================================== --- /trunk/ippScripts/Build.PL (revision 31434) +++ /trunk/ippScripts/Build.PL (revision 31435) @@ -117,4 +117,5 @@ scripts/skycell_jpeg.pl scripts/diffphot.pl + scripts/lap_science.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: /trunk/ippScripts/MANIFEST =================================================================== --- /trunk/ippScripts/MANIFEST (revision 31434) +++ /trunk/ippScripts/MANIFEST (revision 31435) @@ -45,3 +45,4 @@ scripts/ipp_cluster_load_monitor.pl scripts/skycell_jpeg.pl +scripts/lap_science.pl t/00_distribution.t Index: /trunk/ippScripts/scripts/diff_skycell.pl =================================================================== --- /trunk/ippScripts/scripts/diff_skycell.pl (revision 31434) +++ /trunk/ippScripts/scripts/diff_skycell.pl (revision 31435) @@ -31,4 +31,5 @@ my $ppSub = can_run('ppSub') or (warn "Can't find ppSub" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $nebInsert = can_run('neb-insert') or (warn "Can't find neb-insert" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -265,4 +266,7 @@ my $do_photom = 1; my $dump_config = 1; +if ($reduction eq 'NOCONVDIFF') { + $do_photom = 0; +} if ($run_state eq 'new') { $configuration = prepare_output("PPSUB.CONFIG", $outroot, 1); @@ -380,4 +384,17 @@ } } + if ($reduction eq 'NOCONVDIFF') { + my $refConv = prepare_output("PPSUB.REF.CONV", $outroot, 0); + my $templateFile = $ipprc->file_resolve($template,0); + $command = "$nebInsert $refConv $templateFile"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform neb-insert: $error_code", $diff_id, $skycell_id, $error_code); + } + } + + } elsif ($run_state eq 'update') { &my_die("Update resulted in poor quality image: $quality", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR); Index: /trunk/ippScripts/scripts/lap_science.pl =================================================================== --- /trunk/ippScripts/scripts/lap_science.pl (revision 31435) +++ /trunk/ippScripts/scripts/lap_science.pl (revision 31435) @@ -0,0 +1,907 @@ +#!/usr/bin/env perl + + +use warnings; +use strict; +use Carp; +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use DateTime; + +# +# Set up +################################################################################ + +my $missing_tools = 0; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); +my $laptool = can_run('laptool') or (warn "Can't find laptool" and $missing_tools = 1); + +my ( $help, $verbose, $debug, $do_nothing); +my ( $camera, $dbname); +my ( $lap_id ); +my ( $chip_mode, $monitor_mode, $cleanup_mode); + +GetOptions( + 'help|h' => \$help, + 'verbose' => \$verbose, + 'debug' => \$debug, + 'do_nothing' => \$do_nothing, + + 'camera=s' => \$camera, + 'dbname=s' => \$dbname, + + 'lap_id=s' => \$lap_id, + + 'chip_mode' => \$chip_mode, + 'monitor_mode' => \$monitor_mode, + 'cleanup_mode' => \$cleanup_mode, + ) or pod2usage ( 2 ); +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Choose a mode: --chip_mode --monitor_mode --cleanup_mode", + -exitval => 3, + ) unless + defined $chip_mode or defined $monitor_mode or defined $cleanup_mode; +pod2usage( + -msg => "--lap_id is required", + -exitval => 3, + ) unless + defined $lap_id; + +unless (defined $dbname) { + $dbname = 'gpc1'; +} + +my $mdcParser = PS::IPP::Metadata::Config->new; + +# Fetch all the information about this run +my %lapRunInfo = get_lapRun_info($lap_id); + +# Run the appropriate mode + +if (defined($chip_mode)) { + unless ($lapRunInfo{state} eq 'new') { + &my_die("Cannot run chip_mode if lapRun.state != new!",$lap_id); + } + my $status = chip_mode($lap_id); + exit $status; +} +if (defined($monitor_mode)) { + unless ($lapRunInfo{state} eq 'run') { + &my_die("Cannot run monitor_mode if lapRun != run!", $lap_id); + } + my $status = monitor_mode($lap_id); + exit $status; +} +if (defined($cleanup_mode)) { + unless (($lapRunInfo{state} eq 'full')|| + ($lapRunInfo{state} eq 'drop')) { + &my_die("Cannot run cleanup_mode if lapRun != done!", $lap_id); + } + my $status = cleanup_mode($lap_id); + exit $status; +} + +# +# Chip +################################################################################ + +sub chip_mode { + my $lap_id = shift; + my $status = queue_chips($lap_id); + + if ($status) { + my $command = "$laptool -updaterun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_state run "; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + } + return($status); +} + +# retrieve the chip_id for the exposure inserted +sub get_chip_id_from_metadata { + my $exp_id = shift; + my $label = shift; + my $data_group = shift; + # This is a puzzler... chiptool doesn't actually return a useful metadata. We'll just scrape it from the database for now. + + my $command = "$chiptool -listrun -exp_id $exp_id -label $label -data_group $data_group"; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -listrun: $error_code", $exp_id, $data_group); + } + my $chips = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from chiptool -listrun", $exp_id, $data_group); + # There should be only one. + my $chip = ${ $chips }[0]; + my $chip_id = 0; + if ($chip) { + $chip_id = $chip->{chip_id}; + } + + return($chip_id); +} + +sub get_lapRun_info { + my $lap_id = shift; + my $command = "$laptool -pendingrun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -pendingrun: $error_code", $lap_id); + } + my $Runs = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -pendingrun", $lap_id); + # There should be only one. + my $Run = ${ $Runs }[0]; + my %info = %{ $Run }; + return(%info); +} + + +# Queue a chipRun for this exposure with the appropriate parameters. +sub remake_this_exposure { + my $exposure = shift; + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + + my $label = $exposure->{label}; + + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + + my $chip_cmd = "$chiptool"; + $chip_cmd .= " -pretend " if defined $debug; + $chip_cmd .= " -dbname $dbname " if defined $dbname; + $chip_cmd .= " -definebyquery -exp_id $exposure->{exp_id} -set_end_stage warp -set_tess_id $exposure->{tess_id} "; + $chip_cmd .= " -set_label $exposure->{label} -set_data_group $data_group "; + $chip_cmd .= " -set_workdir $workdir -set_dist_group $exposure->{dist_group} "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $chip_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -definebyquery: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + + $exposure->{chip_id} = get_chip_id_from_metadata($exposure->{exp_id},$exposure->{label},$data_group); + $exposure->{data_state} = 'new'; + update_this_exposure($exposure); + return($exposure); +} + +# This is the "user level" subroutine. +sub queue_chips { + my $lap_id = shift; + + my $command = "$laptool -pendingexp -lap_id $lap_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -pendingexp: $error_code", $lap_id); + } + + if (@$stdout_buf == 0) { + # Nothing to do. + return(0); + } + + my $exposures = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -pendingexp", $lap_id); + + my $counter = 0; + my %matching = (); + my %indexing = (); + # Determine which exposures need a chipRun queued. + foreach my $exposure (@$exposures) { + # $lap_id = $exposure->{lap_id}; # This should be already known. + my $tess_id = $exposure->{tess_id}; + my $exp_id = $exposure->{exp_id}; + my $chip_id = $exposure->{chip_id}; + my $pair_id = $exposure->{pair_id}; + my $private = $exposure->{private}; + my $pairwise = $exposure->{pairwise}; + my $active = $exposure->{active}; + my $data_state = $exposure->{data_state}; + my $dateobs = $exposure->{dateobs}; + my $object = $exposure->{object}; + my $comment = $exposure->{comment}; + + if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id + if (($pairwise) && !($pair_id)) { + # This is an error. + &my_die("Exposure $exp_id for $lap_id is declared pairwise without a defined pair", $lap_id); + } + $matching{$object}{$comment} = $exp_id; + $indexing{$exp_id} = $counter; + $counter++; + next; + } + else { # We do not already have a chip_id. + # Make a chipRun, and update the exposure. + $exposure = remake_this_exposure($exposure); + + # Save our information for diff pairing. + $matching{$object}{$comment} = $exp_id; + $indexing{$exp_id} = $counter; + $counter++; + } + } + + # Determine which exposures have a pair for diffing. + foreach my $object (keys %matching) { + my @exp_ids_to_diff = (); + foreach my $comment (keys %{ $matching{$object} }) { + push @exp_ids_to_diff, $matching{$object}{$comment}; + } + @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; + + if (( $#exp_ids_to_diff + 1) % 2 != 0) { # We have an odd number of exposures, even after comment filtering + pop(@exp_ids_to_diff); # dump the last entry. + } + + while ($#exp_ids_to_diff > -1) { # Save the pair information to the opposite exposure. + my $exp_id_A = shift(@exp_ids_to_diff); + my $exp_id_B = shift(@exp_ids_to_diff); + + my $exp_A = ${ $exposures }[$indexing{$exp_id_A}]; + my $exp_B = ${ $exposures }[$indexing{$exp_id_B}]; + + $exp_A->{pairwise} = 1; + $exp_A->{private} = 0; + $exp_A->{pair_id} = $exp_B->{chip_id}; + + $exp_B->{pairwise} = 1; + $exp_B->{private} = 0; + $exp_B->{pair_id} = $exp_A->{chip_id}; + + if ($verbose) { + print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n"; + } + } + } + + # Scan all exposures, and ensure that pairwise and private are set correctly + foreach my $exposure (@$exposures) { + if ($exposure->{pairwise} && !($exposure->{pair_id})) { + $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. + } + if (!($exposure->{pairwise})) { + $exposure->{private} = 1; # If this isn't being pairwise diffed, it needs to be private + } + + $exposure = update_this_exposure($exposure); + } + return(1); +} + + +# +# Decide Things +################################################################################ +sub monitor_mode { + my $lap_id = shift; + return(check_status($lap_id)); +} + + +sub check_stack_status { # look at lapRun associated stacks, and confirm their states. + my $lap_id = shift; + + my ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack); + + my $command = "$laptool -stacks -lap_id $lap_id"; + $command .= " -dbname $dbname" if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -stacks: $error_code", $lap_id, ""); + } + + ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack) = (0,0,0,0); + if (@$stdout_buf == 0) { + # Nothing to do. Possibly an error, but for now, just accept it + return($defined_qstack,$have_qstack,$defined_fstack,$have_fstack); + } + + my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -stacks", $lap_id, ""); + + my $total_qstacks = 0; + my $complete_qstacks = 0; + my $total_fstacks = 0; + my $complete_fstacks = 0; + foreach my $stack (@$stacks) { + if (($stack->{quick_stack_id})&& + (S64_IS_NOT_NULL($stack->{quick_stack_id}))) { + $defined_qstack = 1; + $total_qstacks++; + } + if (($stack->{final_stack_id})&& + (S64_IS_NOT_NULL($stack->{final_stack_id}))) { + $defined_fstack = 1; + $total_fstacks++; + } + + if (($stack->{quick_state})&& + ($stack->{quick_state} eq 'full')) { + $complete_qstacks++; + } + elsif (($stack->{quick_state})&&($stack->{quick_state} eq 'new')&& + ($stack->{quick_fault} >= 4)&&($stack->{quick_fault} != 32767)) { + printf STDERR "Faulted quick stack: $stack->{quick_stack_id} $stack->{quick_fault}\n"; + $complete_qstacks++;# This is not the best solution, but if they continually fail, there's not much we can do. + } + if (($stack->{final_state})&& + ($stack->{final_state} eq 'full')) { + $complete_fstacks++; + } + elsif (($stack->{final_state})&&($stack->{final_state} eq 'new')&& + ($stack->{final_fault} >= 4)&&($stack->{final_fault} != 32767)) { + printf STDERR "Faulted final stack: $stack->{final_stack_id} $stack->{final_fault}\n"; + $complete_fstacks++; # This is not the best solution, but if they continually fail, there's not much we can do. + } + + } + if (($complete_qstacks > 0)&& + ($complete_qstacks == $total_qstacks)) { + $have_qstack = 1; + } + if (($complete_fstacks > 0)&& + ($complete_fstacks == $total_fstacks)) { + $have_fstack = 1; + } + + + return($defined_qstack,$have_qstack,$defined_fstack,$have_fstack); +} +sub check_status { + my $lap_id = shift; + + my ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack) = check_stack_status($lap_id); + + my $command = "$laptool -exposures -lap_id $lap_id"; + $command .= " -dbname $dbname" if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -exposures: $error_code", $lap_id); + } + + if (@$stdout_buf == 0) { + # Nothing to do. However, this is likely an error. + return(0); + } + + my $exposures = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -exposures", $lap_id); + + # Need to prescan to see who matches whom. + my %match_hash = (); + my $index = 0; + foreach my $exposure (@$exposures) { + if ($exposure->{pair_id}) { + $match_hash{$exposure->{pair_id}} = $index; # Tell the companion who we are, so they can find us next time. + } + $index++; + } + + # Things I want to know before I'm through + my $needs_qstack = 0; + my $needs_something_remade = 0; + my $needs_something_private = 0; + my $can_qstack = 0; + my $have_diff = 0; + my $can_diff = 0; + my $can_fstack = 0; + my $total_exposures = 0; + foreach my $exposure (@$exposures) { + $total_exposures++; + my $companion; + + if ($exposure->{pair_id}) { # Load companion exposure information + if (exists($match_hash{$exposure->{chip_id}})) { + $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; # Match! + } + } + + if ($exposure->{private}) { # I've declared this exposure private to this lapRun. + $needs_qstack = 1; + } + + if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun + $needs_something_remade = 1; + $exposure = remake_this_exposure($exposure); + } + if ($exposure->{cam_quality}) { + $needs_qstack = 1; + $needs_something_private = 1; + if ($companion) { + $companion->{private} = 1; + $companion->{pairwise} = 0; + &update_this_exposure($companion); + } + $exposure->{private} = 1; + $exposure->{pairwise} = 0; + $exposure->{data_state} = 'drop'; + &update_this_exposure($exposure); + + } +# if ($companion) { # Validate that there are no problems with the companion exposure +# if ($companion->{cam_quality}) { # Maybe other things here? +# $exposure->{private} = 1; +# $exposure->{data_state} = 'drop'; +# &update_this_exposure($exposure); +# $needs_qstack = 1; +# } +# } + if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. + $can_qstack ++; + $can_diff ++; + $have_diff ++; + $can_fstack ++; + next; + } + + if (($exposure->{warpRun_state})&& # This exposure has a warp + ($exposure->{warpRun_state} eq 'full')) { # This exposure's warp is done. + $can_qstack ++; + $can_diff ++; + } + if (($exposure->{magicked}&& + &S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. + $can_fstack ++; + } + if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) { + $have_diff ++; + } + } + + + print "STATUS: DEFINED_QSTACK: $defined_qstack\n"; + print "STATUS: HAVE_QSTACK: $have_qstack\n"; + print "STATUS: DEFINED_FSTACK: $defined_fstack\n"; + print "STATUS: HAVE_FSTACK: $have_fstack\n"; + + print "STATUS: NEEDS_QSTACK: $needs_qstack\n"; + print "STATUS: NEEDS_REMADE: $needs_something_remade\n"; + print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n"; + print "STATUS: CAN_QSTACK: $can_qstack\n"; + print "STATUS: CAN_DIFF: $can_diff\n"; + print "STATUS: HAVE_DIFF: $have_diff\n"; + print "STATUS: CAN_FSTACK: $can_fstack\n"; + + print "STATUS: TOTAL_EXPOSURES: $total_exposures\n"; + if ($do_nothing) { + exit(0); + } + if (($needs_qstack == 1)&& # Do we need the quick stack? + ($defined_qstack == 0)&& # Have we not made it already? + ($can_qstack == $total_exposures)&& # Are all warps done? + ($needs_something_remade == 0) # Do we need to wait for a stolen exposure? + ) { + print "STATUS: Will now queue quickstacks\n"; + &queue_quickstack($exposures); + } + + if (($can_diff == $total_exposures)&& # Are all warps done? + ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? + ($have_diff < $can_diff)&& # We haven't already done them all. + (($needs_qstack == 0)|| # Are we just doing pairwise diffs? + ($needs_qstack == 1)&&($have_qstack == 1)) ) { # If we need a quickstack, do we have it? + print "STATUS: Will now queue diffs\n"; + &queue_diffs($exposures); + } + + if (($can_fstack == $total_exposures)&& # Are all warps done? + ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? + ($defined_fstack == 0)) { # Have we not made it already? + print "STATUS: Will now queue final stacks\n"; + &queue_finalstack($exposures); + } + + if (($have_fstack == 1)) { # Are we all done making the final product? + print "STATUS: Will now deactivate exposures for this complete lapRun.\n"; + &deactivate_exposures($exposures); + $command = "$laptool -updaterun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_state full "; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + } + +} + +sub queue_quickstack { + my $exposures = shift; # reference to exposure array; + my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information + + my $lap_id = $exposure->{lap_id}; + my $label = $exposure->{label}; + my $filter = $exposure->{filter}; + my $proj_cell = $exposure->{projection_cell}; + + unless (defined($label) && defined($filter) && defined($proj_cell)) { + &my_die("Unable to perform stacktool. Insufficient information.", $lap_id); + } + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + my $command = "$stacktool "; + $command .= " -pretend " if defined $debug; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; + $command .= " -set_label ${label} -set_data_group ${proj_cell}.quick.${date} "; + $command .= " -set_workdir $workdir -set_dist_group NODIST "; + $command .= " -min_num 2 -set_reduction QUICKSTACK "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); + } + + $command = "$stacktool "; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -sassskyfile -data_group ${proj_cell}.quick.${date} "; + $command .= " -filter $filter -projection_cell ${proj_cell} "; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -sassskyfile: $error_code", $lap_id); + } + + my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stacktool -sassskyfile", $lap_id, ""); + + my $stack = ${ $stacks }[0]; + my $sass_id = $stack->{sass_id}; + unless (defined($sass_id)) { + &my_die("Unable to parse metadata from stacktool for sass_id", $lap_id, ""); + } + + print "QUICK_SASS_ID: $sass_id\n"; + $command = "$laptool -updaterun -lap_id $lap_id -set_quick_sass_id $sass_id"; + $command .= " -dbname $dbname " if defined $dbname; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + + +} +sub queue_finalstack { + my $exposures = shift; # reference to exposure array; + my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information + + my $lap_id = $exposure->{lap_id}; + my $label = $exposure->{label}; + my $filter = $exposure->{filter}; + my $proj_cell = $exposure->{projection_cell}; + + unless (defined($label) && defined($filter) && defined($proj_cell)) { + &my_die("Unable to perform stacktool. Insufficient information.", $lap_id); + } + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + my $command = "$stacktool "; + $command .= " -pretend " if defined $debug; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; + $command .= " -set_label ${label} -set_workdir $workdir -set_data_group ${proj_cell}.final.${date} "; + $command .= " -min_num 2 -set_reduction THREEPI_STACK "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); + } + + + $command = "$stacktool "; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -sassskyfile -data_group ${proj_cell}.final.${date} "; + $command .= " -filter $filter -projection_cell ${proj_cell} "; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -sassskyfile: $error_code", $lap_id); + } + + my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stacktool -sassskyfile", $lap_id, ""); + + my $stack = ${ $stacks }[0]; + my $sass_id = $stack->{sass_id}; + unless (defined($sass_id)) { + &my_die("Unable to parse metadata from stacktool for sass_id", $lap_id, ""); + } + print "FINAL_SASS_ID: $sass_id\n"; + $command = "$laptool -updaterun -lap_id $lap_id -set_final_sass_id $sass_id"; + $command .= " -dbname $dbname " if defined $dbname; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + +} +sub queue_diffs { + my $exposures = shift; + + # Need to prescan to see who matches whom. + my %match_hash = (); + my $index = 0; + foreach my $exposure (@$exposures) { + if ($exposure->{pair_id}) { + $match_hash{$exposure->{pair_id}} = $index; # Tell the companion who we are, so they can find us next time. + } + $index++; + } + + my %already_queued = (); + foreach my $exposure (@$exposures) { + if ($exposure->{data_state} eq 'drop') { # Kick out unusable exposures + next; + } + if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # Not sure how this would happen, but still. ## This happens when we inherit a complete exposure. + next; + } + if ($already_queued{$exposure->{warp_id}}) { + print "STATUS: Have already queued a diff containing $exposure->{exp_id} $exposure->{chip_id} $exposure->{warp_id}\n"; + next; + } + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + + my $label = $exposure->{label}; + my $dist_group = $exposure->{dist_group}; + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + + my $command = "$difftool "; + $command .= " -pretend " if defined $debug; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_label $label -set_workdir $workdir -set_data_group $data_group "; + if ($exposure->{dist_group}) { + $command .= " -set_dist_group $exposure->{dist_group} "; + } + + if ($exposure->{pairwise}) { # warpwarp + my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; + $command .= " -definewarpwarp "; + $command .= "-input_label $label -template_label $label -backwards "; + $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} "; + $already_queued{$exposure->{warp_id}} = 1; + $already_queued{$companion->{warp_id}} = 1; + + } + else { # warp-qstack + $command .= " -definewarpstack -set_reduction NOCONVDIFF -available -good_frac 0.2 "; + $command .= " -warp_id $exposure->{warp_id} -stack_label ${label}.quick "; + $already_queued{$exposure->{warp_id}} = 1; + } + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform difftool -definewarp(warp|stack): $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + + my $diffs = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from difftool -definewarp(warp|stack)", $lap_id, ""); + + my $diff = ${ $diffs }[0]; + my $diff_id = $diff->{diff_id}; + unless (defined($diff_id)) { + $exposure->{data_state} = 'drop'; + &update_this_exposure($exposure); + } + + } +} + +# Deactivate all exposures in this run. +sub deactivate_exposures { + my $exposures = shift; + foreach my $exposure (@$exposures) { + $exposure->{active} = 0; + update_this_exposure($exposure); + } +} + + +# +# Cleanup +################################################################################ + +sub cleanup_mode { + my $lap_id = shift; + my $command = "$laptool -inactiveexp -lap_id $lap_id "; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -inactiveexp: $error_code", "none", "none"); + } + if (@$stdout_buf == 0) { + # Nothing to do. + return(0); + } + + my $exposures = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -inactiveexp", $lap_id); + + my @clean_modes = ( + 'chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -chip_id @CHIP_ID@', + 'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@', + 'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@', + 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage chip -stage_id @CHIP_ID@', + 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage warp -stage_id @WARP_ID@', + 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage diff -stage_id @DIFF_ID@'); + foreach my $exposure (@$exposures) { + if ($exposure->{is_in_use}) { + next; + } + foreach my $clean_mode (@clean_modes) { + my $command = $clean_mode; + $command =~ s/\@DBNAME\@/$dbname/g; + $command =~ s/\@CHIP_ID\@/$exposure->{chip_id}/; + $command =~ s/\@WARP_ID\@/$exposure->{warp_id}/; + $command =~ s/\@DIFF_ID\@/$exposure->{diff_id}/; + $command =~ s/\@LABEL\@/$exposure->{label}/; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform cleantool: $command : $error_code", "none", "none"); + } + } + $exposure->{data_state} = 'cleaned'; + update_this_exposure($exposure); + } +} + + + + +# +# Utilities +################################################################################ + +sub my_die { + my $msg = shift; # Warning message on die + my $lap_id = shift; # identifier + my $optional = shift; + my $exit_code = shift; + unless (defined $exit_code) { + $exit_code = $PS_EXIT_PROG_ERROR; + } + carp($msg); + exit $exit_code; +} + +# Check to see if a 64bit integer is NULL or not. This is probably fragile, but I don't know how to get a S64 NULL out. +sub S64_IS_NOT_NULL { + if ($_[0] == 9223372036854775807) { + return(0); + } + else { + return(1); + } +} + +# Take the current exposure structure, and save it to the database. +sub update_this_exposure { + my $exposure = shift; + + # These are required + my $lap_id = $exposure->{lap_id}; + my $exp_id = $exposure->{exp_id}; + + my $command = "$laptool -updateexp -lap_id $lap_id -exp_id $exp_id "; + if (($exposure->{chip_id})&&(S64_IS_NOT_NULL($exposure->{chip_id}))) { + $command .= " -set_chip_id $exposure->{chip_id} "; + } + if (($exposure->{pair_id})&&(S64_IS_NOT_NULL($exposure->{pair_id}))) { + $command .= " -set_pair_id $exposure->{pair_id} "; + } + + if ($exposure->{private}) { + $command .= " -private "; + } + else { + $command .= " -public "; + } + if ($exposure->{pairwise}) { + $command .= " -pairwise "; + } + else { + $command .= " -nopairwise "; + } + if ($exposure->{active}) { + $command .= " -active "; + } + else { + $command .= " -inactive "; + } + if ($exposure->{data_state}) { + $command .= " -set_data_state $exposure->{data_state} "; + } + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updateexp: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } +} + Index: /trunk/ippScripts/scripts/magic_destreak.pl =================================================================== --- /trunk/ippScripts/scripts/magic_destreak.pl (revision 31434) +++ /trunk/ippScripts/scripts/magic_destreak.pl (revision 31435) @@ -360,5 +360,10 @@ $command .= " -chip_mask $ch_mask" if defined $ch_mask; $command .= " -weight $weight" if defined $weight; - $command .= " -sources $sources" if defined $sources; + if ((defined($sources))&&($ipprc->file_exists($sources))) { + $command .= " -sources $sources" if defined $sources; + } + else { + print "Did not add sources because they do not appear to exist. This may be an error.\n"; + } $command .= " -skycelllist $skycell_list" if defined $skycell_list; $command .= " -replace" if $replace; Index: /trunk/ippTasks/lap.pro =================================================================== --- /trunk/ippTasks/lap.pro (revision 31435) +++ /trunk/ippTasks/lap.pro (revision 31435) @@ -0,0 +1,370 @@ +## lap.pro : -*- sh -*- + +check.globals + +$lapSeq:n = 0 + +book init lapRuns + +macro lap.add.sequence + if ($0 != 2) + echo "USAGE: lap.add.sequence (seq_id)" + break + end + if ($?lapSeq:n == 0) + list lapSeq -add $1 + return + end + + local found + $found = 0 + for i 0 $lapSeq:n + if ($lapSeq:$i == $1) + $found = 1 + echo "$lapSeq:$i set" + last + end + end + + if ($found == 0) + list lapSeq -add $1 + end +end + +macro lap.del.sequence + if ($0 != 2) + echo "USAGE: lap.del.sequence (seq_id)" + break + end + if ($?lapSeq:n == 0) + return + end + + list lapSeq -del $1 +end + + +task lap.initial.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/lap.load.log + + $run = laptool -pendingrun -state new + + if ($lapSeq:n == 0) + break + else + option $lapSeq:$lap_N + $run = $run -seq_id $lapSeq:$lap_N + $lap_N ++ + if ($lap_N >= $lapSeq:n) set lap_N = 0 + end + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$lap_DB + $run = $run -dbname $DB:$lap_DB + $lap_DB ++ + if ($lap_DB >= $DB:n) set lap_DB = 0 + end + + command $run + end + # success + task.exit 0 + book delpage lapNewRuns $options:0 + ipptool2book stdout lapNewRuns -uniq -key proj + + if ($VERBOSE > 2) + book listbook lapNewRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task lap.initial.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 600 +# This can probably be increased and spread over hosts in the future. + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/lap.initial.log + + book npages lapNewRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + + + book getpage lapNewRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword lapNewRuns $pageName pantaskState RUN + book getword lapNewRuns $pageName lap_id -var LAP_ID + book getword lapNewRuns $pageName dbname -var DBNAME + + $run = lap_science.pl --chip_mode --dbname $DBNAME --lap_id $LAP_ID + + command $run + + end + + # success + task.exit 0 + book delpage lapNewRuns $options:0 + ipptool2book stdout lapNewRuns -uniq -key proj + + if ($VERBOSE > 2) + book listbook lapNewRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + + +task lap.monitor.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/lap.load.log + + $run = laptool -pendingrun -state run + + if ($lapSeq:n != 0) + option $lapSeq:$lap_N + $run = $run -seq_id $lapSeq:$lap_N + $lap_N ++ + if ($lap_N >= $lapSeq:n) set lap_N = 0 + end + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$lap_DB + $run = $run -dbname $DB:$lap_DB + $lap_DB ++ + if ($lap_DB >= $DB:n) set lap_DB = 0 + end + + add_poll_labels run + + command $run + end + # success + task.exit 0 + book delpage lapRuRuns $options:0 + ipptool2book stdout lapRunRuns -uniq -key lap_id + + if ($VERBOSE > 2) + book listbook lapRunRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task lap.monitor.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 600 +# This can probably be increased and spread over hosts in the future. + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/lap.initial.log + + book npages lapRunRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + + + book getpage lapNewRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword lapNewRuns $pageName pantaskState RUN + book getword lapNewRuns $pageName lap_id -var LAP_ID + book getword lapNewRuns $pageName dbname -var DBNAME + + $run = lap_science.pl --monitor_mode --dbname $DBNAME --lap_id $LAP_ID + + command $run + + end + + # success + task.exit 0 + book delpage lapRunRuns $options:0 + ipptool2book stdout lapRunRuns -uniq -key proj + + if ($VERBOSE > 2) + book listbook lapRunRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + + +task lap.cleanup.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/lap.load.log + + $run = laptool -pendingrun -state done + + if ($lapSeq:n != 0) + option $lapSeq:$lap_N + $run = $run -seq_id $lapSeq:$lap_N + $lap_N ++ + if ($lap_N >= $lapSeq:n) set lap_N = 0 + end + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$lap_DB + $run = $run -dbname $DB:$lap_DB + $lap_DB ++ + if ($lap_DB >= $DB:n) set lap_DB = 0 + end + + add_poll_labels run + + command $run + end + # success + task.exit 0 + book delpage lapDoneRuns $options:0 + ipptool2book stdout lapDoneRuns -uniq -key lap_id + + if ($VERBOSE > 2) + book listbook lapRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task lap.cleanup.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 600 +# This can probably be increased and spread over hosts in the future. + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/lap.cleanup.log + + book npages lapNewRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + + + book getpage lapNewRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword lapDoneRuns $pageName pantaskState RUN + book getword lapDoneRuns $pageName lap_id -var LAP_ID + book getword lapDoneRuns $pageName dbname -var DBNAME + + $run = lap_science.pl --cleanup_mode --dbname $DBNAME --lap_id $LAP_ID + + command $run + + end + + # success + task.exit 0 + book delpage lapDoneRuns $options:0 + ipptool2book stdout lapDoneRuns -uniq -key proj + + if ($VERBOSE > 2) + book listbook lapDoneRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 31434) +++ /trunk/ippTools/share/Makefile.am (revision 31435) @@ -414,3 +414,12 @@ diffphottool_advance.sql \ diffphottool_revert.sql \ - diffphottool_data.sql + diffphottool_data.sql \ + laptool_definerun.sql \ + laptool_exposures.sql \ + laptool_inactiveexp.sql \ + laptool_listsequence.sql \ + laptool_pendingchipexp.sql \ + laptool_pendingexp.sql \ + laptool_pendingrun.sql \ + laptool_stacks.sql + Index: /trunk/ippTools/share/laptool_definerun.sql =================================================================== --- /trunk/ippTools/share/laptool_definerun.sql (revision 31435) +++ /trunk/ippTools/share/laptool_definerun.sql (revision 31435) @@ -0,0 +1,16 @@ +SELECT want.exp_id, have.chip_id, false as private, true as active, false as pairwise + FROM + (SELECT exp_id FROM rawExp + WHERE rawExp.exp_type= 'OBJECT' AND + rawExp.dateobs >= '2009-04-01T00:00:00.000000' AND +-- Position restriction goes here. + @WHERE@ + ) AS want + LEFT JOIN + (SELECT * + FROM lapExp + where private IS FALSE + AND chip_id IS NOT NULL + AND active = TRUE ) AS have USING(exp_id) + + Index: /trunk/ippTools/share/laptool_exposures.sql =================================================================== --- /trunk/ippTools/share/laptool_exposures.sql (revision 31435) +++ /trunk/ippTools/share/laptool_exposures.sql (revision 31435) @@ -0,0 +1,43 @@ +SELECT DISTINCT + D.*,diffRun.state, + coalesce(CONVERT(sum(others.private),SIGNED),0) AS needs_remade +-- 0 AS needs_remade + FROM ( + SELECT DISTINCT + W.*,CONVERT(IFNULL(diff1.diff_id,diff2.diff_id),SIGNED) AS diff_id FROM ( + SELECT DISTINCT + lap_id,lapRun.tess_id,projection_cell,filter,lapRun.state as lapRun_state, lapRun.registered, lapRun.fault, lapRun.label, lapRun.dist_group, + lapExp.exp_id,lapExp.chip_id,lapExp.pair_id,private,pairwise,active,lapExp.data_state, + chipRun.state as chipRun_state, + coalesce(CONVERT(sum(chipProcessedImfile.fault),SIGNED),0) as chip_faults, + coalesce(CONVERT(sum(chipProcessedImfile.quality),SIGNED),0) as chip_quality, + camRun.cam_id, camRun.state as camRun_state, + coalesce(CONVERT(sum(camProcessedExp.fault),SIGNED),0) AS cam_faults, + coalesce(CONVERT(sum(camProcessedExp.quality),SIGNED),0) AS cam_quality, + fakeRun.fake_id, fakeRun.state as fakeRun_state, + coalesce(CONVERT(sum(fakeProcessedImfile.fault),SIGNED),0) as fake_faults, + warpRun.warp_id, warpRun.state as warpRun_state, + coalesce(CONVERT(sum(warpSkyfile.fault),SIGNED),0) as warp_faults, + coalesce(CONVERT(sum(warpSkyfile.quality),SIGNED),0) as warp_quality, + warpRun.magicked + FROM lapRun JOIN lapExp USING(lap_id) + LEFT JOIN chipRun USING(chip_id) + LEFT JOIN chipProcessedImfile USING(chip_id) + LEFT JOIN camRun USING(chip_id) LEFT JOIN camProcessedExp USING(cam_id) + LEFT JOIN fakeRun USING(cam_id) LEFT JOIN fakeProcessedImfile USING(fake_id) + LEFT JOIN warpRun USING(fake_id) LEFT JOIN warpSkyfile USING(warp_id) + WHERE @WHERE@ + AND (warpSkyfile.quality IS NULL OR + (warpSkyfile.quality != 8007 -- known cases where quality != 0, but everything's fine. + AND warpSkyfile.quality != 3006 -- known cases where quality != 0, but everything's fine. + )) + GROUP BY lap_id,exp_id + ) AS W +-- This was unreasonably slow in testing, so that's why I'm using a subquery here. + LEFT JOIN diffInputSkyfile AS diff1 ON (W.warp_id = diff1.warp1) + LEFT JOIN diffInputSkyfile AS diff2 ON (W.warp_id = diff2.warp2) +) AS D +LEFT JOIN diffRun USING(diff_id) +LEFT JOIN lapExp AS others ON (D.chip_id = others.chip_id AND D.lap_id != others.lap_id) +GROUP BY lap_id,exp_id + Index: /trunk/ippTools/share/laptool_inactiveexp.sql =================================================================== --- /trunk/ippTools/share/laptool_inactiveexp.sql (revision 31435) +++ /trunk/ippTools/share/laptool_inactiveexp.sql (revision 31435) @@ -0,0 +1,41 @@ +SELECT DISTINCT + D.*,diffRun.state AS diff_state,coalesce(CONVERT(sum(others.active),SIGNED),0) AS is_in_use FROM ( + SELECT DISTINCT + W.*,IFNULL(diff1.diff_id,diff2.diff_id) AS diff_id FROM ( + SELECT DISTINCT + lapRun.lap_id, lapRun.seq_id, lapRun.tess_id, lapRun.projection_cell, lapRun.filter, lapRun.state, lapRun.label, + lapRun.dist_group, lapRun.registered, lapRun.fault, lapRun.quick_sass_id, lapRun.final_sass_id, + lapExp.exp_id,lapExp.chip_id,lapExp.pair_id,private,pairwise,active,lapExp.data_state, + chipRun.state as chipRun_state, + CONVERT(sum(chipProcessedImfile.fault),SIGNED) as chip_faults, + CONVERT(sum(chipProcessedImfile.quality),SIGNED) as chip_quality, + camRun.cam_id, camRun.state as camRun_state, + CONVERT(sum(camProcessedExp.fault),SIGNED) AS cam_faults, + CONVERT(sum(camProcessedExp.quality),SIGNED) AS cam_quality, + fakeRun.fake_id, fakeRun.state as fakeRun_state, + CONVERT(sum(fakeProcessedImfile.fault),SIGNED) as fake_faults, + warpRun.warp_id, warpRun.state as warpRun_state, + CONVERT(sum(warpSkyfile.fault),SIGNED) as warp_faults, + CONVERT(sum(warpSkyfile.quality),SIGNED) as warp_quality, + warpRun.magicked + FROM lapRun JOIN lapExp USING(lap_id) + LEFT JOIN chipRun USING(chip_id) + LEFT JOIN chipProcessedImfile USING(chip_id) + LEFT JOIN camRun USING(chip_id) LEFT JOIN camProcessedExp USING(cam_id) + LEFT JOIN fakeRun USING(cam_id) LEFT JOIN fakeProcessedImfile USING(fake_id) + LEFT JOIN warpRun USING(fake_id) LEFT JOIN warpSkyfile USING(warp_id) + WHERE lapExp.active = FALSE + AND @WHERE@ + AND (warpSkyfile.quality IS NULL OR + (warpSkyfile.quality != 8007 -- known cases where quality != 0, but everything's fine. + AND warpSkyfile.quality != 3006 -- known cases where quality != 0, but everything's fine. + )) + GROUP BY lap_id,exp_id + ) AS W +-- This was unreasonably slow in testing, so that's why I'm using a subquery here. + LEFT JOIN diffInputSkyfile AS diff1 ON (W.warp_id = diff1.warp1) + LEFT JOIN diffInputSkyfile AS diff2 ON (W.warp_id = diff2.warp2) +) AS D +LEFT JOIN diffRun USING(diff_id) +LEFT JOIN lapExp AS others ON (D.chip_id = others.chip_id AND D.lap_id != others.lap_id) +GROUP BY lap_id,exp_id Index: /trunk/ippTools/share/laptool_listsequence.sql =================================================================== --- /trunk/ippTools/share/laptool_listsequence.sql (revision 31435) +++ /trunk/ippTools/share/laptool_listsequence.sql (revision 31435) @@ -0,0 +1,1 @@ +select * from lapSequence Index: /trunk/ippTools/share/laptool_pendingchipexp.sql =================================================================== --- /trunk/ippTools/share/laptool_pendingchipexp.sql (revision 31435) +++ /trunk/ippTools/share/laptool_pendingchipexp.sql (revision 31435) @@ -0,0 +1,11 @@ +SELECT re_id,projection_cell,registered,reprocRun.state as reproc_state, tess_id, + exp_id,private,active, + chip_id,chipRun.state AS chip_state, + cam_id,camRun.state AS cam_state, + fake_id,fakeRun.state AS fake_state, + warp_id,warpRun.state AS warp_state + FROM reprocRun JOIN reprocExp USING(re_id) + LEFT JOIN chipRun USING(chip_id,tess_id) + LEFT JOIN camRun USING(chip_id,tess_id) + LEFT JOIN fakeRun USING(fake_id,tess_id) + LEFT JOIN warpRun USING(fake_id,tess_id) Index: /trunk/ippTools/share/laptool_pendingexp.sql =================================================================== --- /trunk/ippTools/share/laptool_pendingexp.sql (revision 31435) +++ /trunk/ippTools/share/laptool_pendingexp.sql (revision 31435) @@ -0,0 +1,10 @@ +select lapRun.lap_id, lapRun.seq_id, lapRun.tess_id, lapRun.projection_cell, lapRun.filter, + lapRun.state, lapRun.label, + lapRun.dist_group, lapRun.registered, lapRun.fault, lapRun.quick_sass_id, lapRun.final_sass_id, + exp_id, chip_id, pair_id, private, pairwise, active, lapExp.data_state, + dateobs, object, comment + FROM lapRun JOIN lapExp USING(lap_id) + JOIN rawExp USING(exp_id,filter) +WHERE active IS TRUE AND lapRun.fault = 0 +-- lap_id restriction here. +-- This probably needs to be sorted by dateobs. Index: /trunk/ippTools/share/laptool_pendingrun.sql =================================================================== --- /trunk/ippTools/share/laptool_pendingrun.sql (revision 31435) +++ /trunk/ippTools/share/laptool_pendingrun.sql (revision 31435) @@ -0,0 +1,1 @@ +SELECT * from lapRun Index: /trunk/ippTools/share/laptool_stacks.sql =================================================================== --- /trunk/ippTools/share/laptool_stacks.sql (revision 31435) +++ /trunk/ippTools/share/laptool_stacks.sql (revision 31435) @@ -0,0 +1,103 @@ +SELECT + lapRun.lap_id,lapRun.seq_id,lapRun.tess_id,lapRun.projection_cell,lapRun.filter, + lapRun.state,lapRun.label,lapRun.dist_group,lapRun.registered,lapRun.fault, + skycell_id, + lapRun.quick_sass_id,quick_data_group,quick_stack_id,quick_state,quick_fault,quick_quality, + lapRun.final_sass_id,final_data_group,final_stack_id,final_state,final_fault,final_quality + FROM + lapRun LEFT JOIN + ( +SELECT DISTINCT + lap_id,tess_id,projection_cell,filter,skycell_id, + quick_sass_id,quick_data_group,quick_stack_id,quick_state,quick_fault,quick_quality, + final_sass_id,final_data_group,final_stack_id,final_state,final_fault,final_quality + FROM + (select lapRun.lap_id, + stackAssociation.sass_id AS quick_sass_id, + stackAssociation.data_group AS quick_data_group, + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssociation.filter, + stackRun.stack_id AS quick_stack_id, + stackRun.skycell_id, + stackRun.state AS quick_state, + stackSumSkyfile.fault AS quick_fault, + stackSumSkyfile.quality AS quick_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.quick_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + WHERE 1 @WHERE@ + AND sass_id = lapRun.quick_sass_id + ) AS quick LEFT JOIN + (select lapRun.lap_id, + stackAssociation.sass_id AS final_sass_id, + stackAssociation.data_group AS final_data_group, + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssociation.filter, + stackRun.stack_id AS final_stack_id, + stackRun.skycell_id, + stackRun.state AS final_state, + stackSumSkyfile.fault AS final_fault, + stackSumSkyfile.quality AS final_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.final_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + WHERE 1 @WHERE@ + AND sass_id = lapRun.final_sass_id + ) AS final USING(lap_id,projection_cell,tess_id,filter,skycell_id) + UNION +SELECT DISTINCT + lap_id,tess_id,projection_cell,filter,skycell_id, + quick_sass_id,quick_data_group,quick_stack_id,quick_state,quick_fault,quick_quality, + final_sass_id,final_data_group,final_stack_id,final_state,final_fault,final_quality + FROM + (select lapRun.lap_id, + stackAssociation.sass_id AS quick_sass_id, + stackAssociation.data_group AS quick_data_group, + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssociation.filter, + stackRun.stack_id AS quick_stack_id, + stackRun.skycell_id, + stackRun.state AS quick_state, + stackSumSkyfile.fault AS quick_fault, + stackSumSkyfile.quality AS quick_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.quick_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + WHERE 1 @WHERE@ + AND sass_id = lapRun.quick_sass_id + ) AS quick RIGHT JOIN + (select lapRun.lap_id, + stackAssociation.sass_id AS final_sass_id, + stackAssociation.data_group AS final_data_group, + stackAssociation.projection_cell, + stackAssociation.tess_id, + stackAssociation.filter, + stackRun.stack_id AS final_stack_id, + stackRun.skycell_id, + stackRun.state AS final_state, + stackSumSkyfile.fault AS final_fault, + stackSumSkyfile.quality AS final_quality FROM + lapRun + LEFT JOIN stackAssociation ON (lapRun.final_sass_id = stackAssociation.sass_id) + LEFT JOIN stackAssociationMap USING (sass_id) + LEFT JOIN stackRun USING(stack_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + WHERE 1 @WHERE@ + AND sass_id = lapRun.final_sass_id + ) AS final USING(lap_id,projection_cell,tess_id,filter,skycell_id) + ) stacks ON (stacks.lap_id = lapRun.lap_id AND + stacks.projection_cell = lapRun.projection_cell AND + stacks.tess_id = lapRun.tess_id AND + stacks.filter = lapRun.filter AND + (lapRun.quick_sass_id IS NULL OR lapRun.quick_sass_id = stacks.quick_sass_id) AND + (lapRun.final_sass_id IS NULL OR lapRun.final_sass_id = stacks.final_sass_id)) +WHERE 1 @WHERE@ Index: /trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_create_tables.sql (revision 31434) +++ /trunk/ippTools/share/pxadmin_create_tables.sql (revision 31435) @@ -1891,4 +1891,59 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +-- Tables for large area processing + +CREATE TABLE lapSequence ( + seq_id BIGINT AUTO_INCREMENT, -- Identifier for the processing sequence + name VARCHAR(64) NOT NULL, -- short name of the sequence + description VARCHAR(255) NOT NULL, -- longer description of the sequence + PRIMARY KEY(seq_id), + KEY(name) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE lapRun ( + lap_id BIGINT AUTO_INCREMENT, -- Identifier for the processing run + seq_id BIGINT NOT NULL, -- Identifier to match to the sequence + tess_id VARCHAR(64) NOT NULL, -- tessellation id to use + projection_cell VARCHAR(64) NOT NULL, -- projection cell from the tessellation to consider + filter VARCHAR(64) NOT NULL, -- filter + state VARCHAR(64) NOT NULL, -- state of run + label VARCHAR(64) NOT NULL, -- processing label + dist_group VARCHAR(64) NOT NULL, -- distribution group for products of this run + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + fault SMALLINT NOT NULL, -- fault code + quick_sass_id BIGINT, -- stackAssociation id for quick stack + final_sass_id BIGINT, -- stackAssociation id for final stack + PRIMARY KEY(lap_id), + KEY(seq_id), + KEY(projection_cell), + KEY(filter), + KEY(state), + KEY(label), + KEY(fault), + FOREIGN KEY(seq_id) REFERENCES lapSequence(seq_id), + FOREIGN KEY(quick_sass_id) REFERENCES stackAssociation(sass_id), + FOREIGN KEY(final_sass_id) REFERENCES stackAssociation(sass_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE lapExp ( + lap_id BIGINT NOT NULL, -- Link back to processing run + exp_id BIGINT NOT NULL, -- exposure definition + chip_id BIGINT, -- processing id from chipRun + pair_id BIGINT, -- companion chip_id + private TINYINT DEFAULT 0, -- denotes this exposure is private + pairwise TINYINT DEFAULT 0, -- denotes if this exposure should be pairwise diffed + active TINYINT DEFAULT 0, -- denotes if this exposure is currently in use + data_state VARCHAR(64) NOT NULL, -- state of exposure + PRIMARY KEY (lap_id), + KEY (exp_id), + KEY (chip_id), + KEY (pair_id), + KEY (data_state), + FOREIGN KEY (lap_id) REFERENCES lapRun(lap_id), + FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id), + FOREIGN KEY (chip_id,exp_id) REFERENCES chipRun(chip_id,exp_id), + FOREIGN KEY (pair_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + -- This comment line is here to avoid empty query error. -- Another way to avoid that problem is to omit the semicolon above but I think that is untidy. Index: /trunk/ippTools/share/stacktool_sassskyfile.sql =================================================================== --- /trunk/ippTools/share/stacktool_sassskyfile.sql (revision 31434) +++ /trunk/ippTools/share/stacktool_sassskyfile.sql (revision 31435) @@ -18,6 +18,6 @@ where stack_id = stackRun.stack_id limit 1 ) as camera -FROM stackRun -JOIN stackSumSkyfile USING(stack_id) -JOIN stackAssociationMap USING(stack_id) -JOIN stackAssociation USING(sass_id) +FROM stackAssociation +JOIN stackAssociationMap USING(sass_id) +JOIN stackRun USING(stack_id) +LEFT JOIN stackSumSkyfile USING(stack_id) Index: /trunk/ippTools/src/Makefile.am =================================================================== --- /trunk/ippTools/src/Makefile.am (revision 31434) +++ /trunk/ippTools/src/Makefile.am (revision 31435) @@ -28,5 +28,6 @@ pubtool \ diffphottool \ - minidvodbtool + minidvodbtool \ + laptool pkginclude_HEADERS = \ @@ -76,5 +77,6 @@ pubtool.h \ diffphottool.h \ - minidvodbtool.h + minidvodbtool.h \ + laptool.h lib_LTLIBRARIES = libpxtools.la @@ -290,4 +292,10 @@ minidvodbtoolConfig.c +laptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +laptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +laptool_SOURCES = \ + laptool.c \ + laptoolConfig.c + clean-local: -rm -f TAGS Index: /trunk/ippTools/src/difftool.c =================================================================== --- /trunk/ippTools/src/difftool.c (revision 31434) +++ /trunk/ippTools/src/difftool.c (revision 31435) @@ -1625,4 +1625,5 @@ PXOPT_COPY_S64(config->args, selectWhere, "-exp_id", "inputRawExp.exp_id", "=="); PXOPT_COPY_S64(config->args, selectWhere, "-template_exp_id", "templateRawExp.exp_id", "=="); + PXOPT_COPY_S64(config->args, selectWhere, "-template_warp_id", "templateWarpRun.warp_id", "=="); PXOPT_COPY_STR(config->args, selectWhere, "-filter", "inputRawExp.filter", "=="); PXOPT_COPY_STR(config->args, selectWhere, "-obs_mode", "inputRawExp.obs_mode", "=="); Index: /trunk/ippTools/src/difftoolConfig.c =================================================================== --- /trunk/ippTools/src/difftoolConfig.c (revision 31434) +++ /trunk/ippTools/src/difftoolConfig.c (revision 31435) @@ -278,4 +278,5 @@ psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-not-bothways", 0, "only do the single-direction subtraction?", false); psMetadataAddS64(definewarpwarpArgs, PS_LIST_TAIL, "-template_exp_id", 0, "search by template exposure ID", 0); + psMetadataAddS64(definewarpwarpArgs, PS_LIST_TAIL, "-template_warp_id", 0, "search by template warp ID", 0); psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-distance", 0, "limit distance between input and template (deg)", NAN); Index: /trunk/ippTools/src/laptool.c =================================================================== --- /trunk/ippTools/src/laptool.c (revision 31435) +++ /trunk/ippTools/src/laptool.c (revision 31435) @@ -0,0 +1,816 @@ +/* + * laptool.c + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "laptool.h" + + +// Sequence level +static bool definesequenceMode(pxConfig *config); +static bool listsequenceMode(pxConfig *config); + +// Run level +static bool definerunMode(pxConfig *config); +static bool pendingrunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +// Exposure level +static bool pendingexpMode(pxConfig *config); +static bool exposuresMode(pxConfig *config); +static bool stacksMode(pxConfig *config); +static bool updateexpMode(pxConfig *config); + +static bool inactiveexpMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = laptoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(LAPTOOL_MODE_DEFINESEQUENCE, definesequenceMode); + MODECASE(LAPTOOL_MODE_LISTSEQUENCE, listsequenceMode); + + MODECASE(LAPTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(LAPTOOL_MODE_PENDINGRUN, pendingrunMode); + MODECASE(LAPTOOL_MODE_UPDATERUN, updaterunMode); + + MODECASE(LAPTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(LAPTOOL_MODE_EXPOSURES, exposuresMode); + MODECASE(LAPTOOL_MODE_STACKS, stacksMode); + MODECASE(LAPTOOL_MODE_UPDATEEXP, updateexpMode); + + MODECASE(LAPTOOL_MODE_INACTIVEEXP, inactiveexpMode); + default: + psAbort("invalid option (this should not happen)"); + } + psTrace("laptool", 9, "Attempting to free config\n"); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(EXIT_SUCCESS); + + FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +// Sequence level + +static bool definesequenceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); + PXOPT_LOOKUP_STR(description, config->args, "-description", true, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + lapSequenceRow *run = lapSequenceRowAlloc(0, // seq_id + name, + description); + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc lapSequence object"); + return(true); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!lapSequenceInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return(true); + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!lapSequencePrintObject(stdout, run, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psFree(run); + + return true; +} + +static bool listsequenceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-seq_name", "name", "LIKE"); + + psString query = pxDataGet("laptool_listsequence.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapSequence", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +// Run level +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(seq_id, config->args, "-seq_id", true, false); + PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + PXOPT_LOOKUP_F64(ra, config->args, "-ra", true, false); + PXOPT_LOOKUP_F64(decl, config->args, "-decl", true, false); + PXOPT_LOOKUP_F32(radius, config->args, "-radius", true, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); + PXOPT_LOOKUP_BOOL(all_obsmode, config->args, "-all_obsmode", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + lapRunRow *run = lapRunRowAlloc(0, // lap_id + seq_id, + tess_id, + projection_cell, + filter, + "new", // state + label, + dist_group, + NULL, // registered + 0, // fault + INT64_MAX, // quick_sass_id + INT64_MAX // final_sass_id + ); + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc lapRun object"); + return(true); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!lapRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return(true); + } + + + if (!lapRunPrintObject(stdout, run, !simple)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psS64 lap_id = psDBLastInsertID(config->dbh); + + // Find the input exposures + psString query = pxDataGet("laptool_definerun.sql"); + if (!query) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return(false); + } + + // Add constraints to the exposure search: + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + + // This seems unnecessarily clunky. + if (!all_obsmode) { + // fprintf(stderr, "Not doing all obsmodes!\n"); + if (!psMetadataLookupStr(NULL,config->args,"-obsmode")) { + // fprintf(stderr, "Not doing all obsmodes and none specified!\n"); + psMetadataAddStr(where,PS_LIST_TAIL,"rawExp.obs_mode",0,"==","3PI"); + } + pxAddLabelSearchArgs (config, where, "-obsmode", "rawExp.obsmode", "=="); + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (!pxspaceAddWhere(config, &whereClause, "rawExp")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return(false); + } + + if (whereClause) { + psStringSubstitute(&query,whereClause,"@WHERE@"); + } + psFree(where); + + // Fetch exposures + if (!p_psDBRunQuery(config->dbh, query)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + // Insert the exposure data + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + // Add default values from this run: + psMetadataAddS64(row, PS_LIST_TAIL, "lap_id", 0, "", lap_id); + psMetadataAddS64(row, PS_LIST_TAIL, "pair_id", 0, "", INT64_MAX); + psMetadataAddStr(row, PS_LIST_TAIL, "data_state", 0, "", "new"); + lapExpRow *lapExp = lapExpObjectFromMetadata(row); + lapExp->lap_id = lap_id; + + if (!lapExpInsertObject(config->dbh,lapExp)) { + if (!lapExpPrintObject(stdout, lapExp, !simple)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + psFree(lapExp); + return(false); + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + psFree(output); + return(true); +} + +static bool pendingrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-fault", "fault", "=="); + + psString query = pxDataGet("laptool_pendingrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_S64(lap_id, config->args, "-lap_id", true, false); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); + + psMetadata *values = psMetadataAlloc(); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_COPY_STR(config->args, values, "-set_state", "state", "=="); + PXOPT_COPY_S16(config->args, values, "-fault", "fault", "=="); + PXOPT_COPY_STR(config->args, values, "-set_label", "label", "=="); + PXOPT_COPY_S64(config->args, values, "-set_quick_sass_id", "quick_sass_id", "=="); + PXOPT_COPY_S64(config->args, values, "-set_final_sass_id", "final_sass_id", "=="); + + long rows = psDBUpdateRows(config->dbh, "lapRun", where, values); + psFree(values); + + if (rows) { + // We're done with these exposures now, so mark them as inactive. + if (state) { + if ((strcmp(state,"drop") == 0)|| + (strcmp(state,"full") == 0)) { + values = psMetadataAlloc(); + psMetadataAddBool(values, PS_LIST_TAIL, "active", 0, "", false); + long exps = psDBUpdateRows(config->dbh, "lapExp", where, values); + + if (exps) { + return(true); + } + else { + return(true); // We shouldn't really fail if we didn't change anything. Maybe there's nothing to change. + } + } + } + } + + return(true); +} + +// Exposure level + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); + + psString query = pxDataGet("laptool_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + } + psStringAppend(&query, " ORDER BY rawExp.dateobs "); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n %s", limitString); + psFree(limitString); + } + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool exposuresMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + psString query = pxDataGet("laptool_exposures.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return(false); + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (whereClause) { + psStringSubstitute(&query,whereClause,"@WHERE@"); + } + + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + psStringPrepend(&limitString, "\n"); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + return(false); + } + psFree(limitString); + psFree(query); + psFree(whereClause); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + if (!ippdbPrintMetadatas(stdout, output, "lapExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return(false); + } + + psFree(output); + return(true); +} + +static bool stacksMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_S64(lap_id, config->args, "-lap_id", true, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lapRun.lap_id", "=="); + + psString query = pxDataGet("laptool_stacks.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return(false); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause, "\n AND "); + psStringSubstitute(&query,whereClause,"@WHERE@"); + psFree(whereClause); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error %d",err); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapRunStacks", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return(true); +} + +static bool updateexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_S64(lap_id, config->args, "-lap_id", true, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", false, false); + + PXOPT_LOOKUP_S64(set_chip_id, config->args, "-set_chip_id", false, false); + PXOPT_LOOKUP_S64(set_pair_id, config->args, "-set_pair_id", false, false); + PXOPT_LOOKUP_STR(set_data_state, config->args, "-set_data_state", false, false); + PXOPT_LOOKUP_BOOL(private, config->args, "-private", false); + PXOPT_LOOKUP_BOOL(public, config->args, "-public", false); + PXOPT_LOOKUP_BOOL(pairwise, config->args, "-pairwise", false); + PXOPT_LOOKUP_BOOL(nopairwise, config->args, "-nopairwise", false); + PXOPT_LOOKUP_BOOL(active, config->args, "-active", false); + PXOPT_LOOKUP_BOOL(inactive, config->args, "-inactive", false); + + + if (private && public) { + psError(PS_ERR_UNKNOWN, false, "only one of -private and -public may be selected"); + } + if (active && inactive) { + psError(PS_ERR_UNKNOWN, false, "only one of -active and -inactive may be selected"); + } + if (pairwise && nopairwise) { + psError(PS_ERR_UNKNOWN, false, "only one of -pairwise and -nopairwise may be selected"); + } + + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + + + psMetadata *values = psMetadataAlloc(); + if (set_chip_id) { + PXOPT_COPY_S64(config->args, values, "-set_chip_id", "chip_id", "=="); + } + if (set_pair_id) { + PXOPT_COPY_S64(config->args, values, "-set_pair_id", "pair_id", "=="); + } + if (private) { + psMetadataAddBool(values, PS_LIST_TAIL, "private", 0, "==", true); + } + else if (public) { + psMetadataAddBool(values, PS_LIST_TAIL, "private", 0, "==", false); + } + if (active) { + psMetadataAddBool(values, PS_LIST_TAIL, "active", 0, "==", true); + } + else if (inactive) { + psMetadataAddBool(values, PS_LIST_TAIL, "active", 0, "==", false); + } + if (pairwise) { + psMetadataAddBool(values, PS_LIST_TAIL, "pairwise", 0, "==", true); + } + else if (nopairwise) { + psMetadataAddBool(values, PS_LIST_TAIL, "pairwise", 0, "==", false); + } + if (set_data_state) { + PXOPT_COPY_STR(config->args, values, "-set_data_state", "data_state", "=="); + } + long rows = psDBUpdateRows(config->dbh,"lapExp",where,values); + if (rows) { + return(true); + } +/* else { */ +/* return(false); */ +/* } */ + return(true); +} + + + + +static bool inactiveexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_S64(lap_id, config->args, "-lap_id", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("laptool_inactiveexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return(false); + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (whereClause) { + psStringSubstitute(&query,whereClause,"@WHERE@"); + } + + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + psStringPrepend(&limitString, "\n"); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + return(false); + } + psFree(limitString); + psFree(query); + psFree(whereClause); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + if (!ippdbPrintMetadatas(stdout, output, "lapExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return(false); + } + + psFree(output); + return(true); +} + Index: /trunk/ippTools/src/laptool.h =================================================================== --- /trunk/ippTools/src/laptool.h (revision 31435) +++ /trunk/ippTools/src/laptool.h (revision 31435) @@ -0,0 +1,28 @@ +/* + * laptool.h + */ + +#ifndef LAPTOOL_H +#define LAPTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + LAPTOOL_MODE_NONE = 0x0, + LAPTOOL_MODE_DEFINESEQUENCE, + LAPTOOL_MODE_LISTSEQUENCE, + LAPTOOL_MODE_DEFINERUN, + LAPTOOL_MODE_PENDINGRUN, + LAPTOOL_MODE_UPDATERUN, + LAPTOOL_MODE_PENDINGEXP, + LAPTOOL_MODE_EXPOSURES, + LAPTOOL_MODE_STACKS, + LAPTOOL_MODE_UPDATEEXP, + LAPTOOL_MODE_INACTIVEEXP +} laptoolMode; + +pxConfig *laptoolConfig(pxConfig *config, int argc, char **argv); + +#endif // LAPTOOL_H + + Index: /trunk/ippTools/src/laptoolConfig.c =================================================================== --- /trunk/ippTools/src/laptoolConfig.c (revision 31435) +++ /trunk/ippTools/src/laptoolConfig.c (revision 31435) @@ -0,0 +1,165 @@ +/* + * laptoolConfig.c + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include +#include "pxtools.h" +#include "laptool.h" + +#define ADD_OPT(TYPE,TARG,NAME,COMMENT,DEFAULT) psMetadataAdd##TYPE(TARG, PS_LIST_TAIL, NAME, 0, COMMENT, DEFAULT) + +pxConfig *laptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definesequence + psMetadata *definesequenceArgs = psMetadataAlloc(); + ADD_OPT(Str, definesequenceArgs, "-name", "short name for this LAP sequence (required)", NULL); + ADD_OPT(Str, definesequenceArgs, "-description", "define the description for this LAP sequence (required)", NULL); + ADD_OPT(Bool,definesequenceArgs, "-simple", "use the simple output format", false); + + // -listsequence + psMetadata *listsequenceArgs = psMetadataAlloc(); + ADD_OPT(S64, listsequenceArgs, "-seq_id", "search by LAP sequence ID", 0); + ADD_OPT(Str, listsequenceArgs, "-seq_name", "search by LAP sequence name", 0); + ADD_OPT(Bool,listsequenceArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, listsequenceArgs, "-limit", "limit result set to N items", 0); + + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + ADD_OPT(S64, definerunArgs, "-seq_id", "define the LAP sequence for this run (required)", 0); + ADD_OPT(Str, definerunArgs, "-projection_cell", "define the projection cell for this run (required)", NULL); + ADD_OPT(Str, definerunArgs, "-tess_id", "define the tessellation used (required)", NULL); + ADD_OPT(F64, definerunArgs, "-ra", "define RA center (required)", NAN); + ADD_OPT(F64, definerunArgs, "-decl", "define DEC center (required)", NAN); + ADD_OPT(F32, definerunArgs, "-radius", "define radius from center to consider (required)", NAN); + ADD_OPT(Str, definerunArgs, "-filter", "define the filter used (required)", NULL); + ADD_OPT(Str, definerunArgs, "-label", "define the label used", NULL); + ADD_OPT(Str, definerunArgs, "-dist_group", "define the distribution group for this data", NULL); + + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-obsmode", PS_META_DUPLICATE_OK, "search by obsmode", NULL); + ADD_OPT(Bool,definerunArgs, "-all_obsmode", "use all science obsmodes", false); + + ADD_OPT(Bool,definerunArgs, "-simple", "use the simple output format", false); + + // -pendingrun + psMetadata *pendingrunArgs = psMetadataAlloc(); + ADD_OPT(S64, pendingrunArgs, "-seq_id", "search by LAP sequence ID", 0); + ADD_OPT(S64, pendingrunArgs, "-lap_id", "search by LAP run ID", 0); + ADD_OPT(Str, pendingrunArgs, "-projection_cell", "search by projection cell", NULL); + ADD_OPT(Str, pendingrunArgs, "-filter", "search by filter", NULL); + ADD_OPT(Str, pendingrunArgs, "-label", "search by LAP run label", NULL); + ADD_OPT(Str, pendingrunArgs, "-state", "search by LAP run state", NULL); + ADD_OPT(Str, pendingrunArgs, "-fault", "search by LAP run fault", NULL); + ADD_OPT(Bool,pendingrunArgs, "-simple", "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + ADD_OPT(S64, updaterunArgs, "-lap_id", "search by lap run ID", 0); + ADD_OPT(Str, updaterunArgs, "-set_state", "set state", NULL); + ADD_OPT(S16, updaterunArgs, "-fault", "set fault code", INT16_MAX); + ADD_OPT(Str, updaterunArgs, "-set_label", "set label", NULL); + ADD_OPT(S64, updaterunArgs, "-set_quick_sass_id", "set quick stack sass_id", 0); + ADD_OPT(S64, updaterunArgs, "-set_final_sass_id", "set final stack sass_id", 0); + + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + ADD_OPT(S64, pendingexpArgs, "-lap_id", "lap run ID", 0); + ADD_OPT(Str, pendingexpArgs, "-projection_cell", "projection cell to consider", NULL); + ADD_OPT(U64, pendingexpArgs, "-limit", "limit result set to N items", 0); + ADD_OPT(Bool,pendingexpArgs, "-simple", "use the simple output format", false); + + // -exposures + psMetadata *exposuresArgs = psMetadataAlloc(); + ADD_OPT(S64, exposuresArgs, "-lap_id", "search by lap run ID", 0); + ADD_OPT(S64, exposuresArgs, "-exp_id", "search by exp_id", 0); + ADD_OPT(Bool,exposuresArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, exposuresArgs, "-limit", "limit result set to N items", 0); + + // -stacks + psMetadata *stacksArgs = psMetadataAlloc(); + ADD_OPT(S64, stacksArgs, "-lap_id", "search by lap run ID", 0); + ADD_OPT(Bool,stacksArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, stacksArgs, "-limit", "limit result set to N items", 0); + + // -updateexp + psMetadata *updateexpArgs = psMetadataAlloc(); + ADD_OPT(S64, updateexpArgs, "-lap_id", "search by lap run ID", 0); + ADD_OPT(S64, updateexpArgs, "-exp_id", "search by exposure ID", 0); + ADD_OPT(S64, updateexpArgs, "-chip_id", "search by chip ID", 0); + ADD_OPT(S64, updateexpArgs, "-set_chip_id", "set the chip ID", 0); + ADD_OPT(S64, updateexpArgs, "-set_pair_id", "set the pair ID", 0); + ADD_OPT(Str, updateexpArgs, "-set_data_state", "set the lapExp data_state", NULL); + ADD_OPT(Bool,updateexpArgs, "-private", "set this exposure as private", 0); + ADD_OPT(Bool,updateexpArgs, "-public", "set this exposure as public", 0); + ADD_OPT(Bool,updateexpArgs, "-pairwise", "set this exposure to be pairwise", 0); + ADD_OPT(Bool,updateexpArgs, "-nopairwise", "set this exposure to not be pairwise", 0); + ADD_OPT(Bool,updateexpArgs, "-active", "set this exposure to active", 0); + ADD_OPT(Bool,updateexpArgs, "-inactive", "set this exposure to active", 0); + + // -inactiveexp + psMetadata *inactiveexpArgs = psMetadataAlloc(); + ADD_OPT(S64, inactiveexpArgs, "-lap_id", "search by lap run ID", 0); + ADD_OPT(Bool,inactiveexpArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, inactiveexpArgs, "-limit", "limit result set to N items", 0); + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definesequence", "", LAPTOOL_MODE_DEFINESEQUENCE, definesequenceArgs); + PXOPT_ADD_MODE("-listsequence", "", LAPTOOL_MODE_LISTSEQUENCE, listsequenceArgs); + PXOPT_ADD_MODE("-definerun", "", LAPTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-pendingrun", "", LAPTOOL_MODE_PENDINGRUN, pendingrunArgs); + PXOPT_ADD_MODE("-updaterun", "", LAPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "", LAPTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-exposures", "", LAPTOOL_MODE_EXPOSURES, exposuresArgs); + PXOPT_ADD_MODE("-stacks", "", LAPTOOL_MODE_STACKS, stacksArgs); + PXOPT_ADD_MODE("-updateexp", "", LAPTOOL_MODE_UPDATEEXP, updateexpArgs); + PXOPT_ADD_MODE("-inactiveexp", "", LAPTOOL_MODE_INACTIVEEXP, inactiveexpArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PXTOOLS_ERR_SYS, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} + + + Index: /trunk/ippconfig/recipes/ppSub.config =================================================================== --- /trunk/ippconfig/recipes/ppSub.config (revision 31434) +++ /trunk/ippconfig/recipes/ppSub.config (revision 31435) @@ -76,4 +76,6 @@ INVERSE BOOL FALSE # Generate inverse subtraction? PHOTOMETRY BOOL FALSE # Perform photometry? +NOCONVOLVE BOOL FALSE # Skip convolution? + FORCED.PHOTOMETRY.BOTH BOOL FALSE # forced photometry on input sources? @@ -141,4 +143,42 @@ FORCED.PHOTOMETRY.BOTH BOOL TRUE # forced photometry on input sources END + +# Difference of warp - quickstack +WARPQSTACK METADATA + DUAL BOOL TRUE # Dual convolution? + INVERSE BOOL FALSE # Generate inverse subtraction? + PHOTOMETRY BOOL TRUE # Perform photometry? + + # penalty of 1.0 + # DUAL convolution is more sensitive to the number of kernels + # do not provide as many orders as for SINGLE + # @ISIS.WIDTHS F32 2.4 5.0 10.0 # Gaussian kernel FWHM values + @ISIS.WIDTHS F32 1.5 3.0 6.0 # Gaussian kernel FWHM values + # @ISIS.WIDTHS F32 2.0 3.0 5.0 # Gaussian kernel FWHM values + # @ISIS.WIDTHS F32 2.1 4.2 8.4 # Gaussian kernel FWHM values + @ISIS.ORDERS S32 2 2 2 # Polynomial orders for ISIS kernels + + SCALE.REF F32 5.0 # FWHM reference for kernel parameter scaling + KERNEL.SIZE S32 15 # Kernel half-size (pixels) + STAMP.FOOTPRINT S32 15 # Size of stamps (pixels) +END + +# Do no convolution +UNCONVOLVED METADATA + DUAL BOOL FALSE # Dual convolution? + INVERSE BOOL FALSE # Generate inverse subtraction? + PHOTOMETRY BOOL FALSE # Perform photometry? + NOCONVOLVE BOOL TRUE + CONVOLVE.TARGET STR SINGLE2 # convolution direction +END + +# Difference of warp - junkstack +WARPJSTACK METADATA + DUAL BOOL FALSE # Dual convolution? + INVERSE BOOL FALSE # Generate inverse subtraction? + PHOTOMETRY BOOL TRUE # Perform photometry? + CONVOLVE.TARGET STR SINGLE1 # convolution direction +END + #PR image class Index: /trunk/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /trunk/ippconfig/recipes/reductionClasses.mdc (revision 31434) +++ /trunk/ippconfig/recipes/reductionClasses.mdc (revision 31435) @@ -218,4 +218,18 @@ END +WARPQSTACK METADATA + DIFF_PPSUB STR WARPQSTACK + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 +END + +NOCONVDIFF METADATA + DIFF_PPSUB STR UNCONVOLVED + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 +END + # reduction classes for different stacks # regular stacks Index: /trunk/ppStack/src/ppStackCombineFinal.c =================================================================== --- /trunk/ppStack/src/ppStackCombineFinal.c (revision 31434) +++ /trunk/ppStack/src/ppStackCombineFinal.c (revision 31435) @@ -1,5 +1,6 @@ #include "ppStack.h" -//#define TESTING // Enable test output +// This is the doomsday switch. +// #define TESTING // Enable test output bool ppStackCombineFinal(ppStackThreadData *stack, psArray *covariances, ppStackOptions *options, @@ -29,9 +30,4 @@ reject->data[i] = pmStackRejectGrow(options->rejected->data[i], numCols, numRows, poorFrac, options->regions->data[i], options->kernels->data[i]); - if (!reject->data[i]) { - psError(psErrorCodeLast(), false, "Unable to grow rejected pixels for image %d", i); - psFree(reject); - return false; - } } else { reject->data[i] = psMemIncrRefCounter(options->rejected->data[i]); Index: /trunk/ppStack/src/ppStackCombineInitial.c =================================================================== --- /trunk/ppStack/src/ppStackCombineInitial.c (revision 31434) +++ /trunk/ppStack/src/ppStackCombineInitial.c (revision 31435) @@ -1,5 +1,6 @@ #include "ppStack.h" -//#define TESTING // Enable test output +// This is the doomsday switch. +// #define TESTING // Enable test output bool ppStackCombineInitial(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config) Index: /trunk/ppStack/src/ppStackLoop.c =================================================================== --- /trunk/ppStack/src/ppStackLoop.c (revision 31434) +++ /trunk/ppStack/src/ppStackLoop.c (revision 31435) @@ -123,6 +123,6 @@ } else { - // Since we haven't convolved, I believe we do need to normalize here. We also cannot grow. - if (!ppStackCombineFinal(stack, options->convCovars, options, config, false, true, false)) { + // Since we haven't convolved, I believe we do need to normalize here. + if (!ppStackCombineFinal(stack, options->convCovars, options, config, false, true, true)) { psError(psErrorCodeLast(), false, "Unable to perform final combination."); psFree(stack); Index: /trunk/ppStack/src/ppStackReject.c =================================================================== --- /trunk/ppStack/src/ppStackReject.c (revision 31434) +++ /trunk/ppStack/src/ppStackReject.c (revision 31435) @@ -1,4 +1,5 @@ #include "ppStack.h" +// This is the doomsday switch. // #define TESTING @@ -13,5 +14,36 @@ /* } */ + int num = options->num; // Number of inputs + + // Construct a small convolution kernel to aid in rejection + if (!options->convolve) { + for (int i = 0; i < num; i++) { + psArray *regions = psArrayAllocEmpty(1); + psRegion *region = psRegionAlloc(0,options->numCols - 1, 0, options->numRows - 1); + regions = psArrayAdd(regions,1, region); + + psArray *kernels = psArrayAllocEmpty(1); + psVector *fwhms = psVectorAllocEmpty(1, PS_TYPE_F32); + psVectorAppend(fwhms,5.0); // Should be a parameter + psVector *orders = psVectorAllocEmpty(1, PS_TYPE_S32); + psVectorAppend(orders,0); + pmSubtractionKernels *kernel = pmSubtractionKernelsISIS(15,0,fwhms,orders,0,*region,PM_SUBTRACTION_MODE_2); + kernels = psArrayAdd(kernels, 1, kernel); + + kernel->solution1 = psVectorAlloc(3, PS_TYPE_F64); + psVectorSet(kernel->solution1, 1, 1.0); + psVectorSet(kernel->solution1, 2, 1.0); + psVectorSet(kernel->solution1, 3, 1.0); + kernel->solution2 = psVectorAlloc(3, PS_TYPE_F64); + psVectorSet(kernel->solution2, 1, 1.0); + psVectorSet(kernel->solution1, 2, 1.0); + psVectorSet(kernel->solution1, 3, 1.0); + + options->kernels->data[i] = kernels; + options->regions->data[i] = regions; + } + } + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe Index: /trunk/ppSub/src/ppSubDefineOutput.c =================================================================== --- /trunk/ppSub/src/ppSubDefineOutput.c (revision 31434) +++ /trunk/ppSub/src/ppSubDefineOutput.c (revision 31435) @@ -44,6 +44,21 @@ // Convolved input images - pmReadout *inConv = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); // Input readout - pmReadout *refConv = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); // Reference readout + psMetadata *recipe = psMetadataLookupPtr(NULL, config->recipes, PPSUB_RECIPE); + bool noConvolve = psMetadataLookupBool(NULL, recipe, "NOCONVOLVE"); // Do not use convolved images. + + pmReadout *inConv; + if (noConvolve) { + inConv = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT"); // Input readout + } + else { + inConv = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); // Input readout + } + pmReadout *refConv; + if (noConvolve) { + refConv = pmFPAfileThisReadout(config->files, view, "PPSUB.REF"); // Reference readout + } + else { + refConv = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); // Reference readout + } psFree(view); @@ -60,5 +75,5 @@ kernels = psMetadataLookupPtr(&mdok, analysis, PM_SUBTRACTION_ANALYSIS_KERNEL); } - if (!kernels) { + if (!kernels && !noConvolve) { psError(PPSUB_ERR_UNKNOWN, true, "Unable to find SUBTRACTION.KERNEL"); psFree(outRO); @@ -66,5 +81,7 @@ } psAssert (hdu, "unable to find HDU"); - psMetadataAddStr(outHDU->header, PS_LIST_TAIL, "PPSUB.KERNEL", 0, "Subtraction kernel", kernels->description); + if (!noConvolve) { + psMetadataAddStr(outHDU->header, PS_LIST_TAIL, "PPSUB.KERNEL", 0, "Subtraction kernel", kernels->description); + } outHDU->header = psMetadataCopy(outHDU->header, hdu->header); Index: /trunk/ppSub/src/ppSubLoop.c =================================================================== --- /trunk/ppSub/src/ppSubLoop.c (revision 31434) +++ /trunk/ppSub/src/ppSubLoop.c (revision 31435) @@ -107,9 +107,16 @@ // XXX if it exists, use the POS1, POS2 successs for the FWHMs - if (!ppSubMatchPSFs(data)) { + psMetadata *recipe = psMetadataLookupPtr(NULL, config->recipes, PPSUB_RECIPE); + bool noConvolve = psMetadataLookupBool(NULL, recipe, "NOCONVOLVE"); // Do not use convolved images. + if (noConvolve) { + psWarning("not matching PSFs because NOCONVOLVE is TRUE\n"); + } else { + if (!ppSubMatchPSFs(data)) { psError(psErrorCodeLast(), false, "Unable to match PSFs."); success = false; goto ESCAPE; - } + } + } + if (data->quality) { @@ -132,4 +139,5 @@ // Close input files + if (!noConvolve) { if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) { psError(PPSUB_ERR_IO, false, "Unable to close input files."); @@ -137,5 +145,5 @@ goto ESCAPE; } - + } if (!ppSubLowThreshold(data)) { psError(psErrorCodeLast(), false, "Unable to threshold images."); @@ -157,4 +165,5 @@ } + if (!noConvolve) { if (!data->quality && !ppSubMakePSF(data)) { psError(psErrorCodeLast(), false, "Unable to generate PSF."); @@ -162,5 +171,5 @@ goto ESCAPE; } - + } // Now we've got a PSF, blow away detections in case they're confused with real output detections { @@ -187,5 +196,12 @@ } // dumpout(config, "diff.2a.fits"); - + if (noConvolve) { + if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) { + psError(PPSUB_ERR_IO, false, "Unable to close input files."); + success = false; + goto ESCAPE; + } + } + // Higher order background subtraction using psphot if (!ppSubBackground(config)) { Index: /trunk/ppSub/src/ppSubMakePSF.c =================================================================== --- /trunk/ppSub/src/ppSubMakePSF.c (revision 31434) +++ /trunk/ppSub/src/ppSubMakePSF.c (revision 31435) @@ -37,15 +37,32 @@ bool reverse = psMetadataLookupBool(NULL, config->arguments, "REVERSE"); // Reverse sense of subtraction? - + bool mdok = false; // Status of MD lookup pmReadout *minuend = NULL; // Image that will be positive following subtraction pmFPAfile *minuendFile = NULL; // File for minuend image pmFPAview *view = ppSubViewReadout(); // View to readout - if (reverse) { + + psMetadata *recipe = psMetadataLookupMetadata(&mdok, config->recipes, PPSUB_RECIPE); // Recipe for ppSub + bool noConvolve = psMetadataLookupBool(&mdok, recipe, "NOCONVOLVE"); // Do not use convolved images. + + if (!noConvolve) { + printf("Using Convolved images because NOCONVOLVE is FALSE\n"); + if (reverse) { minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); - minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.REF.CONV"); - } else { + minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.REF.CONV"); + } else { minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); - minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.INPUT.CONV"); + minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.INPUT.CONV"); + } + } + else { + psWarning("Not using Convolved images because NOCONVOLVE is TRUE\n"); + if (reverse) { + minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF"); + minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.REF"); + } else { + minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT"); + minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.INPUT"); + } } Index: /trunk/ppSub/src/ppSubReadoutPhotometry.c =================================================================== --- /trunk/ppSub/src/ppSubReadoutPhotometry.c (revision 31434) +++ /trunk/ppSub/src/ppSubReadoutPhotometry.c (revision 31435) @@ -60,4 +60,5 @@ // grab the PSF information from the pmFPAfile PSPHOT.PSF.LOAD pmFPAfile *psfFile = psMetadataLookupPtr(&mdok, config->files, "PSPHOT.PSF.LOAD"); // PSF file + ppSubCopyPSF (photFile, psfFile, view); Index: /trunk/ppSub/src/ppSubReadoutSubtract.c =================================================================== --- /trunk/ppSub/src/ppSubReadoutSubtract.c (revision 31434) +++ /trunk/ppSub/src/ppSubReadoutSubtract.c (revision 31435) @@ -31,4 +31,5 @@ bool reverse = psMetadataLookupBool(&mdok, config->arguments, "REVERSE"); // Reverse sense of subtraction? + bool noConvolve = psMetadataLookupBool(&mdok, recipe, "NOCONVOLVE"); // Do not use convolved images. bool addPair = psMetadataLookupBool(&mdok, recipe, "ADD.NOT.SUBTRACT"); // add instead of subtracting @@ -38,10 +39,23 @@ pmReadout *minuend = NULL; // Positive image pmReadout *subtrahend = NULL; // Negative image - if (reverse) { + if (!noConvolve) { + printf("Using Convolved images because NOCONVOLVE is FALSE\n"); + if (reverse) { minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); subtrahend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); - } else { + } else { minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); subtrahend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); + } + } + else { + psWarning("Not using Convolved images because NOCONVOLVE is TRUE\n"); + if (reverse) { + minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF"); + subtrahend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT"); + } else { + minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT"); + subtrahend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF"); + } } Index: /trunk/ppSub/src/ppSubThreshold.c =================================================================== --- /trunk/ppSub/src/ppSubThreshold.c (revision 31434) +++ /trunk/ppSub/src/ppSubThreshold.c (revision 31435) @@ -95,5 +95,13 @@ // Input images - pmReadout *in = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); // Input image + bool noConvolve = psMetadataLookupBool(NULL, recipe, "NOCONVOLVE"); + + pmReadout *in; + if (noConvolve) { + in = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT"); // Input image + } + else { + in = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); // Input image + } if (!in) { psError(PPSUB_ERR_UNKNOWN, false, "Unable to find readout."); @@ -102,5 +110,11 @@ } - pmReadout *ref = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); // Reference image + pmReadout *ref; + if (noConvolve) { + ref = pmFPAfileThisReadout(config->files, view, "PPSUB.REF"); // Reference image + } + else { + ref = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); // Reference image + } if (!ref) { psError(PPSUB_ERR_UNKNOWN, false, "Unable to find readout."); Index: /trunk/psLib/src/db/psDB.c =================================================================== --- /trunk/psLib/src/db/psDB.c (revision 31434) +++ /trunk/psLib/src/db/psDB.c (revision 31435) @@ -1043,5 +1043,5 @@ pType = psDBMySQLToPType(field[i].type, field[i].flags); if (!pType) { - psError(PS_ERR_UNKNOWN, false, "Failed to lookup type."); + psError(PS_ERR_UNKNOWN, false, "Failed to lookup type. %d %d %d",i,field[i].type, field[i].flags); psFree(md); mysql_free_result(result); Index: /trunk/psModules/src/imcombine/pmStackReject.c =================================================================== --- /trunk/psModules/src/imcombine/pmStackReject.c (revision 31434) +++ /trunk/psModules/src/imcombine/pmStackReject.c (revision 31435) @@ -127,4 +127,5 @@ if (!subRegions || !subKernels) { + psTrace("psModules.imcombine",2,"Do not have the necessary kernels and regions, returning input pixels."); psPixels *out = psPixelsCopy(NULL,in); return out;