Index: branches/eam_branches/ipp-20110404/ippScripts/Build.PL =================================================================== --- branches/eam_branches/ipp-20110404/ippScripts/Build.PL (revision 31166) +++ branches/eam_branches/ipp-20110404/ippScripts/Build.PL (revision 31439) @@ -117,4 +117,5 @@ scripts/skycell_jpeg.pl scripts/diffphot.pl + scripts/lap_science.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: branches/eam_branches/ipp-20110404/ippScripts/MANIFEST =================================================================== --- branches/eam_branches/ipp-20110404/ippScripts/MANIFEST (revision 31166) +++ branches/eam_branches/ipp-20110404/ippScripts/MANIFEST (revision 31439) @@ -45,3 +45,4 @@ scripts/ipp_cluster_load_monitor.pl scripts/skycell_jpeg.pl +scripts/lap_science.pl t/00_distribution.t Index: branches/eam_branches/ipp-20110404/ippScripts/scripts/addstar_run.pl =================================================================== --- branches/eam_branches/ipp-20110404/ippScripts/scripts/addstar_run.pl (revision 31166) +++ branches/eam_branches/ipp-20110404/ippScripts/scripts/addstar_run.pl (revision 31439) @@ -37,13 +37,14 @@ } my $minidvodb_path; -my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, +my ( $add_id, $camera, $stage, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( - 'exp_tag=s' => \$exp_tag, # Exposure identifier 'add_id=s' => \$add_id, # Camtool identifier 'camera|c=s' => \$camera, # Camera + 'stage|s=s' => \$stage, # Camera + 'dbname|d=s' => \$dbname, # Database name 'outroot|w=s' => \$outroot, # output file base name - 'camroot|w=s' => \$camroot, # camera stage root name. + 'stageroot|w=s' => \$stageroot, # stage root name. 'reduction=s' => \$reduction, # Reduction class 'dvodb|w=s' => \$dvodb, # output DVO database @@ -61,11 +62,11 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --exp_tag --add_id --camera --outroot --dvodb --camroot", + -msg => "Required options: --add_id --camera --outroot --dvodb --stageroot --stage", -exitval => 3, ) unless - defined $exp_tag and + defined $stage and defined $add_id and defined $outroot and - defined $camroot and + defined $stageroot and defined $dvodb and defined $camera; @@ -86,7 +87,15 @@ # Recipes to use based on reduction class $reduction = 'DEFAULT' unless defined $reduction; - +my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # XXX This is now not used: do we still need it? -my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +if ($stage =~/cam/) { + $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +} +if ($stage =~/stack/) { + $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR_STACK'); # Recipe to use +} +#if ($stage =~/staticsky/) { +# $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR_STATICSKY'); # Recipe to use +#} &my_die("Unrecognised ADDSTAR recipe", $add_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; @@ -97,5 +106,12 @@ # the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs -my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $camroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +# it was PSASTRO.OUTPUT +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +if (($stage =~/staticsky/) || ($stage =~/stack/)) { + $fpaObjects =~ s/smf$/cmf/; + +} my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); @@ -173,6 +189,8 @@ $command .= " $realFile"; $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference - $command .= " -image" if $image_only; - + $command .= " -image" if $image_only; + if ($stage = ~/staticsky/) { + $command .= " -accept-astrom "; + } my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script Index: branches/eam_branches/ipp-20110404/ippScripts/scripts/diff_skycell.pl =================================================================== --- branches/eam_branches/ipp-20110404/ippScripts/scripts/diff_skycell.pl (revision 31166) +++ branches/eam_branches/ipp-20110404/ippScripts/scripts/diff_skycell.pl (revision 31439) @@ -31,4 +31,5 @@ my $ppSub = can_run('ppSub') or (warn "Can't find ppSub" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $nebInsert = can_run('neb-insert') or (warn "Can't find neb-insert" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -265,4 +266,7 @@ my $do_photom = 1; my $dump_config = 1; +if ($reduction eq 'NOCONVDIFF') { + $do_photom = 0; +} if ($run_state eq 'new') { $configuration = prepare_output("PPSUB.CONFIG", $outroot, 1); @@ -380,4 +384,17 @@ } } + if ($reduction eq 'NOCONVDIFF') { + my $refConv = prepare_output("PPSUB.REF.CONV", $outroot, 0); + my $templateFile = $ipprc->file_resolve($template,0); + $command = "$nebInsert $refConv $templateFile"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform neb-insert: $error_code", $diff_id, $skycell_id, $error_code); + } + } + + } elsif ($run_state eq 'update') { &my_die("Update resulted in poor quality image: $quality", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR); Index: branches/eam_branches/ipp-20110404/ippScripts/scripts/lap_science.pl =================================================================== --- branches/eam_branches/ipp-20110404/ippScripts/scripts/lap_science.pl (revision 31439) +++ branches/eam_branches/ipp-20110404/ippScripts/scripts/lap_science.pl (revision 31439) @@ -0,0 +1,907 @@ +#!/usr/bin/env perl + + +use warnings; +use strict; +use Carp; +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use DateTime; + +# +# Set up +################################################################################ + +my $missing_tools = 0; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); +my $laptool = can_run('laptool') or (warn "Can't find laptool" and $missing_tools = 1); + +my ( $help, $verbose, $debug, $do_nothing); +my ( $camera, $dbname); +my ( $lap_id ); +my ( $chip_mode, $monitor_mode, $cleanup_mode); + +GetOptions( + 'help|h' => \$help, + 'verbose' => \$verbose, + 'debug' => \$debug, + 'do_nothing' => \$do_nothing, + + 'camera=s' => \$camera, + 'dbname=s' => \$dbname, + + 'lap_id=s' => \$lap_id, + + 'chip_mode' => \$chip_mode, + 'monitor_mode' => \$monitor_mode, + 'cleanup_mode' => \$cleanup_mode, + ) or pod2usage ( 2 ); +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Choose a mode: --chip_mode --monitor_mode --cleanup_mode", + -exitval => 3, + ) unless + defined $chip_mode or defined $monitor_mode or defined $cleanup_mode; +pod2usage( + -msg => "--lap_id is required", + -exitval => 3, + ) unless + defined $lap_id; + +unless (defined $dbname) { + $dbname = 'gpc1'; +} + +my $mdcParser = PS::IPP::Metadata::Config->new; + +# Fetch all the information about this run +my %lapRunInfo = get_lapRun_info($lap_id); + +# Run the appropriate mode + +if (defined($chip_mode)) { + unless ($lapRunInfo{state} eq 'new') { + &my_die("Cannot run chip_mode if lapRun.state != new!",$lap_id); + } + my $status = chip_mode($lap_id); + exit $status; +} +if (defined($monitor_mode)) { + unless ($lapRunInfo{state} eq 'run') { + &my_die("Cannot run monitor_mode if lapRun != run!", $lap_id); + } + my $status = monitor_mode($lap_id); + exit $status; +} +if (defined($cleanup_mode)) { + unless (($lapRunInfo{state} eq 'full')|| + ($lapRunInfo{state} eq 'drop')) { + &my_die("Cannot run cleanup_mode if lapRun != done!", $lap_id); + } + my $status = cleanup_mode($lap_id); + exit $status; +} + +# +# Chip +################################################################################ + +sub chip_mode { + my $lap_id = shift; + my $status = queue_chips($lap_id); + + if ($status) { + my $command = "$laptool -updaterun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_state run "; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + } + return($status); +} + +# retrieve the chip_id for the exposure inserted +sub get_chip_id_from_metadata { + my $exp_id = shift; + my $label = shift; + my $data_group = shift; + # This is a puzzler... chiptool doesn't actually return a useful metadata. We'll just scrape it from the database for now. + + my $command = "$chiptool -listrun -exp_id $exp_id -label $label -data_group $data_group"; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -listrun: $error_code", $exp_id, $data_group); + } + my $chips = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from chiptool -listrun", $exp_id, $data_group); + # There should be only one. + my $chip = ${ $chips }[0]; + my $chip_id = 0; + if ($chip) { + $chip_id = $chip->{chip_id}; + } + + return($chip_id); +} + +sub get_lapRun_info { + my $lap_id = shift; + my $command = "$laptool -pendingrun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -pendingrun: $error_code", $lap_id); + } + my $Runs = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -pendingrun", $lap_id); + # There should be only one. + my $Run = ${ $Runs }[0]; + my %info = %{ $Run }; + return(%info); +} + + +# Queue a chipRun for this exposure with the appropriate parameters. +sub remake_this_exposure { + my $exposure = shift; + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + + my $label = $exposure->{label}; + + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + + my $chip_cmd = "$chiptool"; + $chip_cmd .= " -pretend " if defined $debug; + $chip_cmd .= " -dbname $dbname " if defined $dbname; + $chip_cmd .= " -definebyquery -exp_id $exposure->{exp_id} -set_end_stage warp -set_tess_id $exposure->{tess_id} "; + $chip_cmd .= " -set_label $exposure->{label} -set_data_group $data_group "; + $chip_cmd .= " -set_workdir $workdir -set_dist_group $exposure->{dist_group} "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $chip_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -definebyquery: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + + $exposure->{chip_id} = get_chip_id_from_metadata($exposure->{exp_id},$exposure->{label},$data_group); + $exposure->{data_state} = 'new'; + update_this_exposure($exposure); + return($exposure); +} + +# This is the "user level" subroutine. +sub queue_chips { + my $lap_id = shift; + + my $command = "$laptool -pendingexp -lap_id $lap_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -pendingexp: $error_code", $lap_id); + } + + if (@$stdout_buf == 0) { + # Nothing to do. + return(0); + } + + my $exposures = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -pendingexp", $lap_id); + + my $counter = 0; + my %matching = (); + my %indexing = (); + # Determine which exposures need a chipRun queued. + foreach my $exposure (@$exposures) { + # $lap_id = $exposure->{lap_id}; # This should be already known. + my $tess_id = $exposure->{tess_id}; + my $exp_id = $exposure->{exp_id}; + my $chip_id = $exposure->{chip_id}; + my $pair_id = $exposure->{pair_id}; + my $private = $exposure->{private}; + my $pairwise = $exposure->{pairwise}; + my $active = $exposure->{active}; + my $data_state = $exposure->{data_state}; + my $dateobs = $exposure->{dateobs}; + my $object = $exposure->{object}; + my $comment = $exposure->{comment}; + + if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id + if (($pairwise) && !($pair_id)) { + # This is an error. + &my_die("Exposure $exp_id for $lap_id is declared pairwise without a defined pair", $lap_id); + } + $matching{$object}{$comment} = $exp_id; + $indexing{$exp_id} = $counter; + $counter++; + next; + } + else { # We do not already have a chip_id. + # Make a chipRun, and update the exposure. + $exposure = remake_this_exposure($exposure); + + # Save our information for diff pairing. + $matching{$object}{$comment} = $exp_id; + $indexing{$exp_id} = $counter; + $counter++; + } + } + + # Determine which exposures have a pair for diffing. + foreach my $object (keys %matching) { + my @exp_ids_to_diff = (); + foreach my $comment (keys %{ $matching{$object} }) { + push @exp_ids_to_diff, $matching{$object}{$comment}; + } + @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; + + if (( $#exp_ids_to_diff + 1) % 2 != 0) { # We have an odd number of exposures, even after comment filtering + pop(@exp_ids_to_diff); # dump the last entry. + } + + while ($#exp_ids_to_diff > -1) { # Save the pair information to the opposite exposure. + my $exp_id_A = shift(@exp_ids_to_diff); + my $exp_id_B = shift(@exp_ids_to_diff); + + my $exp_A = ${ $exposures }[$indexing{$exp_id_A}]; + my $exp_B = ${ $exposures }[$indexing{$exp_id_B}]; + + $exp_A->{pairwise} = 1; + $exp_A->{private} = 0; + $exp_A->{pair_id} = $exp_B->{chip_id}; + + $exp_B->{pairwise} = 1; + $exp_B->{private} = 0; + $exp_B->{pair_id} = $exp_A->{chip_id}; + + if ($verbose) { + print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n"; + } + } + } + + # Scan all exposures, and ensure that pairwise and private are set correctly + foreach my $exposure (@$exposures) { + if ($exposure->{pairwise} && !($exposure->{pair_id})) { + $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. + } + if (!($exposure->{pairwise})) { + $exposure->{private} = 1; # If this isn't being pairwise diffed, it needs to be private + } + + $exposure = update_this_exposure($exposure); + } + return(1); +} + + +# +# Decide Things +################################################################################ +sub monitor_mode { + my $lap_id = shift; + return(check_status($lap_id)); +} + + +sub check_stack_status { # look at lapRun associated stacks, and confirm their states. + my $lap_id = shift; + + my ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack); + + my $command = "$laptool -stacks -lap_id $lap_id"; + $command .= " -dbname $dbname" if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -stacks: $error_code", $lap_id, ""); + } + + ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack) = (0,0,0,0); + if (@$stdout_buf == 0) { + # Nothing to do. Possibly an error, but for now, just accept it + return($defined_qstack,$have_qstack,$defined_fstack,$have_fstack); + } + + my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -stacks", $lap_id, ""); + + my $total_qstacks = 0; + my $complete_qstacks = 0; + my $total_fstacks = 0; + my $complete_fstacks = 0; + foreach my $stack (@$stacks) { + if (($stack->{quick_stack_id})&& + (S64_IS_NOT_NULL($stack->{quick_stack_id}))) { + $defined_qstack = 1; + $total_qstacks++; + } + if (($stack->{final_stack_id})&& + (S64_IS_NOT_NULL($stack->{final_stack_id}))) { + $defined_fstack = 1; + $total_fstacks++; + } + + if (($stack->{quick_state})&& + ($stack->{quick_state} eq 'full')) { + $complete_qstacks++; + } + elsif (($stack->{quick_state})&&($stack->{quick_state} eq 'new')&& + ($stack->{quick_fault} >= 4)&&($stack->{quick_fault} != 32767)) { + printf STDERR "Faulted quick stack: $stack->{quick_stack_id} $stack->{quick_fault}\n"; + $complete_qstacks++;# This is not the best solution, but if they continually fail, there's not much we can do. + } + if (($stack->{final_state})&& + ($stack->{final_state} eq 'full')) { + $complete_fstacks++; + } + elsif (($stack->{final_state})&&($stack->{final_state} eq 'new')&& + ($stack->{final_fault} >= 4)&&($stack->{final_fault} != 32767)) { + printf STDERR "Faulted final stack: $stack->{final_stack_id} $stack->{final_fault}\n"; + $complete_fstacks++; # This is not the best solution, but if they continually fail, there's not much we can do. + } + + } + if (($complete_qstacks > 0)&& + ($complete_qstacks == $total_qstacks)) { + $have_qstack = 1; + } + if (($complete_fstacks > 0)&& + ($complete_fstacks == $total_fstacks)) { + $have_fstack = 1; + } + + + return($defined_qstack,$have_qstack,$defined_fstack,$have_fstack); +} +sub check_status { + my $lap_id = shift; + + my ($defined_qstack,$have_qstack,$defined_fstack,$have_fstack) = check_stack_status($lap_id); + + my $command = "$laptool -exposures -lap_id $lap_id"; + $command .= " -dbname $dbname" if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -exposures: $error_code", $lap_id); + } + + if (@$stdout_buf == 0) { + # Nothing to do. However, this is likely an error. + return(0); + } + + my $exposures = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -exposures", $lap_id); + + # Need to prescan to see who matches whom. + my %match_hash = (); + my $index = 0; + foreach my $exposure (@$exposures) { + if ($exposure->{pair_id}) { + $match_hash{$exposure->{pair_id}} = $index; # Tell the companion who we are, so they can find us next time. + } + $index++; + } + + # Things I want to know before I'm through + my $needs_qstack = 0; + my $needs_something_remade = 0; + my $needs_something_private = 0; + my $can_qstack = 0; + my $have_diff = 0; + my $can_diff = 0; + my $can_fstack = 0; + my $total_exposures = 0; + foreach my $exposure (@$exposures) { + $total_exposures++; + my $companion; + + if ($exposure->{pair_id}) { # Load companion exposure information + if (exists($match_hash{$exposure->{chip_id}})) { + $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; # Match! + } + } + + if ($exposure->{private}) { # I've declared this exposure private to this lapRun. + $needs_qstack = 1; + } + + if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun + $needs_something_remade = 1; + $exposure = remake_this_exposure($exposure); + } + if ($exposure->{cam_quality}) { + $needs_qstack = 1; + $needs_something_private = 1; + if ($companion) { + $companion->{private} = 1; + $companion->{pairwise} = 0; + &update_this_exposure($companion); + } + $exposure->{private} = 1; + $exposure->{pairwise} = 0; + $exposure->{data_state} = 'drop'; + &update_this_exposure($exposure); + + } +# if ($companion) { # Validate that there are no problems with the companion exposure +# if ($companion->{cam_quality}) { # Maybe other things here? +# $exposure->{private} = 1; +# $exposure->{data_state} = 'drop'; +# &update_this_exposure($exposure); +# $needs_qstack = 1; +# } +# } + if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. + $can_qstack ++; + $can_diff ++; + $have_diff ++; + $can_fstack ++; + next; + } + + if (($exposure->{warpRun_state})&& # This exposure has a warp + ($exposure->{warpRun_state} eq 'full')) { # This exposure's warp is done. + $can_qstack ++; + $can_diff ++; + } + if (($exposure->{magicked}&& + &S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. + $can_fstack ++; + } + if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) { + $have_diff ++; + } + } + + + print "STATUS: DEFINED_QSTACK: $defined_qstack\n"; + print "STATUS: HAVE_QSTACK: $have_qstack\n"; + print "STATUS: DEFINED_FSTACK: $defined_fstack\n"; + print "STATUS: HAVE_FSTACK: $have_fstack\n"; + + print "STATUS: NEEDS_QSTACK: $needs_qstack\n"; + print "STATUS: NEEDS_REMADE: $needs_something_remade\n"; + print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n"; + print "STATUS: CAN_QSTACK: $can_qstack\n"; + print "STATUS: CAN_DIFF: $can_diff\n"; + print "STATUS: HAVE_DIFF: $have_diff\n"; + print "STATUS: CAN_FSTACK: $can_fstack\n"; + + print "STATUS: TOTAL_EXPOSURES: $total_exposures\n"; + if ($do_nothing) { + exit(0); + } + if (($needs_qstack == 1)&& # Do we need the quick stack? + ($defined_qstack == 0)&& # Have we not made it already? + ($can_qstack == $total_exposures)&& # Are all warps done? + ($needs_something_remade == 0) # Do we need to wait for a stolen exposure? + ) { + print "STATUS: Will now queue quickstacks\n"; + &queue_quickstack($exposures); + } + + if (($can_diff == $total_exposures)&& # Are all warps done? + ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? + ($have_diff < $can_diff)&& # We haven't already done them all. + (($needs_qstack == 0)|| # Are we just doing pairwise diffs? + ($needs_qstack == 1)&&($have_qstack == 1)) ) { # If we need a quickstack, do we have it? + print "STATUS: Will now queue diffs\n"; + &queue_diffs($exposures); + } + + if (($can_fstack == $total_exposures)&& # Are all warps done? + ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? + ($defined_fstack == 0)) { # Have we not made it already? + print "STATUS: Will now queue final stacks\n"; + &queue_finalstack($exposures); + } + + if (($have_fstack == 1)) { # Are we all done making the final product? + print "STATUS: Will now deactivate exposures for this complete lapRun.\n"; + &deactivate_exposures($exposures); + $command = "$laptool -updaterun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_state full "; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + } + +} + +sub queue_quickstack { + my $exposures = shift; # reference to exposure array; + my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information + + my $lap_id = $exposure->{lap_id}; + my $label = $exposure->{label}; + my $filter = $exposure->{filter}; + my $proj_cell = $exposure->{projection_cell}; + + unless (defined($label) && defined($filter) && defined($proj_cell)) { + &my_die("Unable to perform stacktool. Insufficient information.", $lap_id); + } + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + my $command = "$stacktool "; + $command .= " -pretend " if defined $debug; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; + $command .= " -set_label ${label} -set_data_group ${proj_cell}.quick.${date} "; + $command .= " -set_workdir $workdir -set_dist_group NODIST "; + $command .= " -min_num 2 -set_reduction QUICKSTACK "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); + } + + $command = "$stacktool "; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -sassskyfile -data_group ${proj_cell}.quick.${date} "; + $command .= " -filter $filter -projection_cell ${proj_cell} "; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -sassskyfile: $error_code", $lap_id); + } + + my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stacktool -sassskyfile", $lap_id, ""); + + my $stack = ${ $stacks }[0]; + my $sass_id = $stack->{sass_id}; + unless (defined($sass_id)) { + &my_die("Unable to parse metadata from stacktool for sass_id", $lap_id, ""); + } + + print "QUICK_SASS_ID: $sass_id\n"; + $command = "$laptool -updaterun -lap_id $lap_id -set_quick_sass_id $sass_id"; + $command .= " -dbname $dbname " if defined $dbname; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + + +} +sub queue_finalstack { + my $exposures = shift; # reference to exposure array; + my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information + + my $lap_id = $exposure->{lap_id}; + my $label = $exposure->{label}; + my $filter = $exposure->{filter}; + my $proj_cell = $exposure->{projection_cell}; + + unless (defined($label) && defined($filter) && defined($proj_cell)) { + &my_die("Unable to perform stacktool. Insufficient information.", $lap_id); + } + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + my $command = "$stacktool "; + $command .= " -pretend " if defined $debug; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; + $command .= " -set_label ${label} -set_workdir $workdir -set_data_group ${proj_cell}.final.${date} "; + $command .= " -min_num 2 -set_reduction THREEPI_STACK "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); + } + + + $command = "$stacktool "; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -sassskyfile -data_group ${proj_cell}.final.${date} "; + $command .= " -filter $filter -projection_cell ${proj_cell} "; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -sassskyfile: $error_code", $lap_id); + } + + my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stacktool -sassskyfile", $lap_id, ""); + + my $stack = ${ $stacks }[0]; + my $sass_id = $stack->{sass_id}; + unless (defined($sass_id)) { + &my_die("Unable to parse metadata from stacktool for sass_id", $lap_id, ""); + } + print "FINAL_SASS_ID: $sass_id\n"; + $command = "$laptool -updaterun -lap_id $lap_id -set_final_sass_id $sass_id"; + $command .= " -dbname $dbname " if defined $dbname; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + +} +sub queue_diffs { + my $exposures = shift; + + # Need to prescan to see who matches whom. + my %match_hash = (); + my $index = 0; + foreach my $exposure (@$exposures) { + if ($exposure->{pair_id}) { + $match_hash{$exposure->{pair_id}} = $index; # Tell the companion who we are, so they can find us next time. + } + $index++; + } + + my %already_queued = (); + foreach my $exposure (@$exposures) { + if ($exposure->{data_state} eq 'drop') { # Kick out unusable exposures + next; + } + if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # Not sure how this would happen, but still. ## This happens when we inherit a complete exposure. + next; + } + if ($already_queued{$exposure->{warp_id}}) { + print "STATUS: Have already queued a diff containing $exposure->{exp_id} $exposure->{chip_id} $exposure->{warp_id}\n"; + next; + } + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + + my $label = $exposure->{label}; + my $dist_group = $exposure->{dist_group}; + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + + my $command = "$difftool "; + $command .= " -pretend " if defined $debug; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_label $label -set_workdir $workdir -set_data_group $data_group "; + if ($exposure->{dist_group}) { + $command .= " -set_dist_group $exposure->{dist_group} "; + } + + if ($exposure->{pairwise}) { # warpwarp + my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; + $command .= " -definewarpwarp "; + $command .= "-input_label $label -template_label $label -backwards "; + $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} "; + $already_queued{$exposure->{warp_id}} = 1; + $already_queued{$companion->{warp_id}} = 1; + + } + else { # warp-qstack + $command .= " -definewarpstack -set_reduction NOCONVDIFF -available -good_frac 0.2 "; + $command .= " -warp_id $exposure->{warp_id} -stack_label ${label}.quick "; + $already_queued{$exposure->{warp_id}} = 1; + } + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform difftool -definewarp(warp|stack): $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + + my $diffs = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from difftool -definewarp(warp|stack)", $lap_id, ""); + + my $diff = ${ $diffs }[0]; + my $diff_id = $diff->{diff_id}; + unless (defined($diff_id)) { + $exposure->{data_state} = 'drop'; + &update_this_exposure($exposure); + } + + } +} + +# Deactivate all exposures in this run. +sub deactivate_exposures { + my $exposures = shift; + foreach my $exposure (@$exposures) { + $exposure->{active} = 0; + update_this_exposure($exposure); + } +} + + +# +# Cleanup +################################################################################ + +sub cleanup_mode { + my $lap_id = shift; + my $command = "$laptool -inactiveexp -lap_id $lap_id "; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -inactiveexp: $error_code", "none", "none"); + } + if (@$stdout_buf == 0) { + # Nothing to do. + return(0); + } + + my $exposures = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -inactiveexp", $lap_id); + + my @clean_modes = ( + 'chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -chip_id @CHIP_ID@', + 'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@', + 'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@', + 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage chip -stage_id @CHIP_ID@', + 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage warp -stage_id @WARP_ID@', + 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage diff -stage_id @DIFF_ID@'); + foreach my $exposure (@$exposures) { + if ($exposure->{is_in_use}) { + next; + } + foreach my $clean_mode (@clean_modes) { + my $command = $clean_mode; + $command =~ s/\@DBNAME\@/$dbname/g; + $command =~ s/\@CHIP_ID\@/$exposure->{chip_id}/; + $command =~ s/\@WARP_ID\@/$exposure->{warp_id}/; + $command =~ s/\@DIFF_ID\@/$exposure->{diff_id}/; + $command =~ s/\@LABEL\@/$exposure->{label}/; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform cleantool: $command : $error_code", "none", "none"); + } + } + $exposure->{data_state} = 'cleaned'; + update_this_exposure($exposure); + } +} + + + + +# +# Utilities +################################################################################ + +sub my_die { + my $msg = shift; # Warning message on die + my $lap_id = shift; # identifier + my $optional = shift; + my $exit_code = shift; + unless (defined $exit_code) { + $exit_code = $PS_EXIT_PROG_ERROR; + } + carp($msg); + exit $exit_code; +} + +# Check to see if a 64bit integer is NULL or not. This is probably fragile, but I don't know how to get a S64 NULL out. +sub S64_IS_NOT_NULL { + if ($_[0] == 9223372036854775807) { + return(0); + } + else { + return(1); + } +} + +# Take the current exposure structure, and save it to the database. +sub update_this_exposure { + my $exposure = shift; + + # These are required + my $lap_id = $exposure->{lap_id}; + my $exp_id = $exposure->{exp_id}; + + my $command = "$laptool -updateexp -lap_id $lap_id -exp_id $exp_id "; + if (($exposure->{chip_id})&&(S64_IS_NOT_NULL($exposure->{chip_id}))) { + $command .= " -set_chip_id $exposure->{chip_id} "; + } + if (($exposure->{pair_id})&&(S64_IS_NOT_NULL($exposure->{pair_id}))) { + $command .= " -set_pair_id $exposure->{pair_id} "; + } + + if ($exposure->{private}) { + $command .= " -private "; + } + else { + $command .= " -public "; + } + if ($exposure->{pairwise}) { + $command .= " -pairwise "; + } + else { + $command .= " -nopairwise "; + } + if ($exposure->{active}) { + $command .= " -active "; + } + else { + $command .= " -inactive "; + } + if ($exposure->{data_state}) { + $command .= " -set_data_state $exposure->{data_state} "; + } + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updateexp: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } +} + Index: branches/eam_branches/ipp-20110404/ippScripts/scripts/magic_destreak.pl =================================================================== --- branches/eam_branches/ipp-20110404/ippScripts/scripts/magic_destreak.pl (revision 31166) +++ branches/eam_branches/ipp-20110404/ippScripts/scripts/magic_destreak.pl (revision 31439) @@ -360,5 +360,10 @@ $command .= " -chip_mask $ch_mask" if defined $ch_mask; $command .= " -weight $weight" if defined $weight; - $command .= " -sources $sources" if defined $sources; + if ((defined($sources))&&($ipprc->file_exists($sources))) { + $command .= " -sources $sources" if defined $sources; + } + else { + print "Did not add sources because they do not appear to exist. This may be an error.\n"; + } $command .= " -skycelllist $skycell_list" if defined $skycell_list; $command .= " -replace" if $replace; Index: branches/eam_branches/ipp-20110404/ippScripts/scripts/nightly_science.pl =================================================================== --- branches/eam_branches/ipp-20110404/ippScripts/scripts/nightly_science.pl (revision 31166) +++ branches/eam_branches/ipp-20110404/ippScripts/scripts/nightly_science.pl (revision 31439) @@ -996,4 +996,5 @@ $cmd .= " -min_num 4"; $cmd .= " -select_good_frac_min 0.05"; + $cmd .= " -set_reduction NIGHTLY_STACK "; $cmd .= " $select "; if ($debug == 1) { Index: branches/eam_branches/ipp-20110404/ippScripts/scripts/stack_skycell.pl =================================================================== --- branches/eam_branches/ipp-20110404/ippScripts/scripts/stack_skycell.pl (revision 31166) +++ branches/eam_branches/ipp-20110404/ippScripts/scripts/stack_skycell.pl (revision 31439) @@ -73,5 +73,4 @@ PPSTACK.OUTPUT.EXPNUM PPSTACK.OUTPUT.EXPWT -PPSTACK.TARGET.PSF PPSTACK.OUTPUT.JPEG1 PPSTACK.OUTPUT.JPEG2 @@ -84,4 +83,5 @@ # extra outputs when convolving my @outputListUnconv = qw( +PPSTACK.TARGET.PSF PPSTACK.UNCONV PPSTACK.UNCONV.MASK @@ -181,6 +181,7 @@ my $recipe_psphot = $ipprc->reduction($reduction, 'STACK_PSPHOT'); # Recipe to use for psphot my $recipe_ppstats = 'STACKSTATS'; -if ($run_state eq 'update') { - $recipe_ppstats = 'WARPSTATS'; +if ($ +run_state eq 'update') { + $recipe_ppstats = 'WARPSTATS'; } unless ($recipe_ppStack and $recipe_ppSub and $recipe_psphot) { @@ -208,4 +209,6 @@ &my_die("Unable to not compress and logflux compress simultaneously. Check config.",$stack_id, $PS_EXIT_CONFIG_ERROR); } +my $stack_type = metadataLookupStr($recipe, 'STACK.TYPE'); +&my_die("STACK.TYPE not found in recipe. Check config.",$stack_id, $PS_EXIT_CONFIG_ERROR) unless $stack_type; # Generate MDC file with the inputs @@ -281,4 +284,5 @@ $command .= " -recipe PSPHOT $recipe_psphot"; $command .= " -recipe PPSTATS $recipe_ppstats" if $do_stats;; + $command .= " -stack-type $stack_type"; $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; Index: branches/eam_branches/ipp-20110404/ippScripts/scripts/staticsky.pl =================================================================== --- branches/eam_branches/ipp-20110404/ippScripts/scripts/staticsky.pl (revision 31166) +++ branches/eam_branches/ipp-20110404/ippScripts/scripts/staticsky.pl (revision 31439) @@ -32,5 +32,7 @@ my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); my $psphotStack = can_run('psphotStack') or (warn "Can't find psphotStack" and $missing_tools = 1); +my $psphot = can_run('psphot') or (warn "Can't find psphot" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -96,143 +98,229 @@ } +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + + # generate the input print $listFile "INPUT MULTI\n"; my $nInputs = @$files; -my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot); - -foreach my $file (@$files) { - print $listFile "INPUT METADATA\n"; - - # XXX if we take the input from 'warp', we will need to make different selections here - my $path_base = $file->{path_base}; - print "input: $path_base\n"; - - my $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $path_base ); # Image name - my $maskCnv = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base ); # Mask name - my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name - - my $imageRaw = $ipprc->filename("PPSTACK.UNCONV", $path_base ); # Image name - my $maskRaw = $ipprc->filename("PPSTACK.UNCONV.MASK", $path_base ); # Mask name - my $weightRaw = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $path_base ); # Weight name - - my $sources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $path_base ); # Sources name - - # XXX is this the correct PSF file? - my $psfCnv = $ipprc->filename("PPSTACK.TARGET.PSF", $path_base ); # PSF name - - # XXX we could make some different choices if some inputs do not exist... - &my_die("Couldn't find input: $imageRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageRaw"); - &my_die("Couldn't find input: $maskRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskRaw"); - &my_die("Couldn't find input: $weightRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightRaw"); - &my_die("Couldn't find input: $imageCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv"); - &my_die("Couldn't find input: $maskCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv"); - &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv"); - &my_die("Couldn't find input: $psfCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); - &my_die("Couldn't find input: $sources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources"); - - print $listFile " RAW:IMAGE STR " . $imageRaw . "\n"; - print $listFile " RAW:MASK STR " . $maskRaw . "\n"; - print $listFile " RAW:VARIANCE STR " . $weightRaw . "\n"; - - print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; - print $listFile " CNV:MASK STR " . $maskCnv . "\n"; - print $listFile " CNV:VARIANCE STR " . $weightCnv . "\n"; - print $listFile " CNV:PSF STR " . $psfCnv . "\n"; - - print $listFile " SOURCES STR " . $sources . "\n"; - - print $listFile "END\n\n"; -} - -# Recipes to use based on reduction class -$reduction = 'DEFAULT' unless defined $reduction; -my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_PSPHOT'); # Recipe to use for psphot -my $recipe_ppsub = $ipprc->reduction($reduction, 'STACKPHOT_PPSUB'); # Recipe to use for ppsub -my $recipe_ppstack = $ipprc->reduction($reduction, 'STACKPHOT_PPSTACK'); # Recipe to use for ppstack -unless ($recipe_psphot) { - &my_die("Couldn't find selected reduction for STACKPHOT: $reduction\n", $sky_id, $PS_EXIT_CONFIG_ERROR); -} - -print "reduction: $reduction\n"; -print "recipe_psphot: $recipe_psphot\n"; -print "recipe_ppsub: $recipe_ppsub\n"; -print "recipe_ppstack: $recipe_ppstack\n"; - -# my $cmdflags; - -# Perform stack photometry analysis -{ - my $command = "$psphotStack $outroot"; - $command .= " -input $listName"; - $command .= " -threads $threads" if defined $threads; - $command .= " -recipe PSPHOT $recipe_psphot"; - $command .= " -recipe PPSUB $recipe_ppsub"; - $command .= " -recipe PPSTACK $recipe_ppstack"; - $command .= " -dumpconfig $configuration"; - $command .= " -tracedest $traceDest -log $logDest"; - # $command .= " -dbname $dbname" if defined $dbname; - # $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id; - # $command .= " -source_id $source_id" if defined $source_id; - - unless ($no_op) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform psphotStack: $error_code", $sky_id, $error_code); - } - - # my $outputStatsReal = $ipprc->file_resolve($outputStats); - # &my_die("Couldn't find expected output file: $outputStats", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); - - # measure chip stats - # $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL"; - # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - # run(command => $command, verbose => $verbose); - # unless ($success) { - # $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - # &my_die("Unable to perform ppStatsFromMetadata: $error_code", $sky_id, $error_code); - # } - # foreach my $line (@$stdout_buf) { - # $cmdflags .= " $line"; - # } - # chomp $cmdflags; - - # my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag - - my $quality = 0; - if (!$quality) { - - # Get the output filenames - # we have one set of output files per input file set - for (my $i = 0; $i < @$files; $i++) { - my $outputName = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot, $i); - my $outputMask = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot, $i); - my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot, $i); - my $outputSources = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot, $i); - - # XXX these are optional and not generated by default - # &my_die("Couldn't find expected output file: $outputName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); - # &my_die("Couldn't find expected output file: $outputMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); - # &my_die("Couldn't find expected output file: $outputVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance); - &my_die("Couldn't find expected output file: $outputSources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); - } - - #my $configuration = $ipprc->filename("PPSUB.CONFIG", $outroot); - #my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); - #my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); - - my $chisqName = $ipprc->filename("PSPHOT.CHISQ.IMAGE", $outroot); - my $chisqMask = $ipprc->filename("PSPHOT.CHISQ.MASK", $outroot); - my $chisqVariance = $ipprc->filename("PSPHOT.CHISQ.VARIANCE", $outroot); - - # XXX check the recipe -- should we expect these to exist? - # &my_die("Couldn't find expected output file: $chisqName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqName); - # &my_die("Couldn't find expected output file: $chisqMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqMask); - # &my_die("Couldn't find expected output file: $chisqVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqVariance); - } - } else { - print "Not executing: $command\n"; - } +if ($nInputs > 1) { + my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_PSPHOT'); # Recipe to use for psphot + my $recipe_ppsub = $ipprc->reduction($reduction, 'STACKPHOT_PPSUB'); # Recipe to use for ppsub + my $recipe_ppstack = $ipprc->reduction($reduction, 'STACKPHOT_PPSTACK'); # Recipe to use for ppstack + unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for STACKPHOT: $reduction\n", $sky_id, $PS_EXIT_CONFIG_ERROR); + } + + print "reduction: $reduction\n"; + print "recipe_psphot: $recipe_psphot\n"; + print "recipe_ppsub: $recipe_ppsub\n"; + print "recipe_ppstack: $recipe_ppstack\n"; + my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot); + + foreach my $file (@$files) { + print $listFile "INPUT METADATA\n"; + + # XXX if we take the input from 'warp', we will need to make different selections here + my $path_base = $file->{path_base}; + print "input: $path_base\n"; + + my $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $path_base ); # Image name + my $maskCnv = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base ); # Mask name + my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name + + my $imageRaw = $ipprc->filename("PPSTACK.UNCONV", $path_base ); # Image name + my $maskRaw = $ipprc->filename("PPSTACK.UNCONV.MASK", $path_base ); # Mask name + my $weightRaw = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $path_base ); # Weight name + + my $sources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $path_base ); # Sources name + + # XXX is this the correct PSF file? + my $psfCnv = $ipprc->filename("PPSTACK.TARGET.PSF", $path_base ); # PSF name + + # XXX we could make some different choices if some inputs do not exist... + &my_die("Couldn't find input: $imageRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageRaw"); + &my_die("Couldn't find input: $maskRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskRaw"); + &my_die("Couldn't find input: $weightRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightRaw"); + &my_die("Couldn't find input: $imageCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv"); + &my_die("Couldn't find input: $maskCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv"); + &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv"); + &my_die("Couldn't find input: $psfCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); + &my_die("Couldn't find input: $sources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources"); + + print $listFile " RAW:IMAGE STR " . $imageRaw . "\n"; + print $listFile " RAW:MASK STR " . $maskRaw . "\n"; + print $listFile " RAW:VARIANCE STR " . $weightRaw . "\n"; + + print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; + print $listFile " CNV:MASK STR " . $maskCnv . "\n"; + print $listFile " CNV:VARIANCE STR " . $weightCnv . "\n"; + print $listFile " CNV:PSF STR " . $psfCnv . "\n"; + + print $listFile " SOURCES STR " . $sources . "\n"; + + print $listFile "END\n\n"; + } + + # my $cmdflags; + + # Perform stack photometry analysis + { + my $command = "$psphotStack $outroot"; + $command .= " -input $listName"; + $command .= " -threads $threads" if defined $threads; + $command .= " -recipe PSPHOT $recipe_psphot"; + $command .= " -recipe PPSUB $recipe_ppsub"; + $command .= " -recipe PPSTACK $recipe_ppstack"; + $command .= " -dumpconfig $configuration"; + $command .= " -tracedest $traceDest -log $logDest"; + # $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psphotStack: $error_code", $sky_id, $error_code); + } + + # my $outputStatsReal = $ipprc->file_resolve($outputStats); + # &my_die("Couldn't find expected output file: $outputStats", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); + + # measure chip stats + # $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL"; + # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + # run(command => $command, verbose => $verbose); + # unless ($success) { + # $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + # &my_die("Unable to perform ppStatsFromMetadata: $error_code", $sky_id, $error_code); + # } + # foreach my $line (@$stdout_buf) { + # $cmdflags .= " $line"; + # } + # chomp $cmdflags; + + # my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + + my $quality = 0; + if (!$quality) { + + # Get the output filenames + # we have one set of output files per input file set + for (my $i = 0; $i < @$files; $i++) { + my $outputName = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot, $i); + my $outputMask = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot, $i); + my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot, $i); + my $outputSources = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot, $i); + + # XXX these are optional and not generated by default + # &my_die("Couldn't find expected output file: $outputName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + # &my_die("Couldn't find expected output file: $outputMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + # &my_die("Couldn't find expected output file: $outputVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance); + &my_die("Couldn't find expected output file: $outputSources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + } + + #my $configuration = $ipprc->filename("PPSUB.CONFIG", $outroot); + #my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); + #my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); + + my $chisqName = $ipprc->filename("PSPHOT.CHISQ.IMAGE", $outroot); + my $chisqMask = $ipprc->filename("PSPHOT.CHISQ.MASK", $outroot); + my $chisqVariance = $ipprc->filename("PSPHOT.CHISQ.VARIANCE", $outroot); + + # XXX check the recipe -- should we expect these to exist? + # &my_die("Couldn't find expected output file: $chisqName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqName); + # &my_die("Couldn't find expected output file: $chisqMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqMask); + # &my_die("Couldn't find expected output file: $chisqVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqVariance); + } + } else { + print "Not executing: $command\n"; + } + } +} else { + # single input. Run psphot + # find the recipe + my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_SINGLE_PSPHOT'); # Recipe to use for psphot + unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for STACKPHOT: $reduction\n", $sky_id, $PS_EXIT_CONFIG_ERROR); + } + + print "reduction: $reduction\n"; + print "recipe_psphot: $recipe_psphot\n"; + + my $configuration = $ipprc->filename("PSPHOT.SKY.CONFIG", $outroot); + + my $file = $files->[0]; + + # XXX if we take the input from 'warp', we will need to make different selections here + my $path_base = $file->{path_base}; + print "input: $path_base\n"; + + # examine the recipe to determine whether to analyze the "raw" or convolved images + my $command = "$ppConfigDump -camera $camera -dump-recipe PSPHOT -recipe PSPHOT $recipe_psphot -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $sky_id, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $sky_id, $PS_EXIT_SYS_ERROR); + + my $use_raw = metadataLookupBool($recipeData, 'PSPHOT.STACK.USE.RAW'); + + my ($image, $mask, $variance); + if ($use_raw) { + $image = $ipprc->filename("PPSTACK.UNCONV", $path_base ); # Image name + $mask = $ipprc->filename("PPSTACK.UNCONV.MASK", $path_base ); # Mask name + $variance = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $path_base ); # Weight name + } else { + $image = $ipprc->filename("PPSTACK.OUTPUT", $path_base ); # Image name + $mask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base ); # Mask name + $variance = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name + } + + my $output_sources_filerule = "PSPHOT.OUT.CMF.MEF"; + my $output_psf_filerule = "PSPHOT.PSF.SKY.SAVE"; + + # XXX we could make some different choices if some inputs do not exist... + &my_die("Couldn't find input: $image", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$image"); + &my_die("Couldn't find input: $mask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$mask"); + &my_die("Couldn't find input: $variance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$variance"); + + # Perform stack photometry analysis + { + my $command = "$psphot $outroot"; + $command .= " -file $image"; + $command .= " -mask $mask"; + $command .= " -variance $variance"; + $command .= " -threads $threads" if defined $threads; + $command .= " -recipe PSPHOT $recipe_psphot"; + $command .= " -dumpconfig $configuration" if $configuration; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -F PSPHOT.OUTPUT $output_sources_filerule"; + $command .= " -F PSPHOT.PSF.SAVE $output_psf_filerule"; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psphot: $error_code", $sky_id, $error_code); + } + + my $quality = 0; + if (!$quality) { + my $outputSources = $ipprc->filename($output_sources_filerule, $outroot); + &my_die("Couldn't find expected output file: $outputSources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + my $outputPSF = $ipprc->filename($output_psf_filerule, $outroot); + &my_die("Couldn't find expected output file: $outputPSF", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF); + } + + &my_die("Couldn't find expected output file: $configuration", $sky_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($configuration); + } else { + print "Not executing: $command\n"; + } + } }