Index: trunk/ippScripts/scripts/lap_science.pl =================================================================== --- trunk/ippScripts/scripts/lap_science.pl (revision 31941) +++ trunk/ippScripts/scripts/lap_science.pl (revision 31945) @@ -1,4 +1,3 @@ #!/usr/bin/env perl - use warnings; @@ -30,6 +29,8 @@ my ( $chip_mode, $monitor_mode, $cleanup_mode); -# Global configuration constants that probably should be read from elsewhere. -my $qstack_threshold = 0.05; # Only make a quickstack if more than 5% of the exposures require it. +# +# Global configuration constants that probably should be read from elsewhere. +my $qstack_threshold = 0.05; # Only make a quickstack if more than 5% of the exposures require it. +my $minimum_stack_inputs = 6; # We can avoid magicking stack inputs if we have more than this number. GetOptions( @@ -263,5 +264,4 @@ } - if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id if (($pairwise) && !($pair_id)) { @@ -283,5 +283,7 @@ $counter++; } - print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; + if ($verbose) { + print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; + } } @@ -322,5 +324,7 @@ # Scan all exposures, and ensure that pairwise and private are set correctly foreach my $exposure (@$exposures) { - print "YY: $exposure\n"; + if ($verbose) { + print "YY: $exposure\n"; + } if ($exposure->{pairwise} && !($exposure->{pair_id})) { $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. @@ -451,13 +455,17 @@ # Things I want to know before I'm through - my $needs_qstack = 0; +# my $needs_qstack = 0; my $needs_something_remade = 0; my $needs_something_private = 0; - my $can_qstack = 0; - my $have_diff = 0; - my $can_diff = 0; - my $can_fstack = 0; +# my $can_qstack = 0; + my $have_diff = 0; + my $can_diff = 0; +# my $can_fstack = 0; my $total_exposures = 0; my @lonely_exposures = (); + + my $are_warped = 0; + my $are_magicked = 0; + foreach my $exposure (@$exposures) { $total_exposures++; @@ -477,5 +485,5 @@ if ($exposure->{private}) { # I've declared this exposure private to this lapRun. - $needs_qstack++; +# $needs_qstack++; push @lonely_exposures, $exposure; # $lonely_exposures++; @@ -516,5 +524,6 @@ unless ((defined($exposure->{diffRun_state}))&& ($exposure->{diffRun_state} eq 'full')) { - $needs_qstack++; + $are_warped++; +# $needs_qstack++; $needs_something_private = 1; if ($companion) { @@ -530,12 +539,13 @@ $exposure->{data_state} = 'drop'; &update_this_exposure($exposure); - } if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. - $can_qstack ++; + $are_warped++; +# $can_qstack ++; $can_diff ++; $have_diff ++; - $can_fstack ++; + $are_magicked ++; +# $can_fstack ++; next; } @@ -543,10 +553,12 @@ if (($exposure->{warpRun_state})&& # This exposure has a warp ($exposure->{warpRun_state} eq 'full')) { # This exposure's warp is done. - $can_qstack ++; + $are_warped++; +# $can_qstack ++; $can_diff ++; } if (($exposure->{magicked}&& &S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. - $can_fstack ++; + $are_magicked++; +# $can_fstack ++; } if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) { @@ -561,11 +573,13 @@ print "STATUS: HAVE_FSTACK: $have_fstack\n"; - print "STATUS: NEEDS_QSTACK: $needs_qstack\n"; +# print "STATUS: NEEDS_QSTACK: $needs_qstack\n"; print "STATUS: NEEDS_REMADE: $needs_something_remade\n"; print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n"; - print "STATUS: CAN_QSTACK: $can_qstack\n"; + print "STATUS: ARE WARPED: $are_warped\n"; + print "STATUS: ARE MAGICKED: $are_magicked\n"; +# print "STATUS: CAN_QSTACK: $can_qstack\n"; print "STATUS: CAN_DIFF: $can_diff\n"; print "STATUS: HAVE_DIFF: $have_diff\n"; - print "STATUS: CAN_FSTACK: $can_fstack\n"; +# print "STATUS: CAN_FSTACK: $can_fstack\n"; print "STATUS: TOTAL_EXPOSURES: $total_exposures\n"; @@ -574,51 +588,73 @@ exit(0); } - if (($needs_qstack > 0)&& # Do we need the quick stack? - ($defined_qstack == 0)&& # Have we not made it already? - ($can_qstack == $total_exposures)&& # Are all warps done? - ($needs_something_remade == 0) # Do we need to wait for a stolen exposure? - ) { - if ($needs_qstack < $total_exposures * $qstack_threshold) { # Drop any exposures that are requiring the qstack - print "STATUS: Insufficient unpaired exposures to justify quickstack. Dropping unpaired exposures\n"; - foreach my $exposure (@lonely_exposures) { - $exposure->{data_state} = 'drop'; - &update_this_exposure($exposure); - } - } - else { - print "STATUS: Will now queue quickstacks\n"; - &queue_quickstack($exposures); - } - } - - if (($can_diff == $total_exposures)&& # Are all warps done? - ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? - ($have_diff < $can_diff)&& # We haven't already done them all. - (($needs_qstack == 0)|| # Are we just doing pairwise diffs? - ($needs_qstack == 1)&&($have_qstack == 1)) ) { # If we need a quickstack, do we have it? - print "STATUS: Will now queue diffs\n"; - &queue_diffs($exposures); - } - - if (($can_fstack == $total_exposures)&& # Are all warps done? - ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? - ($defined_fstack == 0)) { # Have we not made it already? - print "STATUS: Will now queue final stacks\n"; - &queue_finalstack($exposures); - } - - if (($have_fstack == 1)) { # Are we all done making the final product? - print "STATUS: Will now deactivate exposures for this complete lapRun.\n"; - &deactivate_exposures($exposures); - $command = "$laptool -updaterun -lap_id $lap_id"; - $command .= " -dbname $dbname " if defined $dbname; - $command .= " -set_state full "; - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); - } - } + + # Do the decision making based on what we've learned + if ($are_warped == $total_exposures) { # Are all the warps done? + if ($defined_fstack == 0) { # We have not yet made any stacks. + print "STATUS: We can attempt to queue deep final stacks.\n"; + my $Nstacks_made = &queue_muggle_finalstack($exposures); + print "STATUS: Made $Nstacks_made final stacks.\n"; + exit($Nstacks_made); + # If this doesn't do anything, we may end up stuck here. Therefore, we need a more robust test. + } + else { # We have made a stack before for this lapRun. + if ($have_fstack == $defined_fstack) { # We have made all the stacks we have asked for (so far) + if ($are_magicked < $total_exposures) { # But we have not yet made diffs + if (($can_diff == $total_exposures)&& # And we have exposures that should be diffed. + ($have_diff < $can_diff)) { # and we haven't made them all yet, either. + print "STATUS: Dropping unpaired exposures and making pairwise diffs.\n"; + foreach my $exposure (@lonely_exposures) { + $exposure->{data_state} = 'drop'; + &update_this_exposure($exposure); + } + &queue_diffs($exposures); + exit(0); + } + else { + # This is apparently the case where we've queued diffs to run, but they have not yet + # completed though the destreak phase. + print "STATUS: Waiting for diffs/magic/destreak to complete.\n"; + exit(0); + } + } + elsif ($are_magicked == $total_exposures) { # We have made all of the diffs, and propagated through magic + print "STATUS: We can create all remaining stacks possible.\n"; + my $Nstacks_made = &queue_magic_finalstack($exposures); + if ($Nstacks_made == 0) { # We have made all the stacks that we will ever make. + print "STATUS: No more stacks generated. Deactivating exposures for this complete lapRun.\n"; + &deactivate_exposures($exposures); + $command = "$laptool -updaterun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_state full "; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + exit(0); + } + else { + print "STATUS: $Nstacks_made stacks made.\n"; + exit(0); + } + } + else { # What? + print "STATUS: More exposures magicked than exist. This is a mistake.\n"; + exit(0); + } + } + else { # We have not yet made all the stacks we've asked for. + print "STATUS: Defined $defined_fstack, completed $have_fstack. Waiting for stacking to finish.\n"; + exit(0); + } + } + } + else { # Not all warps finished. + # do nothing + print "STATUS: Not all warps are finished, waiting for that to finish.\n"; + exit(0); + } + print "\n"; return(0); @@ -657,5 +693,6 @@ $command .= " -pretend " if defined $debug; $command .= " -dbname $dbname " if defined $dbname; - $command .= " -definebyquery -select_label ${label}% -select_skycell_id ${proj_cell}.% -select_filter $filter "; + $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; +# $command .= " -select_label $label "; # Removed to allow exposure sharing to work $command .= " -set_label ${label} -set_data_group $data_group "; $command .= " -set_workdir $workdir -set_dist_group NODIST "; @@ -669,4 +706,6 @@ &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); } + my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); $command = "$stacktool "; @@ -702,7 +741,7 @@ } - -} -sub queue_finalstack { + return($#{ $stacks_made } + 1); +} +sub queue_muggle_finalstack { my $exposures = shift; # reference to exposure array; my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information @@ -720,5 +759,5 @@ foreach $exposure (@$exposures) { if (($exposure->{data_state} ne 'drop')&& - (S64_IS_NOT_NULL($exposure->{magicked}))&& +# (S64_IS_NOT_NULL($exposure->{magicked}))&& (S64_IS_NOT_NULL($exposure->{warp_id}))) { $warps .= " -warp_id $exposure->{warp_id} "; @@ -737,7 +776,8 @@ $command .= " -pretend " if defined $debug; $command .= " -dbname $dbname " if defined $dbname; - $command .= " -definebyquery -select_label ${label}% -select_skycell_id ${proj_cell}.% -select_filter $filter "; + $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; +# $command .= " -select_label $label "; # Removed to allow exposure sharing to work $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group "; - $command .= " -min_num 2 -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} "; + $command .= " -min_num ${minimum_stack_inputs} -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} "; $command .= " $warps "; @@ -748,5 +788,6 @@ &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); } - + my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); $command = "$stacktool "; @@ -781,4 +822,85 @@ } + return($#{ $stacks_made } + 1); +} +sub queue_magic_finalstack { + my $exposures = shift; # reference to exposure array; + my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information + + my $lap_id = $exposure->{lap_id}; + my $label = $exposure->{label}; + my $filter = $exposure->{filter}; + my $proj_cell = $exposure->{projection_cell}; + + unless (defined($label) && defined($filter) && defined($proj_cell)) { + &my_die("Unable to perform stacktool. Insufficient information.", $lap_id); + } + + my $warps = ''; + foreach $exposure (@$exposures) { + if (($exposure->{data_state} ne 'drop')&& + (S64_IS_NOT_NULL($exposure->{magicked}))&& + (S64_IS_NOT_NULL($exposure->{warp_id}))) { + $warps .= " -warp_id $exposure->{warp_id} "; + } + } + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${proj_cell}.final.${date}"; + + my $command = "$stacktool "; + $command .= " -pretend " if defined $debug; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; +# $command .= " -select_label $label "; # Removed to allow exposure sharing to work + $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group "; + $command .= " -min_num 2 -min_new 2 -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} "; + $command .= " $warps "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); + } + my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); + + $command = "$stacktool "; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -sassskyfile -data_group $data_group "; + $command .= " -filter $filter -projection_cell ${proj_cell} "; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -sassskyfile: $error_code", $lap_id); + } + + my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stacktool -sassskyfile", $lap_id, ""); + + my $stack = ${ $stacks }[0]; + my $sass_id = $stack->{sass_id}; + unless (defined($sass_id)) { + &my_die("Unable to parse metadata from stacktool for sass_id", $lap_id, ""); + } + print "FINAL_SASS_ID: $sass_id\n"; + $command = "$laptool -updaterun -lap_id $lap_id -set_final_sass_id $sass_id"; + $command .= " -dbname $dbname " if defined $dbname; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + return($#{ $stacks_made } + 1); } sub queue_diffs { @@ -831,5 +953,5 @@ my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; $command .= " -definewarpwarp "; - $command .= "-input_label $label -template_label $label "; +# $command .= "-input_label $label -template_label $label "; $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} "; $retry_command = $command; @@ -840,7 +962,9 @@ } else { # warp-qstack - $command .= " -definewarpstack -set_reduction NOCONVDIFF -available -good_frac 0.2 "; - $command .= " -warp_id $exposure->{warp_id} -stack_label ${label}.quick "; - $already_queued{$exposure->{warp_id}} = 1; + next; # We no longer wish to make these, and should never get here. + $command .= '-pretend'; +# $command .= " -definewarpstack -set_reduction NOCONVDIFF -available -good_frac 0.2 "; +# $command .= " -warp_id $exposure->{warp_id} -stack_label ${label}.quick "; +# $already_queued{$exposure->{warp_id}} = 1; }