Index: trunk/ippScripts/scripts/lap_science.pl =================================================================== --- trunk/ippScripts/scripts/lap_science.pl (revision 32195) +++ trunk/ippScripts/scripts/lap_science.pl (revision 32422) @@ -200,4 +200,111 @@ return($exposure); } + +sub remake_this_exposure_by_update { + my $exposure = shift; + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + + my $label = $exposure->{label}; + + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + my $chiptool_info_cmd = "chiptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} "; + $chiptool_info_cmd .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $chiptool_info_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -listrun: $error_code", $exposure->{exp_id}, $data_group); + } + my $chips = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from chiptool -listrun", $exposure->{exp_id}, $data_group); + # There should be only one. + my $chip = ${ $chips }[0]; + my $chip_magicDS_id = 0; + if ($chip) { + $chip_magicDS_id = $chip->{magic_ds_id}; + } + if ($chip_magicDS_id == 0) { + return(&remake_this_exposure($exposure)); + } + + my $warptool_info_cmd = "warptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} "; + $warptool_info_cmd .= " -dbname $dbname " if defined $dbname; + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $warptool_info_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -listrun: $error_code", $exposure->{exp_id}, $data_group); + } + my $warps = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from warptool -listrun", $exposure->{exp_id}, $data_group); + # There should be only one. + my $warp = ${ $warps }[0]; + my $warp_id = 0; + if ($warp) { + $warp_id = $warp->{warp_id}; + } + else { + $exposure->{data_state} = 'drop'; + return($exposure); + } + if ($warp_id == 0) { + # Handle this correctly. Probably want to drop the exposure here. + $exposure->{data_state} = 'drop'; + return($exposure); + } + + my $chiptool_update_cmd = "chiptool -setimfiletoupdate -chip_id $exposure->{chip_id} -set_label $label"; + $chiptool_update_cmd .= " -dbname $dbname " if defined $dbname; + my $magicDS_update_cmd = "magicdstool -setfiletoupdate -magic_ds_id $chip_magicDS_id -set_label $label"; + $magicDS_update_cmd .= " -dbname $dbname " if defined $dbname; + my $warptool_update_cmd = "warptool -setskyfiletoupdate -warp_id $warp_id -set_label $label"; + $warptool_update_cmd .= " -dbname $dbname " if defined $dbname; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $chiptool_update_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -setimfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); + } + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $magicDS_update_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); + } + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $warptool_update_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -setskyfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); + } + + $exposure->{active} = 1; + return($exposure); +} + +sub determine_if_can_update { + my $exposure = shift; + + if (S64_IS_NOT_NULL($exposure->{chip_id})) { + if (($exposure->{chip_state} eq 'cleaned')|| + ($exposure->{chip_state} eq 'error_cleaned')) { + return(1); + } + } + + return(0); +} + # This is the "user level" subroutine. @@ -240,5 +347,7 @@ my $comment = $exposure->{comment}; - # This is a hack to fix old exposures that have no object. + my $updateable = determine_if_can_update($exposure); + + # This is a hack to fix old exposures that have no usable object/comment data. unless(defined($comment)) { $comment = ''; @@ -247,22 +356,41 @@ if ($comment =~ /3pi_/) { $object = $comment; - $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/$1/; + $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/F1 $1/; } elsif ($comment =~ / ps1_/) { $object = $comment; - $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/$1/; - } - else { - $exposure->{data_state} = 'new'; + $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/F2 $1/; + } + elsif ($comment =~ /ThreePi.*3pi_/) { + $object = $comment; + $object =~ s/^ThreePi . (\S+? 3pi_\d\d_\d\d\d\d) .*$/F3 $1/; + } + elsif ($comment =~ /ThreePi.*ps1_/) { + $object = $comment; + $object =~ s/^ThreePi . (\S+? ps1_\d\d_\d\d\d\d) .*$/F4 $1/; + } + elsif ($comment =~ /ThreePi /) { + $object = $comment; + $object =~ s/^ThreePi . (\S+? \d\d\d\d) .*$/F5 $1/; + } + elsif (($comment =~ /focus/i)|| + ($comment =~ /test/i)|| + ($comment =~ /hyster/i)|| + ($comment =~ /dither/i)|| + ($comment =~ /camera/i) + ){ + # This is junk that shouldn't exist. + $object = 'DROP'; + $exposure->{data_state} = 'drop'; $exposure->{pairwise} = 0; - $exposure->{private} = 1; - $exposure->{pair_id} = 9223372036854775807; - update_this_exposure($exposure); - $exposure = remake_this_exposure($exposure); - $counter++; + $exposure->{pair_id} = 9223372036854775807; + $exposure->{active} = 0; + &update_this_exposure($exposure); + $counter++; # To ensure everyone else is consistent next; } } + # Determine the current state of chipRuns for these exposures, and update/remake as needed. if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id if (($pairwise) && !($pair_id)) { @@ -270,18 +398,25 @@ &my_die("Exposure $exp_id for $lap_id is declared pairwise without a defined pair", $lap_id); } + $exposure->{data_state} = 'exists'; + if ($updateable) { # We know about this run, but the data needs to be regenerated. + $exposure->{data_state} = 'update'; + $exposure = remake_this_exposure_by_update($exposure); + } + &update_this_exposure($exposure); $matching{$object}{$comment} = $exp_id; $indexing{$exp_id} = $counter; $counter++; - next; } else { # We do not already have a chip_id. # Make a chipRun, and update the exposure. + $exposure->{data_state} = 'run'; $exposure = remake_this_exposure($exposure); - # Save our information for diff pairing. + &update_this_exposure($exposure); $matching{$object}{$comment} = $exp_id; $indexing{$exp_id} = $counter; $counter++; } + if ($verbose) { print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; @@ -296,5 +431,5 @@ print "$object $comment $matching{$object}{$comment} $indexing{$matching{$object}{$comment}} $exp_ids_to_diff[-1]\n"; } - @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; + @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; # This is effectively a sort by dateobs. if (( $#exp_ids_to_diff + 1) % 2 != 0) { # We have an odd number of exposures, even after comment filtering @@ -310,9 +445,7 @@ print "$exp_A $exp_B $exp_id_A $exp_id_B $indexing{$exp_id_A} $indexing{$exp_id_B}\n"; $exp_A->{pairwise} = 1; - $exp_A->{private} = 0; $exp_A->{pair_id} = $exp_B->{chip_id}; $exp_B->{pairwise} = 1; - $exp_B->{private} = 0; $exp_B->{pair_id} = $exp_A->{chip_id}; @@ -331,8 +464,5 @@ $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. } - if (!($exposure->{pairwise})) { - $exposure->{private} = 1; # If this isn't being pairwise diffed, it needs to be private - } - + update_this_exposure($exposure); } @@ -457,6 +587,6 @@ # Things I want to know before I'm through # my $needs_qstack = 0; - my $needs_something_remade = 0; - my $needs_something_private = 0; + my $needs_something_remade = 0; # I never need something remade, because we're committed to one-exp=one-chip + my $needs_something_private = 0; # I never need something private, as that is calculated by the difference engine. # my $can_qstack = 0; my $have_diff = 0; @@ -480,5 +610,4 @@ else { # We claimed to be pairwise, but do not have a valid pair_id. $exposure->{pairwise} = 0; - $exposure->{private} = 1; &update_this_exposure($exposure); } @@ -487,22 +616,18 @@ if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. $are_warped++; -# $can_qstack ++; $can_diff ++; $have_diff ++; $are_magicked ++; -# $can_fstack ++; next; } if ($exposure->{private}) { # I've declared this exposure private to this lapRun. -# $needs_qstack++; push @lonely_exposures, $exposure; -# $lonely_exposures++; } - if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun - $needs_something_remade = 1; - $exposure = remake_this_exposure($exposure); - } +# if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun +# $needs_something_remade = 1; +# $exposure = remake_this_exposure($exposure); +# } # Do quality checks here my $is_bad_quality = 0; @@ -536,14 +661,9 @@ ($exposure->{diffRun_state} eq 'full')) { $are_warped++; -# $needs_qstack++; - $needs_something_private = 1; if ($companion) { - $companion->{private} = 1; $companion->{pairwise} = 0; &update_this_exposure($companion); - push @lonely_exposures, $companion; -# $lonely_exposures++; +# push @lonely_exposures, $companion; } - $exposure->{private} = 1; $exposure->{pairwise} = 0; } @@ -558,13 +678,19 @@ # $can_qstack ++; $can_diff ++; - } - if (($exposure->{magicked}&& - &S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. - $are_magicked++; -# $can_fstack ++; + $exposure->{data_state} = 'to_diff'; } if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) { $have_diff ++; - } + $exposure->{data_state} = 'to_magic'; + } + if (($exposure->{magicked})&& + ($exposure->{warpRun_state})&& + ($exposure->{warpRun_state} eq 'full')&& + (&S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. + $are_magicked++; + $exposure->{data_state} = 'full'; +# $can_fstack ++; + } + &update_this_exposure($exposure); } @@ -575,13 +701,10 @@ print "STATUS: HAVE_FSTACK: $have_fstack\n"; -# print "STATUS: NEEDS_QSTACK: $needs_qstack\n"; print "STATUS: NEEDS_REMADE: $needs_something_remade\n"; print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n"; print "STATUS: ARE WARPED: $are_warped\n"; print "STATUS: ARE MAGICKED: $are_magicked\n"; -# print "STATUS: CAN_QSTACK: $can_qstack\n"; print "STATUS: CAN_DIFF: $can_diff\n"; print "STATUS: HAVE_DIFF: $have_diff\n"; -# print "STATUS: CAN_FSTACK: $can_fstack\n"; print "STATUS: TOTAL_EXPOSURES: $total_exposures\n"; @@ -924,5 +1047,5 @@ next; } - if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # Not sure how this would happen, but still. ## This happens when we inherit a complete exposure. + if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # This happens when we inherit a complete exposure. next; } @@ -964,9 +1087,21 @@ } else { # warp-qstack + # We need to decide if we can make a warp-stack diff: + if (&can_warp_stack_diff_be_made($exposure)) { # next; # $command .= '-pretend'; - $command .= " -definewarpstack -available -good_frac 0.2 "; - $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} "; - $already_queued{$exposure->{warp_id}} = 1; + $command .= " -definewarpstack -available -good_frac 0.2 "; + $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} "; + $already_queued{$exposure->{warp_id}} = 1; + $exposure->{private} = 0; + &update_this_exposure($exposure); + } + else { + $exposure->{private} = 1; + $exposure->{data_state} = 'drop'; + $already_queued{$exposure->{warp_id}} = 1; + &update_this_exposure($exposure); + next; + } } @@ -1006,10 +1141,46 @@ } +sub can_warp_stack_diff_be_made { + my $exposure = shift; + my $lap_id = $exposure->{lap_id}; + my $warp_id = $exposure->{warp_id}; + + my $command = "$laptool -diffcheck -lap_id $lap_id -warp_id $warp_id"; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -diffcheck: $error_code", $lap_id, $warp_id); + } + if (@$stdout_buf == 0) { + return(0); + } + my $skycells = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -diffcheck", $lap_id, $warp_id); + foreach my $skycell (@$skycells) { + if ($skycell->{stack_state} ne 'full') { + return(0); + } + if ($skycell->{warp_state} ne 'full') { + return(0); + } + unless (S64_IS_NOT_NULL($skycell->{stack_id})) { + return(0); + } + } + return(1); +} + + # Deactivate all exposures in this run. sub deactivate_exposures { my $exposures = shift; foreach my $exposure (@$exposures) { - $exposure->{active} = 0; - update_this_exposure($exposure); + unless ($exposure->{private} == 1) { + $exposure->{active} = 0; + update_this_exposure($exposure); + } } } @@ -1043,7 +1214,9 @@ 'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@', 'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@', + 'magictool -dbname @DBNAME@ -updaterun -set_state full -set_label @LABEL@.old -magic_id @MAGIC_ID@', 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage chip -stage_id @CHIP_ID@', 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage warp -stage_id @WARP_ID@', 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage diff -stage_id @DIFF_ID@'); + foreach my $exposure (@$exposures) { if ($exposure->{is_in_use}) { @@ -1077,4 +1250,13 @@ } } + if ($command =~ /\@MAGIC_ID\@/) { + if ((S64_IS_NOT_NULL($exposure->{magicked}))&& + ($exposure->{magicked} > 0)) { + $command =~ s/\@MAGIC_ID\@/$exposure->{magicked}/; + } + else { + next; + } + } $command =~ s/\@LABEL\@/$exposure->{label}/;