Index: branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl =================================================================== --- branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl (revision 30586) +++ branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl (revision 30587) @@ -325,4 +325,5 @@ # get the UNIX version of the (possible) neb: or path: filename my $uriReal = $ipprc->file_resolve( $uri ); + &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal; # funpack into the temp file. Index: branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl =================================================================== --- branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl (revision 30586) +++ branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl (revision 30587) @@ -15,5 +15,7 @@ my $REALRUN = 1; -my ( $class_id, $exp_id, $this_uri, $previous_uri, $imfile_state, $camera, $dbname, $logfile, $verbose, $save_temps, $rerun ); +my ( $class_id, $exp_id, $this_uri, $previous_uri, $imfile_state, $camera, $dbname, $logfile, $verbose, $save_temps ); +my ( $rerun, $fixit ); +my $continue = 0; GetOptions( 'exp_id=s' => \$exp_id, # exposure identifier @@ -26,4 +28,6 @@ 'logfile=s' => \$logfile, 'rerun' => \$rerun, +# 'fix|f' => \$fixit, + 'continue=i' => \$continue, 'verbose' => \$verbose, # Print to stdout 'save-temps' => \$save_temps, # Save temporary files? @@ -33,9 +37,10 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --class_id --this_uri --dbname", + -msg => "Required options: --class_id --this_uri --exp_id --dbname", -exitval => 3, ) unless defined $class_id and defined $this_uri and + defined $exp_id and defined $dbname; @@ -93,68 +98,157 @@ print ">>$burntoolStateGood<<\n"; +# Determine what to do if we're fixing. +# if ($fixit) { +# ($exp_id,$this_uri,$previous_uri,$continue) = fix_burntool($exp_id,$this_uri,$class_id); +# } + + +# Precalculate what we should be able to do if we're continuing +my @continue_uris; +my @continue_exp_ids; +my @continue_states; +if ($continue > 0) { + my $regtool_get_date = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -dbname $dbname"; + my $dateobs; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $regtool_get_date, verbose => $verbose ); + unless ($success) { + $continue = 0; + last; + } + my $imfile_data = $mdcParser->parse(join "", @$stdout_buf); + my $imfile_data2 = parse_md_list($imfile_data); + if ($#{ $imfile_data2 } > 0) { + &my_die("Too many entries returned for date query", $exp_id, $class_id, $PS_EXIT_SYS_ERROR); + } + foreach my $entry (@$imfile_data2) { + $dateobs = $entry->{dateobs}; + } + my $date = $dateobs; $date =~ s/T.*//; + + my $regtool_status_cmd = "$regtool -checkstatus -dateobs_begin $dateobs -dateobs_end ${date}T17:30:00 -class_id $class_id -dbname $dbname"; + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $regtool_status_cmd, verbose => $verbose ); + unless ($success) { + $continue = 0; + } + if ($continue != 0) { + my $night_data = $mdcParser->parse(join "", @$stdout_buf); + my $night_data2 = parse_md_list($night_data); + my $addable = 0; + + foreach my $entry (@$night_data2) { + my ($this_exp_id,$this_uri,$this_state); + if ($entry->{exp_id} == $exp_id) { + $addable = 1; + next; + } + if (($addable == 1)&&($#continue_exp_ids < $continue)) { + $this_exp_id = $entry->{exp_id}; + $this_uri = $entry->{uri}; + $this_state = $entry->{imfile_state}; + print ">> $this_exp_id $this_uri $this_state\n"; + if ($this_state eq 'pending_burntool') { + push @continue_exp_ids, $this_exp_id; + push @continue_uris, $this_uri; + push @continue_states, $this_state; + } + else { + last; + } + } + if ($#continue_exp_ids >= $continue) { + last; + } + } + } +} +$continue = $#continue_exp_ids + 1; + +# Look over at least one uri/imfile/exp_id and do burntool on it. +my $next_exp_id; +for (my $iteration = 0; $iteration <= $continue; $iteration ++) { # Set up files -my $rawImfile = $this_uri; -my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0); - -my $outTable = $this_uri; -$outTable =~ s/fits$/burn.tbl/; -my $outTableReal = $ipprc->file_resolve($outTable, 1); - -my $previousTable; -my $previousTableReal; -if ($previous_uri) { - $previousTable = $previous_uri; - $previousTable =~ s/fits$/burn.tbl/; - $previousTableReal = $ipprc->file_resolve($previousTable, 0); -} - + my $rawImfile = $this_uri; + my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0); + + my $outTable = $this_uri; + $outTable =~ s/fits$/burn.tbl/; + my $outTableReal = $ipprc->file_resolve($outTable, 1); + + my $previousTable; + my $previousTableReal; + if ($previous_uri) { + $previousTable = $previous_uri; + $previousTable =~ s/fits$/burn.tbl/; + $previousTableReal = $ipprc->file_resolve($previousTable, 0); + } + # Set state to processing: -my $burntool_state = 0; - -my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN); -if ($status) { - &my_die("failed to update imfile"); -} -$burntool_state = 1; - - + my $burntool_state = 0; + + my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN); + if ($status) { + &my_die("failed to update imfile"); + } + $burntool_state = 1; + + # funpack -my $tempfile = new File::Temp ( TEMPLATE => "burntool.${exp_id}.${class_id}.XXXX", - DIR => '/tmp/', - UNLINK => !$save_temps, - SUFFIX => '.fits'); -my $tempPixels = $tempfile->filename; - -$status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN); -if ($status) { - &my_die("failed on funpack",$exp_id,$class_id); -} - + my $tempfile = new File::Temp ( TEMPLATE => "burntool.${exp_id}.${class_id}.XXXX", + DIR => '/tmp/', + UNLINK => !$save_temps, + SUFFIX => '.fits'); + my $tempPixels = $tempfile->filename; + + $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN); + if ($status) { + &my_die("failed on funpack",$exp_id,$class_id); + } + # burntool -if (($previousTableReal)&&($previousTableReal ne '')) { - $status = vsystem ("$burntool $tempPixels in=$previousTableReal out=$outTableReal tableonly=t persist=t", $REALRUN); -} -else { - $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN); -} -if ($status) { - &my_die("failed on burntool",$exp_id,$class_id); -} -&my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal); - + if (($previousTableReal)&&($previousTableReal ne '')) { + $status = vsystem ("$burntool $tempPixels in=$previousTableReal out=$outTableReal tableonly=t persist=t", $REALRUN); + } + else { + $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN); + } + if ($status) { + &my_die("failed on burntool",$exp_id,$class_id); + } + &my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal); + # Replicate files -$status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN); -$status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN); - + $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN); + $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN); + +# Prep for next iteration if necessary. + if ($iteration < $continue) { + $next_exp_id = shift(@continue_exp_ids); + my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $next_exp_id -class_id $class_id -set_state check_burntool"; + $status = vsystem ($command, $REALRUN); + if ($status) { + $continue = 0; + } + } # Set state to finished. -my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState"; -if ($imfile_state ne 'full') { - $command .= " -set_state full "; -} -$status = vsystem ($command, $REALRUN); -if ($status) { - &my_die("failed to update imfile"); -} - + my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState"; + if ($imfile_state ne 'full') { + $command .= " -set_state full "; + } + $status = vsystem ($command, $REALRUN); + if ($status) { + &my_die("failed to update imfile"); + } + +# Set values for the next iteration if we are doing another iteration. + $previous_uri = $this_uri; + if ($iteration < $continue) { + $this_uri = shift (@continue_uris); + $exp_id = $next_exp_id; + $imfile_state = shift(@continue_states); + } + +} exit 0; @@ -183,5 +277,145 @@ exit 1; } - - - + +# sub fix_burntool { +# my ($exp_id,$this_uri,$class_id,$burntoolGoodState) = @_; +# my ($dateobs,$exp_name); +# # Calculate the date for this exposure, and calculate back to find the burntool period. +# my $regtool_get_date = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -dbname $dbname"; +# my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $regtool_get_date, verbose => $verbose ); +# unless ($success) { +# $continue = 0; +# return($exp_id,$this_uri,undef,0); +# } +# my $imfile_data = $mdcParser->parse(join "", @$stdout_buf); +# foreach my $entry (@$imfile_data) { +# if ($entry->{name} eq 'dateobs') { +# $dateobs = $entry->{value}; +# } +# if ($entry->{name} eq 'exp_name') { +# $exp_name = $entry->{value}; +# } +# } +# my $date = $dateobs; $date =~ s/T.*//; +# my $time = $dateobs; $time =~ s/.*T//; +# my ($hour,$min,$sec) = split /\:/, $time; #/; +# if ($min >= 30) { +# $min -= 30; +# } +# else { +# $min += 30; +# $hour -= 1; +# } +# my $dateobs_begin = sprintf("%sT%02d:%02d:%02d",$date,$hour,$min,$sec); + + +# # Get the night data for this date range. +# my $regtool_status_cmd = "$regtool -checkstatus -dateobs_begin $dateobs_begin -dateobs_end $dateobs -class_id $class_id -dbname $dbname"; +# ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $regtool_get_date, verbose => $verbose ); +# unless ($success) { +# return($exp_id,$this_uri,undef,0); +# } + +# my $night_data = $mdcParser->parse(join "", @$stdout_buf); + +# my $mode; +# my ($row_exp_id,$row_uri,$row_data_state,$row_bt_state); +# my $entry = @$night_data[-1]; +# foreach my $row (@$entry) { # This is a single exposure, so let's get everything we'll need here. +# if ($row->{name} eq 'exp_id') { +# $row_exp_id = $row->{value}; +# } +# elsif ($row->{name} eq 'uri') { +# $row_uri = $row->{uri}; +# } +# elsif ($row->{name} eq 'data_state') { +# $row_data_state = $row->{data_state}; +# } +# elsif ($row->{name} eq 'burntool_state') { +# $row_bt_state = $row->{burntool_state}; +# } +# } +# # Determine what to do. +# if ($row_exp_id == $exp_id) { +# if (($row_bt_state != $burntoolStateGood)) { +# # Never been run. +# $mode = 1; +# } +# else { +# $mode = 0; +# } +# } +# if ($mode == 0) { +# # If we're here, then the database thinks we have a table for this uri on disk. Let's see if that's true. +# # Zero byte file/good alternate +# # Inconsistent md5sums +# # Missing this table only. +# my $neb = $ipprc->nebulous(); +# my $table_key = $this_uri; $table_key =~ s/fits/burn.tbl/; + +# my $neb_instances = $neb->find_instances($table_key, 'any'); + +# my $Ninstances = $#{ $neb_instances } + 1; + +# if (($Ninstances == 0)||($Ninstances == 1)) { +# #Missing this table only. This should fall through to do a regular pass on this uri +# $continue = 0; +# $entry = @$night_data[-2]; +# foreach my $row (@$entry) { # This is a single exposure, so let's get everything we'll need here. +# if ($row->{name} eq 'exp_id') { +# $row_exp_id = $row->{value}; +# } +# elsif ($row->{name} eq 'uri') { +# $row_uri = $row->{uri}; +# } +# elsif ($row->{name} eq 'data_state') { +# $row_data_state = $row->{data_state}; +# } +# elsif ($row->{name} eq 'burntool_state') { +# $row_bt_state = $row->{burntool_state}; +# } +# } +# $previous_uri = $row_uri; +# # Go through to regular processing of this exposure. +# } +# elsif ($Ninstances == 2) { +# $md5sum_A = md5sum(${ $neb_instances }[0]); +# $md5sum_B = md5sum(${ $neb_instances }[1]); +# if ($md5sum_A != $md5sum_B) { +# # md5sums are different +# if ($md5sum_A = 'd41d8cd98f00b204e9800998ecf8427e') { +# # copy B -> A +# } +# elsif ($md5sum_B = 'd41d8cd98f00b204e9800998ecf8427e') { +# # copy A -> B +# } +# else { +# # different but unknown why +# } +# } +# } +# } + +# if ($mode == 1) { +# foreach my $entry (reverse(@$night_data)) { # reverse it so we can start with the exposure we have and go backwards +# foreach my $row (@$entry) { # This is a single exposure, so let's get everything we'll need here. +# if ($row->{name} eq 'exp_id') { +# $row_exp_id = $row->{value}; +# } +# elsif ($row->{name} eq 'uri') { +# $row_uri = $row->{uri}; +# } +# elsif ($row->{name} eq 'data_state') { +# $row_data_state = $row->{data_state}; +# } +# elsif ($row->{name} eq 'burntool_state') { +# $row_bt_state = $row->{burntool_state}; +# } +# } +# } +# } + +# return($exp_id,$this_uri,$previous_uri,$continue); +# } Index: branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl (revision 30586) +++ branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl (revision 30587) @@ -207,14 +207,5 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + set_destreak_goto_cleaned(); } else { @@ -324,14 +315,5 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + set_destreak_goto_cleaned(); } else { @@ -469,14 +451,5 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + set_destreak_goto_cleaned(); } else { @@ -603,14 +576,5 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + set_destreak_goto_cleaned(); } else { @@ -759,14 +723,5 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + set_destreak_goto_cleaned(); } else { @@ -2062,4 +2017,23 @@ } +# this gets set to 1 the first time we set the corresponding destreak run to be cleaned +my $ds_done = 0; +sub set_destreak_goto_cleaned { + + return if $ds_done; + + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + my $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } + $ds_done = 1; +} + # XXX we currently do not set the error state in the db on my_die sub my_die Index: branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl (revision 30586) +++ branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl (revision 30587) @@ -159,5 +159,4 @@ my $component = $comp->{component}; my $backup_path_base = $comp->{backup_path_base}; - my $recovery_path_base = $comp->{recovery_path_base}; my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom); my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom); @@ -197,17 +196,4 @@ $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $component); } - if ($recovery_path_base) { - $rimage = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $component); - - if ($dynamicMasks) { - my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $component); - # This is kludgey but correct - $rmask = dirname($recovery_path_base) . "/" . basename($mask); - $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component); - } else { - $rmask = $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component); - } - $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $component); - } } elsif ($stage eq "camera") { if ($backup_path_base) { @@ -221,12 +207,20 @@ $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base); } - if ($recovery_path_base) { - $rimage = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base); - $rmask = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base); - $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base); - $rsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $recovery_path_base); - } } elsif ($stage eq "diff") { my $name = "PPSUB.OUTPUT"; + if ($backup_path_base) { + $bimage = $ipprc->filename($name, $backup_path_base); + $bmask = $ipprc->filename("$name.MASK", $backup_path_base); + $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); + # bills 2011-01-24 + # don't clean up the uncensored Diff sources file + # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); + } + } + + delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask); + + if ($stage eq "diff" and $warp_warp) { + my $name = "PPSUB.INVERSE"; if ($backup_path_base) { $bimage = $ipprc->filename($name, $backup_path_base); @@ -237,32 +231,5 @@ # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); } - if ($recovery_path_base) { - $rimage = $ipprc->filename($name, $recovery_path_base); - $rmask = $ipprc->filename("$name.MASK", $recovery_path_base); - $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base); - $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base); - } - } - - delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask); - - if ($stage eq "diff" and $warp_warp) { - my $name = "PPSUB.INVERSE"; - if ($backup_path_base) { - $bimage = $ipprc->filename($name, $backup_path_base); - $bmask = $ipprc->filename("$name.MASK", $backup_path_base); - $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); - # bills 2011-01-24 - # don't clean up the uncensored sources file - # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); - } - if ($recovery_path_base) { - $rimage = $ipprc->filename($name, $recovery_path_base); - $rmask = $ipprc->filename("$name.MASK", $recovery_path_base); - $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base); - $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base); - } - # undef is not necessary, it's just to keep the lists in the same order. - delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources); + delete_files($bimage, $bmask, $bweight, $bsources); } my $command = "$magicdstool -tocleanedfile -magic_ds_id $magic_ds_id -component $component"; Index: branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl =================================================================== --- branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl (revision 30586) +++ branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl (revision 30587) @@ -1266,4 +1266,27 @@ } +sub multi_date_verify_uniqueness_diff { + my $warp_id_1 = shift; + my $warp_id_2 = shift; + my $date = shift; + my $target = shift; + + my $db = init_gpc_db(); + $date =~ s/-//g; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $count = 0; + + my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_1 AND warp2 = $warp_id_2"; + my $data_ref = $db->selectall_arrayref( $sth ); + $count += $#{ $data_ref } + 1; + + $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_2 AND warp2 = $warp_id_1"; + $data_ref = $db->selectall_arrayref( $sth ); + $count += $#{ $data_ref } + 1; + + return($count); +} + sub pre_diff_queue { my $date = shift; @@ -1375,5 +1398,13 @@ $metadata_out{nsDiffState} = 'DIFFING'; } - + if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') { + foreach my $target (sort (keys %science_config)) { + if ($science_config{$target}{DIFFABLE} == 1) { + foreach my $filter (@filter_list) { + multi_date_diff_queue($date,$target,$filter,$pretend); + } + } + } + } # if (($Npotential == $Nnoexp)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)) { # $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; @@ -1511,4 +1542,137 @@ } + +sub multi_date_diff_queue { + my $date = shift; + my $target = shift; + my $filter = shift; + my $pretend = shift; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $db = init_gpc_db(); + + my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; + $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND rawExp.filter = '$filter' ORDER BY rawExp.object"; + + my $object_ref = $db->selectall_arrayref( $obj_sth ); + + my $Npotential = 0; + my $Nqueued = 0; + + foreach my $object_row (@{ $object_ref }) { + my $this_object = shift @{ $object_row }; +# my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; +# $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; +# $input_sth .= " ORDER BY dateobs "; + + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM "; + $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; + $input_sth .= " WHERE warpRun.label = '$label' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; + $input_sth .= " ORDER BY dateobs "; + + my $warps = $db->selectall_arrayref( $input_sth ); + + # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. + my %comment_hash = (); + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + $comment_hash{$this_comment} = $this_exp_id; + } + + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "md_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; + if ($#{ $warps} + 1 == 1) { + print STDERR ": I can do no diffs with only one exposure.\n"; + next; + } + else { + print STDERR ": I should declare an exposure to be faulty.\n"; + my @keep_warps = (); +# print "@{ $warps }\n"; + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + if ($comment_hash{$this_comment} == $this_exp_id) { + push @keep_warps, $this_warp; + } + else { + print STDERR "md_diff_queue: excluding $this_exp_id for $this_object\n"; + } + } + @{ $warps } = @keep_warps; +# print "@{ $warps }\n"; + } + } + + while ($#{ $warps } > -1) { + my $input_warp = shift @{ $warps }; + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + + my $template_warp = shift @{ $warps }; + + my $template_exp_id = ${ $template_warp }[0]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + unless(defined($template_warp)&& defined($template_exp_id)) { + print STDERR "md_diff received an undef! $input_exp_id $input_comment $this_object T: $template_warp V: @$template_warp\n"; + next; + } + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "md_Diff for this $date $target $input_exp_id ($input_warp_id $input_warp_state) $template_exp_id ($template_warp_id $template_warp_state) not fully processed\n"; + next; + } + + if (multi_date_verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "md_Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n"; + next; + } + + my $new_data_group = "${data_group}.multi"; + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $new_data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; +# $cmd .= " -pretend "; + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "md_Diffs would like to queue for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n"; + print STDERR "md_diff_queue: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } + } + } + $metadata_out{nsDiffPotential} += $Npotential; + $metadata_out{nsDiffQueued} += $Nqueued; +# if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) { +# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; +# } + +} Index: branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl =================================================================== --- branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl (revision 30586) +++ branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl (revision 30587) @@ -255,5 +255,5 @@ if (exists($nightlyscience_config{$target}{OBSMODE})) { $possible++; - if ($obsmode =~ /$nightlyscience_config{$target}{OBSMODE}/) { + if ($obsmode =~ /^$nightlyscience_config{$target}{OBSMODE}$/) { $match++; } @@ -261,5 +261,5 @@ if (exists($nightlyscience_config{$target}{OBJECT})) { $possible++; - if ($object =~ /$nightlyscience_config{$target}{OBJECT}/) { + if ($object =~ /^$nightlyscience_config{$target}{OBJECT}$/) { $match++; } @@ -267,5 +267,5 @@ if (exists($nightlyscience_config{$target}{COMMENT})) { $possible++; - if ($comment =~ /$nightlyscience_config{$target}{COMMENT}/) { + if ($comment =~ /^$nightlyscience_config{$target}{COMMENT}$/) { $match++; } Index: branches/czw_branch/20101203/ippScripts/scripts/warp_skycell.pl =================================================================== --- branches/czw_branch/20101203/ippScripts/scripts/warp_skycell.pl (revision 30586) +++ branches/czw_branch/20101203/ippScripts/scripts/warp_skycell.pl (revision 30587) @@ -71,6 +71,20 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id) or my_die( "Unable to get log filename", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ); -$logDest .= ".update" if ($run_state eq 'update'); +my ($logDest, $traceDest); +if ($run_state eq 'new') { + $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, 0); + $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1); +} elsif ($run_state eq 'update') { + $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 0); + $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, $skycell_id, 1); +} else { + &my_die( "invalid run_state: $run_state", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR ); +} + +my $neb; +my $scheme = file_scheme($outroot); +if ($scheme and $scheme eq 'neb') { + $neb = $ipprc->nebulous(); +} $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect; @@ -117,8 +131,8 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $skycell_id, $tess_dir, $error_code); } my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); } @@ -141,21 +155,43 @@ -my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outroot, $skycell_id ); -my $outputMask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outroot, $skycell_id); -my $outputWeight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id); -my $outputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id); -my $outputPSF = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $outroot); -my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outroot, $skycell_id ); -my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outroot, $skycell_id ); -my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot, $skycell_id ); -my $traceDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id); -my $configuration = $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id); - -if ($run_state eq 'update') { - $traceDest .= ".update"; - $outputStats .= ".update"; -} - -my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id ); +my $outputImage = prepare_output ("PSWARP.OUTPUT", $outroot, $skycell_id, 1); +my $outputMask = prepare_output ("PSWARP.OUTPUT.MASK", $outroot, $skycell_id, 1); +my $outputWeight = prepare_output ("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id, 1); +my $outputSources = prepare_output ("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id, 1); +my $outputPSF = prepare_output ("PSPHOT.PSF.SKY.SAVE", $outroot, 1); +my $outputBin1 = prepare_output ("PSWARP.BIN1", $outroot, $skycell_id, 1); +my $outputBin2 = prepare_output ("PSWARP.BIN2", $outroot, $skycell_id, 1); +my $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1); +my $configuration; + +my $dump_config = 1; +if ($run_state eq 'new') { + $configuration = prepare_output ("PSWARP.CONFIG", $outroot, $skycell_id, 1); +} else { + $configuration = $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id) or + &my_die("Missing entry from camera config PSWARP.CONFIG", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR); + if ($ipprc->file_exists($configuration)) { + $dump_config = 0; + } else { + print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n"; + + # XXX: should we create a new config dump file? + # I vote yes but only if we can distingusing between temporarily unavailable and GONE. + my $gone = 0; + if (storage_object_exists($configuration, \$gone)) { + if ($gone) { + $configuration = prepare_output('PSWARP.CONFIG', $outroot, $skycell_id, 1); + # if we dump the config we need to insure that the config dump represents + # the full processing + } else { + # file is temporarily not available. Don't dump config. + $dump_config = 0; + } + } + } +} + + +my $skyFile = prepare_output ("SKYCELL.TEMPLATE", $outroot, $skycell_id, 1); $ipprc->skycell_file( $tess_dir, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); ## XXX this seems to have insufficient error checking: dvoImageExtract can fail to write and still return a valid exit status @@ -171,11 +207,11 @@ foreach my $imfile (@$imfiles) { my $image = $imfile->{uri}; # Image name - my $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name + my $weight = $ipprc->filename ("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name my $mask; # Mask name if ($dynamicMasks) { - $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); + $mask = $ipprc->filename ("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); } else { - $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id}); + $mask = $ipprc->filename ("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id}); } @@ -186,5 +222,5 @@ # Astrometry file: astrometry is done at the camera stage, and always results in a MEF file # XXX allow an option to use the image header astrometry? - my $astrom = $ipprc->filename($astromSource, $imfile->{cam_path_base}); + my $astrom = $ipprc->filename ($astromSource, $imfile->{cam_path_base}); &my_die("Couldn't find input file: $astrom", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless defined $astrom and $ipprc->file_exists($astrom); @@ -245,7 +281,5 @@ $do_stats = 1; } else { - #$command .= " -ipprc $configuration"; - my $resolved = $ipprc->file_resolve($configuration); - $command .= " -ipprc $resolved"; + $command .= " -ipprc $configuration"; } if ($do_stats) { @@ -263,9 +297,10 @@ if ($do_stats) { # Check first for the stats file + check_output($outputStats, 0); my $outputStatsReal = $ipprc->file_resolve($outputStats); - &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); +# &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); &my_die("Stats file has zero size: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless -s $outputStatsReal; - # measure chip stats + # measure skycell stats $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL"; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -283,11 +318,18 @@ if (!$quality) { - &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); - &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); - &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); - &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); - &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF); - if ($run_state eq 'new') { - &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration); + # &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); + # &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + # &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + # &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + # &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF); + + check_output($outputImage, 0); + check_output($outputMask, 0); + check_output($outputWeight, 0); + check_output($outputSources, 1); + check_output($outputPSF, 1) if metadataLookupBool($recipe, 'PSF') ; + if ($dump_config) { + check_output($configuration, 1); +# &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration); } } @@ -335,4 +377,117 @@ } } + +exit 0; + +# Prepare to write to an output file +# Lookup the filename in the rules. +# Make sure that if file exists and is a nebulous file that there is only one instance +# Deal with files that have been lost. +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $skycell_id = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, $skycell_id, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $warp_id, $skycell_id, $tess_dir, $error); + return $output; +} + +sub check_output +{ + my $file = shift; + my $replicate = shift; + + if (!defined $file) { + return; + } + + &my_die("Couldn't find expected output file: $file", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + if ($replicate and $neb) { + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); + } +} + +# subroutine to check the status of a nebulous file. Used to distinguish between a storage object that +# does not exist and one that all of the instances have been lost. +# XXXX This should be implemented properly in Nebulous +# For now uses Bill's script 'whichnode' which queries the nebulous database directly + +my $whichnode; +sub storage_object_exists +{ + return 0 if !$neb; + + my $file = shift; + my $ref_all_gone = shift; + + my $exists = $neb->storage_object_exists($file); + if (!$exists) { + return 0; + } + + if (!$whichnode) { + $whichnode = can_run('whichnode') or + &my_die("Can't find whichnode", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR); + } + + my $command = "$whichnode $file"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform whichnode: $error_code", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR); + } + + my @lines = split "\n", (join "", @$stdout_buf); + + if (scalar @lines == 0) { + # no output the file is really and truely gone + # XXX: this is now caught above + print STDERR "storage object for $file does not exist\n"; + return 0; + } + + my $numGone = 0; + my $numNotGone = 0; + foreach my $line (@lines) { + chomp $line; + + # output lines are either + # "volume available" + # or + # "volume not available" + + my ($volume, $answer, undef) = split " ", $line; + # our hack is if the volume has an X in the name it's gone + if ($volume =~ /X/) { + print STDERR "$file is on $volume which is gone\n"; + $numGone++; + } elsif ($answer eq 'available') { + $numNotGone++; + } elsif ($answer eq 'not') { + print STDERR "$file is on $volume which is not available\n"; + $numNotGone++; + } else { + print STDERR "unexpected output from whichnode: $line\n"; + } + } + # if there are any instances that are not on a gone volume set all_gone to 0 + if ($numNotGone == 0 and $numGone > 0) { + $$ref_all_gone = 1; + } else { + $$ref_all_gone = 0; + } + + # storage object exists so return true + return 1; +} + sub my_die