Index: trunk/pstamp/scripts/pstamp_check_dependents.pl =================================================================== --- trunk/pstamp/scripts/pstamp_check_dependents.pl (revision 27670) +++ (revision ) @@ -1,228 +1,0 @@ -#!/bin/env perl - -# pstamp_finish.pl - -use warnings; -use strict; - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); -use Carp; - -use Time::Local; -use Sys::Hostname; -use IPC::Cmd 0.36 qw( can_run run ); -use File::Temp qw( tempfile ); -use File::Copy; -use File::Basename qw(dirname); - -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; -use PS::IPP::Metadata::List qw( parse_md_list ); - -use PS::IPP::Config qw( :standard ); -use PS::IPP::PStamp::RequestFile qw( :standard ); -use PS::IPP::PStamp::Job qw( :standard ); - -my ( $dep_id, $dep_state, $stage, $stage_id, $imagedb, $no_magic, $dbname, $dbserver); -my ( $no_update, $verbose, $save_temps, $logfile); - -GetOptions( - 'dep_id=s' => \$dep_id, - 'state=s' => \$dep_state, - 'stage=s' => \$stage, - 'stage_id=s' => \$stage_id, - 'imagedb=s' => \$imagedb, - 'no-magic=s' => \$no_magic, - 'dbname=s' => \$dbname, - 'dbserver=s' => \$dbserver, - 'verbose' => \$verbose, - 'no-update' => \$no_update, - 'save-temps' => \$save_temps, - 'logfile' => \$logfile, -) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; - -die "--dep_id --state --stage --stage_id and --imagedb are requried \n" - if !$dep_id or !$dep_state or !$stage or !$stage_id or !$imagedb; - -my $ipprc = PS::IPP::Config->new(); # IPP Configuration -if ($logfile) { - $ipprc->redirect_output($logfile); -} - -if (!$dbname) { - $dbname = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBNAME'); -} -if (!$dbserver) { - $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); -} - -# XXX: for testing -$no_update = 1; - -my $missing_tools; - -my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); -my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); -my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); -my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); -my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); -my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); -if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); -} - -$pstamptool .= " -dbname $dbname -dbserver $dbserver"; - -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - -my $done = 0; -if ($stage eq "chip") { - my $checkcmd = "$chiptool -runstatus -chip_id $stage_id -dbname $imagedb"; - - my $run_state = get_state($checkcmd); - - my $state = $run_state->{state}; - my $magic_ds_state = $run_state->{magic_ds_state}; - if ($state eq "full") { - print STDERR "chipRun $stage_id is full\n" if $verbose; - if ($no_magic or ($run_state->{magicked} > 0)) { - # It's all done set the jobs free - $done = 1; - } - } elsif (($state eq "purged") or ($state eq "goto_purged") - or ($magic_ds_state eq "censored") or ($magic_ds_state eq "goto_censored")) { - - # Can't update the run. fault all pstampJobs depending on this - runcommand("$pstamptool -updatejob -set_state stop -dep_id $dep_id -fault $PSTAMP_GONE"); - - } else { - - # don't update if in runs are in state goto_cleaned, wait for clean to finish - if ($state eq "cleaned") { - runcommand("$chiptool -dbname $imagedb -chip_id $stage_id -updaterun -set_state update -set_label update"); - } - - # don't update yet if magicDSRuns are in state goto_cleaned, wait for purge to finish - if ($magic_ds_state eq "cleaned") { - my $magic_ds_id = $run_state->{magic_ds_id}; - runcommand("$magicdstool -dbname $imagedb -magic_ds_id $magic_ds_id -updaterun -set_state new"); - } - } - -} elsif ($stage eq "warp") { - # perhaps -runstatus is a better name for this mode - my $checkcmd = "$warptool -runstate -dbname $imagedb -warp_id $stage_id"; - - my $run_state = get_state($checkcmd); - - my_die("unable to get run_state for $stage $stage_id", $dep_id, $PS_EXIT_CONFIG_ERROR) if !$run_state; - - my $state = $run_state->{'state'}; - my $chip_state = $run_state->{chip_state}; - my $magic_ds_state = $run_state->{magic_ds_state}; - -# $state = 'purged'; - if ($state eq 'full') { - print STDERR "warpRun $stage_id is full\n" if $verbose; - if ($no_magic or ($run_state->{magicked} > 0)) { - # It's all done set the jobs free - $done = 1; - } - } elsif (($state eq 'purged') or ($state eq 'goto_purged') - or ($chip_state eq 'purged') or ($chip_state eq 'goto_purged') - or ($magic_ds_state eq 'censored') or ($magic_ds_state eq 'goto_censored')) { - - # Can't update the run. fault all pstampJobs depending on this - runcommand("$pstamptool -updatejob -set_state stop -dep_id $dep_id -fault $PSTAMP_GONE"); - - } else { - - # don't update if in runs are in state goto_cleaned, wait for clean to finish - if ($state eq 'cleaned') { - runcommand("$warptool -dbname $imagedb -warp_id $stage_id -updaterun -set_state update -set_label update"); - } - - if ($chip_state eq 'cleaned') { - my $chip_id = $run_state->{chip_id}; - runcommand("$chiptool -dbname $imagedb -chip_id $chip_id -updaterun -set_state update -set_label update"); - } - - # don't update yet if magicDSRuns are in state goto_cleaned, wait for purge to finish - if ($magic_ds_state eq 'cleaned') { - my $magic_ds_id = $run_state->{magic_ds_id}; - runcommand("$magicdstool -dbname $imagedb -magic_ds_id $magic_ds_id -updaterun -set_state new"); - } - } - -} elsif ($stage eq "diff") { - my $checkcmd = "$difftool -runstate -diff_id $stage_id -dbname $imagedb"; - my $run_state = get_state($checkcmd); - -} elsif ($stage eq "stack") { - my $checkcmd = "$stacktool -runstate -stack_id $stage_id -dbname $imagedb"; - my $run_state = get_state($checkcmd); -} else { - my_die("Invalid stage value found: $stage", $dep_id, $PS_EXIT_CONFIG_ERROR); -} - -if ($done) { - runcommand("$pstamptool -updatedependent -dep_id $dep_id -set_state full"); -} - -sub runcommand -{ - my $command = shift; - my $command_does_not_update = shift; - if ($command_does_not_update or !$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("Unable to perform $command error code: $error_code", $dep_id, $PS_EXIT_UNKNOWN_ERROR); - } - return join "", @$stdout_buf if defined wantarray(); - return $success; - } else { - print STDERR "skipping $command\n"; - } - return 1; -} - -sub get_state -{ - my $command = shift; - my $output = runcommand($command, 1); - - if ($output) { - my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc"); - - my $files = parse_md_list($metadata); - if (scalar @$files eq 1) { - return $files->[0]; - } - } - return undef; -} - - -sub my_die { - my $msg = shift; - my $req_id = shift; - my $fault = shift; - - carp($msg); - - exit $fault; - - # XXX TODO: add a fault column - my $command = "$pstamptool -updatedependent -dep_id $dep_id -set_state fault_$fault"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - die("Unable to perform $command error code: $error_code"); - } - exit $fault; -} Index: trunk/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 27703) +++ trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 27703) @@ -0,0 +1,439 @@ +#!/bin/env perl +# +# pstamp_checkdependent.pl +# +# Check the status of a pending pstampDependent insuring that +# the dependent processing has been queued and whether it is +# finished or not + +use warnings; +use strict; + +use Sys::Hostname; +use Getopt::Long qw( GetOptions ); +use File::Basename qw(basename); +use Carp; +use POSIX; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( :standard ); +use PS::IPP::PStamp::RequestFile qw( :standard ); +use Carp; + + +my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic); +my ($dbname, $dbserver, $verbose, $save_temps, $no_update); + +GetOptions( + 'dep_id=i' => \$dep_id, + 'stage=s' => \$stage, + 'stage_id=i' => \$stage_id, + 'component=s' => \$component, + 'imagedb=s' => \$imagedb, # dbname for images + 'rlabel=s' => \$rlabel, + 'need_magic' => \$need_magic, + 'dbname=s' => \$dbname, # postage stamp server dbname + 'dbserver=s' => \$dbserver, # postage stamp server dbserver + 'verbose' => \$verbose, + 'save-temps' => \$save_temps, + 'no-update' => \$no_update, +); + +die "--dep_id --stage --stage_id --component --imagedb are required" + if !(defined $dep_id and defined $stage and defined $stage_id and + defined $component and defined $imagedb); + +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); +my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); +} + +$chiptool .= " -dbname $imagedb"; +$warptool .= " -dbname $imagedb"; +$difftool .= " -dbname $imagedb"; +$magicdstool .= " -dbname $imagedb"; + + +my $ipprc = PS::IPP::Config->new(); + +if (!$dbserver) { + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); +} + + + +my $tool; +my $cmd; +my $dsRun_state = ""; +my $whole_run = ($component eq 'all'); + +if ($whole_run) { + if ($stage eq "chip") { + $cmd = "$chiptool -listrun -chip_id $stage_id"; + } elsif ($stage eq "warp") { + $cmd = "$warptool -listrun -warp_id $stage_id"; + } elsif ($stage eq "diff") { + $cmd = "$difftool -listrun -diff_id $stage_id"; + } else { + my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR); + } +} else { + if ($stage eq "chip") { + $cmd = "$chiptool -processedimfile -chip_id $stage_id -class_id $component"; + } elsif ($stage eq "warp") { + $cmd = "$warptool -warped -warp_id $stage_id -skycell_id $component"; + } elsif ($stage eq "diff") { + $cmd = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $component"; + } else { + my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR); + } +} + +my $components = runToolAndParse($cmd, $verbose); + +my $n = scalar @$components; +my_die("Unexpected number of components found $n", $PS_EXIT_PROG_ERROR) if ($n != 1); + +# Got "it" +my $it = $components->[0]; + +my $status = 0; +if (($it->{state} eq 'full') or ($it->{state} eq 'update') and ($whole_run or ($it->{data_state} eq 'full')) + and (!$need_magic or $it->{magicked} > 0)) { + + # This Dependency is satisfied. all done + my $command = "$pstamptool -updatedependent -set_state full -dep_id $dep_id"; + $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to set pstampDependent.state to 'full' dep_id: $dep_id", + $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } +} elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update') and + ($whole_run or ($it->{data_state} eq 'cleaned'))) { + + # XXX: if $whole_run this is going to repeatedly call -setimfiletoupdate. This will be a no-op + # but is wasteful. + # Maybe we should deal with having three states for pstampDependent.state. 'new', 'run', 'full' + # need to queue the updates + if ($stage eq 'chip') { + # queue_update_chip takes an array so that queue_update_warp can pass a set of + # chips to it + my $chips = [$it]; + $status = queue_update_chip($it->{chip_id}, $whole_run, $chips, $rlabel, $need_magic); + } elsif ($stage eq 'warp') { + $status = queue_update_warp($it, $whole_run, $rlabel, $need_magic); + } elsif ($stage eq 'diff') { + $status = queue_update_diff($it, $whole_run, $rlabel, $need_magic); + } else { + my_die("Unexpected stage found $stage", $PS_EXIT_PROG_ERROR); + } +} else { + print "${stage}Run $stage_id state is $it->{state} $component data_state is $it->{data_state}\n"; + + # detect states that cannot be updated and update the job state + # XXX: we should be more assertive here and check for the specific states where we can continue + my $state = $it->{state}; + my $job_fault = 0; + if (($state =~ /scrub/) or ($state =~ /purge/)) { + # jobs must have changed state since depenency was made + print STDERR "Dependency cannot be satisfied\n"; + $job_fault = $PSTAMP_GONE; + } elsif ($state =~ /new/) { + # Dependency never should have been inserted + my_die ("Unexpected state for ${stage}Run $stage_id $state", $PS_EXIT_PROG_ERROR); + } + if ($job_fault) { + faultJobs($state, $stage, $stage_id, $job_fault); + } +} + +exit $status; + + +sub queue_update_chip { + my $chip_id = shift; + my $whole_run = shift; # if true queue entire run for update + my $metadatas = shift; # an array of hashes, either from -processedimfile or -listrun + my $rlabel = shift; # if defined a new label for the chipRun + my $need_magic = shift; + + my $dsRun_state; + my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked + if (!$whole_run) { + foreach my $chip (@$metadatas) { + $dsRun_state = $chip->{dsRun_state}; + $raw_all_magicked &= ($chip->{raw_magicked} > 0); + + if ($chip->{data_state} ne 'update') { + my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id -class_id $chip->{class_id}"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } + } + } else { + my $run = $metadatas->[0]; + $dsRun_state = $run->{dsRun_state}; + $raw_all_magicked = ($run->{raw_magicked} > 0); + + # providing no -class_id arguments changes all imfiles with data_state = 'cleaned' to 'update' + my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } + + # if called from queue_update_warp dsRun_state is unknown. Go find it. + if (!$dsRun_state) { + my $command = "$chiptool -listrun -chip_id $chip_id"; + my $data = runToolAndParse($command, $verbose); + my $chipRun = $data->[0]; + $dsRun_state = $chipRun->{dsRun_state}; + } + + # if the input file is already magicked no need to queue destreaking for this chipRun + if ($need_magic and !$raw_all_magicked) { + if ($dsRun_state eq 'cleaned') { + my $command = "$magicdstool -updaterun -set_state new -stage $stage -stage_id $stage_id"; + $command .= " -set_label $rlabel" if $rlabel; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to set destreak run to 'new' for ${stage}Run $stage_id", + $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } else { + print "magicDSRun.state = $dsRun_state for chipRun $stage_id"; + print " cannot update yet" if $dsRun_state ne "new"; + print ".\n"; + } + } + + # return value may be the return status of script so zero is good + return 0; +} + +sub queue_update_warp { + # check status of input chips. If state is not updatable set error code for job + + # if chipProcessedImfile.state is cleaned call queue_update_chip + + # need to code warptool -setskyfiletoupdate + my $metadata = shift; + my $whole_run = shift; # if true queue entire run for update + my $rlabel = shift; # if defined a new label for the chipRun + my $need_magic = shift; + + my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked + + my $warp_id = $metadata->{warp_id}; + if (!$whole_run) { + my $skycell = $metadata; + my $skycell_id = $skycell->{skycell_id}; + + my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id"; + my $data = runToolAndParse($command, $verbose); + my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) + if !$data or scalar @$data == 0; + + my $good_to_go = 1; + my @chipsToUpdate; + my $chip_id; + foreach my $chip (@$data) { + $chip_id = $chip->{chip_id}; + if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { + $good_to_go = 0; + push @chipsToUpdate, $chip; + } else { + # this chip is good to go + } + } + + if ($good_to_go) { + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif (scalar @chipsToUpdate > 0) { + return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic); + } + } else { + my $warpRun = $metadata; + my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}"; + my $data = runToolAndParse($command, $verbose); + my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR) + if !$data or scalar @$data != 1; + + my $chipRun = $data->[0]; + + my $chipRunState = $chipRun->{state}; + if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) { + print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n"; + faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE); + return 0; + } + my $warpRunState = $warpRun->{state}; + if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) { + # The inputs and outputs are ready. Queue the warpRun for update. + + # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update' + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif ($chipRunState eq 'cleaned' or + (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) { + my $data = [$chipRun]; + return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic); + } + } + + # return value may be the return status of script so zero is good + return 0; +} +sub queue_update_diff { + return 0; +} + + +# run a command that produces metadata output and parse the results into an array of objects +sub runToolAndParse { + my $command = shift; + my $verbose = shift; + + my ($program) = split " ", $command; + $program = basename($program); + + print "Running $command\n" if !$verbose; + my $start_tool = DateTime->now->mjd; + # run the command and parse the output + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf if !$verbose; + return undef; + } + + my $now = DateTime->now->mjd; + my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.; + print "Time to run $program: $dtime_tool\n"; + + my $buf = join "", @$stdout_buf; + if (!$buf) { + return undef; + } + + my $start_parse = DateTime->now->mjd; + + my $mdcParser = PS::IPP::Metadata::Config->new; + my $results = parse_md_fast($mdcParser, $buf) + or my_die ("Unable to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR); + + my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.; + print "Time to parse results from $program: $dtime_parse\n"; + + return $results; +} + +# splits meta data config input stream into single units to work around the pathalogically +# slow parser. This is similar to and adapted from code in various ippScripts. +sub parse_md_fast { + my $mdcParser = shift; + my $input = shift; + my $output = (); + + my @whole = split /\n/, $input; + my @single = (); + + my $n; + while ( ($n = @whole) > 0) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join("\n", @single ) ) ) or + print STDERR "Unable to parse metdata config doc" and return undef; + push @$output, $list->[0]; + + @single = (); + } + } + return $output; +} +sub faultJobs { + my ($state, $stage, $stage_id, $job_fault) = @_; + + my $command = "$pstamptool -updatejob -state stop -fault $job_fault -dep_id $dep_id"; + $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to set pstamJob.fault for dep_id: $dep_id", + $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } +} + +sub my_die +{ + my $msg = shift; + my $fault = shift; + + carp $msg; + + return $fault; +}