Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_retry_return.pl
===================================================================
--- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_retry_return.pl	(revision 37264)
+++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_retry_return.pl	(revision 37264)
@@ -0,0 +1,444 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+
+use File::Basename;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Hard coded values
+my $DMZ_HOST = 'wtrw';
+my $SEC_HOST = 'mu-fe';
+my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
+my $remote_root  = '/scratch3/watersc1/';
+
+# tools
+my $missing_tools;
+my $ssh        = can_run('ssh')  or (warn "Can't find ssh" and $missing_tools = 1);
+my $scp        = can_run('scp')  or (warn "Can't find scp" and $missing_tools = 1);
+my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
+
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Options
+my ($remote_id,$path_base,$policy,$poll,$force,$job_id,$dbname,$verbose,$no_update,$camera);
+$force = 0;
+$verbose = 0;
+GetOptions(
+    'remote_id=s'   => \$remote_id,
+    'job_id=s'      => \$job_id,
+    'path_base=s'   => \$path_base,
+    'policy=s'      => \$policy,
+    'poll'          => \$poll,
+    'force-poll'    => \$force,
+    'camera=s'      => \$camera,
+    'dbname=s'      => \$dbname,
+    'verbose'       => \$verbose,
+    'no_update'     => \$no_update,
+    ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --remote_id --path_base --job_id", -exitval => 3) unless
+    defined($path_base) and
+    defined($remote_id) and
+    defined($job_id);
+
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id);
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+# Phase 1: See if we can actually do anything.
+# If the link is down, there's no benefit in trying to do anything else.
+&check_ssh_connection();
+
+print "passed authentication challenge.\n";
+
+# Phase 1.5: Grab the information about this run;
+my $rt_cmd = "$remotetool -listrun -remote_id $remote_id ";
+$rt_cmd   .= " -dbname $dbname " if defined($dbname);
+
+my ( $rt_success, $rt_error_code, $rt_full_buf, $rt_stdout_buf, $rt_stderr_buf ) =
+    run(command => $rt_cmd, verbose => $verbose);
+unless ($rt_success) {
+    $rt_error_code = (($rt_error_code >> 8) or $PS_EXIT_PROG_ERROR);
+    &my_die("Unable to run remotetool to determine remote run status",
+            $remote_id,$rt_error_code);
+}
+
+my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or
+    &my_die("Unable to run remotetool to determine remote run status",
+            $remote_id,$rt_error_code);
+my $rrData2 = parse_md_list($rrData);
+my $runData = ${ $rrData2 }[0]; # There should be only one
+
+# keep this for reference for the moment, but this block should not be in this function
+if (0) {
+# Phase 2: Ensure files are in place:
+# Copy command files
+    my @files = ();
+
+    my $uri_command = $path_base . ".cmd";
+    my $uri_transfer= $path_base . ".transfer";
+    my $uri_check   = $path_base . ".check";
+    my $uri_config  = $path_base . ".config";
+    my $uri_generate= $path_base . ".generate";
+    my $uri_return  = $path_base . ".return";
+
+    my $disk_command = $ipprc->file_resolve($uri_command);
+    my $disk_transfer= $ipprc->file_resolve($uri_transfer);
+    my $disk_check   = $ipprc->file_resolve($uri_check);
+    my $disk_config  = $ipprc->file_resolve($uri_config);
+    my $disk_generate= $ipprc->file_resolve($uri_generate);
+    my $disk_return  = $ipprc->file_resolve($uri_return);
+
+    scp_put($disk_command,uri_local_to_remote($uri_command));
+    scp_put($disk_transfer,uri_local_to_remote($uri_transfer));
+    scp_put($disk_check,uri_local_to_remote($uri_check));
+    scp_put($disk_config,uri_local_to_remote($uri_config));
+    scp_put($disk_generate,uri_local_to_remote($uri_generate));
+    scp_put($disk_return, uri_local_to_remote($uri_return));
+
+# Run check command
+    my (undef,$remote_check) = uri_convert($uri_check);
+    my (undef,$remote_transfer)= uri_convert($uri_transfer);
+
+    my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer");
+    # We no longer need to parse this output, as it retrieves files itself.
+    foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) {
+        print "$l\n";
+    }
+
+# Run real command
+    my (undef,$remote_command) = uri_convert($uri_command);
+    my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command");
+    if ($#{ $ssh_exec_stdout } != -1) { # Parse the output
+        $job_id = ${ $ssh_exec_stdout }[0];
+        chomp($job_id);
+        $job_id =~ s/\s+//g;
+    }
+
+    unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly.
+        &my_die("No job_id returned.  Sorry.",$remote_id,$PS_EXIT_PROG_ERROR);
+    }
+
+    # Notify the database that this entry is currently running.
+    my $command = "$remotetool -updaterun -remote_id $remote_id ";
+    $command .= " -set_state run ";
+    $command .= " -job_id $job_id ";
+    $command .= " -dbname $dbname " if defined $dbname;
+
+    print STDERR "$command\n";
+    system($command);
+
+} # End of exec phase
+
+# Poll the job status?
+my $poll_response = 0;
+my $poll_word  = "nothing";
+
+if ($force) {
+    $poll_response = 1;
+    $poll_word     = "Completed";
+}
+
+($poll_response, $poll_word) = poll_job($job_id);
+
+if ($poll_response == -1) { # This job has an error
+    warn("The job exited incorrectly: $remote_id $job_id $poll_word.");
+    exit (2);
+}
+
+if ($poll_response != 1) { # This job has NOT completed
+
+}
+if ($verbose) {
+    print "Stopped Polling code.  $poll_count of $poll_max $poll_response\n";
+}
+if ($poll_response != 1) {
+    warn("Job hasn't completed: $remote_id $job_id $poll_response $poll_word");
+    exit (1);
+}
+
+# Initialize the remote side to sync things
+my $uri_return  = $path_base . ".return";
+my ($ipp_return, $remote_return) = uri_convert($uri_return);
+
+# Construct list of dbinfo files we'll need to execute.
+my @returned_files = ();
+my @dbinfo_files   = ();
+
+# get a list of the dbinfo files
+open(RETURN, "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id, $PS_EXIT_SYS_ERROR);
+while (<RETURN>) {
+    chomp;
+    if ($_ =~ /dbinfo/) {
+        push @dbinfo_files, $_;
+    }
+}
+close(RETURN);
+print STDERR "Expect $#dbinfo_files dbinfo files\n";
+
+# Feed the return list into the transfer tool
+# If this fails, we should go to an auth wait point.
+{
+    my $offset = 0;  # Offset to start from.
+    my $iter = 0;
+    my $transfer_success = 1;
+    do {
+        $transfer_success = 1;
+        $iter++;
+        check_ssh_connection();
+        my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
+        my $return_push_output_print = join "\n", @{ $return_push_output };
+        if ($return_push_output_print =~ /exited with value 255/) { $transfer_success = 0; }
+        print "$return_push_output_print\n";
+    } while (($transfer_success != 1) &&  ($iter < 10));  # Try harder;
+}
+
+foreach my $file (@dbinfo_files) {
+    # use the filename to get the stage id, check if we have already updated this entry:
+
+
+
+    open(DBF,"$file") || warn "Missing file $file\n";
+    my $cmd = <DBF>;
+    close(DBF);
+    chomp($cmd);
+#    print STDERR "$file $cmd\n";
+    $cmd =~ s/-/ -/g; # This is just a safety check for missing space.
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $cmd, verbose => $verbose);
+    unless ($success) {
+
+        # This shouldn't fail, but we also can't suddenly abort if something does fail.
+        # Now we're going to drop the component that owns this dbinfo, which should force a retry.
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code");
+        if ($poll_word ne 'Completed') {  # If the job claims it completed, then we're fighting the network.
+            drop_component($remote_id,$cmd,$file);
+        }
+    }
+
+}
+
+# We're done, so set the state and exit.
+&my_die("Finished",$remote_id,0,"full");
+
+
+# END PROGRAM
+
+
+sub check_ssh_connection {
+    my $cmd = "$ssh -O check $DMZ_HOST";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Authorization check failed.",$remote_id,0,'auth');
+    }
+}
+
+sub scp_put {
+    my $file = shift;
+    my $destination = shift;
+    my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}";
+
+    my $directory = dirname($destination);
+    ssh_exec_command("mkdir -p $directory");
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die();
+    }
+}
+
+sub scp_get {
+    my $destination = shift;
+    my $file = shift;
+
+    my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file ";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die();
+    }
+}
+
+sub ssh_exec_command {
+    my $cmd = shift;
+    $cmd = "$ssh  $DMZ_HOST ssh  ${SEC_HOST} $cmd";
+    print "EXEC: $cmd\n";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        # If we failed, see if we failed due to authorization.
+        check_ssh_connection();
+        &my_die("Failed to execute command: >>$cmd<<");
+    }
+    return ($stdout_buf);
+}
+
+sub poll_job {
+    my $job_id = shift;
+    my $response_array = ssh_exec_command("checkjob -v $job_id");
+    my $response = join "\n", @$response_array;
+
+    if ($response =~ /State: Running/) {
+        return(0,"Running");
+    }
+    elsif ($response =~ /State: Completed/) {
+        return(1,"Completed");
+    }
+    elsif ($response =~ /State: Idle/) {
+        return(-2,"Idle");
+    }
+    else {
+        my $response_word = $response;
+        $response_word =~ s/.*State: //;
+        $response_word =~ s/\n.*//;
+        return(-1,$response_word);
+    }
+}
+
+
+sub uri_convert {
+    my $neb_uri = shift;
+    my $ipp_disk= $ipprc->file_resolve( $neb_uri );
+    my $remote_disk = $ipp_disk;
+
+    unless(defined($ipp_disk)) {
+        my_die();
+    }
+
+    $remote_disk =~ s%^.*/%%;   # Remove nebulous path
+    $remote_disk =~ s%^\d+\.%%; # Remove ins_id
+    $remote_disk =~ s%:%/%g;    # Replace colons with directories
+    $remote_disk = "${remote_root}/${remote_disk}";
+    return($ipp_disk,$remote_disk);
+}
+
+sub uri_to_outputs {
+    my $neb_uri = shift;
+    my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
+
+#    print TRANSFER "$ipp_disk\n";
+#    print CHECK    "$remote_disk\n";
+    return($ipp_disk,$remote_disk);
+}
+
+sub uri_local_to_remote {
+    # This needs to replace the nebulous tag with the remote root.
+    my $local_uri = shift;
+    $local_uri =~ s%^.*?/%%; # neb:/
+    $local_uri =~ s%^.*?/%%; # /
+    $local_uri =~ s%^.*?/%%; # @HOST@.0/
+    my $remote_uri = "${remote_root}/" . $local_uri;
+
+    return($remote_uri);
+}
+
+sub uri_remote_to_local {
+    # This needs to replace the remote root directory with the nebulous tag.
+    my $remote_uri = shift;
+    $remote_uri =~ s%${remote_root}%%;
+    my $local_uri  = "neb:///" . $remote_uri;
+
+    return($local_uri);
+}
+
+# assumes $runData->{stage} exists
+sub get_stage_id_from_dbinfo {
+    my $file = shift;
+
+    # Steal the stage_id from the filename.  This isn't something I'm happy about.
+    my $stage = $runData->{stage};
+    if ($stage eq 'chip') {
+        ($stage_id, $class_id) = $file =~ /ch\.(\d*).(XY\d\d).dbinfo/;
+    }
+    elsif ($stage eq 'camera') {
+        $stage_id =~ s/^.*cm\.//;
+    }
+    elsif ($stage eq 'warp') {
+        $stage_id =~ s/^.*wrp\.//;
+    }
+    elsif ($stage eq 'stack') {
+        $stage_id =~ s/^.*stk\.//;
+    }
+
+    return ($stage_id, $class_id);
+}
+
+sub drop_component {
+    my $remote_id = shift;
+    my $cmd = shift;
+    my $file = shift;
+
+    if ($cmd eq '') {
+        # Zero byte file.  This dbinfo was never constructed properly.
+
+        # Steal the stage_id from the filename.  This isn't something I'm happy about.
+        my $stage = $runData->{stage};
+        my $stage_id = $file;
+        if ($stage eq 'chip') {
+            $stage_id =~ s/^.*ch\.//;
+        }
+        elsif ($stage eq 'camera') {
+            $stage_id =~ s/^.*cm\.//;
+        }
+        elsif ($stage eq 'warp') {
+            $stage_id =~ s/^.*wrp\.//;
+        }
+        elsif ($stage eq 'stack') {
+            $stage_id =~ s/^.*stk\.//;
+        }
+        $stage_id =~ s/\..*$//;
+
+        # Drop this from the database
+        my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id ";
+        $drop_cmd   .= " -dbname $dbname " if defined($dbname);
+        system($drop_cmd);
+    }
+
+}
+
+sub my_die {
+    my $msg = shift;
+    my $id  = shift;
+    my $exit_code = shift;
+    my $exit_state = shift;
+    my $jobid = shift;
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+    carp($msg);
+
+    if (defined $id and not $no_update) {
+        my $command = "$remotetool -updaterun -remote_id $id";
+        $command .= " -fault $exit_code " if defined $exit_code;
+        $command .= " -job_id $job_id " if defined $jobid;
+        $command .= " -set_state $exit_state " if defined $exit_state;
+        $command .= " -dbname $dbname " if defined $dbname;
+
+        system($command);
+    }
+
+    exit($exit_code);
+}
+
+# Quick review:
+# new -> pending -> run -> full
+# auth