Changeset 37187

Timestamp:

Aug 5, 2014, 5:03:52 PM (12 years ago)

Author:

watersc1

Message:

Updated version with the improvements at retrying.

File:

• trunk/ippScripts/scripts/sc_remote_exec.pl (modified) (12 diffs)

trunk/ippScripts/scripts/sc_remote_exec.pl

@@ -18 +18 @@
 my $SEC_HOST = 'mu-fe4';
 my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
-#my $remote_root  = '/scratch3/watersc1/';
-my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/';
-$remote_root = '/scratch3/watersc1/';
+my $remote_root  = '/scratch3/watersc1/';
+
 # tools
 my $missing_tools;
@@ -31 +30 @@
     exit($PS_EXIT_CONFIG_ERROR);
 }
-
 
 # Options
@@ -56 +54 @@
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id);
-
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files                                    
 
@@ -152 +149 @@
 my $poll_sleep = 60;
 my $poll_response = 0;
+my $poll_word  = "nothing";
+
 if ($force) {
     $poll_response = 1;
-}
-my $poll_word  = "nothing";
+    $poll_word     = "Completed";
+}
+
 while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)&&($force != 1)) {
     if ($verbose) {
@@ -171 +171 @@
         warn("The job exited incorrectly: $remote_id $job_id $poll_word.  Attempting to recover what we can.");
         $poll_response = 1; # To fool the check later into proceding
-#       &my_die("The job exited incorrectly.",$remote_id, $PS_EXIT_PROG_ERROR);
     }
     $poll_count++;
@@ -181 +180 @@
     &my_die("Job hasn't completed.  Ending polling.", $remote_id, 0, "run", $job_id);
 }
-# Retrieve validation summary
-my $fault = 0;
+
 
 # Initialize the remote side to sync things
@@ -193 +191 @@
 #scp_get($remote_return,$ipp_return);
 
-# Check to see if we can resume.  Pack the files into an array
+# Construct list of dbinfo files we'll need to execute.
 my @returned_files = ();
 my @dbinfo_files   = ();
@@ -211 +209 @@
 close(RETURN);
 print STDERR "Expect $#dbinfo_files dbinfo files\n";
-# Binary search it.  BUT: We're binary searching binary values.  So,
-# this has to be far more careful than a fully sorted case would be.
-# Only scan the dbinfo files as those are the things that are
-# generated after the processing, and so must be the last things to
-# exist.
 
 my $first = 0;
@@ -222 +215 @@
 my $range = 20;  # How wide of a chunk to look through to catch threading drops
 my $offset = 0;  # Offset to start from.
-# This doesn't work as well as I'd like, and with the rsync, it's less needed.
-# if (( stat($dbinfo_files[$last]) )[7] != 0) { $offset = $#dbinfo_files;}
-# else {
-#     my $count = 0;
-#     while ($first < $last - $range) {
-
-#       $mid = int(($last + $first)/2);
-#       for (my $i = 0; $i <= $range; $i++) {
-#           my $size = (stat($dbinfo_files[$mid + $i]))[7];
-# #         print STDERR "$first $last $mid $i $size $dbinfo_files[$mid + $i]\n";
-#           if ($size == 0) { 
-#               if ($last == $mid + $i) { 
-#                   $first = $last - 1;
-#               }
-#               $last = $mid + $i; 
-#               $i = $range + 10;
-#           }
-#       }
-#       if ($last > $mid + $range) { $first = $mid + $range; }
-#       $count++;
-#       if ($count > 200) { 
-#           &my_die("The iteration loop couldn't converge after 200 rounds, suggesting something has gone wrong.",
-#                   $remote_id,4,"error_run");
-#       }
-#     }
-#     $offset = $mid - $range; # Because I want to make sure I get to the beginning of the thread group.
-#     $offset = $dbindex[$offset]; # Because we need to convert from dbinfo space to returned file space.
-# }
-
-# Feed teh return list into the transfer tool
+
+# Feed the return list into the transfer tool
 # If this fails, we should go to an auth wait point.
-if ($poll_word eq 'Completed') { 
+if ($poll_word eq 'Completed') {  
+    # We're complete.  All files should exist on disk
     my $iter = 0;
     my $transfer_success = 1;
@@ -267 +233 @@
 }
 else {
+    # Just try and fetch things, because we didn't complete correctly, it seems.
     my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
     my $return_push_output_print = join "\n", @{ $return_push_output };
@@ -282 +249 @@
         run(command => $cmd, verbose => $verbose);
     unless ($success) {
+
         # This shouldn't fail, but we also can't suddenly abort if something does fail.  
         # Now we're going to drop the component that owns this dbinfo, which should force a retry.
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
         warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code");
-        drop_component($remote_id,$cmd,$file);
+        if ($poll_word ne 'Completed') {  # If the job claims it completed, then we're fighting the network.
+            drop_component($remote_id,$cmd,$file);
+        }
     }
 
@@ -342 +312 @@
     my $cmd = shift;
     $cmd = "$ssh  $DMZ_HOST ssh  ${SEC_HOST} $cmd";
+    print "EXEC: $cmd\n";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $cmd, verbose => $verbose);