Changeset 37242 for branches/eam_branches/ipp-20140717/ippScripts/scripts/sc_remote_exec.pl

Timestamp:

Aug 12, 2014, 3:55:38 PM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

File:

• branches/eam_branches/ipp-20140717/ippScripts/scripts/sc_remote_exec.pl (modified) (16 diffs, 1 prop)

branches/eam_branches/ipp-20140717/ippScripts/scripts/sc_remote_exec.pl

@@ -16 +16 @@
 # Hard coded values
 my $DMZ_HOST = 'wtrw';
-my $SEC_HOST = 'mu-fe4';
+my $SEC_HOST = 'mu-fe';
 my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
 my $remote_root  = '/scratch3/watersc1/';
@@ -31 +31 @@
 }
 
-
 # Options
-my ($remote_id,$path_base,$policy,$poll,$job_id,$dbname,$verbose,$no_update,$camera);
-
+my ($remote_id,$path_base,$policy,$poll,$force,$job_id,$dbname,$verbose,$no_update,$camera);
+$force = 0;
+$verbose = 0;
 GetOptions(
     'remote_id=s'   => \$remote_id,
@@ -41 +41 @@
     'policy=s'      => \$policy,
     'poll'          => \$poll,
+    'force-poll'    => \$force,
     'camera=s'      => \$camera,
     'dbname=s'      => \$dbname,
@@ -53 +54 @@
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id);
-
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files                                    
 
@@ -61 +61 @@
 
 print "passed authentication challenge.\n";
+
+# Phase 1.5: Grab the information about this run;
+my $rt_cmd = "$remotetool -listrun -remote_id $remote_id ";
+$rt_cmd   .= " -dbname $dbname " if defined($dbname);
+my ( $rt_success, $rt_error_code, $rt_full_buf, $rt_stdout_buf, $rt_stderr_buf ) =
+    run(command => $rt_cmd, verbose => $verbose);
+unless ($rt_success) {
+    $rt_error_code = (($rt_error_code >> 8) or $PS_EXIT_PROG_ERROR);
+    &my_die("UNable to run remotetool to determine remote run status",
+            $remote_id,$rt_error_code);
+}
+my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or 
+    &my_die("UNable to run remotetool to determine remote run status",
+            $remote_id,$rt_error_code);
+my $rrData2 = parse_md_list($rrData);
+my $runData = ${ $rrData2 }[0]; # There should be only one
 
 # These are operations we only need to do on the first call, not on subsequent poll operations.
@@ -92 +108 @@
     my (undef,$remote_check) = uri_convert($uri_check);
     my (undef,$remote_transfer)= uri_convert($uri_transfer);
-#    my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_validate_files.pl --check_list $remote_check --transfer_list $remote_transfer");
+
     my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer");
     # We no longer need to parse this output, as it retrieves files itself.
@@ -130 +146 @@
 # Poll the job status?
 my $poll_count = 0;
-my $poll_max   = 5;
+my $poll_max   = 1; # If it isn't finished now, it's unlikely to magically finish in the next five minutes.
 my $poll_sleep = 60;
 my $poll_response = 0;
-while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)) {
+my $poll_word  = "nothing";
+
+if ($force) {
+    $poll_response = 1;
+    $poll_word     = "Completed";
+}
+
+while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)&&($force != 1)) {
     if ($verbose) {
         print "Polling code.  Sleeping.  $poll_count of $poll_max $poll_response\n";
@@ -140 +163 @@
         sleep($poll_sleep);
     }
-    $poll_response = poll_job($job_id);
+    ($poll_response,$poll_word) = poll_job($job_id);
 
     if ($poll_response == 1) { # This job has completed
@@ -146 +169 @@
     }
     elsif ($poll_response == -1) { # This job has an error
-        &my_die("The job exited incorrectly.",$remote_id, $PS_EXIT_PROG_ERROR);
+        warn("The job exited incorrectly: $remote_id $job_id $poll_word.  Attempting to recover what we can.");
+        $poll_response = 1; # To fool the check later into proceding
     }
     $poll_count++;
@@ -156 +180 @@
     &my_die("Job hasn't completed.  Ending polling.", $remote_id, 0, "run", $job_id);
 }
-# Retrieve validation summary
-my $fault = 0;
+
 
 # Initialize the remote side to sync things
@@ -165 +188 @@
 my $uri_return  = $path_base . ".return";
 my ($ipp_return,$remote_return) = uri_convert($uri_return);
-scp_get($remote_return,$ipp_return);
-# Feed teh return list into the 
-my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return");
-print "$return_push_output\n";
-
-
+# We shouldn't need to get this file.  We made that file.
+#scp_get($remote_return,$ipp_return);
+
+# Construct list of dbinfo files we'll need to execute.
+my @returned_files = ();
+my @dbinfo_files   = ();
+my @dbindex        = ();
+my $iter           = 0;
 open(RETURN,   "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id,$PS_EXIT_SYS_ERROR);
-while(<RETURN>) {
+while (<RETURN>) {
     chomp;
+    push @returned_files, $_;
     if ($_ =~ /dbinfo/) {
-        my $file = $_;
-#       $file =~ s/${remote_root}.tmp//; THis shouldn't be needed, as the transfer tool munges the files on that side.
-        open(DBF,"$file") || warn "Missing file $file\n";
-        my $cmd = <DBF>;
-        chomp($cmd);
-        $cmd =~ s/-/ -/g; # This is just a safety check for missing space.
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $cmd, verbose => $verbose);
-        unless ($success) {
-            # This shouldn't fail, but we also can't suddenly abort if something does fail.  Maybe retry if this looks to be a continuing issue?
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            warn("The command that shouldn't fail has failed: $cmd $remote_id $error_code");##,$remote_id,$error_code);
+        push @dbinfo_files, $_;
+        push @dbindex, $iter;
+    }
+    $iter++;
+
+}
+close(RETURN);
+print STDERR "Expect $#dbinfo_files dbinfo files\n";
+
+my $first = 0;
+my $last  = $#dbinfo_files;
+my $mid   = 0;
+my $range = 20;  # How wide of a chunk to look through to catch threading drops
+my $offset = 0;  # Offset to start from.
+
+# Feed the return list into the transfer tool
+# If this fails, we should go to an auth wait point.
+if ($poll_word eq 'Completed') {  
+    # We're complete.  All files should exist on disk
+    my $iter = 0;
+    my $transfer_success = 1;
+    do {
+        $transfer_success = 1;
+        $iter++;
+        check_ssh_connection(); 
+        my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
+        my $return_push_output_print = join "\n", @{ $return_push_output };
+        if ($return_push_output_print =~ /exited with value 255/) { $transfer_success = 0; }
+        print "$return_push_output_print\n";
+    } while (($transfer_success != 1) &&  ($iter < 10));  # Try harder;
+}
+else {
+    # Just try and fetch things, because we didn't complete correctly, it seems.
+    my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
+    my $return_push_output_print = join "\n", @{ $return_push_output };
+    print "$return_push_output_print\n";
+}
+
+foreach my $file (@dbinfo_files) {
+    open(DBF,"$file") || warn "Missing file $file\n";
+    my $cmd = <DBF>;
+    close(DBF);
+    chomp($cmd);
+#    print STDERR "$file $cmd\n";
+    $cmd =~ s/-/ -/g; # This is just a safety check for missing space.
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $cmd, verbose => $verbose);
+    unless ($success) {
+
+        # This shouldn't fail, but we also can't suddenly abort if something does fail.  
+        # Now we're going to drop the component that owns this dbinfo, which should force a retry.
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code");
+        if ($poll_word ne 'Completed') {  # If the job claims it completed, then we're fighting the network.
+            drop_component($remote_id,$cmd,$file);
         }
-        close(DBF);
-    }
-}
-close(RETURN);
+    }
+
+}
 
 # We're done, so set the state and exit.
 &my_die("Finished",$remote_id,0,"full");
-
 
 
@@ -209 +276 @@
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
         &my_die("Authorization check failed.",$remote_id,0,'auth');
-
-#       my $update_command = "$remotetool -updaterun -remote_id $remote_id ";
-#       $update_command .= " -set_state auth ";
-#       $update_command .= " -dbname $dbname " if defined $dbname;
-#       system($update_command);
-#       exit(0);  # This isn't an error.  It's a state we monitor and move to automatically.
     }
 }
@@ -250 +311 @@
 sub ssh_exec_command {
     my $cmd = shift;
-    $cmd = "$ssh $DMZ_HOST ssh ${SEC_HOST} $cmd";
+    $cmd = "$ssh  $DMZ_HOST ssh  ${SEC_HOST} $cmd";
+    print "EXEC: $cmd\n";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $cmd, verbose => $verbose);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        # If we failed, see if we failed due to authorization.
+        check_ssh_connection(); 
         &my_die("Failed to execute command: >>$cmd<<"); 
     }
@@ -266 +330 @@
     
     if ($response =~ /State: Running/) { 
-        return(0);
+        return(0,"Running");
     }
     elsif ($response =~ /State: Completed/) {
-        return(1);
+        return(1,"Completed");
     }
     elsif ($response =~ /State: Idle/) {
-        return(-2);
+        return(-2,"Idle");
     }
     else {
-        return(-1);
+        my $response_word = $response;
+        $response_word =~ s/.*State: //;
+        $response_word =~ s/\n.*//;
+        return(-1,$response_word);
     }
 }
@@ -324 +391 @@
     return($local_uri);
 }
+
+sub drop_component {
+    my $remote_id = shift;
+    my $cmd = shift;
+    my $file = shift;
+
+    if ($cmd eq '') { 
+        # Zero byte file.  This dbinfo was never constructed properly.
+        
+        # Steal the stage_id from the filename.  This isn't something I'm happy about.
+        my $stage = $runData->{stage};
+        my $stage_id = $file;
+        if ($stage eq 'chip') { 
+            $stage_id =~ s/^.*ch\.//;
+        }
+        elsif ($stage eq 'camera') {
+            $stage_id =~ s/^.*cm\.//;
+        }
+        elsif ($stage eq 'warp') {
+            $stage_id =~ s/^.*wrp\.//;
+        }
+        elsif ($stage eq 'stack') {
+            $stage_id =~ s/^.*stk\.//;
+        }
+        $stage_id =~ s/\..*$//;
+        
+        # Drop this from the database
+        my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id ";
+        $drop_cmd   .= " -dbname $dbname " if defined($dbname);
+        system($drop_cmd);
+    }
+
+}
+
 sub my_die {
     my $msg = shift;
@@ -347 +448 @@
 }
 
-# I don't think I need this anymore, but I don't want to burn it until I"m sure.
-
-# exit(1);
-# # Read this as a metadata object
-# my $ssh_exec_stdout = ssh_exec_command("sc_validate_processing.pl ${path_base}.out");
-# my $retrieveData = $mdcParser->parse(join "", @$ssh_exec_stdout);
-# my $retrieveData2= parse_md_list($retrieveData);
-
-# foreach my $retrieveEntry (@$retrieveData2) {
-#     my $process_fault = $retrieveEntry->{fault};
-#     my $dbcommand = $retrieveEntry->{dbcommand};
-#     my $filelist  = $retrieveEntry->{filelist};
-
-#     scp_get("${filelist}",${filelist});
-#     open(FL,${filelist});
-#     while(<FL>) {
-#       chomp;
-#       my $f = $_;
-#       scp_get("${f}",$f);
-#     }
-#     close(FL);
-
-#     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-#       run(command => $dbcommand, verbose => $verbose);
-#     unless ($success) {
-#       $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-#       &my_die("The database command returned did not execute correctly. $dbcommand $process_fault",
-#               $remote_id,$error_code); 
-#     }
-
-#     if ($process_fault != 0) {
-#       $fault = 1;
-#     }
-# }
-
-# if (($policy ne 'ignore')&&($fault != 0)) {
-#     &my_die("This job exited without all components finishing successfully.  Halting.",
-#           $remote_id,$fault,"halt");
-# }
-# else {
-#     print "This job exited without all components finishing successfully. $fault\n" if $fault;
-#     &my_die("This job finished.  Setting remote job to full.",
-#           $remote_id,0,"full");
-# }
-
-# Promote to finished.
-#&my_die(); # Not really, but I don't have the tool interface finished, so these are all just placeholders.
 # Quick review:
 # new -> pending -> run -> full