IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 37289


Ignore:
Timestamp:
Aug 22, 2014, 12:11:54 PM (12 years ago)
Author:
eugene
Message:

finish sc_remote_exec.pl split from sc_remote_poll.pl

File:
1 edited

Legend:

Unmodified
Added
Removed

  • tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_exec.pl

    r37269 r37289  
    3232
    3333# Options
    34 my ($remote_id,$path_base,$policy,$poll,$force,$job_id,$dbname,$verbose,$no_update,$camera);
    35 $force = 0;
     34my ($remote_id,$path_base,$dbname,$verbose,$no_update,$camera);
    3635$verbose = 0;
    3736GetOptions(
    3837    'remote_id=s'   => \$remote_id,
    39     'job_id=s'      => \$job_id,
    4038    'path_base=s'   => \$path_base,
    41     'policy=s'      => \$policy,
    42     'poll'          => \$poll,
    43     'force-poll'    => \$force,
    4439    'camera=s'      => \$camera,
    4540    'dbname=s'      => \$dbname,
    … …  
    7873my $runData = ${ $rrData2 }[0]; # There should be only one
    7974
    80 # These are operations we only need to do on the first call, not on subsequent poll operations.
    81 unless ($poll) {
     75my ($remote_path) = uri_convert($path_base);
     76print "REMOTE PATH: $remote_path\n";
     77
    8278# Phase 2: Ensure files are in place:
    8379# Copy command files
    84     my @files = ();
    85 
    86     my $uri_command = $path_base . ".cmd";
    87     my $uri_transfer= $path_base . ".transfer";
    88     my $uri_check   = $path_base . ".check";
    89     my $uri_config  = $path_base . ".config";
    90     my $uri_generate= $path_base . ".generate";
    91     my $uri_return  = $path_base . ".return";
    92 
    93     my $disk_command = $ipprc->file_resolve($uri_command);
    94     my $disk_transfer= $ipprc->file_resolve($uri_transfer);
    95     my $disk_check   = $ipprc->file_resolve($uri_check);
    96     my $disk_config  = $ipprc->file_resolve($uri_config);
    97     my $disk_generate= $ipprc->file_resolve($uri_generate);
    98     my $disk_return  = $ipprc->file_resolve($uri_return);
    99 
    100     scp_put($disk_command,uri_local_to_remote($uri_command));
    101     scp_put($disk_transfer,uri_local_to_remote($uri_transfer));
    102     scp_put($disk_check,uri_local_to_remote($uri_check));
    103     scp_put($disk_config,uri_local_to_remote($uri_config));
    104     scp_put($disk_generate,uri_local_to_remote($uri_generate));
    105     scp_put($disk_return, uri_local_to_remote($uri_return));
     80my @files = ();
     81
     82my $uri_command = $path_base . ".cmd";
     83my $uri_transfer= $path_base . ".transfer";
     84my $uri_check   = $path_base . ".check";
     85my $uri_config  = $path_base . ".config";
     86my $uri_generate= $path_base . ".generate";
     87my $uri_return  = $path_base . ".return";
     88
     89my $disk_command = $ipprc->file_resolve($uri_command);
     90my $disk_transfer= $ipprc->file_resolve($uri_transfer);
     91my $disk_check   = $ipprc->file_resolve($uri_check);
     92my $disk_config  = $ipprc->file_resolve($uri_config);
     93my $disk_generate= $ipprc->file_resolve($uri_generate);
     94my $disk_return  = $ipprc->file_resolve($uri_return);
     95
     96scp_put($disk_command,uri_local_to_remote($uri_command));
     97scp_put($disk_transfer,uri_local_to_remote($uri_transfer));
     98scp_put($disk_check,uri_local_to_remote($uri_check));
     99scp_put($disk_config,uri_local_to_remote($uri_config));
     100scp_put($disk_generate,uri_local_to_remote($uri_generate));
     101scp_put($disk_return, uri_local_to_remote($uri_return));
    106102
    107103# Run check command
    108     my (undef,$remote_check) = uri_convert($uri_check);
    109     my (undef,$remote_transfer)= uri_convert($uri_transfer);
    110 
    111     my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer");
    112     # We no longer need to parse this output, as it retrieves files itself.
    113     foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) {
    114         print "$l\n";
    115     }
     104# my (undef,$remote_check) = uri_convert($uri_check);
     105# my (undef,$remote_transfer)= uri_convert($uri_transfer);
     106
     107my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_path --fetch");
     108
     109# We no longer need to parse this output, as it retrieves files itself.
     110foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) {
     111    print "$l\n";
     112}
    116113
    117114# Run real command
    118     my (undef,$remote_command) = uri_convert($uri_command);
    119     my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command");
    120     if ($#{ $ssh_exec_stdout } != -1) { # Parse the output
    121         $job_id = ${ $ssh_exec_stdout }[0];
    122         chomp($job_id);
    123         $job_id =~ s/\s+//g;
    124     }
    125 
    126     unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly.
    127         &my_die("No job_id returned.  Sorry.",$remote_id,$PS_EXIT_PROG_ERROR);
    128     }
    129 
    130     # Notify the database that this entry is currently running.
    131     my $command = "$remotetool -updaterun -remote_id $remote_id ";
    132     $command .= " -set_state run ";
    133     $command .= " -job_id $job_id ";
    134     $command .= " -dbname $dbname " if defined $dbname;
    135 
    136     print STDERR "$command\n";
    137     system($command);
    138 
    139 } # End of exec phase
    140 
    141 # Check that we have a valid job_id, either from the command line, or from the msub command.
     115my (undef,$remote_command) = uri_convert($uri_command);
     116my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command");
     117if ($#{ $ssh_exec_stdout } != -1) { # Parse the output
     118    $job_id = ${ $ssh_exec_stdout }[0];
     119    chomp($job_id);
     120    $job_id =~ s/\s+//g;
     121}
     122
    142123unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly.
    143124    &my_die("No job_id returned.  Sorry.",$remote_id,$PS_EXIT_PROG_ERROR);
    144125}
    145126
    146 
    147 # Poll the job status?
    148 my $poll_count = 0;
    149 my $poll_max   = 1; # If it isn't finished now, it's unlikely to magically finish in the next five minutes.
    150 my $poll_sleep = 60;
    151 my $poll_response = 0;
    152 my $poll_word  = "nothing";
    153 
    154 if ($force) {
    155     $poll_response = 1;
    156     $poll_word     = "Completed";
    157 }
    158 
    159 while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)&&($force != 1)) {
    160     if ($verbose) {
    161         print "Polling code.  Sleeping.  $poll_count of $poll_max $poll_response\n";
    162     }
    163     unless ($poll_count == 0) {
    164         sleep($poll_sleep);
    165     }
    166     ($poll_response,$poll_word) = poll_job($job_id);
    167 
    168     if ($poll_response == 1) { # This job has completed
    169         last;
    170     }
    171     elsif ($poll_response == -1) { # This job has an error
    172         warn("The job exited incorrectly: $remote_id $job_id $poll_word.  Attempting to recover what we can.");
    173         $poll_response = 1; # To fool the check later into proceding
    174     }
    175     $poll_count++;
    176 }
    177 if ($verbose) {
    178     print "Stopped Polling code.  $poll_count of $poll_max $poll_response\n";
    179 }
    180 if ($poll_response != 1) {
    181     &my_die("Job hasn't completed.  Ending polling.", $remote_id, 0, "run", $job_id);
    182 }
    183 
    184 
    185 # Initialize the remote side to sync things
    186 
    187 # Once sync completes, execute all the database commands
    188 # Grab the return list
    189 my $uri_return  = $path_base . ".return";
    190 my ($ipp_return,$remote_return) = uri_convert($uri_return);
    191 # We shouldn't need to get this file.  We made that file.
    192 #scp_get($remote_return,$ipp_return);
    193 
    194 # Construct list of dbinfo files we'll need to execute.
    195 my @returned_files = ();
    196 my @dbinfo_files   = ();
    197 my @dbindex        = ();
    198 my $iter           = 0;
    199 open(RETURN,   "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id,$PS_EXIT_SYS_ERROR);
    200 while (<RETURN>) {
    201     chomp;
    202     push @returned_files, $_;
    203     if ($_ =~ /dbinfo/) {
    204         push @dbinfo_files, $_;
    205         push @dbindex, $iter;
    206     }
    207     $iter++;
    208 
    209 }
    210 close(RETURN);
    211 print STDERR "Expect $#dbinfo_files dbinfo files\n";
    212 
    213 my $first = 0;
    214 my $last  = $#dbinfo_files;
    215 my $mid   = 0;
    216 my $range = 20;  # How wide of a chunk to look through to catch threading drops
    217 my $offset = 0;  # Offset to start from.
    218 
    219 # Feed the return list into the transfer tool
    220 # If this fails, we should go to an auth wait point.
    221 if ($poll_word eq 'Completed') {
    222     # We're complete.  All files should exist on disk
    223     my $iter = 0;
    224     my $transfer_success = 1;
    225     do {
    226         $transfer_success = 1;
    227         $iter++;
    228         check_ssh_connection();
    229         my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
    230         my $return_push_output_print = join "\n", @{ $return_push_output };
    231         if ($return_push_output_print =~ /exited with value 255/) { $transfer_success = 0; }
    232         print "$return_push_output_print\n";
    233     } while (($transfer_success != 1) &&  ($iter < 10));  # Try harder;
    234 }
    235 else {
    236     # Just try and fetch things, because we didn't complete correctly, it seems.
    237     my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
    238     my $return_push_output_print = join "\n", @{ $return_push_output };
    239     print "$return_push_output_print\n";
    240 }
    241 
    242 foreach my $file (@dbinfo_files) {
    243     open(DBF,"$file") || warn "Missing file $file\n";
    244     my $cmd = <DBF>;
    245     close(DBF);
    246     chomp($cmd);
    247 #    print STDERR "$file $cmd\n";
    248     $cmd =~ s/-/ -/g; # This is just a safety check for missing space.
    249     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    250         run(command => $cmd, verbose => $verbose);
    251     unless ($success) {
    252 
    253         # This shouldn't fail, but we also can't suddenly abort if something does fail.
    254         # Now we're going to drop the component that owns this dbinfo, which should force a retry.
    255         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    256         warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code");
    257         if ($poll_word ne 'Completed') {  # If the job claims it completed, then we're fighting the network.
    258             drop_component($remote_id,$cmd,$file);
    259         }
    260     }
    261 
    262 }
    263 
    264 # We're done, so set the state and exit.
    265 &my_die("Finished",$remote_id,0,"full");
    266 
     127# Notify the database that this entry is currently running.
     128&my_die("Finished",$remote_id,0,"run",$job_id);
    267129
    268130# END PROGRAM
    269 
    270131
    271132sub check_ssh_connection {
Note: See TracChangeset for help on using the changeset viewer.