Changeset 37833 for trunk/ippScripts/scripts/sc_remote_exec.pl
- Timestamp:
- Jan 12, 2015, 12:53:36 PM (12 years ago)
- Location:
- trunk/ippScripts/scripts
- Files:
-
- 2 edited
-
. (modified) (1 prop)
-
sc_remote_exec.pl (modified) (12 diffs, 1 prop)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts
- Property svn:mergeinfo set to
-
trunk/ippScripts/scripts/sc_remote_exec.pl
- Property svn:mergeinfo changed
r37812 r37833 1 #!/usr/bin/env perl 1 #!/usr/bin/env perl 2 2 3 3 use Carp; … … 16 16 # Hard coded values 17 17 my $DMZ_HOST = 'wtrw'; 18 my $SEC_HOST = 'mu-fe'; 18 my @SEC_HOSTS= ('mu-fe1','mu-fe2','mu-fe3' 19 # ,'mu-fe4' 20 ); 21 my $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; # 'mu-fe'; 19 22 my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/'; 20 23 my $remote_root = '/scratch3/watersc1/'; … … 32 35 33 36 # Options 34 my ($remote_id,$path_base,$policy,$poll,$force,$job_id,$dbname,$verbose,$no_update,$camera); 35 $force = 0; 37 my ($remote_id,$job_id,$path_base,$dbname,$verbose,$no_update,$camera); 36 38 $verbose = 0; 37 39 GetOptions( 38 40 'remote_id=s' => \$remote_id, 39 'job_id=s' => \$job_id,40 41 'path_base=s' => \$path_base, 41 'policy=s' => \$policy,42 'poll' => \$poll,43 'force-poll' => \$force,44 42 'camera=s' => \$camera, 45 43 'dbname=s' => \$dbname, … … 54 52 55 53 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id); 56 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files 54 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files 57 55 58 56 # Phase 1: See if we can actually do anything. … … 69 67 unless ($rt_success) { 70 68 $rt_error_code = (($rt_error_code >> 8) or $PS_EXIT_PROG_ERROR); 71 &my_die("UNable to run remotetool to determine remote run status", 72 $remote_id,$rt_error_code); 73 } 74 my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or 75 &my_die("UNable to run remotetool to determine remote run status", 76 $remote_id,$rt_error_code); 69 &my_die("Unable to run remotetool to determine remote run status", $remote_id, $rt_error_code); 70 } 71 my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or 72 &my_die("Unable to run remotetool to determine remote run status", $remote_id, $rt_error_code); 77 73 my $rrData2 = parse_md_list($rrData); 78 74 my $runData = ${ $rrData2 }[0]; # There should be only one 79 75 80 # These are operations we only need to do on the first call, not on subsequent poll operations. 81 unless ($poll) { 76 my ($remote_path) = &uri_local_to_remote($path_base); 77 print "REMOTE PATH: $remote_path\n"; 78 82 79 # Phase 2: Ensure files are in place: 83 80 # Copy command files 84 my @files = (); 85 86 my $uri_command = $path_base . ".cmd"; 87 my $uri_transfer= $path_base . ".transfer"; 88 my $uri_check = $path_base . ".check"; 89 my $uri_config = $path_base . ".config"; 90 my $uri_generate= $path_base . ".generate"; 91 my $uri_return = $path_base . ".return"; 92 93 my $disk_command = $ipprc->file_resolve($uri_command); 94 my $disk_transfer= $ipprc->file_resolve($uri_transfer); 95 my $disk_check = $ipprc->file_resolve($uri_check); 96 my $disk_config = $ipprc->file_resolve($uri_config); 97 my $disk_generate= $ipprc->file_resolve($uri_generate); 98 my $disk_return = $ipprc->file_resolve($uri_return); 99 100 scp_put($disk_command,uri_local_to_remote($uri_command)); 101 scp_put($disk_transfer,uri_local_to_remote($uri_transfer)); 102 scp_put($disk_check,uri_local_to_remote($uri_check)); 103 scp_put($disk_config,uri_local_to_remote($uri_config)); 104 scp_put($disk_generate,uri_local_to_remote($uri_generate)); 105 scp_put($disk_return, uri_local_to_remote($uri_return)); 106 107 # Run check command 108 my (undef,$remote_check) = uri_convert($uri_check); 109 my (undef,$remote_transfer)= uri_convert($uri_transfer); 110 111 my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer"); 112 # We no longer need to parse this output, as it retrieves files itself. 113 foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) { 114 print "$l\n"; 115 } 81 my @files = (); 82 83 my $uri_command = $path_base . ".cmd"; 84 my $uri_transfer= $path_base . ".transfer"; 85 my $uri_check = $path_base . ".check"; 86 my $uri_config = $path_base . ".config"; 87 my $uri_generate= $path_base . ".generate"; 88 my $uri_return = $path_base . ".return"; 89 90 my $disk_command = $ipprc->file_resolve($uri_command); 91 my $disk_transfer= $ipprc->file_resolve($uri_transfer); 92 my $disk_check = $ipprc->file_resolve($uri_check); 93 my $disk_config = $ipprc->file_resolve($uri_config); 94 my $disk_generate= $ipprc->file_resolve($uri_generate); 95 my $disk_return = $ipprc->file_resolve($uri_return); 96 97 &scp_put($disk_command, &uri_local_to_remote($uri_command)); 98 &scp_put($disk_transfer, &uri_local_to_remote($uri_transfer)); 99 &scp_put($disk_check, &uri_local_to_remote($uri_check)); 100 &scp_put($disk_config, &uri_local_to_remote($uri_config)); 101 &scp_put($disk_generate, &uri_local_to_remote($uri_generate)); 102 &scp_put($disk_return, &uri_local_to_remote($uri_return)); 103 104 my $ssh_check_stdout = &ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_path --fetch"); 105 106 # We no longer need to parse this output, as it retrieves files itself. 107 foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) { 108 print "$l\n"; 109 } 116 110 117 111 # Run real command 118 my (undef,$remote_command) = uri_convert($uri_command); 119 my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command"); 120 if ($#{ $ssh_exec_stdout } != -1) { # Parse the output 121 $job_id = ${ $ssh_exec_stdout }[0]; 122 chomp($job_id); 123 $job_id =~ s/\s+//g; 124 } 125 126 unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. 127 &my_die("No job_id returned. Sorry.",$remote_id,$PS_EXIT_PROG_ERROR); 128 } 129 130 # Notify the database that this entry is currently running. 131 my $command = "$remotetool -updaterun -remote_id $remote_id "; 132 $command .= " -set_state run "; 133 $command .= " -job_id $job_id "; 134 $command .= " -dbname $dbname " if defined $dbname; 135 136 system($command); 137 138 } # End of exec phase 139 140 # Check that we have a valid job_id, either from the command line, or from the msub command. 112 my (undef,$remote_command) = &uri_convert($uri_command); 113 my $ssh_exec_stdout = &ssh_exec_command("msub -V $remote_command"); 114 if ($#{ $ssh_exec_stdout } != -1) { # Parse the output 115 $job_id = ${ $ssh_exec_stdout }[0]; 116 chomp($job_id); 117 $job_id =~ s/\s+//g; 118 } 119 141 120 unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. 142 &my_die("No job_id returned. Sorry.",$remote_id,$PS_EXIT_PROG_ERROR); 143 } 144 145 146 # Poll the job status? 147 my $poll_count = 0; 148 my $poll_max = 1; # If it isn't finished now, it's unlikely to magically finish in the next five minutes. 149 my $poll_sleep = 60; 150 my $poll_response = 0; 151 my $poll_word = "nothing"; 152 153 if ($force) { 154 $poll_response = 1; 155 $poll_word = "Completed"; 156 } 157 158 while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)&&($force != 1)) { 159 if ($verbose) { 160 print "Polling code. Sleeping. $poll_count of $poll_max $poll_response\n"; 161 } 162 unless ($poll_count == 0) { 163 sleep($poll_sleep); 164 } 165 ($poll_response,$poll_word) = poll_job($job_id); 166 167 if ($poll_response == 1) { # This job has completed 168 last; 169 } 170 elsif ($poll_response == -1) { # This job has an error 171 warn("The job exited incorrectly: $remote_id $job_id $poll_word. Attempting to recover what we can."); 172 $poll_response = 1; # To fool the check later into proceding 173 } 174 $poll_count++; 175 } 176 if ($verbose) { 177 print "Stopped Polling code. $poll_count of $poll_max $poll_response\n"; 178 } 179 if ($poll_response != 1) { 180 &my_die("Job hasn't completed. Ending polling.", $remote_id, 0, "run", $job_id); 181 } 182 183 184 # Initialize the remote side to sync things 185 186 # Once sync completes, execute all the database commands 187 # Grab the return list 188 my $uri_return = $path_base . ".return"; 189 my ($ipp_return,$remote_return) = uri_convert($uri_return); 190 # We shouldn't need to get this file. We made that file. 191 #scp_get($remote_return,$ipp_return); 192 193 # Construct list of dbinfo files we'll need to execute. 194 my @returned_files = (); 195 my @dbinfo_files = (); 196 my @dbindex = (); 197 my $iter = 0; 198 open(RETURN, "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id,$PS_EXIT_SYS_ERROR); 199 while (<RETURN>) { 200 chomp; 201 push @returned_files, $_; 202 if ($_ =~ /dbinfo/) { 203 push @dbinfo_files, $_; 204 push @dbindex, $iter; 205 } 206 $iter++; 207 208 } 209 close(RETURN); 210 print STDERR "Expect $#dbinfo_files dbinfo files\n"; 211 212 my $first = 0; 213 my $last = $#dbinfo_files; 214 my $mid = 0; 215 my $range = 20; # How wide of a chunk to look through to catch threading drops 216 my $offset = 0; # Offset to start from. 217 218 # Feed the return list into the transfer tool 219 # If this fails, we should go to an auth wait point. 220 if ($poll_word eq 'Completed') { 221 # We're complete. All files should exist on disk 222 my $iter = 0; 223 my $transfer_success = 1; 224 do { 225 $transfer_success = 1; 226 $iter++; 227 check_ssh_connection(); 228 my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset"); 229 my $return_push_output_print = join "\n", @{ $return_push_output }; 230 if ($return_push_output_print =~ /exited with value 255/) { $transfer_success = 0; } 231 print "$return_push_output_print\n"; 232 } while (($transfer_success != 1) && ($iter < 10)); # Try harder; 233 } 234 else { 235 # Just try and fetch things, because we didn't complete correctly, it seems. 236 my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset"); 237 my $return_push_output_print = join "\n", @{ $return_push_output }; 238 print "$return_push_output_print\n"; 239 } 240 241 foreach my $file (@dbinfo_files) { 242 open(DBF,"$file") || warn "Missing file $file\n"; 243 my $cmd = <DBF>; 244 close(DBF); 245 chomp($cmd); 246 # print STDERR "$file $cmd\n"; 247 $cmd =~ s/-/ -/g; # This is just a safety check for missing space. 248 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 249 run(command => $cmd, verbose => $verbose); 250 unless ($success) { 251 252 # This shouldn't fail, but we also can't suddenly abort if something does fail. 253 # Now we're going to drop the component that owns this dbinfo, which should force a retry. 254 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 255 warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code"); 256 if ($poll_word ne 'Completed') { # If the job claims it completed, then we're fighting the network. 257 drop_component($remote_id,$cmd,$file); 258 } 259 } 260 261 } 262 263 # We're done, so set the state and exit. 264 &my_die("Finished",$remote_id,0,"full"); 265 121 &my_die("No job_id returned. Sorry.", $remote_id, $PS_EXIT_PROG_ERROR); 122 } 123 124 # Notify the database that this entry is currently running. 125 &my_die("Finished", $remote_id, 0, "run", $job_id); 266 126 267 127 # END PROGRAM 268 269 128 270 129 sub check_ssh_connection { … … 272 131 273 132 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 274 275 unless ($success) { 276 277 &my_die("Authorization check failed.",$remote_id,0,'auth');133 run(command => $cmd, verbose => $verbose); 134 unless ($success) { 135 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 136 &my_die("Authorization check failed.", $remote_id, 0, 'pending'); 278 137 } 279 138 } … … 285 144 286 145 my $directory = dirname($destination); 287 ssh_exec_command("mkdir -p $directory");288 289 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 290 291 unless ($success) { 292 293 &my_die(); 146 &ssh_exec_command("mkdir -p $directory"); 147 148 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 149 run(command => $cmd, verbose => $verbose); 150 unless ($success) { 151 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 152 &my_die("failed to send file $file to LANL", $remote_id, $error_code); 294 153 } 295 154 } … … 302 161 303 162 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 304 305 unless ($success) { 306 307 &my_die(); 163 run(command => $cmd, verbose => $verbose); 164 unless ($success) { 165 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 166 &my_die("failed to get file $file from LANL", $remote_id, $error_code); 308 167 } 309 168 } … … 311 170 sub ssh_exec_command { 312 171 my $cmd = shift; 313 $cmd = "$ssh $DMZ_HOST ssh ${SEC_HOST} $cmd";172 $cmd = "$ssh -n $DMZ_HOST ssh ${SEC_HOST} $cmd"; 314 173 print "EXEC: $cmd\n"; 315 174 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 316 317 unless ($success) { 318 319 320 check_ssh_connection(); 321 &my_die("Failed to execute command: >>$cmd<<"); 175 run(command => $cmd, verbose => $verbose); 176 unless ($success) { 177 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 178 # If we failed, see if we failed due to authorization. 179 check_ssh_connection(); 180 &my_die("Failed to execute remote command: >>$cmd<<", $remote_id, $error_code); 322 181 } 323 182 return ($stdout_buf); 324 } 325 326 sub poll_job { 327 my $job_id = shift; 328 my $response_array = ssh_exec_command("checkjob -v $job_id"); 329 my $response = join "\n", @$response_array; 330 331 if ($response =~ /State: Running/) { 332 return(0,"Running"); 333 } 334 elsif ($response =~ /State: Completed/) { 335 return(1,"Completed"); 336 } 337 elsif ($response =~ /State: Idle/) { 338 return(-2,"Idle"); 339 } 340 else { 341 my $response_word = $response; 342 $response_word =~ s/.*State: //; 343 $response_word =~ s/\n.*//; 344 return(-1,$response_word); 345 } 346 } 347 348 183 } 184 185 # neb URI to (ipp_disk,remote_disk) 349 186 sub uri_convert { 350 187 my $neb_uri = shift; 351 188 my $ipp_disk= $ipprc->file_resolve( $neb_uri ); 352 189 my $remote_disk = $ipp_disk; 353 190 354 191 unless(defined($ipp_disk)) { 355 my_die();192 &my_die("Failed to generate or find uri $neb_uri", $remote_id, $PS_EXIT_SYS_ERROR); 356 193 } 357 194 … … 363 200 } 364 201 365 sub uri_to_outputs { 366 my $neb_uri = shift; 367 my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); 368 369 # print TRANSFER "$ipp_disk\n"; 370 # print CHECK "$remote_disk\n"; 371 return($ipp_disk,$remote_disk); 372 } 373 202 # neb URI to remote URI-on-disk 374 203 sub uri_local_to_remote { 375 204 # This needs to replace the nebulous tag with the remote root. … … 382 211 return($remote_uri); 383 212 } 384 385 sub uri_remote_to_local {386 # This needs to replace the remote root directory with the nebulous tag.387 my $remote_uri = shift;388 $remote_uri =~ s%${remote_root}%%;389 my $local_uri = "neb:///" . $remote_uri;390 391 return($local_uri);392 }393 394 sub drop_component {395 my $remote_id = shift;396 my $cmd = shift;397 my $file = shift;398 399 if ($cmd eq '') {400 # Zero byte file. This dbinfo was never constructed properly.401 402 # Steal the stage_id from the filename. This isn't something I'm happy about.403 my $stage = $runData->{stage};404 my $stage_id = $file;405 if ($stage eq 'chip') {406 $stage_id =~ s/^.*ch\.//;407 }408 elsif ($stage eq 'camera') {409 $stage_id =~ s/^.*cm\.//;410 }411 elsif ($stage eq 'warp') {412 $stage_id =~ s/^.*wrp\.//;413 }414 elsif ($stage eq 'stack') {415 $stage_id =~ s/^.*stk\.//;416 }417 $stage_id =~ s/\..*$//;418 419 # Drop this from the database420 my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id ";421 $drop_cmd .= " -dbname $dbname " if defined($dbname);422 system($drop_cmd);423 }424 425 }426 213 427 214 sub my_die { … … 434 221 435 222 carp($msg); 436 223 437 224 if (defined $id and not $no_update) { 438 439 440 441 442 443 444 225 my $command = "$remotetool -updaterun -remote_id $id"; 226 $command .= " -fault $exit_code " if defined $exit_code; 227 $command .= " -job_id $job_id " if defined $jobid; 228 $command .= " -set_state $exit_state " if defined $exit_state; 229 $command .= " -dbname $dbname " if defined $dbname; 230 231 system($command); 445 232 } 446 233
Note:
See TracChangeset
for help on using the changeset viewer.
