Changeset 37269 for tags/ipp-pv3-20140717
- Timestamp:
- Aug 20, 2014, 11:59:03 AM (12 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_exec.pl
r37203 r37269 1 #!/usr/bin/env perl 1 #!/usr/bin/env perl 2 2 3 3 use Carp; … … 54 54 55 55 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id); 56 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files 56 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files 57 57 58 58 # Phase 1: See if we can actually do anything. … … 69 69 unless ($rt_success) { 70 70 $rt_error_code = (($rt_error_code >> 8) or $PS_EXIT_PROG_ERROR); 71 &my_die("U Nable to run remotetool to determine remote run status",72 73 } 74 my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or 75 &my_die("U Nable to run remotetool to determine remote run status",76 71 &my_die("Unable to run remotetool to determine remote run status", 72 $remote_id,$rt_error_code); 73 } 74 my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or 75 &my_die("Unable to run remotetool to determine remote run status", 76 $remote_id,$rt_error_code); 77 77 my $rrData2 = parse_md_list($rrData); 78 78 my $runData = ${ $rrData2 }[0]; # There should be only one … … 83 83 # Copy command files 84 84 my @files = (); 85 85 86 86 my $uri_command = $path_base . ".cmd"; 87 87 my $uri_transfer= $path_base . ".transfer"; … … 97 97 my $disk_generate= $ipprc->file_resolve($uri_generate); 98 98 my $disk_return = $ipprc->file_resolve($uri_return); 99 99 100 100 scp_put($disk_command,uri_local_to_remote($uri_command)); 101 101 scp_put($disk_transfer,uri_local_to_remote($uri_transfer)); … … 104 104 scp_put($disk_generate,uri_local_to_remote($uri_generate)); 105 105 scp_put($disk_return, uri_local_to_remote($uri_return)); 106 106 107 107 # Run check command 108 108 my (undef,$remote_check) = uri_convert($uri_check); … … 112 112 # We no longer need to parse this output, as it retrieves files itself. 113 113 foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) { 114 114 print "$l\n"; 115 115 } 116 116 … … 119 119 my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command"); 120 120 if ($#{ $ssh_exec_stdout } != -1) { # Parse the output 121 122 123 121 $job_id = ${ $ssh_exec_stdout }[0]; 122 chomp($job_id); 123 $job_id =~ s/\s+//g; 124 124 } 125 125 126 126 unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. 127 127 &my_die("No job_id returned. Sorry.",$remote_id,$PS_EXIT_PROG_ERROR); 128 128 } 129 129 … … 133 133 $command .= " -job_id $job_id "; 134 134 $command .= " -dbname $dbname " if defined $dbname; 135 135 136 print STDERR "$command\n"; 136 137 system($command); 137 138 138 139 } # End of exec phase 139 140 … … 158 159 while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)&&($force != 1)) { 159 160 if ($verbose) { 160 161 print "Polling code. Sleeping. $poll_count of $poll_max $poll_response\n"; 161 162 } 162 163 unless ($poll_count == 0) { 163 164 sleep($poll_sleep); 164 165 } 165 166 ($poll_response,$poll_word) = poll_job($job_id); 166 167 167 168 if ($poll_response == 1) { # This job has completed 168 169 last; 169 170 } 170 171 elsif ($poll_response == -1) { # This job has an error 171 172 172 warn("The job exited incorrectly: $remote_id $job_id $poll_word. Attempting to recover what we can."); 173 $poll_response = 1; # To fool the check later into proceding 173 174 } 174 175 $poll_count++; … … 201 202 push @returned_files, $_; 202 203 if ($_ =~ /dbinfo/) { 203 204 204 push @dbinfo_files, $_; 205 push @dbindex, $iter; 205 206 } 206 207 $iter++; … … 218 219 # Feed the return list into the transfer tool 219 220 # If this fails, we should go to an auth wait point. 220 if ($poll_word eq 'Completed') { 221 if ($poll_word eq 'Completed') { 221 222 # We're complete. All files should exist on disk 222 223 my $iter = 0; 223 224 my $transfer_success = 1; 224 225 do { 225 226 227 check_ssh_connection(); 228 229 230 231 226 $transfer_success = 1; 227 $iter++; 228 check_ssh_connection(); 229 my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset"); 230 my $return_push_output_print = join "\n", @{ $return_push_output }; 231 if ($return_push_output_print =~ /exited with value 255/) { $transfer_success = 0; } 232 print "$return_push_output_print\n"; 232 233 } while (($transfer_success != 1) && ($iter < 10)); # Try harder; 233 234 } … … 247 248 $cmd =~ s/-/ -/g; # This is just a safety check for missing space. 248 249 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 249 250 run(command => $cmd, verbose => $verbose); 250 251 unless ($success) { 251 252 252 # This shouldn't fail, but we also can't suddenly abort if something does fail. 253 254 255 256 257 258 253 # This shouldn't fail, but we also can't suddenly abort if something does fail. 254 # Now we're going to drop the component that owns this dbinfo, which should force a retry. 255 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 256 warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code"); 257 if ($poll_word ne 'Completed') { # If the job claims it completed, then we're fighting the network. 258 drop_component($remote_id,$cmd,$file); 259 } 259 260 } 260 261 … … 272 273 273 274 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 274 275 run(command => $cmd, verbose => $verbose); 275 276 unless ($success) { 276 277 277 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 278 &my_die("Authorization check failed.",$remote_id,0,'auth'); 278 279 } 279 280 } … … 288 289 289 290 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 290 291 run(command => $cmd, verbose => $verbose); 291 292 unless ($success) { 292 293 &my_die(); 293 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 294 &my_die(); 294 295 } 295 296 } … … 302 303 303 304 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 304 305 run(command => $cmd, verbose => $verbose); 305 306 unless ($success) { 306 307 &my_die(); 307 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 308 &my_die(); 308 309 } 309 310 } … … 314 315 print "EXEC: $cmd\n"; 315 316 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 316 317 run(command => $cmd, verbose => $verbose); 317 318 unless ($success) { 318 319 320 check_ssh_connection(); 321 &my_die("Failed to execute command: >>$cmd<<"); 319 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 320 # If we failed, see if we failed due to authorization. 321 check_ssh_connection(); 322 &my_die("Failed to execute command: >>$cmd<<"); 322 323 } 323 324 return ($stdout_buf); 324 } 325 } 325 326 326 327 sub poll_job { … … 328 329 my $response_array = ssh_exec_command("checkjob -v $job_id"); 329 330 my $response = join "\n", @$response_array; 330 331 if ($response =~ /State: Running/) { 332 331 332 if ($response =~ /State: Running/) { 333 return(0,"Running"); 333 334 } 334 335 elsif ($response =~ /State: Completed/) { 335 336 return(1,"Completed"); 336 337 } 337 338 elsif ($response =~ /State: Idle/) { 338 339 return(-2,"Idle"); 339 340 } 340 341 else { 341 342 343 344 342 my $response_word = $response; 343 $response_word =~ s/.*State: //; 344 $response_word =~ s/\n.*//; 345 return(-1,$response_word); 345 346 } 346 347 } … … 351 352 my $ipp_disk= $ipprc->file_resolve( $neb_uri ); 352 353 my $remote_disk = $ipp_disk; 353 354 354 355 unless(defined($ipp_disk)) { 355 356 my_die(); 356 357 } 357 358 … … 382 383 return($remote_uri); 383 384 } 384 385 385 386 sub uri_remote_to_local { 386 387 # This needs to replace the remote root directory with the nebulous tag. … … 388 389 $remote_uri =~ s%${remote_root}%%; 389 390 my $local_uri = "neb:///" . $remote_uri; 390 391 391 392 return($local_uri); 392 393 } … … 397 398 my $file = shift; 398 399 399 if ($cmd eq '') { 400 401 402 403 404 405 if ($stage eq 'chip') { 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 400 if ($cmd eq '') { 401 # Zero byte file. This dbinfo was never constructed properly. 402 403 # Steal the stage_id from the filename. This isn't something I'm happy about. 404 my $stage = $runData->{stage}; 405 my $stage_id = $file; 406 if ($stage eq 'chip') { 407 $stage_id =~ s/^.*ch\.//; 408 } 409 elsif ($stage eq 'camera') { 410 $stage_id =~ s/^.*cm\.//; 411 } 412 elsif ($stage eq 'warp') { 413 $stage_id =~ s/^.*wrp\.//; 414 } 415 elsif ($stage eq 'stack') { 416 $stage_id =~ s/^.*stk\.//; 417 } 418 $stage_id =~ s/\..*$//; 419 420 # Drop this from the database 421 my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id "; 422 $drop_cmd .= " -dbname $dbname " if defined($dbname); 423 system($drop_cmd); 423 424 } 424 425 … … 434 435 435 436 carp($msg); 436 437 437 438 if (defined $id and not $no_update) { 438 439 440 441 442 443 444 439 my $command = "$remotetool -updaterun -remote_id $id"; 440 $command .= " -fault $exit_code " if defined $exit_code; 441 $command .= " -job_id $job_id " if defined $jobid; 442 $command .= " -set_state $exit_state " if defined $exit_state; 443 $command .= " -dbname $dbname " if defined $dbname; 444 445 system($command); 445 446 } 446 447
Note:
See TracChangeset
for help on using the changeset viewer.
