Index: trunk/ippScripts/scripts/sc_remote_exec.pl =================================================================== --- trunk/ippScripts/scripts/sc_remote_exec.pl (revision 36583) +++ trunk/ippScripts/scripts/sc_remote_exec.pl (revision 36844) @@ -16,5 +16,5 @@ # Hard coded values my $DMZ_HOST = 'wtrw'; -my $SEC_HOST = 'mu-fe'; +my $SEC_HOST = 'mu-fe4'; my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/'; my $remote_root = '/scratch3/watersc1/'; @@ -24,5 +24,5 @@ my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); -my $remotetool = can_run('echo') or (warn "Can't find remotetool" and $missing_tools = 1); +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); if ($missing_tools) { @@ -72,4 +72,6 @@ my $uri_check = $path_base . ".check"; my $uri_config = $path_base . ".config"; + my $uri_generate= $path_base . ".generate"; + my $uri_return = $path_base . ".return"; my $disk_command = $ipprc->file_resolve($uri_command); @@ -77,4 +79,6 @@ my $disk_check = $ipprc->file_resolve($uri_check); my $disk_config = $ipprc->file_resolve($uri_config); + my $disk_generate= $ipprc->file_resolve($uri_generate); + my $disk_return = $ipprc->file_resolve($uri_return); scp_put($disk_command,uri_local_to_remote($uri_command)); @@ -82,52 +86,16 @@ scp_put($disk_check,uri_local_to_remote($uri_check)); scp_put($disk_config,uri_local_to_remote($uri_config)); + scp_put($disk_generate,uri_local_to_remote($uri_generate)); + scp_put($disk_return, uri_local_to_remote($uri_return)); # Run check command my (undef,$remote_check) = uri_convert($uri_check); - my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_validate_files.pl --check_list $remote_check"); - -# Parse stdout for files to transfer. - if ($#{ $ssh_check_stdout } != -1) { # There were files not found. - # Someone was missing. Use the check and transfer lists to identify the local version, and send it. - # What was missing? - my %missing_files = (); - foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) { - print "Missing!: $l\n"; - $missing_files{$l} = 1; - } - - # Do we have that file? - my @check_files = (); - open(CHECK,$disk_check); - while() { - chomp; - push @check_files, $_; - } - close(CHECK); - - my @trans_files = (); - open(TRANS,$disk_transfer); - while() { - chomp; - push @trans_files, $_; - } - close(TRANS); - - if ($#trans_files != $#check_files) { &my_die("The transfer file list and check file list are different sizes", - $remote_id,$PS_EXIT_DATA_ERROR); } - - for (my $i = 0; $i <= $#check_files; $i++) { - if (exists($missing_files{$check_files[$i]})) { - if ($trans_files[$i] ne "REMOTE_ONLY") { - scp_put($trans_files[$i],"$check_files[$i]"); - } - else { - &my_die("There is a file that should be there, but the file isn't there, and I don't have a copy of that file here. The file is: $check_files[$i]", - $remote_id,$PS_EXIT_DATA_ERROR); # This seems a bit harsh. - } - } - } - } - + my (undef,$remote_transfer)= uri_convert($uri_transfer); +# my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_validate_files.pl --check_list $remote_check --transfer_list $remote_transfer"); + my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer"); + # We no longer need to parse this output, as it retrieves files itself. + foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) { + print "$l\n"; + } # Run real command @@ -147,10 +115,10 @@ my $command = "$remotetool -updaterun -remote_id $remote_id "; $command .= " -set_state run "; - $command .= " -set_job_id $job_id "; + $command .= " -job_id $job_id "; $command .= " -dbname $dbname " if defined $dbname; system($command); -} +} # End of exec phase # Check that we have a valid job_id, either from the command line, or from the msub command. @@ -185,55 +153,48 @@ print "Stopped Polling code. $poll_count of $poll_max $poll_response\n"; } +if ($poll_response != 1) { + &my_die("Job hasn't completed. Ending polling.", $remote_id, 0, "run", $job_id); +} # Retrieve validation summary my $fault = 0; -exit(1); -# Read this as a metadata object -my $ssh_exec_stdout = ssh_exec_command("sc_validate_processing.pl ${path_base}.out"); -my $retrieveData = $mdcParser->parse(join "", @$ssh_exec_stdout); -my $retrieveData2= parse_md_list($retrieveData); - -foreach my $retrieveEntry (@$retrieveData2) { - my $process_fault = $retrieveEntry->{fault}; - my $dbcommand = $retrieveEntry->{dbcommand}; - my $filelist = $retrieveEntry->{filelist}; - - scp_get("${filelist}",${filelist}); - open(FL,${filelist}); - while() { - chomp; - my $f = $_; - scp_get("${f}",$f); - } - close(FL); - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $dbcommand, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("The database command returned did not execute correctly. $dbcommand $process_fault", - $remote_id,$error_code); - } - - if ($process_fault != 0) { - $fault = 1; - } -} - -if (($policy ne 'ignore')&&($fault != 0)) { - &my_die("This job exited without all components finishing successfully. Halting.", - $remote_id,$fault,"halt"); -} -else { - print "This job exited without all components finishing successfully. $fault\n" if $fault; - &my_die("This job finished. Setting remote job to full.", - $remote_id,0,"full"); -} - -# Promote to finished. -&my_die(); # Not really, but I don't have the tool interface finished, so these are all just placeholders. -# Quick review: -# new -> pending -> run -> full -# auth +# Initialize the remote side to sync things + +# Once sync completes, execute all the database commands +# Grab the return list +my $uri_return = $path_base . ".return"; +my ($ipp_return,$remote_return) = uri_convert($uri_return); +scp_get($remote_return,$ipp_return); +# Feed teh return list into the +my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return"); +print "$return_push_output\n"; + + +open(RETURN, "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id,$PS_EXIT_SYS_ERROR); +while() { + chomp; + if ($_ =~ /dbinfo/) { + my $file = $_; +# $file =~ s/${remote_root}.tmp//; THis shouldn't be needed, as the transfer tool munges the files on that side. + open(DBF,"$file") || warn "Missing file $file\n"; + my $cmd = ; + chomp($cmd); + $cmd =~ s/-/ -/g; # This is just a safety check for missing space. + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + # This shouldn't fail, but we also can't suddenly abort if something does fail. Maybe retry if this looks to be a continuing issue? + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("The command that shouldn't fail has failed: $cmd $remote_id $error_code");##,$remote_id,$error_code); + } + close(DBF); + } +} +close(RETURN); + +# We're done, so set the state and exit. +&my_die("Finished",$remote_id,0,"full"); + + # END PROGRAM @@ -294,5 +255,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die(); + &my_die("Failed to execute command: >>$cmd<<"); } return ($stdout_buf); @@ -368,5 +329,5 @@ my $exit_code = shift; my $exit_state = shift; - + my $jobid = shift; $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; @@ -374,6 +335,7 @@ if (defined $id and not $no_update) { - my $command = "$remotetool -updaterun -stage_id $id"; + my $command = "$remotetool -updaterun -remote_id $id"; $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -job_id $job_id " if defined $jobid; $command .= " -set_state $exit_state " if defined $exit_state; $command .= " -dbname $dbname " if defined $dbname; @@ -384,2 +346,53 @@ exit($exit_code); } + +# I don't think I need this anymore, but I don't want to burn it until I"m sure. + +# exit(1); +# # Read this as a metadata object +# my $ssh_exec_stdout = ssh_exec_command("sc_validate_processing.pl ${path_base}.out"); +# my $retrieveData = $mdcParser->parse(join "", @$ssh_exec_stdout); +# my $retrieveData2= parse_md_list($retrieveData); + +# foreach my $retrieveEntry (@$retrieveData2) { +# my $process_fault = $retrieveEntry->{fault}; +# my $dbcommand = $retrieveEntry->{dbcommand}; +# my $filelist = $retrieveEntry->{filelist}; + +# scp_get("${filelist}",${filelist}); +# open(FL,${filelist}); +# while() { +# chomp; +# my $f = $_; +# scp_get("${f}",$f); +# } +# close(FL); + +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $dbcommand, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("The database command returned did not execute correctly. $dbcommand $process_fault", +# $remote_id,$error_code); +# } + +# if ($process_fault != 0) { +# $fault = 1; +# } +# } + +# if (($policy ne 'ignore')&&($fault != 0)) { +# &my_die("This job exited without all components finishing successfully. Halting.", +# $remote_id,$fault,"halt"); +# } +# else { +# print "This job exited without all components finishing successfully. $fault\n" if $fault; +# &my_die("This job finished. Setting remote job to full.", +# $remote_id,0,"full"); +# } + +# Promote to finished. +#&my_die(); # Not really, but I don't have the tool interface finished, so these are all just placeholders. +# Quick review: +# new -> pending -> run -> full +# auth