Changeset 36844 for trunk/ippScripts

Timestamp:

Jun 9, 2014, 6:17:18 PM (12 years ago)

Author:

watersc1

Message:

These have been tested and seem to work correctly.

Location:

trunk/ippScripts/scripts

Files:

• sc_mk_stack_mdc.pl (added)
• sc_prepare_camera.pl (added)
• sc_prepare_chip.pl (modified) (13 diffs)
• sc_prepare_warp.pl (added)
• sc_remote_exec.pl (modified) (11 diffs)
• sc_transfer_tool.pl (added)

trunk/ippScripts/scripts/sc_prepare_chip.pl

@@ -19 +19 @@
 # Hard coded values
 my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
+my $remote_raw    = "${remote_root}/raw/";  # Directory to find raw data in.
 my $threads       = 2;                      # How many threads are we going to use?
 my $job_cost      = 150 / 60 / 60;          # Estimate of how long a job runs, in hours.
@@ -150 +151 @@
 #print Dumper(%detrends);
 
+my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","TRACE.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN");
 
 #
@@ -157 +159 @@
 my $uri_check   = $path_base . ".check";
 my $uri_config  = $path_base . ".config";
+my $uri_generate= $path_base . ".generate";
+my $uri_return  = $path_base . ".return";
 
 my $disk_command = $ipprc->file_resolve($uri_command,1);
@@ -162 +166 @@
 my $disk_check   = $ipprc->file_resolve($uri_check,1);
 my $disk_config  = $ipprc->file_resolve($uri_config,1);
+my $disk_generate= $ipprc->file_resolve($uri_generate,1);
+my $disk_return  = $ipprc->file_resolve($uri_return,1);
 
 my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it.
@@ -168 +174 @@
 open(CHECK,    ">$disk_check")  || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR);
 open(CONFIG,   ">$disk_config")  || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR);
+open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR);
+open(RETURN,   ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR);
+
 
 #
@@ -185 +194 @@
     
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
+        run(command => $command, verbose => 0);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -254 +263 @@
     
     my $chipData = $mdcParser->parse(join "", @$stdout_buf) or 
-        &my_die("Unable to parse chiptool -pendingimfile information.",
-                $chip_id,$error_code);
+        next;
+#       &my_die("Unable to parse chiptool -pendingimfile information.",
+#               $chip_id,$error_code);
     my $chipData2= parse_md_list($chipData);
 
@@ -270 +280 @@
         
         # Process the image and burntool table
-        my (undef,$remote_uri) = uri_to_outputs($uri);
+        my ($ipp_uri,$remote_uri) = uri_to_outputs_raw($uri);
         my $btt = $uri;
         $btt =~ s/fits$/burn.tbl/;
-        my (undef,$remote_btt) = uri_to_outputs($btt);
+        my ($ipp_btt,$remote_btt) = uri_to_outputs_raw($btt);
         
         # Initialize the ppI command
@@ -287 +297 @@
             my $det_code = $det_types{$det};
             my $duri = $detrends_to_use{$det}->{$class_id};
-            my (undef, $remote_det_uri) = uri_to_outputs($duri);
+            my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri);
             $ppImage_command .= " $det_code $remote_det_uri ";
         }
@@ -310 +320 @@
         $ppImage_command .= " -log ${remote_outroot}.${class_id}.log ";
 
-        push @main_commands, $ppImage_command;
+#       push @main_commands, $ppImage_command;
 
         # Calculate pre and post commands
         my $remote_outroot_dir = dirname($remote_outroot);
-        push @pre_commands, "mkdir -p $remote_outroot_dir";
+        my $pre_command =  "mkdir -p $remote_outroot_dir";
         
         my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id ";
@@ -321 +331 @@
         $post_commandA   .= " -dbname $dbname " if defined $dbname;
 
-        my $post_commandB = "echo -n $post_commandA > ${remote_outroot}.${class_id}.dbinfo && ";
+        my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo  ";
         my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo";
         
-        push @post_commands, "$post_commandB $post_commandC";
+#       push @post_commands, "$post_commandB $post_commandC";
 
         $job_index++;
+
+        print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}";
+
+
+        # Determine which output files need to be returned.
+        foreach my $component (@return_component_list) {
+#           uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) );
+            my $filename = $ipprc->filename($component, $ipp_outroot, $class_id);
+            my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename );
+            my $remote_outroot_dir = dirname($ipp_disk);
+            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} ";
+#           print "  $filename $ipp_disk $remote_disk\n";
+#           die;
+        }
+
+        print CONFIG "\n";
     }
 }
 
 # Actually put the commands into the output file.
-my %unique_cmds = ();
-foreach my $cmd (@pre_commands) {
-    unless(exists($unique_cmds{$cmd})) {
-        print CONFIG $cmd . "\n";
-        $unique_cmds{$cmd} = 1;
-    }
-}
-foreach my $cmd (@main_commands) {
-    unless(exists($unique_cmds{$cmd})) {
-        print CONFIG $cmd . "\n";
-        $unique_cmds{$cmd} = 1;
-    }
-}
-foreach my $cmd (@post_commands) {
-    unless(exists($unique_cmds{$cmd})) {
-        print CONFIG $cmd . "\n";
-        $unique_cmds{$cmd} = 1;
-    }
-}
+# my %unique_cmds = ();
+# foreach my $cmd (@pre_commands) {
+#     unless(exists($unique_cmds{$cmd})) {
+#       print CONFIG $cmd . "\n";
+#       $unique_cmds{$cmd} = 1;
+#     }
+# }
+# foreach my $cmd (@main_commands) {
+#     unless(exists($unique_cmds{$cmd})) {
+#       print CONFIG $cmd . "\n";
+#       $unique_cmds{$cmd} = 1;
+#     }
+# }
+# foreach my $cmd (@post_commands) {
+#     unless(exists($unique_cmds{$cmd})) {
+#       print CONFIG $cmd . "\n";
+#       $unique_cmds{$cmd} = 1;
+#     }
+# }
 
 close(CONFIG);
 close(TRANSFER);
 close(CHECK);
+close(RETURN);
+close(GENERATE);
 
 #
@@ -425 +453 @@
 }
 
+sub uri_convert_and_create {
+    my $neb_uri = shift;
+    my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1);
+    my $remote_disk = $ipp_disk;
+    
+    unless(defined($ipp_disk)) {
+        my_die();
+    }
+
+    $remote_disk =~ s%^.*/%%;   # Remove nebulous path
+    $remote_disk =~ s%^\d+\.%%; # Remove ins_id
+    $remote_disk =~ s%:%/%g;    # Replace colons with directories
+    $remote_disk = "${remote_root}/${remote_disk}";
+    return($ipp_disk,$remote_disk);
+}
+
 sub uri_to_outputs {
     my $neb_uri = shift;
@@ -433 +477 @@
         print TRANSFER "$ipp_disk\n";
         print CHECK    "$remote_disk\n";
+    }
+    return($ipp_disk,$remote_disk);
+}
+
+sub uri_to_outputs_raw {
+    my $neb_uri = shift;
+    my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
+    $remote_disk = $remote_raw . $ipp_disk;
+    unless (exists($file_filter{$neb_uri})) {
+        $file_filter{$neb_uri} = 1;
+        print TRANSFER "$ipp_disk\n";
+        print CHECK    "$remote_disk\n";
+    }
+    return($ipp_disk,$remote_disk);
+}
+
+sub uri_to_outputs_for_return {
+    my $neb_uri = shift;
+    my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri );
+    unless (exists($file_filter{$neb_uri})) {
+        $file_filter{$neb_uri} = 1;
+        print RETURN "$ipp_disk\n";
+        print GENERATE "$remote_disk\n";
     }
     return($ipp_disk,$remote_disk);

trunk/ippScripts/scripts/sc_remote_exec.pl

@@ -16 +16 @@
 # Hard coded values
 my $DMZ_HOST = 'wtrw';
-my $SEC_HOST = 'mu-fe';
+my $SEC_HOST = 'mu-fe4';
 my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
 my $remote_root  = '/scratch3/watersc1/';
@@ -24 +24 @@
 my $ssh        = can_run('ssh')  or (warn "Can't find ssh" and $missing_tools = 1);
 my $scp        = can_run('scp')  or (warn "Can't find scp" and $missing_tools = 1);
-my $remotetool = can_run('echo') or (warn "Can't find remotetool" and $missing_tools = 1);
+my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -72 +72 @@
     my $uri_check   = $path_base . ".check";
     my $uri_config  = $path_base . ".config";
+    my $uri_generate= $path_base . ".generate";
+    my $uri_return  = $path_base . ".return";
 
     my $disk_command = $ipprc->file_resolve($uri_command);
@@ -77 +79 @@
     my $disk_check   = $ipprc->file_resolve($uri_check);
     my $disk_config  = $ipprc->file_resolve($uri_config);
+    my $disk_generate= $ipprc->file_resolve($uri_generate);
+    my $disk_return  = $ipprc->file_resolve($uri_return);
     
     scp_put($disk_command,uri_local_to_remote($uri_command));
@@ -82 +86 @@
     scp_put($disk_check,uri_local_to_remote($uri_check));
     scp_put($disk_config,uri_local_to_remote($uri_config));
+    scp_put($disk_generate,uri_local_to_remote($uri_generate));
+    scp_put($disk_return, uri_local_to_remote($uri_return));
     
 # Run check command
     my (undef,$remote_check) = uri_convert($uri_check);
-    my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_validate_files.pl --check_list $remote_check");
-    
-# Parse stdout for files to transfer.
-    if ($#{ $ssh_check_stdout } != -1) { # There were files not found.
-        # Someone was missing.  Use the check and transfer lists to identify the local version, and send it.
-        # What was missing?
-        my %missing_files = ();
-        foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) {
-            print "Missing!: $l\n";
-            $missing_files{$l} = 1;
-        }
-        
-        # Do we have that file?
-        my @check_files = ();
-        open(CHECK,$disk_check);
-        while(<CHECK>) {
-            chomp;
-            push @check_files, $_;
-        }
-        close(CHECK);
-        
-        my @trans_files = ();
-        open(TRANS,$disk_transfer);
-        while(<TRANS>) {
-            chomp;
-            push @trans_files, $_;
-        }
-        close(TRANS);
-        
-        if ($#trans_files != $#check_files) { &my_die("The transfer file list and check file list are different sizes",
-                                                      $remote_id,$PS_EXIT_DATA_ERROR); }
-        
-        for (my $i = 0; $i <= $#check_files; $i++) {
-            if (exists($missing_files{$check_files[$i]})) {
-                if ($trans_files[$i] ne "REMOTE_ONLY") {
-                    scp_put($trans_files[$i],"$check_files[$i]");
-                }
-                else {
-                    &my_die("There is a file that should be there, but the file isn't there, and I don't have a copy of that file here.  The file is: $check_files[$i]",
-                        $remote_id,$PS_EXIT_DATA_ERROR); # This seems a bit harsh.
-                }
-            }
-        }
-    }
-
+    my (undef,$remote_transfer)= uri_convert($uri_transfer);
+#    my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_validate_files.pl --check_list $remote_check --transfer_list $remote_transfer");
+    my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer");
+    # We no longer need to parse this output, as it retrieves files itself.
+    foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) {
+        print "$l\n";
+    }
 
 # Run real command
@@ -147 +115 @@
     my $command = "$remotetool -updaterun -remote_id $remote_id ";
     $command .= " -set_state run ";
-    $command .= " -set_job_id $job_id ";
+    $command .= " -job_id $job_id ";
     $command .= " -dbname $dbname " if defined $dbname;
     
     system($command);
     
-}
+} # End of exec phase
 
 # Check that we have a valid job_id, either from the command line, or from the msub command.
@@ -185 +153 @@
     print "Stopped Polling code.  $poll_count of $poll_max $poll_response\n";
 }
+if ($poll_response != 1) {
+    &my_die("Job hasn't completed.  Ending polling.", $remote_id, 0, "run", $job_id);
+}
 # Retrieve validation summary
 my $fault = 0;
 
-exit(1);
-# Read this as a metadata object
-my $ssh_exec_stdout = ssh_exec_command("sc_validate_processing.pl ${path_base}.out");
-my $retrieveData = $mdcParser->parse(join "", @$ssh_exec_stdout);
-my $retrieveData2= parse_md_list($retrieveData);
-
-foreach my $retrieveEntry (@$retrieveData2) {
-    my $process_fault = $retrieveEntry->{fault};
-    my $dbcommand = $retrieveEntry->{dbcommand};
-    my $filelist  = $retrieveEntry->{filelist};
-
-    scp_get("${filelist}",${filelist});
-    open(FL,${filelist});
-    while(<FL>) {
-        chomp;
-        my $f = $_;
-        scp_get("${f}",$f);
-    }
-    close(FL);
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $dbcommand, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("The database command returned did not execute correctly. $dbcommand $process_fault",
-                $remote_id,$error_code); 
-    }
-
-    if ($process_fault != 0) {
-        $fault = 1;
-    }
-}
-
-if (($policy ne 'ignore')&&($fault != 0)) {
-    &my_die("This job exited without all components finishing successfully.  Halting.",
-            $remote_id,$fault,"halt");
-}
-else {
-    print "This job exited without all components finishing successfully. $fault\n" if $fault;
-    &my_die("This job finished.  Setting remote job to full.",
-            $remote_id,0,"full");
-}
-
-# Promote to finished.
-&my_die(); # Not really, but I don't have the tool interface finished, so these are all just placeholders.
-# Quick review:
-# new -> pending -> run -> full
-# auth
+# Initialize the remote side to sync things
+
+# Once sync completes, execute all the database commands
+# Grab the return list
+my $uri_return  = $path_base . ".return";
+my ($ipp_return,$remote_return) = uri_convert($uri_return);
+scp_get($remote_return,$ipp_return);
+# Feed teh return list into the 
+my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return");
+print "$return_push_output\n";
+
+
+open(RETURN,   "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id,$PS_EXIT_SYS_ERROR);
+while(<RETURN>) {
+    chomp;
+    if ($_ =~ /dbinfo/) {
+        my $file = $_;
+#       $file =~ s/${remote_root}.tmp//; THis shouldn't be needed, as the transfer tool munges the files on that side.
+        open(DBF,"$file") || warn "Missing file $file\n";
+        my $cmd = <DBF>;
+        chomp($cmd);
+        $cmd =~ s/-/ -/g; # This is just a safety check for missing space.
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $cmd, verbose => $verbose);
+        unless ($success) {
+            # This shouldn't fail, but we also can't suddenly abort if something does fail.  Maybe retry if this looks to be a continuing issue?
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            warn("The command that shouldn't fail has failed: $cmd $remote_id $error_code");##,$remote_id,$error_code);
+        }
+        close(DBF);
+    }
+}
+close(RETURN);
+
+# We're done, so set the state and exit.
+&my_die("Finished",$remote_id,0,"full");
+
+
 
 # END PROGRAM
@@ -294 +255 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die(); 
+        &my_die("Failed to execute command: >>$cmd<<"); 
     }
     return ($stdout_buf);
@@ -368 +329 @@
     my $exit_code = shift;
     my $exit_state = shift;
-
+    my $jobid = shift;
     $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
 
@@ -374 +335 @@
     
     if (defined $id and not $no_update) {
-        my $command = "$remotetool -updaterun -stage_id $id";
+        my $command = "$remotetool -updaterun -remote_id $id";
         $command .= " -fault $exit_code " if defined $exit_code;
+        $command .= " -job_id $job_id " if defined $jobid;
         $command .= " -set_state $exit_state " if defined $exit_state;
         $command .= " -dbname $dbname " if defined $dbname;
@@ -384 +346 @@
     exit($exit_code);
 }
+
+# I don't think I need this anymore, but I don't want to burn it until I"m sure.
+
+# exit(1);
+# # Read this as a metadata object
+# my $ssh_exec_stdout = ssh_exec_command("sc_validate_processing.pl ${path_base}.out");
+# my $retrieveData = $mdcParser->parse(join "", @$ssh_exec_stdout);
+# my $retrieveData2= parse_md_list($retrieveData);
+
+# foreach my $retrieveEntry (@$retrieveData2) {
+#     my $process_fault = $retrieveEntry->{fault};
+#     my $dbcommand = $retrieveEntry->{dbcommand};
+#     my $filelist  = $retrieveEntry->{filelist};
+
+#     scp_get("${filelist}",${filelist});
+#     open(FL,${filelist});
+#     while(<FL>) {
+#       chomp;
+#       my $f = $_;
+#       scp_get("${f}",$f);
+#     }
+#     close(FL);
+
+#     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#       run(command => $dbcommand, verbose => $verbose);
+#     unless ($success) {
+#       $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+#       &my_die("The database command returned did not execute correctly. $dbcommand $process_fault",
+#               $remote_id,$error_code); 
+#     }
+
+#     if ($process_fault != 0) {
+#       $fault = 1;
+#     }
+# }
+
+# if (($policy ne 'ignore')&&($fault != 0)) {
+#     &my_die("This job exited without all components finishing successfully.  Halting.",
+#           $remote_id,$fault,"halt");
+# }
+# else {
+#     print "This job exited without all components finishing successfully. $fault\n" if $fault;
+#     &my_die("This job finished.  Setting remote job to full.",
+#           $remote_id,0,"full");
+# }
+
+# Promote to finished.
+#&my_die(); # Not really, but I don't have the tool interface finished, so these are all just placeholders.
+# Quick review:
+# new -> pending -> run -> full
+# auth