Changeset 37403 for branches/eam_branches/ps2-tc3-20130727/ippScripts/scripts/sc_prepare_chip.pl

Timestamp:

Sep 19, 2014, 4:05:27 PM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ps2-tc3-20130727

Files:

• . (modified) (1 prop)
• ippScripts/scripts/sc_prepare_chip.pl (modified) (16 diffs)

branches/eam_branches/ps2-tc3-20130727/ippScripts/scripts/sc_prepare_chip.pl

@@ -18 +18 @@
 
 # Hard coded values
-my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
+# my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
+my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+$remote_root = '/scratch3/watersc1/';
+my $remote_raw    = "${remote_root}/raw/";  # Directory to find raw data in.
 my $threads       = 2;                      # How many threads are we going to use?
 my $job_cost      = 150 / 60 / 60;          # Estimate of how long a job runs, in hours.
@@ -126 +129 @@
         $detselect_command2   .= " -det_id $det_id ";
         $detselect_command2   .= " -iteration $det_iter ";
+        $detselect_command2   .= " -dbname $dbname " if defined($dbname);
         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $detselect_command2, verbose => 0);
@@ -150 +154 @@
 #print Dumper(%detrends);
 
+my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN");
 
 #
@@ -157 +162 @@
 my $uri_check   = $path_base . ".check";
 my $uri_config  = $path_base . ".config";
+my $uri_generate= $path_base . ".generate";
+my $uri_return  = $path_base . ".return";
 
 my $disk_command = $ipprc->file_resolve($uri_command,1);
@@ -162 +169 @@
 my $disk_check   = $ipprc->file_resolve($uri_check,1);
 my $disk_config  = $ipprc->file_resolve($uri_config,1);
+my $disk_generate= $ipprc->file_resolve($uri_generate,1);
+my $disk_return  = $ipprc->file_resolve($uri_return,1);
 
 my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it.
@@ -168 +177 @@
 open(CHECK,    ">$disk_check")  || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR);
 open(CONFIG,   ">$disk_config")  || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR);
+open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR);
+open(RETURN,   ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR);
+
 
 #
@@ -185 +197 @@
     
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
+        run(command => $command, verbose => 0);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -254 +266 @@
     
     my $chipData = $mdcParser->parse(join "", @$stdout_buf) or 
-        &my_die("Unable to parse chiptool -pendingimfile information.",
-                $chip_id,$error_code);
+        next;
+#       &my_die("Unable to parse chiptool -pendingimfile information.",
+#               $chip_id,$error_code);
     my $chipData2= parse_md_list($chipData);
 
@@ -270 +283 @@
         
         # Process the image and burntool table
-        my (undef,$remote_uri) = uri_to_outputs($uri);
+        my ($ipp_uri,$remote_uri) = uri_to_outputs_raw($uri);
         my $btt = $uri;
         $btt =~ s/fits$/burn.tbl/;
-        my (undef,$remote_btt) = uri_to_outputs($btt);
+        my ($ipp_btt,$remote_btt) = uri_to_outputs_raw($btt);
         
         # Initialize the ppI command
@@ -287 +300 @@
             my $det_code = $det_types{$det};
             my $duri = $detrends_to_use{$det}->{$class_id};
-            my (undef, $remote_det_uri) = uri_to_outputs($duri);
+            my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri);
             $ppImage_command .= " $det_code $remote_det_uri ";
         }
@@ -305 +318 @@
         $ppImage_command .= " -recipe PSPHOT $recipe_psphot ";
         $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats ";
-        $ppImage_command .= " -threads $threads ";
+        $ppImage_command .= " -threads 4 ";
         $ppImage_command .= " -image_id $chip_imfile_id ";
         $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace ";
         $ppImage_command .= " -log ${remote_outroot}.${class_id}.log ";
 
-        push @main_commands, $ppImage_command;
+#       push @main_commands, $ppImage_command;
 
         # Calculate pre and post commands
         my $remote_outroot_dir = dirname($remote_outroot);
-        push @pre_commands, "mkdir -p $remote_outroot_dir";
+        my $pre_command =  "mkdir -p $remote_outroot_dir";
         
         my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id ";
@@ -321 +334 @@
         $post_commandA   .= " -dbname $dbname " if defined $dbname;
 
-        my $post_commandB = "echo -n $post_commandA > ${remote_outroot}.${class_id}.dbinfo && ";
+        my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo  ";
         my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo";
         
-        push @post_commands, "$post_commandB $post_commandC";
+#       push @post_commands, "$post_commandB $post_commandC";
 
         $job_index++;
+
+        print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}";
+
+
+        # Determine which output files need to be returned.
+        foreach my $component (@return_component_list) {
+#           uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) );
+            my $filename = $ipprc->filename($component, $ipp_outroot, $class_id);
+            my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename );
+            my $remote_outroot_dir = dirname($ipp_disk);
+            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
+#           print "  $filename $ipp_disk $remote_disk\n";
+#           die;
+        }
+
+        print CONFIG "\n";
     }
 }
 
 # Actually put the commands into the output file.
-my %unique_cmds = ();
-foreach my $cmd (@pre_commands) {
-    unless(exists($unique_cmds{$cmd})) {
-        print CONFIG $cmd . "\n";
-        $unique_cmds{$cmd} = 1;
-    }
-}
-foreach my $cmd (@main_commands) {
-    unless(exists($unique_cmds{$cmd})) {
-        print CONFIG $cmd . "\n";
-        $unique_cmds{$cmd} = 1;
-    }
-}
-foreach my $cmd (@post_commands) {
-    unless(exists($unique_cmds{$cmd})) {
-        print CONFIG $cmd . "\n";
-        $unique_cmds{$cmd} = 1;
-    }
-}
+# my %unique_cmds = ();
+# foreach my $cmd (@pre_commands) {
+#     unless(exists($unique_cmds{$cmd})) {
+#       print CONFIG $cmd . "\n";
+#       $unique_cmds{$cmd} = 1;
+#     }
+# }
+# foreach my $cmd (@main_commands) {
+#     unless(exists($unique_cmds{$cmd})) {
+#       print CONFIG $cmd . "\n";
+#       $unique_cmds{$cmd} = 1;
+#     }
+# }
+# foreach my $cmd (@post_commands) {
+#     unless(exists($unique_cmds{$cmd})) {
+#       print CONFIG $cmd . "\n";
+#       $unique_cmds{$cmd} = 1;
+#     }
+# }
 
 close(CONFIG);
 close(TRANSFER);
 close(CHECK);
+close(RETURN);
+close(GENERATE);
 
 #
@@ -376 +407 @@
     $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1;
 }
+$time_req += 1; # Safety addition.
 
 open(COMMAND,  ">$disk_command") || &my_die("Couldn't open file? $disk_command",$chip_id,$PS_EXIT_SYS_ERROR);
@@ -425 +457 @@
 }
 
+sub uri_convert_and_create {
+    my $neb_uri = shift;
+    my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1);
+    my $remote_disk = $ipp_disk;
+    
+    unless(defined($ipp_disk)) {
+        my_die();
+    }
+
+    $remote_disk =~ s%^.*/%%;   # Remove nebulous path
+    $remote_disk =~ s%^\d+\.%%; # Remove ins_id
+    $remote_disk =~ s%:%/%g;    # Replace colons with directories
+    $remote_disk = "${remote_root}/${remote_disk}";
+    return($ipp_disk,$remote_disk);
+}
+
 sub uri_to_outputs {
     my $neb_uri = shift;
@@ -433 +481 @@
         print TRANSFER "$ipp_disk\n";
         print CHECK    "$remote_disk\n";
+    }
+    return($ipp_disk,$remote_disk);
+}
+
+sub uri_to_outputs_raw {
+    my $neb_uri = shift;
+    my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
+    $remote_disk = $remote_raw . $ipp_disk;
+    unless (exists($file_filter{$neb_uri})) {
+        $file_filter{$neb_uri} = 1;
+        print TRANSFER "$ipp_disk\n";
+        print CHECK    "$remote_disk\n";
+    }
+    return($ipp_disk,$remote_disk);
+}
+
+sub uri_to_outputs_for_return {
+    my $neb_uri = shift;
+    my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri );
+    unless (exists($file_filter{$neb_uri})) {
+        $file_filter{$neb_uri} = 1;
+        print RETURN "$ipp_disk\n";
+        print GENERATE "$remote_disk\n";
     }
     return($ipp_disk,$remote_disk);
@@ -467 +538 @@
     
     if (defined $id and not $no_update) {
-        my $command = "remotetool -updaterun -stage_id $id";
+        my $command = "remotetool -updaterun -remote_id $id";
         $command .= " -fault $exit_code " if defined $exit_code;
         $command .= " -set_state $exit_state " if defined $exit_state;