Changeset 37332

Timestamp:

Aug 29, 2014, 3:18:55 PM (12 years ago)

Author:

watersc1

Message:

stack_skycell.pl: We should never use the uri value, the path_base should be used instead.
sc_chip: use the ...tmp directory tree instead of the ...raw directory tree.
sc_transfer: same, remove -t option
sc_prep_run: implement job subscription values in the stask call, to prevent stacks overloading a node.
sc_warp: write a valid uri entry.
sc_stack: write a component map file that contains the locations of both possible locations for the warp.
sc_stack_mdc: read that map and use it to construct a useful mdc file with the most useful files.

sc_exec,sc_camera,queuestaticsky: mergeinfo things that svn insists I need to commit.

Location:

tags/ipp-pv3-20140717/ippScripts/scripts

Files:

• . (modified) (1 prop)
• queuestaticsky.pl (modified) (1 prop)
• sc_mk_stack_mdc.pl (modified) (1 diff)
• sc_prepare_camera.pl (modified) (1 prop)
• sc_prepare_chip.pl (modified) (1 diff)
• sc_prepare_run.pl (modified) (3 diffs)
• sc_prepare_stack.pl (modified) (2 diffs, 1 prop)
• sc_prepare_warp.pl (modified) (1 diff, 1 prop)
• sc_remote_exec.pl (modified) (1 prop)
• sc_transfer_tool.pl (modified) (4 diffs, 1 prop)
• sc_validate_files.pl (deleted)
• sc_validate_processing.pl (deleted)
• stack_skycell.pl (modified) (1 diff)

tags/ipp-pv3-20140717/ippScripts/scripts/sc_mk_stack_mdc.pl

@@ -1 +1 @@
 #! /usr/bin/env perl                                                                                                                                                       
-
 use Carp;
 use warnings;
 use strict;
 
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+
+my $remote_root = '/scratch3/watersc1/';
+my 
+
+my ($compmap_file);
+GetOptions(
+    'compmap=s'   => \$compmap_file,
+    ) or pod2usage( 2 );
+
+pod2usage( -msg => "Required options: --compmap", -exitval => 3) unless
+    defined($compmap_file);
+
+my %component_map = ();
+open(CM,$compmap_file) || die "Cannot find requested compmap file";
+while(<CM>) {
+    chomp;
+    my ($symbolic_name,$disk_name) = split /\s+/;
+    $component_map{$symbolic_name} = $disk_name;
+}
+close(CM);
+
 my $counter = 0;
 
 foreach my $path_base (@ARGV) {
-    print "INPUT${counter}  METADATA\n";
-    print "   IMAGE           STR     ${path_base}.fits\n";
-    print "   MASK            STR     ${path_base}.mask.fits\n";
-    print "   VARIANCE        STR     ${path_base}.wt.fits\n";
-    print "   PSF             STR     ${path_base}.psf\n";
-    print "   SOURCES         STR     ${path_base}.cmf\n";
-    print "   BKGMODEL        STR     ${path_base}.mdl.fits\n";
-    print "END\n\n";
+    my $mdc_content = "";
+    my $file = "";
+    my $error = 0;
+    #header
+    $mdc_content .= "INPUT${counter}  METADATA\n";
+
+    # check each file against its possible locations, incrementing the error counter if not found
+    $file = do_checks("${path_base}.fits",$error);
+    $mdc_content .=  "   IMAGE           STR     $file\n";
+
+    $file = do_checks("${path_base}.mask.fits",$error);
+    $mdc_content .=  "   MASK            STR     $file\n";
+
+    $file = do_checks("${path_base}.wt.fits",$error);
+    $mdc_content .=  "   VARIANCE        STR     $file\n";
+    
+    $file = do_checks("${path_base}.psf",$error);
+    $mdc_content .=  "   PSF             STR     $file\n";
+    
+    $file = do_checks("${path_base}.cmf",$error);
+    $mdc_content .=  "   SOURCES         STR     $file\n";
+
+    # This is static, and isn't used by ppStack, but I believe needs to exist in the mdc. 2014-08-28 czw
+    $mdc_content .=  "   BKGMODEL        STR     ${path_base}.mdl.fits\n";
+    $mdc_content .=  "END\n\n";
+
+    # If we had no errors, print it and move on.
+    if ($error != 0) {
+        print $mdc_content;
+    }
+
     $counter++;
 }
+
+sub do_checks {
+    my $file = shift(@ARGV);
+    if (! ((-e $file)&&(-s $file))) { # This file does not exist or is zero size.
+        $file = "${remote_root}/tmp/$component_map{$file}";
+        if (! ((-e $file)&&(-s $file))) { # The other expected file did not exist.
+            $file = '';
+            die "Missing file!";
+        }
+    }
+    return($file);
+}
+    

tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl

@@ -22 +22 @@
 my $remote_root     = '/scratch3/watersc1/';  # Far side destination base location
 my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
-my $remote_raw      = "${remote_root}/raw/";  # Directory to find raw data in.
+my $remote_raw      = "${remote_root}/tmp/";  # Directory to find raw data in.
 my $threads_req     = 4;
 

tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl

@@ -24 +24 @@
 # my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
 my $remote_root = '/scratch3/watersc1/';
-my $remote_raw    = "${remote_root}/raw/";  # Directory to find raw data in.
+my $remote_raw    = "${remote_root}/tmp/";  # Directory to find raw data in.
 
 my $threads       = 2;                      # How many threads are we going to use?
@@ -36 +36 @@
 $job_cost{"warp"}   = 110 / 60 / 60;
 $job_cost{"stack"}  = 1500 / 60 / 60;
+
+# This object holds how "expensive" a given job is in terms of cores on a node.  
+# Passed to stask to limit how many jobs run simultaneously
+my %job_subscription = ();  
+$job_subscription{"camera"} = 1;
+$job_subscription{"chip"}   = 1;
+$job_subscription{"warp"}   = 1;
+$job_subscription{"stack"}  = 3;
 
 my $proc_per_node = 24;                     # processors per node
@@ -236 +244 @@
 print COMMAND "date\n";
 print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n";
-print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n";
+print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} '  . $job_subscription{$stage} . "\n";
 print COMMAND "date\n";
 close(COMMAND);

tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_stack.pl

@@ -136 +136 @@
 # STEP 3: Iterate over the sub-components
 {
+    # Generate a file mapping the symbolic and "disk" representations of each needed file.
+    my $uri_compmap   = ${remote_outroot}.compmap;
+    my $component_map = $ipprc->file_resolve($uri_compmap,1);    
+    my ($ipp_compmap, $remote_compmap) = &uri_to_outputs($component_map);
+    open(COMPMAP,">$ipp_compmap") || &my_die("Couldn't open file? $ipp_compmap",  $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state);
+    
+    # Loop over all needed components for this stack.
     my $warp_path_base_string = "";
     foreach my $warpEntry ( @{ $warpData } ) {
@@ -145 +152 @@
 
         # Append file names to transfer lists so we can recover if warps are deleted.
-        &uri_to_outputs($warp_path_base . ".fits");
-        &uri_to_outputs($warp_path_base . ".wt.fits");
-        &uri_to_outputs($warp_path_base . ".mask.fits");
-        &uri_to_outputs($warp_path_base . ".cmf");
-#       &uri_to_outputs($warp_path_base . ".psf");
-    }
-    my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl $warp_path_base_string > ${remote_outroot}.in.mdc";
+        my $remote_file;
+        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".fits");
+        print COMPMAP "${warp_path_base}.fits $remote_file\n";
+        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".wt.fits");
+        print COMPMAP "${warp_path_base}.wt.fits $remote_file\n";
+        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".mask.fits");
+        print COMPMAP "${warp_path_base}.maskfits $remote_file\n";
+        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".cmf");
+        print COMPMAP "${warp_path_base}.cmf $remote_file\n";
+        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".psf");
+        print COMPMAP "${warp_path_base}.psf $remote_file\n";
+    }
+
+    close(COMPMAP);
+
+    my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl --compmap $remote_compmap $warp_path_base_string > ${remote_outroot}.in.mdc";
     my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use
     my $ppstack_command  = " ppStack -input ${remote_outroot}.in.mdc ";

tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl

@@ -184 +184 @@
 
         my $post_cmd_echo = " echo -n \"warptool  -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id ";
-        $post_cmd_echo   .= " -path_base $remote_outroot -uri UNKNOWN ";
         $post_cmd_echo   .=  " -dbname $dbname " if defined $dbname;
+        $post_cmd_echo   .= " -uri ${ipp_outroot}.fits ";
         $post_cmd_echo   .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo ";
 

tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl

@@ -16 +16 @@
    $remote_root = '/scratch3/watersc1/';
 
-my $local_raw = "${remote_root}/raw/";
+#my $local_raw = "${remote_root}/tmp/";
 my $local_tmp = "${remote_root}/tmp/";
 my $threads = 11;
@@ -79 +79 @@
 # split in input file list
 unless(-d $local_tmp) { system("mkdir -p $local_tmp"); }
-unless(-d $local_raw) { system("mkdir -p $local_raw"); }
+#unless(-d $local_raw) { system("mkdir -p $local_raw"); }
 
 # we have two modes: fetch (-> LANL) and return (-> IPP)
@@ -212 +212 @@
     my $option = $server_options{$destination_host};
 
-    my $command = "rsync -Lt --size-only --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
+    my $command = "rsync -L --size-only --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
     print STDERR "$command\n";
     
@@ -248 +248 @@
     my $option = $server_options{$destination_host};
 
-    my $command = "rsync --size-only -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${destination_host}:/ ${local_raw}";
+    my $command = "rsync --size-only -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${destination_host}:/ ${local_tmp}";
     print STDERR "$command\n";
 

tags/ipp-pv3-20140717/ippScripts/scripts/stack_skycell.pl

@@ -225 +225 @@
     $num++;
 
-    my $image = $file->{uri};   # Image name
+#    my $image = $file->{uri};   # Image name
+    my $image = $ipprc->filename("PSWARP.OUTPUT", $file->{path_base} ); # Image name
     my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name
     my $weight = $ipprc->filename( "PSWARP.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name