Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_mk_stack_mdc.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_mk_stack_mdc.pl (revision 37331) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_mk_stack_mdc.pl (revision 37332) @@ -1,19 +1,78 @@ #! /usr/bin/env perl - use Carp; use warnings; use strict; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +my $remote_root = '/scratch3/watersc1/'; +my + +my ($compmap_file); +GetOptions( + 'compmap=s' => \$compmap_file, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Required options: --compmap", -exitval => 3) unless + defined($compmap_file); + +my %component_map = (); +open(CM,$compmap_file) || die "Cannot find requested compmap file"; +while() { + chomp; + my ($symbolic_name,$disk_name) = split /\s+/; + $component_map{$symbolic_name} = $disk_name; +} +close(CM); + my $counter = 0; foreach my $path_base (@ARGV) { - print "INPUT${counter} METADATA\n"; - print " IMAGE STR ${path_base}.fits\n"; - print " MASK STR ${path_base}.mask.fits\n"; - print " VARIANCE STR ${path_base}.wt.fits\n"; - print " PSF STR ${path_base}.psf\n"; - print " SOURCES STR ${path_base}.cmf\n"; - print " BKGMODEL STR ${path_base}.mdl.fits\n"; - print "END\n\n"; + my $mdc_content = ""; + my $file = ""; + my $error = 0; + #header + $mdc_content .= "INPUT${counter} METADATA\n"; + + # check each file against its possible locations, incrementing the error counter if not found + $file = do_checks("${path_base}.fits",$error); + $mdc_content .= " IMAGE STR $file\n"; + + $file = do_checks("${path_base}.mask.fits",$error); + $mdc_content .= " MASK STR $file\n"; + + $file = do_checks("${path_base}.wt.fits",$error); + $mdc_content .= " VARIANCE STR $file\n"; + + $file = do_checks("${path_base}.psf",$error); + $mdc_content .= " PSF STR $file\n"; + + $file = do_checks("${path_base}.cmf",$error); + $mdc_content .= " SOURCES STR $file\n"; + + # This is static, and isn't used by ppStack, but I believe needs to exist in the mdc. 2014-08-28 czw + $mdc_content .= " BKGMODEL STR ${path_base}.mdl.fits\n"; + $mdc_content .= "END\n\n"; + + # If we had no errors, print it and move on. + if ($error != 0) { + print $mdc_content; + } + $counter++; } + +sub do_checks { + my $file = shift(@ARGV); + if (! ((-e $file)&&(-s $file))) { # This file does not exist or is zero size. + $file = "${remote_root}/tmp/$component_map{$file}"; + if (! ((-e $file)&&(-s $file))) { # The other expected file did not exist. + $file = ''; + die "Missing file!"; + } + } + return($file); +} + Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl (revision 37331) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl (revision 37332) @@ -22,5 +22,5 @@ my $remote_root = '/scratch3/watersc1/'; # Far side destination base location my $remote_hostname = "LANL/Mustang"; # Name of the remote node. -my $remote_raw = "${remote_root}/raw/"; # Directory to find raw data in. +my $remote_raw = "${remote_root}/tmp/"; # Directory to find raw data in. my $threads_req = 4; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl (revision 37331) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl (revision 37332) @@ -24,5 +24,5 @@ # my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location my $remote_root = '/scratch3/watersc1/'; -my $remote_raw = "${remote_root}/raw/"; # Directory to find raw data in. +my $remote_raw = "${remote_root}/tmp/"; # Directory to find raw data in. my $threads = 2; # How many threads are we going to use? @@ -36,4 +36,12 @@ $job_cost{"warp"} = 110 / 60 / 60; $job_cost{"stack"} = 1500 / 60 / 60; + +# This object holds how "expensive" a given job is in terms of cores on a node. +# Passed to stask to limit how many jobs run simultaneously +my %job_subscription = (); +$job_subscription{"camera"} = 1; +$job_subscription{"chip"} = 1; +$job_subscription{"warp"} = 1; +$job_subscription{"stack"} = 3; my $proc_per_node = 24; # processors per node @@ -236,5 +244,5 @@ print COMMAND "date\n"; print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; -print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; +print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} ' . $job_subscription{$stage} . "\n"; print COMMAND "date\n"; close(COMMAND); Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_stack.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_stack.pl (revision 37331) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_stack.pl (revision 37332) @@ -136,4 +136,11 @@ # STEP 3: Iterate over the sub-components { + # Generate a file mapping the symbolic and "disk" representations of each needed file. + my $uri_compmap = ${remote_outroot}.compmap; + my $component_map = $ipprc->file_resolve($uri_compmap,1); + my ($ipp_compmap, $remote_compmap) = &uri_to_outputs($component_map); + open(COMPMAP,">$ipp_compmap") || &my_die("Couldn't open file? $ipp_compmap", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); + + # Loop over all needed components for this stack. my $warp_path_base_string = ""; foreach my $warpEntry ( @{ $warpData } ) { @@ -145,11 +152,20 @@ # Append file names to transfer lists so we can recover if warps are deleted. - &uri_to_outputs($warp_path_base . ".fits"); - &uri_to_outputs($warp_path_base . ".wt.fits"); - &uri_to_outputs($warp_path_base . ".mask.fits"); - &uri_to_outputs($warp_path_base . ".cmf"); -# &uri_to_outputs($warp_path_base . ".psf"); - } - my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl $warp_path_base_string > ${remote_outroot}.in.mdc"; + my $remote_file; + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".fits"); + print COMPMAP "${warp_path_base}.fits $remote_file\n"; + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".wt.fits"); + print COMPMAP "${warp_path_base}.wt.fits $remote_file\n"; + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".mask.fits"); + print COMPMAP "${warp_path_base}.maskfits $remote_file\n"; + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".cmf"); + print COMPMAP "${warp_path_base}.cmf $remote_file\n"; + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".psf"); + print COMPMAP "${warp_path_base}.psf $remote_file\n"; + } + + close(COMPMAP); + + my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl --compmap $remote_compmap $warp_path_base_string > ${remote_outroot}.in.mdc"; my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use my $ppstack_command = " ppStack -input ${remote_outroot}.in.mdc "; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl (revision 37331) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl (revision 37332) @@ -184,6 +184,6 @@ my $post_cmd_echo = " echo -n \"warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id "; - $post_cmd_echo .= " -path_base $remote_outroot -uri UNKNOWN "; $post_cmd_echo .= " -dbname $dbname " if defined $dbname; + $post_cmd_echo .= " -uri ${ipp_outroot}.fits "; $post_cmd_echo .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl (revision 37331) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl (revision 37332) @@ -16,5 +16,5 @@ $remote_root = '/scratch3/watersc1/'; -my $local_raw = "${remote_root}/raw/"; +#my $local_raw = "${remote_root}/tmp/"; my $local_tmp = "${remote_root}/tmp/"; my $threads = 11; @@ -79,5 +79,5 @@ # split in input file list unless(-d $local_tmp) { system("mkdir -p $local_tmp"); } -unless(-d $local_raw) { system("mkdir -p $local_raw"); } +#unless(-d $local_raw) { system("mkdir -p $local_raw"); } # we have two modes: fetch (-> LANL) and return (-> IPP) @@ -212,5 +212,5 @@ my $option = $server_options{$destination_host}; - my $command = "rsync -Lt --size-only --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/"; + my $command = "rsync -L --size-only --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/"; print STDERR "$command\n"; @@ -248,5 +248,5 @@ my $option = $server_options{$destination_host}; - my $command = "rsync --size-only -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${destination_host}:/ ${local_raw}"; + my $command = "rsync --size-only -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${destination_host}:/ ${local_tmp}"; print STDERR "$command\n"; Index: gs/ipp-pv3-20140717/ippScripts/scripts/sc_validate_files.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_validate_files.pl (revision 37331) +++ (revision ) @@ -1,68 +1,0 @@ -#! /usr/bin/env perl - -use Carp; -use warnings; -use strict; - -use File::Basename; -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - - -# Hard coded values -my $remote_root = '/scratch3/watersc1/'; -my $host_iter = 0; -my @ipp_hosts = ('166.122.172.41','166.122.172.42','166.122.172.43','166.122.172.44', - '166.122.172.45','166.122.172.46','166.122.172.47','166.122.172.48', - '166.122.172.49','166.122.172.50','166.122.172.51','166.122.172.52','166.122.172.53'); -my @remote_hosts = ('mu-fe1.lanl.gov','mu-fe2.lanl.gov','mu-fe3.lanl.gov','mu-fe4.lanl.gov'); -my $check_list; -my $transfer_list; -GetOptions( - 'check_list=s' => \$check_list, - 'transfer_list=s' => \$transfer_list - ) or pod2usage( 2 ); - -my @check_files = (); -my @transfer_files = (); - -open(LIST,$check_list); -while() { - chomp; - push @check_files, $_; -} -close(LIST); -open(LIST2,$transfer_list); -while() { - chomp; - push @transfer_files, $_; -} -close(LIST2); - -unless ($#check_files == $#transfer_files) { - die "The two lists are of unequal length."; -} - -for (my $i = 0; $i <= $#check_files; $i++) { - my $file = $check_files[$i]; - unless (-e $file) { - print "$file\n"; - my $fetch_file = $transfer_files[$i]; - my $directory = dirname($file); - unless (-d $directory) { - system("mkdir -p $directory"); - } - - scp_pull($fetch_file,$file); - } -} - -sub scp_pull { - my $remote_file = shift; - my $local_file = shift; - - my $host = $ipp_hosts[$host_iter]; - $host_iter++; - if ($host_iter > $#ipp_hosts) { $host_iter = 0; } - system("scp ${host}:${remote_file} ${local_file}"); -} Index: gs/ipp-pv3-20140717/ippScripts/scripts/sc_validate_processing.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_validate_processing.pl (revision 37331) +++ (revision ) @@ -1,79 +1,0 @@ -#! /usr/bin/env perl - -use Carp; -use warnings; -use strict; - -use IPC::Cmd 0.36 qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::List qw( parse_md_list ); -use PS::IPP::Config 1.01 qw( :standard ); - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - - -# Hard coded values -my $remote_root = '/scratch3/watersc1/'; - -my $command_output -GetOptions( - 'command_output=s' => \$command_output; - ) or pod2usage( 2 ); - -open(LIST,$command_output); -while() { - chomp; - - if ($_ =~ /ppImage/) { - my $base = (split /\s+/)[15]; # This is super fragile. - my $stats= $base . "." . $class_id . ".stats"; # no good idea how to get that. - my $command = ""; - if (-e $stats) { - my $stat_cmd = "ppStatsFromMetadata $stats - CHIP_IMFILE"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppStatsFromMetadata: $error_code", $exp_id, $chip_id, $class_id, $error_code); - } - foreach my $line (@$stdout_buf) { - $cmdflags .= " $line"; - } - chomp $cmdflags; - - # Need to get all these parameters. - $command = "$chiptool -addprocessedimfile"; - $command .= " -exp_id $exp_id"; - $command .= " -chip_id $chip_id"; - $command .= " -class_id $class_id"; - $command .= " -uri $outputImage"; - $command .= " -path_base $outroot"; - $command .= " -magicked $magicked" if $magicked; - $command .= " -hostname $host" if defined $host; - $command .= " -dbname $dbname" if defined $dbname; - $command .= " $cmdflags"; - $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); - - } - else { # This job failed permanently - $command = "$chiptool -addprocessedimfile"; - $command .= " -exp_id $exp_id"; - $command .= " -uri $outputImage" if defined $outputImage; - $command .= " -path_base $outroot"; - $command .= " -hostname $host" if defined $host; - $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $command .= " -chip_id $chip_id"; - $command .= " -class_id $class_id"; - $command .= " -fault $exit_code"; - $command .= " -dbname $dbname" if defined $dbname; - } - - -# if ($_ =~ /stask: error: .*: task .*: Exited with exit code/) { -# print " "; # This needs to be smarter. -# exit(1); -# } -} -close(LIST); - Index: /tags/ipp-pv3-20140717/ippScripts/scripts/stack_skycell.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/stack_skycell.pl (revision 37331) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/stack_skycell.pl (revision 37332) @@ -225,5 +225,6 @@ $num++; - my $image = $file->{uri}; # Image name +# my $image = $file->{uri}; # Image name + my $image = $ipprc->filename("PSWARP.OUTPUT", $file->{path_base} ); # Image name my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name my $weight = $ipprc->filename( "PSWARP.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name