Index: /tags/ipp-20150329/ippScripts/scripts/sc_check_diff.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_check_diff.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_check_diff.pl (revision 38170) @@ -8,7 +8,7 @@ -my $remote_root = '/scratch3/watersc1/'; +#my $remote_root = '/scratch3/watersc1/'; -my ($diff_id,$skycell_id,$dbname,$out_path_base); +my ($diff_id,$skycell_id,$dbname,$out_path_base, $remote_root); GetOptions( 'diff_id=s' => \$diff_id, @@ -16,10 +16,12 @@ 'dbname=s' => \$dbname, 'out_path_base=s' => \$out_path_base, + 'remote_root=s' => \$remote_root, ) or pod2usage( 2 ); -pod2usage( -msg => "Required options: --out_path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --out_path_base --remote_root", -exitval => 3) unless defined($out_path_base) && defined($diff_id) && defined($skycell_id) && + defined($remote_root) && defined($dbname); Index: /tags/ipp-20150329/ippScripts/scripts/sc_mk_stack_mdc.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_mk_stack_mdc.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_mk_stack_mdc.pl (revision 38170) @@ -8,12 +8,14 @@ -my $remote_root = '/scratch3/watersc1/'; +#my $remote_root = '/scratch3/watersc1/'; -my ($compmap_file); +my ($compmap_file,$remote_root); GetOptions( 'compmap=s' => \$compmap_file, + 'remote_root=s' => \$remote_root, ) or pod2usage( 2 ); pod2usage( -msg => "Required options: --compmap", -exitval => 3) unless + defined($remote_root) and defined($compmap_file); Index: /tags/ipp-20150329/ippScripts/scripts/sc_mk_staticsky_mdc.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_mk_staticsky_mdc.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_mk_staticsky_mdc.pl (revision 38170) @@ -8,12 +8,14 @@ -my $remote_root = '/scratch3/watersc1/'; +#my $remote_root = '/scratch3/watersc1/'; -my ($compmap_file); +my ($compmap_file, $remote_root); GetOptions( 'compmap=s' => \$compmap_file, + 'remote_root=s' => \$remote_root, ) or pod2usage( 2 ); pod2usage( -msg => "Required options: --compmap", -exitval => 3) unless + defined($remote_root) and defined($compmap_file); Index: /tags/ipp-20150329/ippScripts/scripts/sc_prepare_camera.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_prepare_camera.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_prepare_camera.pl (revision 38170) @@ -19,12 +19,4 @@ use Pod::Usage qw( pod2usage ); -# Hard coded values -# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $remote_hostname = "LANL/Mustang"; # Name of the remote node. -my $threads_req = 0; # How many threads are we going to use? - -my $fail_state = "prep_fail"; - # Look for programs we need my $missing_tools; @@ -32,4 +24,5 @@ my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); if ($missing_tools) { @@ -38,5 +31,5 @@ } -my ($remote_id,$cam_id,$camera,$dbname,$verbose,$path_base,$no_update); +my ($remote_id,$cam_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe); GetOptions( 'remote_id=s' => \$remote_id, @@ -44,4 +37,5 @@ 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, + 'recipe=s' => \$cmd_recipe, 'path_base=s' => \$path_base, 'no_update' => \$no_update, @@ -50,10 +44,67 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --cam_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id --cam_id --camera --dbname --path_base --recipe", -exitval => 3) unless defined($remote_id) and defined($cam_id) and defined($camera) and defined($path_base) and + defined($cmd_recipe) and defined($dbname); + + +# Hard coded values +# Now accessible from a recipe +my %remote_recipe = (); +{ + my $verbose = 0; + my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + + my $active_recipe = ''; + my %recipes = (); + +# print Dumper($metadata); + foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'ACTIVE') { + $active_recipe = ${ $entry }{value}; # Not actually used + } + else { + if (${ $entry }{class} eq 'metadata') { # A real recipe + my $name = ${ $entry }{name}; + foreach my $tentry (@{ ${ $entry }{value} }) { + if (${ $tentry }{class} eq 'scalar') { # A recipe value + $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value}; + } + elsif (${ $tentry }{class} eq 'metadata') { # A recipe array + foreach my $arr_entry (@{ ${ $tentry }{value} }) { + push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value}; + } + } + } + } + } + } + + unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) }; +# print Dumper(%recipes); + %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe. +# print Dumper(\%remote_recipe); +} + + +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = $remote_recipe{REMOTE_ROOT}; +my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME}; # Name of the remote node. +my $threads_req = 0; # How many threads are we going to use? + +my $fail_state = "prep_fail"; + my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $cam_id, $PS_EXIT_CONFIG_ERROR, $fail_state); Index: /tags/ipp-20150329/ippScripts/scripts/sc_prepare_chip.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_prepare_chip.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_prepare_chip.pl (revision 38170) @@ -18,13 +18,4 @@ use Pod::Usage qw( pod2usage ); -# Hard coded values -# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $remote_hostname = "LANL/Mustang"; # Name of the remote node. -my $remote_raw = "${remote_root}/tmp/"; # Directory to find raw data in. -my $threads_req = 4; - -my $fail_state = "prep_fail"; -my $hard_fail_state = "fail"; # Look for programs we need @@ -34,4 +25,5 @@ my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); my $ipp_burntool_fix = can_run('ipp_apply_burntool_fix.pl') or (warn "Can't find ipp_apply_burntool_fix.pl" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); if ($missing_tools) { @@ -41,5 +33,5 @@ # Options -my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose,$dbverbose); +my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose,$dbverbose,$cmd_recipe); GetOptions( 'remote_id=s' => \$remote_id, @@ -47,4 +39,5 @@ 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, + 'recipe=s' => \$cmd_recipe, 'path_base=s' => \$path_base, 'no_update' => \$no_update, @@ -54,10 +47,68 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --chip_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id --chip_id --camera --dbname --path_base --recipe", -exitval => 3) unless defined($remote_id) and defined($chip_id) and defined($camera) and defined($path_base) and + defined($cmd_recipe) and defined($dbname); + + +# Hard coded values +# Now accessible from a recipe +my %remote_recipe = (); +{ + my $verbose = 0; + my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + + my $active_recipe = ''; + my %recipes = (); + +# print Dumper($metadata); + foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'ACTIVE') { + $active_recipe = ${ $entry }{value}; # Not actually used + } + else { + if (${ $entry }{class} eq 'metadata') { # A real recipe + my $name = ${ $entry }{name}; + foreach my $tentry (@{ ${ $entry }{value} }) { + if (${ $tentry }{class} eq 'scalar') { # A recipe value + $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value}; + } + elsif (${ $tentry }{class} eq 'metadata') { # A recipe array + foreach my $arr_entry (@{ ${ $tentry }{value} }) { + push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value}; + } + } + } + } + } + } + + unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) }; +# print Dumper(%recipes); + %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe. +# print Dumper(\%remote_recipe); +} + + +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = $remote_recipe{REMOTE_ROOT}; +my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME}; # Name of the remote node. +my $remote_raw = "${remote_root}/tmp/"; # Directory to find raw data in. +my $threads_req = 4; + +my $fail_state = "prep_fail"; +my $hard_fail_state = "fail"; my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); Index: /tags/ipp-20150329/ippScripts/scripts/sc_prepare_diff.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_prepare_diff.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_prepare_diff.pl (revision 38170) @@ -18,16 +18,9 @@ use Pod::Usage qw( pod2usage ); -# Hard coded values -# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -my $remote_root = '/scratch3/watersc1/'; -my $remote_hostname = "LANL/Mustang"; # Name of the remote node. -my $threads_req = 4; # How many threads are we going to use? - -my $fail_state = "prep_fail"; - # Look for programs we need my $missing_tools; my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); if ($missing_tools) { @@ -36,5 +29,5 @@ } -my ($remote_id,$diff_id,$camera,$dbname,$verbose,$path_base,$no_update); +my ($remote_id,$diff_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe); GetOptions( 'remote_id=s' => \$remote_id, @@ -42,4 +35,5 @@ 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, + 'recipe=s' => \$cmd_recipe, 'path_base=s' => \$path_base, 'no_update' => \$no_update, @@ -48,10 +42,66 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id -diff_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id -diff_id --camera --dbname --path_base --recipe", -exitval => 3) unless defined($remote_id) and defined($diff_id) and defined($camera) and defined($path_base) and + defined($cmd_recipe) and defined($dbname); + + +# Hard coded values +# Now accessible from a recipe +my %remote_recipe = (); +{ + my $verbose = 0; + my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + + my $active_recipe = ''; + my %recipes = (); + +# print Dumper($metadata); + foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'ACTIVE') { + $active_recipe = ${ $entry }{value}; # Not actually used + } + else { + if (${ $entry }{class} eq 'metadata') { # A real recipe + my $name = ${ $entry }{name}; + foreach my $tentry (@{ ${ $entry }{value} }) { + if (${ $tentry }{class} eq 'scalar') { # A recipe value + $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value}; + } + elsif (${ $tentry }{class} eq 'metadata') { # A recipe array + foreach my $arr_entry (@{ ${ $tentry }{value} }) { + push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value}; + } + } + } + } + } + } + + unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) }; +# print Dumper(%recipes); + %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe. +# print Dumper(\%remote_recipe); +} + +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = $remote_recipe{REMOTE_ROOT}; +my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME}; # Name of the remote node. +my $threads_req = 4; # How many threads are we going to use? + +my $fail_state = "prep_fail"; + my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id, $diff_id, $PS_EXIT_CONFIG_ERROR, $fail_state); @@ -198,5 +248,5 @@ my $remote_templsourc = &uri_local_to_remote($ipp_templsourc); - my $pre_command = " sc_check_diff.pl --diff_id ${diff_id} --skycell_id ${skycell_id} --dbname ${dbname} --out_path_base ${remote_outroot} $remote_inimage $remote_inmask $remote_inwt $remote_insourc $remote_templimage $remote_templmask $remote_templwt $remote_templsourc "; + my $pre_command = " sc_check_diff.pl --diff_id ${diff_id} --remote_root ${remote_root} --skycell_id ${skycell_id} --dbname ${dbname} --out_path_base ${remote_outroot} $remote_inimage $remote_inmask $remote_inwt $remote_insourc $remote_templimage $remote_templmask $remote_templwt $remote_templsourc "; # make any directory we may need Index: /tags/ipp-20150329/ippScripts/scripts/sc_prepare_ff.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_prepare_ff.pl (revision 38170) +++ /tags/ipp-20150329/ippScripts/scripts/sc_prepare_ff.pl (revision 38170) @@ -0,0 +1,362 @@ +#! /usr/bin/env perl + +# generate the input & output files lists and commands for a single ffRun + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use DateTime; +use Data::Dumper; +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($remote_id,$ff_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe); +GetOptions( + 'remote_id=s' => \$remote_id, + 'ff_id=s' => \$ff_id, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, + 'recipe=s' => \$cmd_recipe, + 'path_base=s' => \$path_base, + 'no_update' => \$no_update, + 'verbose' => \$verbose, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id -ff_id --camera --dbname --path_base --recipe", -exitval => 3) unless + defined($remote_id) and + defined($ff_id) and + defined($camera) and + defined($path_base) and + defined($cmd_recipe) and + defined($dbname); + + +# Hard coded values +# Now accessible from a recipe +my %remote_recipe = (); +{ + my $verbose = 0; + my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + + my $active_recipe = ''; + my %recipes = (); + +# print Dumper($metadata); + foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'ACTIVE') { + $active_recipe = ${ $entry }{value}; # Not actually used + } + else { + if (${ $entry }{class} eq 'metadata') { # A real recipe + my $name = ${ $entry }{name}; + foreach my $tentry (@{ ${ $entry }{value} }) { + if (${ $tentry }{class} eq 'scalar') { # A recipe value + $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value}; + } + elsif (${ $tentry }{class} eq 'metadata') { # A recipe array + foreach my $arr_entry (@{ ${ $tentry }{value} }) { + push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value}; + } + } + } + } + } + } + + unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) }; +# print Dumper(%recipes); + %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe. +# print Dumper(\%remote_recipe); +} + +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = $remote_recipe{REMOTE_ROOT}; +my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME}; # Name of the remote node. +my $remote_raw = "${remote_root}/tmp/"; # Directory to find raw data in. + +my $threads_req = 4; # How many threads are we going to use? + +my $fail_state = "prep_fail"; + + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id, $ff_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my @return_component_list = ("DBINFO.EXP","LOG.EXP","SKYCELL.STATS","TRACE.EXP", + "PSPHOT.OUT.CMF.MEF","PSPHOT.SKY.CONFIG"); + + +# STEP 0: Open output files +my $uri_transfer= $path_base . ".transfer"; +my $uri_check = $path_base . ".check"; +my $uri_config = $path_base . ".config"; +my $uri_generate= $path_base . ".generate"; +my $uri_return = $path_base . ".return"; + +my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); +my $disk_check = $ipprc->file_resolve($uri_check,1); +my $disk_config = $ipprc->file_resolve($uri_config,1); +my $disk_generate= $ipprc->file_resolve($uri_generate,1); +my $disk_return = $ipprc->file_resolve($uri_return,1); + +my (undef, $remote_config) = &uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id, $ff_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check", $remote_id, $ff_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config", $remote_id, $ff_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id, $ff_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $ff_id, $PS_EXIT_SYS_ERROR, $fail_state); + +my $job_index = 0; + +# STEP 1: Get exposure level information from the ffRun we're working from. +my ($ffRun); +{ + # Get the Run level information first. + my $command = "$fftool -todo -ff_id $ff_id "; + $command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run fftool -todo ", $remote_id, $ff_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine ff run information.", $remote_id, $ff_id, $PS_EXIT_PROG_ERROR, $fail_state); + $ffRun = parse_md_list($MDlist); +} + +# STEP 2: Iterate over the sub-components +{ + foreach my $ffRun_entry (@{ $ffRun }) { + my $skycell_id = $ffRun_entry->{skycell_id}; + my $tess_id = $ffRun_entry->{tess_id}; + my $workdir = $ffRun_entry->{workdir}; + my $warp_id = $ffRun_entry->{warp_id}; + my $source_pb = $ffRun_entry->{sources_path_base}; + my $warp_pb = $ffRun_entry->{warp_path_base}; + my $reduction = $ffRun_entry->{reduction}; + my $camera = $ffRun_entry->{camera}; + + my $ipp_outroot = sprintf("%s/%s/%s/%s.%s.wrp.%s.ff.%s",$workdir,$tess_id,$skycell_id,$tess_id,$skycell_id,$warp_id,$ff_id); + print "$ipp_outroot\n"; + my $remote_outroot = &uri_local_to_remote($ipp_outroot); + + my $ipp_inimage = $ipprc->filename("PSWARP.OUTPUT",$warp_pb); + my $ipp_inmask = $ipprc->filename("PSWARP.OUTPUT.MASK",$warp_pb); + my $ipp_inwt = $ipprc->filename("PSWARP.OUTPUT.VARIANCE",$warp_pb); + my $ipp_insourc = $ipprc->filename("PSPHOT.OUTPUT.CFF",$source_pb); + +# my ($ipp_disk_inimage, $remote_inimage) = &uri_to_outputs($ipp_inimage); +# my ($ipp_disk_inmask, $remote_inmask) = &uri_to_outputs($ipp_inmask); +# my ($ipp_disk_inwt, $remote_inwt) = &uri_to_outputs($ipp_inwt); +# my ($ipp_disk_insourc, $remote_insourc) = &uri_to_outputs($ipp_insourc); + + my (undef,$remote_inimage) = uri_to_outputs_raw($ipp_inimage); + my (undef,$remote_inmask) = uri_to_outputs_raw($ipp_inmask); + my (undef,$remote_inwt) = uri_to_outputs_raw($ipp_inwt); + my (undef,$remote_insourc) = uri_to_outputs_raw($ipp_insourc); + + # make any directory we may need + my $remote_outroot_dir = dirname($remote_outroot); + my $dir_command = "mkdir -p $remote_outroot_dir"; + + # Find the recipe + $reduction = 'DEFAULT' unless defined $reduction; + my $recipe_psphot = $ipprc->reduction($reduction, 'FULLFORCE_PSPHOT'); # Recipe to use for psphot + unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for PSPHOT: $reduction\n", + $remote_id,$ff_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + } + + + + # Convert these here + my $psff_command = " psphotFullForce ${remote_outroot} "; + $psff_command .= " -force $remote_insourc "; + $psff_command .= " -file $remote_inimage "; + $psff_command .= " -mask $remote_inmask "; + $psff_command .= " -variance $remote_inwt "; + $psff_command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE "; + $psff_command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF "; + $psff_command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF "; + $psff_command .= " -recipe PSPHOT $recipe_psphot "; + $psff_command .= " -photcode-rule '{DETECTOR}.{FILTER.ID}.ForcedWarp' "; + $psff_command .= " -dumpconfig ${remote_outroot}.psphot.mdc -stats ${remote_outroot}.stats "; + $psff_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; + $psff_command .= " -threads $threads_req "; # -image_id ${image_id} -source_id ${source_id} "; + + my $post_cmd_echo = " echo -n \"fftool -addresult -ff_id $ff_id -warp_id $warp_id "; + $post_cmd_echo .= " -dbname $dbname " if defined $dbname; +# $post_cmd_echo .= " -uri ${ipp_outroot}.fits "; + $post_cmd_echo .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; + + my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - WARP_SKYCELL >> ${remote_outroot}.dbinfo "; + + print CONFIG " ${dir_command} && ${psff_command} && ${post_cmd_echo} && ${post_cmd_SfM} "; + $job_index++; + + # Determine which output files need to be returned + foreach my $component(@return_component_list) { + my $filename = $ipprc->filename($component,$ipp_outroot,$skycell_id); + my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + } + # If we're this far, we succeeded, so delete the input files. + print CONFIG " && rm -f $remote_insourc $remote_inimage $remote_inmask $remote_inwt "; + print CONFIG "\n"; +# die(); + } +} +close(CONFIG); +close(TRANSFER); +close(CHECK); +close(RETURN); +close(GENERATE); + +unless($no_update) { + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $ff_id "; + $command .= " -set_jobs $job_index"; + $command .= " -set_path_base $path_base"; + $command .= " -set_state prep_done"; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); +} +exit (0); + +## Common SC routines + +sub uri_convert { # (ipp_disk,remote_disk) = uri_convert(neb_uri); + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $ff_id, $PS_EXIT_SYS_ERROR, $fail_state); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_convert_and_create { # (ipp_disk,remote_disk) = uri_convert_and_create(neb_uri); ipp_disk is created if it doesn't exist + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $ff_id, $PS_EXIT_SYS_ERROR, $fail_state); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs { # (ipp_disk,remote_disk) = uri_to_output(neb_uri); Appends to TRANSFER and CHECK filehandles + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = &uri_convert( $neb_uri ); + + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs_raw { + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + $remote_disk = $remote_raw . $ipp_disk; + + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs_for_return { # (ipp_disk,remote_disk) = uri_to_outputs_for_return(neb_uri); create ipp_disk, append to RETURN and GENERATE + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = &uri_convert_and_create( $neb_uri ); + + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; + return($ipp_disk,$remote_disk); +} + +sub uri_local_to_remote { #(remote_uri) = uri_local_to_remote(local_neb_uri); + # This needs to replace the nebulous tag with the remote root. + my $local_uri = shift; + $local_uri =~ s%^.*?/%%; # neb:/ + $local_uri =~ s%^.*?/%%; # / + $local_uri =~ s%^.*?/%%; # @HOST@.0/ + my $remote_uri = "${remote_root}/" . $local_uri; + + return($remote_uri); +} + +sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); + # This needs to replace the remote root directory with the nebulous tag. + my $remote_uri = shift; + $remote_uri =~ s%${remote_root}%%; + my $local_uri = "neb:///" . $remote_uri; + + return($local_uri); +} + + +sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status); + my $msg = shift; + my $remote_id = shift; + my $stage_id = shift; + my $exit_code = shift; + my $exit_state = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $remote_id and defined $stage_id and not $no_update) { + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} Index: /tags/ipp-20150329/ippScripts/scripts/sc_prepare_run.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_prepare_run.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_prepare_run.pl (revision 38170) @@ -21,7 +21,85 @@ use Pod::Usage qw( pod2usage ); +#use Data::Dumper; +# Look for programs we need +my $missing_tools; +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Options +my ($remote_id,$cmd_recipe,$stage,$camera,$path_base,$dbname,$verbose,$no_update); +GetOptions( + 'remote_id=s' => \$remote_id, + 'recipe=s' => \$cmd_recipe, + 'stage=s' => \$stage, + 'camera|c=s' => \$camera, + 'path_base=s' => \$path_base, + 'dbname|d=s' => \$dbname, + 'verbose' => \$verbose, + 'no_update' => \$no_update, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --stage --camera --dbname --path_base --recipe", -exitval => 3) unless + defined($remote_id) and + defined($stage) and + defined($camera) and + defined($path_base) and + defined($cmd_recipe) and + defined($dbname) and + defined($dbname); + # Hard coded values -# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -my $remote_root = '/scratch3/watersc1/'; +# Now accessible from a recipe +my %remote_recipe = (); +{ + my $verbose = 0; + my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + + my $active_recipe = ''; + my %recipes = (); + +# print Dumper($metadata); + foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'ACTIVE') { + $active_recipe = ${ $entry }{value}; # Not actually used + } + else { + if (${ $entry }{class} eq 'metadata') { # A real recipe + my $name = ${ $entry }{name}; + foreach my $tentry (@{ ${ $entry }{value} }) { + if (${ $tentry }{class} eq 'scalar') { # A recipe value + $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value}; + } + elsif (${ $tentry }{class} eq 'metadata') { # A recipe array + foreach my $arr_entry (@{ ${ $tentry }{value} }) { + push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value}; + } + } + } + } + } + } + + unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) }; +# print Dumper(%recipes); + %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe. +# print Dumper(\%remote_recipe); +} + +my $remote_root = $remote_recipe{REMOTE_ROOT}; my $remote_raw = "${remote_root}/tmp/"; # Directory to find raw data in. @@ -38,4 +116,5 @@ $job_cost{"staticsky"} = 2; # really? avg time of 5720s? $job_cost{"diff"} = 300 / 60 / 60; +$job_cost{"ff"} = 300 / 60 / 60; # This is an 83%-ile point, but the tail is long. # This object holds how "expensive" a given job is in terms of cores on a node. @@ -48,43 +127,17 @@ $job_subscription{"staticsky"} = 3; $job_subscription{"diff"} = 1; - -my $proc_per_node = 24; # processors per node -my $min_nodes = 1; # smallest allocation to ask for -my $max_nodes = 1000; # largest allocation to ask for -my $min_time = 1; # shortest allocation to ask for -my $max_time = 8; # longest allocation to ask for +$job_subscription{"ff"} = 1; + +my $proc_per_node = $remote_recipe{PROC_PER_NODE}; # processors per node +my $min_nodes = $remote_recipe{MIN_NODES}; # smallest allocation to ask for +my $max_nodes = $remote_recipe{MAX_NODES}; # largest allocation to ask for +my $min_time = $remote_recipe{MIN_TIME}; # shortest allocation to ask for +my $max_time = $remote_recipe{MAX_TIME}; # longest allocation to ask for # We need to ensure we only ever try to transfer a file once. my %file_filter = (); -# Look for programs we need -my $missing_tools; -my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); - -if ($missing_tools) { - warn("Can't find required tools."); - exit($PS_EXIT_CONFIG_ERROR); -} - -# Options -my ($remote_id,$stage,$camera,$path_base,$dbname,$verbose,$no_update); -GetOptions( - 'remote_id=s' => \$remote_id, - 'stage=s' => \$stage, - 'camera|c=s' => \$camera, - 'path_base=s' => \$path_base, - 'dbname|d=s' => \$dbname, - 'verbose' => \$verbose, - 'no_update' => \$no_update, - ) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --stage --camera --dbname --path_base", -exitval => 3) unless - defined($remote_id) and - defined($stage) and - defined($camera) and - defined($path_base) and - defined($dbname); - + +# Finish setup my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $PS_EXIT_CONFIG_ERROR, $fail_state); Index: /tags/ipp-20150329/ippScripts/scripts/sc_prepare_stack.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_prepare_stack.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_prepare_stack.pl (revision 38170) @@ -18,17 +18,9 @@ use Pod::Usage qw( pod2usage ); -# Hard coded values -# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $remote_hostname = "LANL/Mustang"; # Name of the remote node. -my $threads_req = 4; # How many threads are we going to use? -my $have_warps = 0; # Do we have a copy of the warp if we don't find one there? - -my $fail_state = "prep_fail"; - # Look for programs we need my $missing_tools; my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); if ($missing_tools) { @@ -37,5 +29,5 @@ } -my ($remote_id,$stack_id,$camera,$dbname,$verbose,$path_base,$no_update); +my ($remote_id,$stack_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe); GetOptions( 'remote_id=s' => \$remote_id, @@ -43,4 +35,5 @@ 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, + 'recipe=s' => \$cmd_recipe, 'path_base=s' => \$path_base, 'no_update' => \$no_update, @@ -49,10 +42,66 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --stack_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id --stack_id --camera --dbname --path_base --recipe", -exitval => 3) unless defined($remote_id) and defined($stack_id) and defined($camera) and - defined($path_base) and + defined($path_base) and + defined($cmd_recipe) and defined($dbname); + +# Hard coded values +# Now accessible from a recipe +my %remote_recipe = (); +{ + my $verbose = 0; + my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + + my $active_recipe = ''; + my %recipes = (); + +# print Dumper($metadata); + foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'ACTIVE') { + $active_recipe = ${ $entry }{value}; # Not actually used + } + else { + if (${ $entry }{class} eq 'metadata') { # A real recipe + my $name = ${ $entry }{name}; + foreach my $tentry (@{ ${ $entry }{value} }) { + if (${ $tentry }{class} eq 'scalar') { # A recipe value + $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value}; + } + elsif (${ $tentry }{class} eq 'metadata') { # A recipe array + foreach my $arr_entry (@{ ${ $tentry }{value} }) { + push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value}; + } + } + } + } + } + } + + unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) }; +# print Dumper(%recipes); + %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe. +# print Dumper(\%remote_recipe); +} + +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = $remote_recipe{REMOTE_ROOT}; # Far side destination base location +my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME}; # Name of the remote node. +my $threads_req = 4; # How many threads are we going to use? +my $have_warps = $remote_recipe{TRANSFER_WARP_IMAGES}; # Do we have a copy of the warp if we don't find one there? + +my $fail_state = "prep_fail"; + my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $stack_id, $PS_EXIT_CONFIG_ERROR, $fail_state); @@ -158,5 +207,5 @@ # ${remote_root}/tmp/${ipp_disk} - if ($have_warps != 0) { # Check to see if we have a copy, because if we do, then we can send it over if it's missing. + if ($have_warps) { # Check to see if we have a copy, because if we do, then we can send it over if it's missing. (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".fits"); ($ipp_disk,undef) = &uri_convert($warp_path_base . ".fits"); @@ -201,5 +250,5 @@ close(COMPMAP); - my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl --compmap ${remote_root}/tmp/${component_map} $warp_path_base_string > ${remote_outroot}.in.mdc"; + my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl --remote_root ${remote_root} --compmap ${remote_root}/tmp/${component_map} $warp_path_base_string > ${remote_outroot}.in.mdc"; my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use my $ppstack_command = " ppStack -input ${remote_outroot}.in.mdc "; @@ -216,5 +265,5 @@ $ppstack_command .= " -threads $threads_req "; $ppstack_command .= " -dumpconfig ${remote_outroot}.mdc "; - $ppstack_command .= " -stack_id $stack_id -skycell_id $skycell_id -tess_id $tess_id "; + $ppstack_command .= " -stack_id $stack_id -skycell_id $skycell_id -tess_id $tess_id >& ${remote_outroot}.log2 "; my $post_cmd_echo = " echo -n \"stacktool -addsumskyfile -stack_id $stack_id "; Index: /tags/ipp-20150329/ippScripts/scripts/sc_prepare_staticsky.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_prepare_staticsky.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_prepare_staticsky.pl (revision 38170) @@ -18,12 +18,4 @@ use Pod::Usage qw( pod2usage ); -# Hard coded values -# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $remote_hostname = "LANL/Mustang"; # Name of the remote node. -my $threads_req = 4; # How many threads are we going to use? - -my $fail_state = "prep_fail"; - # Look for programs we need my $missing_tools; @@ -31,4 +23,5 @@ my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); my $staticskytool= can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); if ($missing_tools) { @@ -37,5 +30,5 @@ } -my ($remote_id,$sky_id,$camera,$dbname,$verbose,$path_base,$no_update); +my ($remote_id,$sky_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe); GetOptions( 'remote_id=s' => \$remote_id, @@ -43,4 +36,5 @@ 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, + 'recipe=s' => \$cmd_recipe, 'path_base=s' => \$path_base, 'no_update' => \$no_update, @@ -49,10 +43,65 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --sky_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id --sky_id --camera --dbname --path_base --recipe", -exitval => 3) unless defined($remote_id) and defined($sky_id) and defined($camera) and defined($path_base) and + defined($cmd_recipe) and defined($dbname); + +# Hard coded values +# Now accessible from a recipe +my %remote_recipe = (); +{ + my $verbose = 0; + my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + + my $active_recipe = ''; + my %recipes = (); + +# print Dumper($metadata); + foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'ACTIVE') { + $active_recipe = ${ $entry }{value}; # Not actually used + } + else { + if (${ $entry }{class} eq 'metadata') { # A real recipe + my $name = ${ $entry }{name}; + foreach my $tentry (@{ ${ $entry }{value} }) { + if (${ $tentry }{class} eq 'scalar') { # A recipe value + $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value}; + } + elsif (${ $tentry }{class} eq 'metadata') { # A recipe array + foreach my $arr_entry (@{ ${ $tentry }{value} }) { + push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value}; + } + } + } + } + } + } + + unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) }; +# print Dumper(%recipes); + %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe. +# print Dumper(\%remote_recipe); +} + + +my $remote_root = $remote_recipe{REMOTE_ROOT}; # Far side destination base location +my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME}; # Name of the remote node. +my $threads_req = 4; # How many threads are we going to use? + +my $fail_state = "prep_fail"; + my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $sky_id, $PS_EXIT_CONFIG_ERROR, $fail_state); @@ -186,5 +235,5 @@ close(COMPMAP); - my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_staticsky_mdc.pl --compmap ${remote_root}/tmp/${component_map} $input_path_base_string > ${remote_outroot}.in.mdc"; + my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_staticsky_mdc.pl --remote_root ${remote_root} --compmap ${remote_root}/tmp/${component_map} $input_path_base_string > ${remote_outroot}.in.mdc"; my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_PSPHOT'); my $recipe_ppsub = $ipprc->reduction($reduction, 'STACKPHOT_PPSUB'); @@ -198,5 +247,5 @@ $pps_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; $pps_command .= " -threads $threads_req "; - $pps_command .= " -dumpconfig ${remote_outroot}.psphotStack.mdc "; + $pps_command .= " -dumpconfig ${remote_outroot}.psphotStack.mdc >& ${remote_outroot}.log2 "; my $post_cmd_echo = " echo -n \"staticskytool -addresult -sky_id $sky_id "; Index: /tags/ipp-20150329/ippScripts/scripts/sc_prepare_warp.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_prepare_warp.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_prepare_warp.pl (revision 38170) @@ -18,16 +18,9 @@ use Pod::Usage qw( pod2usage ); -# Hard coded values -# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -my $remote_root = '/scratch3/watersc1/'; -my $remote_hostname = "LANL/Mustang"; # Name of the remote node. -my $threads_req = 4; # How many threads are we going to use? - -my $fail_state = "prep_fail"; - # Look for programs we need my $missing_tools; my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); if ($missing_tools) { @@ -36,5 +29,5 @@ } -my ($remote_id,$warp_id,$camera,$dbname,$verbose,$path_base,$no_update); +my ($remote_id,$warp_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe); GetOptions( 'remote_id=s' => \$remote_id, @@ -42,4 +35,5 @@ 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, + 'recipe=s' => \$cmd_recipe, 'path_base=s' => \$path_base, 'no_update' => \$no_update, @@ -48,16 +42,76 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id -warp_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id -warp_id --camera --dbname --path_base --recipe", -exitval => 3) unless defined($remote_id) and defined($warp_id) and defined($camera) and defined($path_base) and + defined($cmd_recipe) and defined($dbname); + +# Hard coded values +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +# Now accessible from a recipe +my %remote_recipe = (); +{ + my $verbose = 0; + my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + + my $active_recipe = ''; + my %recipes = (); + +# print Dumper($metadata); + foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'ACTIVE') { + $active_recipe = ${ $entry }{value}; # Not actually used + } + else { + if (${ $entry }{class} eq 'metadata') { # A real recipe + my $name = ${ $entry }{name}; + foreach my $tentry (@{ ${ $entry }{value} }) { + if (${ $tentry }{class} eq 'scalar') { # A recipe value + $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value}; + } + elsif (${ $tentry }{class} eq 'metadata') { # A recipe array + foreach my $arr_entry (@{ ${ $tentry }{value} }) { + push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value}; + } + } + } + } + } + } + unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) }; +# print Dumper(%recipes); + %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe. +# print Dumper(\%remote_recipe); + +} + +my $remote_root = $remote_recipe{REMOTE_ROOT}; +my $hostname = $remote_recipe{REMOTE_HOSTNAME}; +my $have_warps = $remote_recipe{TRANSFER_WARP_IMAGES}; +my $threads_req = 4; # How many threads are we going to use? + +my $fail_state = "prep_fail"; + + my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id, $warp_id, $PS_EXIT_CONFIG_ERROR, $fail_state); my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SKY.SAVE","LOG.EXP"); +my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SKY.SAVE","LOG.EXP"); +if ($have_warps) { + push @return_component_list, ("PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE"); +} # STEP 0: Open output files Index: /tags/ipp-20150329/ippScripts/scripts/sc_remote_exec.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_remote_exec.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_remote_exec.pl (revision 38170) @@ -13,13 +13,4 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); use Pod::Usage qw( pod2usage ); - -# Hard coded values -my $DMZ_HOST = 'wtrw'; -my @SEC_HOSTS= ('mu-fe1','mu-fe2','mu-fe3' - # ,'mu-fe4' - ); -my $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; # 'mu-fe'; -my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/'; -my $remote_root = '/scratch3/watersc1/'; # tools @@ -28,4 +19,5 @@ my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); if ($missing_tools) { @@ -35,5 +27,5 @@ # Options -my ($remote_id,$job_id,$path_base,$dbname,$verbose,$no_update,$camera); +my ($remote_id,$job_id,$path_base,$dbname,$verbose,$no_update,$camera,$cmd_recipe); $verbose = 0; GetOptions( @@ -42,4 +34,5 @@ 'camera=s' => \$camera, 'dbname=s' => \$dbname, + 'recipe=s' => \$cmd_recipe, 'verbose' => \$verbose, 'no_update' => \$no_update, @@ -47,7 +40,69 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id --path_base --recipe", -exitval => 3) unless defined($path_base) and + defined($camera) and + defined($cmd_recipe) and + defined($dbname) and defined($remote_id); + +# Hard coded values +# Now accessible from a recipe +my %remote_recipe = (); +{ + my $verbose = 0; + my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + + my $active_recipe = ''; + my %recipes = (); + +# print Dumper($metadata); + foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'ACTIVE') { + $active_recipe = ${ $entry }{value}; # Not actually used + } + else { + if (${ $entry }{class} eq 'metadata') { # A real recipe + my $name = ${ $entry }{name}; + foreach my $tentry (@{ ${ $entry }{value} }) { + if (${ $tentry }{class} eq 'scalar') { # A recipe value + $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value}; + } + elsif (${ $tentry }{class} eq 'metadata') { # A recipe array + foreach my $arr_entry (@{ ${ $tentry }{value} }) { + push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value}; + } + } + } + } + } + } + + unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) }; +# print Dumper(%recipes); + %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe. +# print Dumper(\%remote_recipe); +} + +# Hard coded values +my $DMZ_HOST = $remote_recipe{DMZ_HOST}; +my @SEC_HOSTS= @{ $remote_recipe{SEC_HOST} }; +my $SEC_HOST; +if ($#SEC_HOSTS != -1) { + $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; +} +else { + $SEC_HOST = ''; +} +my $IPP_PATH = $remote_recipe{IPP_PATH}; +my $remote_root = $remote_recipe{REMOTE_ROOT}; my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id); @@ -141,5 +196,11 @@ my $file = shift; my $destination = shift; - my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}"; + my $cmd; + if ($SEC_HOST ne '') { + $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}"; + } + else { + $cmd = "$ssh -n $DMZ_HOST $cmd"; + } my $directory = dirname($destination); @@ -157,6 +218,11 @@ my $destination = shift; my $file = shift; - - my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file "; + my $cmd; + if ($SEC_HOST ne '') { + $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file "; + } + else { + $cmd = "$ssh -n $DMZ_HOST $cmd"; + } my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -170,5 +236,11 @@ sub ssh_exec_command { my $cmd = shift; - $cmd = "$ssh -n $DMZ_HOST ssh ${SEC_HOST} $cmd"; + if ($SEC_HOST ne '') { + $cmd = "$ssh -n $DMZ_HOST ssh ${SEC_HOST} $cmd"; + } + else { + $cmd = "$ssh -n $DMZ_HOST $cmd"; + } + print "EXEC: $cmd\n"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = Index: /tags/ipp-20150329/ippScripts/scripts/sc_remote_poll.pl =================================================================== --- /tags/ipp-20150329/ippScripts/scripts/sc_remote_poll.pl (revision 38169) +++ /tags/ipp-20150329/ippScripts/scripts/sc_remote_poll.pl (revision 38170) @@ -26,13 +26,4 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); use Pod::Usage qw( pod2usage ); - -# Hard coded values -my $DMZ_HOST = 'wtrw'; -my @SEC_HOSTS= ('mu-fe1','mu-fe2','mu-fe3' - # ,'mu-fe4' - ); -my $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; # 'mu-fe'; -my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/'; -my $remote_root = '/scratch3/watersc1/'; # tools @@ -44,4 +35,5 @@ my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); if ($missing_tools) { @@ -51,5 +43,5 @@ # Options -my ($remote_id,$path_base,$skip_poll,$skip_transfer,$job_id,$dbname,$retry,$verbose,$no_update,$camera); +my ($remote_id,$path_base,$skip_poll,$skip_transfer,$job_id,$dbname,$retry,$verbose,$no_update,$camera,$cmd_recipe); $verbose = 0; GetOptions( @@ -61,4 +53,5 @@ 'camera=s' => \$camera, 'dbname=s' => \$dbname, + 'recipe=s' => \$cmd_recipe, 'retry' => \$retry, 'verbose' => \$verbose, @@ -67,8 +60,71 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --job_id --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id --job_id --path_base --recipe", -exitval => 3) unless defined($path_base) and defined($remote_id) and + defined($cmd_recipe) and defined($job_id); + + + +# Now accessible from a recipe +my %remote_recipe = (); +{ + my $verbose = 0; + my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR); + } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + + my $active_recipe = ''; + my %recipes = (); + +# print Dumper($metadata); + foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'ACTIVE') { + $active_recipe = ${ $entry }{value}; # Not actually used + } + else { + if (${ $entry }{class} eq 'metadata') { # A real recipe + my $name = ${ $entry }{name}; + foreach my $tentry (@{ ${ $entry }{value} }) { + if (${ $tentry }{class} eq 'scalar') { # A recipe value + $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value}; + } + elsif (${ $tentry }{class} eq 'metadata') { # A recipe array + foreach my $arr_entry (@{ ${ $tentry }{value} }) { + push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value}; + } + } + } + } + } + } + + unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) }; +# print Dumper(%recipes); + %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe. +# print Dumper(\%remote_recipe); +} + + +# Hard coded values +my $DMZ_HOST = $remote_recipe{DMZ_HOST}; +my @SEC_HOSTS= @{ $remote_recipe{SEC_HOST} }; +my $SEC_HOST; +if ($#SEC_HOSTS != -1) { + $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; +} +else { + $SEC_HOST = ''; +} +my $IPP_PATH = $remote_recipe{IPP_PATH}; +my $remote_root = $remote_recipe{REMOTE_ROOT}; + my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id); @@ -351,5 +407,11 @@ my $file = shift; my $destination = shift; - my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}"; + my $cmd; + if ($SEC_HOST ne '') { + $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}"; + } + else { + $cmd = "$scp $file ${DMZ_HOST}:${destination}"; + } my $directory = dirname($destination); @@ -367,6 +429,11 @@ my $destination = shift; my $file = shift; - - my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file "; + my $cmd; + if ($SEC_HOST ne '') { + $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file "; + } + else { + $cmd = "$scp $file ${DMZ_HOST}:${destination}"; + } my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -380,5 +447,12 @@ sub ssh_exec_command { my $cmd = shift; - $cmd = "$ssh -n $DMZ_HOST ssh ${SEC_HOST} $cmd"; + + if ($SEC_HOST ne '') { + $cmd = "$ssh -n $DMZ_HOST ssh ${SEC_HOST} $cmd"; + } + else { + $cmd = "$ssh -n $DMZ_HOST $cmd"; + } + print "EXEC: $cmd\n"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -511,8 +585,11 @@ } elsif ($stage eq 'staticsky') { -# ($class_id, $stage_id) = $file =~ /(/; + ($class_id, $stage_id) = $file =~ /(skycell.\d\d\d\d.\d\d\d).sky.(\d*).dbinfo/; } elsif ($stage eq 'diff') { -# ($class_id,$stage_id) = $file =~ /(/; + ($class_id,$stage_id) = $file =~ /(skycell.\d\d\d\d.\d\d\d).WS.dif.(\d*).dbinfo/; + } + elsif ($stage eq 'ff') { + ($class_id,$stage_id) = $file =~ /wrp\.(\d*).ff.(\d*).dbinfo/; } return ($stage_id, $class_id); @@ -545,4 +622,14 @@ $class_id = 0; } + elsif ($stage eq 'staticsky') { + ($class_id, $stage_id, $component) = $file =~ /(skycell.\d\d\d\d.\d\d\d).sky.(\d*).([\w\.]+)$/; + } + elsif ($stage eq 'diff') { + ($class_id,$stage_id, $component) = $file =~ /(skycell.\d\d\d\d.\d\d\d).WS.dif.(\d*).([\w\.]+)$/; + } + elsif ($stage eq 'ff') { + ($class_id,$stage_id, $component) = $file =~ /wrp\.(\d*).ff.(\d*).([\w\.]+)$/; + } + return($stage_id,$class_id,$component); } Index: /tags/ipp-20150329/ippTasks/remote.pro =================================================================== --- /tags/ipp-20150329/ippTasks/remote.pro (revision 38169) +++ /tags/ipp-20150329/ippTasks/remote.pro (revision 38170) @@ -20,4 +20,6 @@ $remoteExec_DB = 0 $remotePoll_DB = 0 +$REMOTE_RECIPE = 0 + list STAGES @@ -96,4 +98,12 @@ end +macro set.remote.recipe + if ($0 != 2) + echo "USAGE: set.remote.recipe (recipe_name)" + break + end + $REMOTE_RECIPE = $1 +end + task remote.define host local @@ -122,5 +132,10 @@ # min entry limit? - $run = remotetool -definebyquery -label $label -stage $stage -path_base neb://@HOST@.0/remote/$label -limit 500 + if ("$stage" == "stack") + $run = remotetool -definebyquery -label $label -stage $stage -path_base neb://@HOST@.0/remote/$label -limit 50 + else + $run = remotetool -definebyquery -label $label -stage $stage -path_base neb://@HOST@.0/remote/$label -limit 500 + end + if ($DB:n == 0) option DEFAULT @@ -268,4 +283,5 @@ if ($N == 0) break if ($NETWORK == 0) break + if ($REMOTE_RECIPE == 0) break book getpage remotePrepCompRuns 0 -var pageName -key pantaskState INIT @@ -282,21 +298,27 @@ if ("$STAGE" == "chip") - $command = sc_prepare_chip.pl --camera GPC1 --chip_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + $command = sc_prepare_chip.pl --chip_id $STAGE_ID end if ("$STAGE" == "camera") - $command = sc_prepare_camera.pl --camera GPC1 --cam_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + $command = sc_prepare_camera.pl --cam_id $STAGE_ID end if ("$STAGE" == "warp") - $command = sc_prepare_warp.pl --camera GPC1 --warp_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + $command = sc_prepare_warp.pl --warp_id $STAGE_ID end if ("$STAGE" == "stack") - $command = sc_prepare_stack.pl --camera GPC1 --stack_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + $command = sc_prepare_stack.pl --stack_id $STAGE_ID end if ("$STAGE" == "staticsky") - $command = sc_prepare_staticsky.pl --camera GPC1 --sky_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + $command = sc_prepare_staticsky.pl --sky_id $STAGE_ID end if ("$STAGE" == "diff") - $command = sc_prepare_diff.pl --camera GPC1 --diff_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME - end + $command = sc_prepare_diff.pl --diff_id $STAGE_ID + end + if ("$STAGE" == "ff") + $command = sc_prepare_ff.pl --ff_id $STAGE_ID + end + +# Common elements + $command = $command --recipe $REMOTE_RECIPE --camera GPC1 --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME options $pageName @@ -386,4 +408,5 @@ if ($N == 0) break if ($NETWORK == 0) break + if ($REMOTE_RECIPE == 0) break book getpage remotePrepRuns 0 -var pageName -key pantaskState INIT @@ -399,5 +422,5 @@ sprintf outroot "%s/remote_%s.%s" $PATH_BASE $STAGE $REMOTE_ID - $command = sc_prepare_run.pl --camera GPC1 --remote_id $REMOTE_ID --stage $STAGE --path_base $outroot --dbname $DBNAME + $command = sc_prepare_run.pl --camera GPC1 --remote_id $REMOTE_ID --stage $STAGE --path_base $outroot --dbname $DBNAME --recipe $REMOTE_RECIPE options $pageName @@ -486,4 +509,5 @@ if ($N == 0) break if ($NETWORK == 0) break + if ($REMOTE_RECIPE == 0) break book getpage remoteExecRuns 0 -var pageName -key pantaskState INIT @@ -500,5 +524,5 @@ sprintf outroot "%s/remote_%s.%s" $PATH_BASE $STAGE $REMOTE_ID - $command = sc_remote_exec.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 + $command = sc_remote_exec.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 --recipe $REMOTE_RECIPE options $pageName @@ -588,4 +612,5 @@ if ($N == 0) break if ($NETWORK == 0) break + if ($REMOTE_RECIPE == 0) break book getpage remotePollRuns 0 -var pageName -key pantaskState INIT @@ -604,5 +629,5 @@ # This can't have an invalid job_id if ($JOB_ID == -1) break - $command = sc_remote_poll.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 --job_id $JOB_ID + $command = sc_remote_poll.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 --job_id $JOB_ID --recipe $REMOTE_RECIPE options $pageName Index: /tags/ipp-20150329/ippTools/share/Makefile.am =================================================================== --- /tags/ipp-20150329/ippTools/share/Makefile.am (revision 38169) +++ /tags/ipp-20150329/ippTools/share/Makefile.am (revision 38170) @@ -526,4 +526,5 @@ remotetool_definebyquery_staticsky.sql \ remotetool_definebyquery_diff.sql \ + remotetool_definebyquery_ff.sql \ remotetool_dropcomponent.sql \ remotetool_listcomponent.sql \ Index: /tags/ipp-20150329/ippTools/share/remotetool_definebyquery_diff.sql =================================================================== --- /tags/ipp-20150329/ippTools/share/remotetool_definebyquery_diff.sql (revision 38169) +++ /tags/ipp-20150329/ippTools/share/remotetool_definebyquery_diff.sql (revision 38170) @@ -15,5 +15,5 @@ JOIN remoteComponent USING(remote_id) WHERE -remoteRun.stage = 'staticsky' AND remoteComponent.state != 'retry' +remoteRun.stage = 'diff' AND remoteComponent.state != 'retry' -- where hook %s ) AS R Index: /tags/ipp-20150329/ippTools/share/remotetool_definebyquery_ff.sql =================================================================== --- /tags/ipp-20150329/ippTools/share/remotetool_definebyquery_ff.sql (revision 38170) +++ /tags/ipp-20150329/ippTools/share/remotetool_definebyquery_ff.sql (revision 38170) @@ -0,0 +1,21 @@ +SELECT stage_id FROM + +( SELECT ff_id AS stage_id + FROM fullForceRun +WHERE +fullForceRun.state = 'new' +-- where hook %s +GROUP BY ff_id +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING(remote_id) +WHERE +remoteRun.stage = 'ff' AND remoteComponent.state != 'retry' +-- where hook %s +) AS R +USING(stage_id) +WHERE R.remote_id IS NULL Index: /tags/ipp-20150329/ippTools/src/remotetool.c =================================================================== --- /tags/ipp-20150329/ippTools/src/remotetool.c (revision 38169) +++ /tags/ipp-20150329/ippTools/src/remotetool.c (revision 38170) @@ -166,4 +166,15 @@ else if (!strcmp(stage,"diff")) { query = pxDataGet("remotetool_definebyquery_diff.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + if (label) { + psStringAppend(&whereOption, "\n AND (label = '%s')", label); + } + } + else if (!strcmp(stage,"ff")) { + query = pxDataGet("remotetool_definebyquery_ff.sql"); if (!query) { psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); Index: /tags/ipp-20150329/ippconfig/recipes/Makefile.am =================================================================== --- /tags/ipp-20150329/ippconfig/recipes/Makefile.am (revision 38169) +++ /tags/ipp-20150329/ippconfig/recipes/Makefile.am (revision 38170) @@ -33,5 +33,6 @@ nightly_science.config \ ppBackground.mdc \ - pstamp.config + pstamp.config \ + remote_hosts.config install_DATA = $(install_files) Index: /tags/ipp-20150329/ippconfig/recipes/remote_hosts.config =================================================================== --- /tags/ipp-20150329/ippconfig/recipes/remote_hosts.config (revision 38170) +++ /tags/ipp-20150329/ippconfig/recipes/remote_hosts.config (revision 38170) @@ -0,0 +1,46 @@ +ACTIVE STR LANL + +LANL METADATA + DMZ_HOST STR wtrw + + SEC_HOST MULTI + SEC_HOST METADATA + SEC_HOST1 STR mu-fe1 + SEC_HOST2 STR mu-fe2 + SEC_HOST3 STR mu-fe3 + SEC_HOST4 STR mu-fe4 + END + + IPP_PATH STR /turquoise/usr/projects/cosmo/mswarren/ipp/ + REMOTE_ROOT STR /scratch3/watersc1/ + REMOTE_HOSTNAME STR LANL/Mustang + +# This is used to decide if we bring back warp images, and if stack should ask for local copies. + TRANSFER_WARP_IMAGES BOOL FALSE + + PROC_PER_NODE S32 24 + MIN_NODES S32 1 + MAX_NODES S32 1000 + MIN_TIME S32 1 + MAX_TIME S32 8 +END + +UHCRAY METADATA + DMZ_HOST STR 128.171.123.18 + + SEC_HOST MULTI + SEC_HOST METADATA + END + + IPP_PATH STR /home/watersc/src/ + REMOTE_ROOT STR /lus/scratch/watersc/ + REMOTE_HOSTNAME STR UH/Cray + + TRANSFER_WARP_IMAGES BOOL FALSE + + PROC_PER_NODE S32 20 + MIN_NODES S32 1 + MAX_NODES S32 10 + MIN_TIME S32 1 + MAX_TIME S32 8 +END Index: /tags/ipp-20150329/ippconfig/system.config =================================================================== --- /tags/ipp-20150329/ippconfig/system.config (revision 38169) +++ /tags/ipp-20150329/ippconfig/system.config (revision 38170) @@ -66,3 +66,4 @@ PSVIDEOPHOT STR recipes/psvideophot.config # video cell photometry PSTAMP STR recipes/pstamp.config # postage stamp server + REMOTE STR recipes/remote_hosts.config # Remote supercomputer processing END