Index: /trunk/ippScripts/Build.PL =================================================================== --- /trunk/ippScripts/Build.PL (revision 37832) +++ /trunk/ippScripts/Build.PL (revision 37833) @@ -138,9 +138,14 @@ scripts/stack_bkg_mk_mdc.pl scripts/sc_mk_stack_mdc.pl + scripts/sc_mk_staticsky_mdc.pl scripts/sc_prepare_camera.pl scripts/sc_prepare_chip.pl scripts/sc_prepare_stack.pl + scripts/sc_prepare_staticsky.pl scripts/sc_prepare_warp.pl + scripts/sc_prepare_run.pl scripts/sc_remote_exec.pl + scripts/sc_remote_poll.pl + scripts/sc_retry_return.pl scripts/sc_transfer_tool.pl )], Index: /trunk/ippScripts/MANIFEST =================================================================== --- /trunk/ippScripts/MANIFEST (revision 37832) +++ /trunk/ippScripts/MANIFEST (revision 37833) @@ -51,7 +51,9 @@ scripts/regenerate_background.pl scripts/sc_mk_stack_mdc.pl +scripts/sc_mk_staticsky_mdc.pl scripts/sc_prepare_camera.pl scripts/sc_prepare_chip.pl scripts/sc_prepare_stack.pl +scripts/sc_prepare_staticsky.pl scripts/sc_prepare_warp.pl scripts/sc_remote_exec.pl Index: /trunk/ippScripts/scripts/dist_defineruns.pl =================================================================== --- /trunk/ippScripts/scripts/dist_defineruns.pl (revision 37832) +++ /trunk/ippScripts/scripts/dist_defineruns.pl (revision 37833) @@ -90,5 +90,5 @@ push @stages, $stage; } else { - @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle skycal ff); + @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle ff); } Index: /trunk/ippScripts/scripts/lap_science.pl =================================================================== --- /trunk/ippScripts/scripts/lap_science.pl (revision 37832) +++ /trunk/ippScripts/scripts/lap_science.pl (revision 37833) @@ -34,5 +34,5 @@ my $qstack_threshold = 0.05; # Only make a quickstack if more than 5% of the exposures require it. my $minimum_stack_inputs = 2; # We can avoid magicking stack inputs if we have more than this number. -my $maximum_stack_inputs = 25; # This is the place-holder guess on how big a stack we should process locally +my $maximum_stack_inputs = 25; # This is the place-holder guess on how big a stack we should process locally. 25 is the best guess. my $do_updates_pv2 = 0; # Do updates using the calls needed for PV2. @@ -1058,5 +1058,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); + &my_die("Unable to perform stacktool -definebyquery: $error_code :: $command", $lap_id); } my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or @@ -1066,4 +1066,5 @@ ## Queue the "deep" stacks that will run locally. my $deep_minimum = $maximum_stack_inputs + 1; + if ($deep_minimum < 2) { $deep_minimum = 2; } $command = "$stacktool "; $command .= " -pretend " if defined $debug; @@ -1079,11 +1080,11 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); - } - my $stacks_made_deep .= $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to perform stacktool -definebyquery: $error_code :: $command", $lap_id); + } + my $stacks_made_deep = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); # Append deep stacks to stacks so we can get a complete count later. - push @{ $stacks_made}, @{ $stacks_made_deep }; + push @{ $stacks_made }, @{ $stacks_made_deep }; ## Determine the sass_id so we can record it in the database. @@ -1449,5 +1450,5 @@ my @clean_modes = ( -# 'chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -chip_id @CHIP_ID@', + 'chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -chip_id @CHIP_ID@', # 'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@', 'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@', Index: /trunk/ippScripts/scripts/permcheck.pl =================================================================== --- /trunk/ippScripts/scripts/permcheck.pl (revision 37832) +++ /trunk/ippScripts/scripts/permcheck.pl (revision 37833) @@ -62,5 +62,6 @@ if ($stage eq 'warp') { # These things are destined to have their only copy on the stsciXX nodes. %backup_hosts = ('stsci00' => 1, 'stsci01' => 1, 'stsci02' => 1, - 'stsci03' => 1, 'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1, + #'stsci03' => 1, + 'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1, 'stsci07' => 1, 'stsci08' => 1, 'stsci09' => 1, 'stsci10' => 1, 'stsci11' => 1, 'stsci12' => 1, Index: /trunk/ippScripts/scripts/sc_mk_stack_mdc.pl =================================================================== --- /trunk/ippScripts/scripts/sc_mk_stack_mdc.pl (revision 37832) +++ /trunk/ippScripts/scripts/sc_mk_stack_mdc.pl (revision 37833) @@ -1,19 +1,83 @@ #! /usr/bin/env perl - use Carp; use warnings; use strict; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +my $remote_root = '/scratch3/watersc1/'; + +my ($compmap_file); +GetOptions( + 'compmap=s' => \$compmap_file, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Required options: --compmap", -exitval => 3) unless + defined($compmap_file); + +my %component_map = (); +open(CM,$compmap_file) || die "Cannot find requested compmap file"; +while() { + chomp; + my ($symbolic_name,$disk_name) = split /\s+/; + $component_map{$symbolic_name} = $disk_name; +} +close(CM); + my $counter = 0; foreach my $path_base (@ARGV) { - print "INPUT${counter} METADATA\n"; - print " IMAGE STR ${path_base}.fits\n"; - print " MASK STR ${path_base}.mask.fits\n"; - print " VARIANCE STR ${path_base}.wt.fits\n"; - print " PSF STR ${path_base}.psf\n"; - print " SOURCES STR ${path_base}.cmf\n"; - print " BKGMODEL STR ${path_base}.mdl.fits\n"; - print "END\n\n"; + my $mdc_content = ""; + my $file = ""; + my $error = 0; + +# print "PB: $path_base\n"; + + #header + $mdc_content .= "INPUT${counter} METADATA\n"; + + # check each file against its possible locations, incrementing the error counter if not found + $file = do_checks("${path_base}.fits",$error); + $mdc_content .= " IMAGE STR $file\n"; + + $file = do_checks("${path_base}.mask.fits",$error); + $mdc_content .= " MASK STR $file\n"; + + $file = do_checks("${path_base}.wt.fits",$error); + $mdc_content .= " VARIANCE STR $file\n"; + + $file = do_checks("${path_base}.psf",$error); + $mdc_content .= " PSF STR $file\n"; + + $file = do_checks("${path_base}.cmf",$error); + $mdc_content .= " SOURCES STR $file\n"; + + # This is static, and isn't used by ppStack, but I believe needs to exist in the mdc. 2014-08-28 czw + $mdc_content .= " BKGMODEL STR ${path_base}.mdl.fits\n"; + $mdc_content .= "END\n\n"; +# print "MDC: $mdc_content\n"; + # If we had no errors, print it and move on. + if ($error == 0) { + print $mdc_content; + } + $counter++; } + +sub do_checks { + my ($cfile,$error) = @_; +# print ">$cfile<\n"; + if (! ((-e $cfile)&&(-s $cfile))) { # This cfile does not exist or is zero size. + my $disk = $component_map{$cfile}; + $cfile = "${disk}"; + if (! ((-e $cfile)&&(-s $cfile))) { # The other expected cfile did not exist. +# $cfile = ''; + $error++; + die "Missing cfile $cfile!"; + } + } + return($cfile); +} + Index: /trunk/ippScripts/scripts/sc_mk_staticsky_mdc.pl =================================================================== --- /trunk/ippScripts/scripts/sc_mk_staticsky_mdc.pl (revision 37833) +++ /trunk/ippScripts/scripts/sc_mk_staticsky_mdc.pl (revision 37833) @@ -0,0 +1,92 @@ +#! /usr/bin/env perl +use Carp; +use warnings; +use strict; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +my $remote_root = '/scratch3/watersc1/'; + +my ($compmap_file); +GetOptions( + 'compmap=s' => \$compmap_file, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Required options: --compmap", -exitval => 3) unless + defined($compmap_file); + +if (($#ARGV + 1) % 2 != 0) { + die "Do not have a matched pair of path_base/stack_id values $#ARGV"; +} + +my %component_map = (); +open(CM,$compmap_file) || die "Cannot find requested compmap file"; +while() { + chomp; + my ($symbolic_name,$disk_name) = split /\s+/; + $symbolic_name =~ s%//%/%g; + $disk_name =~ s%//%/%g; + $component_map{$symbolic_name} = $disk_name; +} +close(CM); + +my $counter = 0; + + +my $mdc_content .= "INPUT MULTI\n"; +print $mdc_content; + + +for (my $index = 0; $index <= $#ARGV; $index += 2) { + my $path_base = $ARGV[$index]; + my $stack_id = $ARGV[$index+1]; + my $mdc_content = ""; + my $file = ""; + my $error = 0; + $path_base =~ s%//%/%g; + + #header + $mdc_content .= "INPUT METADATA\n"; + $mdc_content .= " STACK_ID S64 " . $stack_id . "\n"; + + # check each file against its possible locations, incrementing the error counter if not found + $file = do_checks("${path_base}.unconv.fits",$error); + $mdc_content .= " RAW:IMAGE STR $file\n"; + + $file = do_checks("${path_base}.unconv.mask.fits",$error); + $mdc_content .= " RAW:MASK STR $file\n"; + + $file = do_checks("${path_base}.unconv.wt.fits",$error); + $mdc_content .= " RAW:VARIANCE STR $file\n"; + + $file = do_checks("${path_base}.unconv.num.fits",$error); + $mdc_content .= " RAW:EXPNUM STR $file\n"; + + $mdc_content .= "END\n\n"; + + # If we had no errors, print it and move on. + if ($error == 0) { + print $mdc_content; + } + + $counter++; +} + +sub do_checks { + my ($cfile,$error) = @_; +# print STDERR ">$cfile<\n"; + if (! ((-e $cfile)&&(-s $cfile))) { # This cfile does not exist or is zero size. + my $disk = $component_map{$cfile}; + $cfile = "${disk}"; + if (! ((-e $cfile)&&(-s $cfile))) { # The other expected cfile did not exist. +# $cfile = ''; +# $cfile = $disk; + $error++; + die "Missing cfile $cfile ($disk) $mdc_content!"; + } + } + return($cfile); +} + Index: /trunk/ippScripts/scripts/sc_prepare_camera.pl =================================================================== --- /trunk/ippScripts/scripts/sc_prepare_camera.pl (revision 37832) +++ /trunk/ippScripts/scripts/sc_prepare_camera.pl (revision 37833) @@ -1,3 +1,5 @@ #! /usr/bin/env perl + +# generate the input & output files lists and commands for a single camRun use Carp; @@ -16,21 +18,14 @@ use Pod::Usage qw( pod2usage ); - # Hard coded values -# my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -$remote_root = '/scratch3/watersc1/'; -my $threads = 2; # How many threads are we going to use? -my $job_cost = 1700 / 60 / 60; # Estimate of how long a job runs, in hours. -my $proc_per_node = 24; # processors per node -my $min_nodes = 1; # smallest allocation to ask for -my $max_nodes = 1000; # largest allocation to ask for -my $min_time = 1; # shortest allocation to ask for -my $max_time = 8; # longest allocation to ask for -my $remote_hostname= "LANL/Mustang"; # Name of the remote node. -my @return_component_list = ("DBINFO.EXP", "PSASTRO.CONFIG", "PSASTRO.OUTPUT", "LOG.EXP", "PSASTRO.STATS"); +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $remote_hostname = "LANL/Mustang"; # Name of the remote node. +my $threads_req = 0; # How many threads are we going to use? + +my $fail_state = "prep_fail"; + # Look for programs we need my $missing_tools; - my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); @@ -41,7 +36,8 @@ } -my ($remote_id,$cam_id,,$camera,$dbname,$verbose,$path_base,$no_update); +my ($remote_id,$cam_id,$camera,$dbname,$verbose,$path_base,$no_update); GetOptions( 'remote_id=s' => \$remote_id, + 'cam_id=s' => \$cam_id, 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, @@ -52,35 +48,18 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id --cam_id --camera --dbname --path_base", -exitval => 3) unless defined($remote_id) and + defined($cam_id) and defined($camera) and defined($path_base) and defined($dbname); -my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id); + +my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $cam_id, $PS_EXIT_CONFIG_ERROR, $fail_state); my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -# -# Step 1: Get a list of the components that make up this remoteRun - -# SHould this call listrun to ensure we're in state new? -my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; -$rt_cmd .= " -dbname $dbname " if defined($dbname); - -my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $rt_cmd, verbose => $verbose); -unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run remotetool to determine stage parameters.", - $remote_id,$error_code); -} -my $compData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine component information.", - $remote_id,$PS_EXIT_PROG_ERROR); -my $compData2= parse_md_list($compData); - -# -# Step 1b: Open output files -my $uri_command = $path_base . ".cmd"; +my @return_component_list = ("DBINFO.EXP", "PSASTRO.CONFIG", "PSASTRO.OUTPUT", "LOG.EXP", "PSASTRO.STATS"); + +# STEP 0: Open output files my $uri_transfer= $path_base . ".transfer"; my $uri_check = $path_base . ".check"; @@ -89,5 +68,4 @@ my $uri_return = $path_base . ".return"; -my $disk_command = $ipprc->file_resolve($uri_command,1); my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); my $disk_check = $ipprc->file_resolve($uri_check,1); @@ -96,93 +74,84 @@ my $disk_return = $ipprc->file_resolve($uri_return,1); -my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. - -open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id,$PS_EXIT_SYS_ERROR); -open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$remote_id,$PS_EXIT_SYS_ERROR); -open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$remote_id,$PS_EXIT_SYS_ERROR); -open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id,$PS_EXIT_SYS_ERROR); -open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $PS_EXIT_SYS_ERROR); +my (undef, $remote_config) = &uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check", $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config", $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); my %file_filter = (); -# -# Step 2: Iterate over all componenets in this remote run. my $job_index = 0; -my @pre_commands = (); -my @main_commands = (); -my @post_commands = (); - -foreach my $compEntry (@$compData2) { - my $cam_id = $compEntry->{stage_id}; - -# Get exposure level information from the camRun we're working from. - + +# STEP 1: Get exposure level information for this camRun +my ($workdir,$exp_tag,$reduction); +{ my $command = "$camtool -pendingexp -cam_id $cam_id "; $command .= " -dbname $dbname " if defined($dbname); - + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => 0); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - - &my_die("Unable to run camtool to determine stage parameters.", - $cam_id,$error_code); - } - - my $camData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine cam component information.", - $cam_id,$PS_EXIT_PROG_ERROR); - my $camData2= parse_md_list($camData); - my $camEntry = ${ $camData2 }[0]; - - my $workdir = $camEntry->{workdir}; - my $exp_tag = $camEntry->{exp_tag}; + + &my_die("Unable to run camtool to determine stage parameters.", $remote_id, $cam_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine cam component information.", $remote_id, $cam_id, $PS_EXIT_PROG_ERROR, $fail_state); + my $metadata = parse_md_list($MDlist); + my $camEntry = $metadata->[0]; + + $workdir = $camEntry->{workdir}; + $exp_tag = $camEntry->{exp_tag}; + $reduction = $camEntry->{reduction}; + $reduction = 'DEFAULT' unless defined $reduction; unless (defined($workdir)) { - while( my ($k, $v) = each %$camEntry ) { - print "key: $k, value: $v.\n"; - } - - print "%{ $camEntry }\n"; - die; - } - my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.cm.${cam_id}"; - my $remote_outroot = uri_local_to_remote($ipp_outroot); - - -# + while( my ($k, $v) = each %$camEntry ) { + print "key: $k, value: $v.\n"; + } + print "%{ $camEntry }\n"; + die; + } +} + +my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.cm.${cam_id}"; +my $remote_outroot = uri_local_to_remote($ipp_outroot); + # Step 3: Iterate over the sub-components - - my $reg_command = "$camtool -pendingimfile -cam_id $cam_id "; - $reg_command .= " -dbname $dbname " if defined($dbname); - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $reg_command, verbose => 0); +{ + my $command = "$camtool -pendingimfile -cam_id $cam_id "; + $command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run camtool -pendingimfile ", - $cam_id,$error_code); - } - + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run camtool -pendingimfile ", $remote_id, $cam_id, $error_code, $fail_state); + } # We don't actually care about the input cam data other than to know which mask files to instantiate. - my $inpData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine cam component information.", - $cam_id, $PS_EXIT_PROG_ERROR); - my $inpData2=parse_md_list($inpData); - my $chipProto= ${ $inpData2 }[0]; + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine cam component information.", $remote_id, $cam_id, $PS_EXIT_PROG_ERROR, $fail_state); + + my $metadata = parse_md_list($MDlist); + my $chipProto = $metadata->[0]; my $chip_path = $chipProto->{path_base}; - my $remote_chip_path = uri_local_to_remote( $chip_path ); + my $remote_chip_path = &uri_local_to_remote( $chip_path ); my $pre_cmd_cmfs = "ls -1 ${remote_chip_path}*.cmf > ${remote_outroot}.cmflist"; my $pre_cmd_masks= "ls -1 ${remote_chip_path}*.mk.fits > ${remote_outroot}.masklist"; - my $reduction = 'DEFAULT' unless defined $camEntry->{reduction}; - my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use + my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use + if ($recipe_psastro eq "") { + &my_die("Unable to determine PSASTRO recipe", $cam_id, $PS_EXIT_PROG_ERROR); + } my $psastro_command = " psastro -list ${remote_outroot}.cmflist "; $psastro_command .= " -masklist ${remote_outroot}.masklist ${remote_outroot} "; -# $psastro_command .= " -refmasklist ${remote_outroot}.masklist ${remote_outroot} "; # This is used to do edge calculations. It should probably be smart enough to know that it can just use the masks I just specified, but it's not. - # Uncomment when remote psastro supports this +# $psastro_command .= " -refmasklist ${remote_outroot}.masklist ${remote_outroot} "; + $psastro_command .= " -refmasklist /turquoise/usr/projects/ps1/watersc1/references/gpc1.refmask.list "; $psastro_command .= " -kh-correct /turquoise/usr/projects/ps1/watersc1/references/khcorrect.20140606.v0.fits "; - $psastro_command .= " -refmasklist /turquoise/usr/projects/ps1/watersc1/references/gpc1.refmask.list "; -# $psastro_command .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20080909.asm "; +# $psastro_command .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20080909.asm "; $psastro_command .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20140505.asm "; $psastro_command .= " -recipe PSASTRO $recipe_psastro "; @@ -190,7 +159,8 @@ $psastro_command .= " -dumpconfig ${remote_outroot}.psastro.mdc -stats ${remote_outroot}.stats "; $psastro_command .= " -recipe PPSTATS CAMSTATS "; - + my $camtool_post_cmd = "camtool -cam_id $cam_id -addprocessedexp -uri UNKNOWN "; - $camtool_post_cmd .= " -dbname $dbname " if defined $dbname; $camtool_post_cmd .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 "; + $camtool_post_cmd .= " -dbname $dbname " if defined $dbname; + $camtool_post_cmd .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 "; my $post_cmd_echo = " echo -n \"$camtool_post_cmd\" > ${remote_outroot}.dbinfo "; my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - CAMERA_EXP_FPA >> ${remote_outroot}.dbinfo "; @@ -201,14 +171,14 @@ # Determine which output files need to be returned foreach my $component(@return_component_list) { - my $filename = $ipprc->filename($component,$ipp_outroot); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); - my $remote_outroot_dir = dirname($ipp_disk); - print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; - } - foreach my $chipInfo (@{ $inpData2 }) { - my $filename = $ipprc->filename("PSASTRO.OUTPUT.MASK",$ipp_outroot,$chipInfo->{class_id}); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); - my $remote_outroot_dir = dirname($ipp_disk); - print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + my $filename = $ipprc->filename($component,$ipp_outroot); + my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + } + foreach my $chipInfo (@{ $metadata }) { + my $filename = $ipprc->filename("PSASTRO.OUTPUT.MASK",$ipp_outroot,$chipInfo->{class_id}); + my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; } print CONFIG "\n"; @@ -220,48 +190,14 @@ close(GENERATE); -# -# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. -my $proc_need = $job_index * $threads; # how many total processors do we need? -my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? -my $time_need = $job_index * $job_cost; # How many seconds will this take? - -my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill -my ($time_req,$node_req); -if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { - $time_req = $min_time; - $node_req = $min_nodes; -} -elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { - $time_req = $max_time; - $node_req = $max_nodes; - print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; -} -else { - $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; - $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; -} -$time_req += 1; # Safety addition. - -open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$remote_id,$PS_EXIT_SYS_ERROR); -print COMMAND "#!/bin/tcsh\n"; -print COMMAND "##### Moab controll lines\n"; -print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES -print COMMAND "#MSUB -j oe\n"; -print COMMAND "#MSUB -V\n"; -print COMMAND "#MSUB -o ${remote_root}cam.${remote_id}.out\n"; -print COMMAND "date\n"; -print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; -print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; -print COMMAND "date\n"; -close(COMMAND); - unless($no_update) { - my $command = "remotetool -updaterun -remote_id $remote_id "; - $command .= " -set_state pending "; - $command .= " -dbname $dbname " if defined $dbname; - + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $cam_id "; + $command .= " -set_jobs $job_index"; + $command .= " -set_path_base $path_base"; + $command .= " -set_state prep_done"; + $command .= " -dbname $dbname " if defined $dbname; system($command); } +exit (0); ## Common SC routines @@ -271,7 +207,7 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri ); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { - my_die(); + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); } @@ -287,7 +223,7 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { - my_die(); + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); } @@ -302,5 +238,5 @@ my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - + unless (exists($file_filter{$neb_uri})) { $file_filter{$neb_uri} = 1; @@ -332,5 +268,5 @@ return($remote_uri); } - + sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); # This needs to replace the remote root directory with the nebulous tag. @@ -338,5 +274,5 @@ $remote_uri =~ s%${remote_root}%%; my $local_uri = "neb:///" . $remote_uri; - + return($local_uri); } @@ -345,5 +281,6 @@ sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status); my $msg = shift; - my $id = shift; + my $remote_id = shift; + my $stage_id = shift; my $exit_code = shift; my $exit_state = shift; @@ -352,8 +289,7 @@ carp($msg); - - if (defined $id and not $no_update) { - my $command = "remotetool -updaterun -remote_id $id"; - $command .= " -fault $exit_code " if defined $exit_code; + + if (defined $remote_id and defined $stage_id and not $no_update) { + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; $command .= " -set_state $exit_state " if defined $exit_state; $command .= " -dbname $dbname " if defined $dbname; Index: /trunk/ippScripts/scripts/sc_prepare_chip.pl =================================================================== --- /trunk/ippScripts/scripts/sc_prepare_chip.pl (revision 37832) +++ /trunk/ippScripts/scripts/sc_prepare_chip.pl (revision 37833) @@ -1,3 +1,5 @@ #! /usr/bin/env perl + +# generate the input & output files lists and commands for a single chipRun use Carp; @@ -16,27 +18,20 @@ use Pod::Usage qw( pod2usage ); - # Hard coded values -# my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -$remote_root = '/scratch3/watersc1/'; -my $remote_raw = "${remote_root}/raw/"; # Directory to find raw data in. -my $threads = 2; # How many threads are we going to use? -my $job_cost = 150 / 60 / 60; # Estimate of how long a job runs, in hours. -my $proc_per_node = 24; # processors per node -my $min_nodes = 1; # smallest allocation to ask for -my $max_nodes = 1000; # largest allocation to ask for -my $min_time = 1; # shortest allocation to ask for -my $max_time = 8; # longest allocation to ask for - -# We need to ensure we only ever try to transfer a file once. -my %file_filter = (); +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $remote_hostname = "LANL/Mustang"; # Name of the remote node. +my $remote_raw = "${remote_root}/tmp/"; # Directory to find raw data in. +my $threads_req = 4; + +my $fail_state = "prep_fail"; +my $hard_fail_state = "fail"; # Look for programs we need my $missing_tools; -my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); my $detselect= can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $ipp_burntool_fix = can_run('ipp_apply_burntool_fix.pl') or (warn "Can't find ipp_apply_burntool_fix.pl" and $missing_tools = 1); if ($missing_tools) { @@ -46,7 +41,8 @@ # Options -my ($exp_id,$exp_name,$remote_id,$chip_id,$detrends,$camera,$dbname,$verbose,$path_base,$no_update); +my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose,$dbverbose); GetOptions( 'remote_id=s' => \$remote_id, + 'chip_id=s' => \$chip_id, 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, @@ -54,109 +50,33 @@ 'no_update' => \$no_update, 'verbose' => \$verbose, + 'dbverbose' => \$dbverbose, ) or pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id --chip_id --camera --dbname --path_base", -exitval => 3) unless defined($remote_id) and + defined($chip_id) and defined($camera) and defined($path_base) and defined($dbname); -my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id); + +my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -# -# Step 1: Get a list of the components that make up this remoteRun - -# SHould this call listrun to ensure we're in state new? -my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; -$rt_cmd .= " -dbname $dbname " if defined($dbname); - -my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $rt_cmd, verbose => $verbose); -unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - - &my_die("Unable to run chiptool to determine stage parameters.", - $remote_id,$error_code); -} -my $compData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine component information.", - $remote_id,$PS_EXIT_PROG_ERROR); -my $compData2= parse_md_list($compData); - -# -# Step 1a: Load up the catalog of detrends -# Detrend concept holders -my %detrends = (); -my %det_types = ('MASK' => '-mask', - 'FLAT' => '-flat', - 'DARK' => '-dark', - 'VIDEODARK' => '-dark', - 'LINEARITY' => '-linearity', - 'FRINGE' => '-fringe', - 'NOISEMAP' => '-noisemap'); - -foreach my $det_type (keys (%det_types)) { - my $dt_cmd = "$detselect -show -camera $camera -det_type $det_type "; - $dt_cmd .= " -dbname $dbname " if defined($dbname); - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $dt_cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("UNable to run detselect to determine detrend catalog", - $remote_id,$error_code); - } - my $detData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine detrend information.", - $remote_id,$PS_EXIT_PROG_ERROR); - my $detData2= parse_md_list($detData); - - # Data structure for this: - # $detrend{NAME} = \@list_of_detselect_shows - # $detrend{NAME}[0] = \%hash_of_detselect_constraint_results (first element) - # $detrend{NAME}[0]{class_id} = uri - - # At this point, I have the list of detselect shows, assign it - $detrends{$det_type} = $detData2; - - for (my $dd = 0; $dd <= $#{ @{ $detrends{$det_type} } }; $dd++) { - my $det_id = ${ $detrends{$det_type} }[$dd]->{det_id}; - my $det_iter=${ $detrends{$det_type} }[$dd]->{iteration}; - - my $detselect_command2 = "detselect -select "; - $detselect_command2 .= " -det_id $det_id "; - $detselect_command2 .= " -iteration $det_iter "; - $detselect_command2 .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $detselect_command2, verbose => 0); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - - &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", - $chip_id,$error_code); - } - - my $detImfile = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", - $chip_id,$error_code); - my $detImfile2= parse_md_list($detImfile); - foreach $detImfile (@$detImfile2) { - my $class_id = $detImfile->{class_id}; - my $duri = $detImfile->{uri}; - - ${ $detrends{$det_type} }[$dd]->{$class_id} = $duri; - } - } -} - -#print Dumper(%detrends); +# Detrend concept holders (name, ppImage option) +my %det_types = ( + 'MASK' => '-mask', + 'FLAT' => '-flat', + 'DARK' => '-dark', + 'VIDEODARK' => '-dark', + 'LINEARITY' => '-linearity', + 'FRINGE' => '-fringe', + 'NOISEMAP' => '-noisemap' + ); my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN"); -# -# Step 1b: Open output files -my $uri_command = $path_base . ".cmd"; +# STEP 0: Open output files my $uri_transfer= $path_base . ".transfer"; my $uri_check = $path_base . ".check"; @@ -165,5 +85,4 @@ my $uri_return = $path_base . ".return"; -my $disk_command = $ipprc->file_resolve($uri_command,1); my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); my $disk_check = $ipprc->file_resolve($uri_check,1); @@ -174,209 +93,230 @@ my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. -open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$chip_id,$PS_EXIT_SYS_ERROR); -open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR); -open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR); -open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR); -open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR); - - -# -# Step 2: Iterate over all componenets in this remote run. +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); + my $job_index = 0; -my @pre_commands = (); -my @main_commands = (); -my @post_commands = (); - -foreach my $compEntry (@$compData2) { - - my $chip_id = $compEntry->{stage_id}; - my $exp_id ; -# Get exposure level information from the chipRun we're working from. + +# STEP 1 : Get exposure level information for this chipRun +my ($exp_id, $filter, $altfilt, $dateobs); +{ my $command = "$chiptool -listrun -chip_id $chip_id "; $command .= " -dbname $dbname " if defined($dbname); - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $dbverbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - - &my_die("Unable to run chiptool to determine stage parameters.", - $chip_id,$error_code); - } - -# Parse chipRun level data to determine detrend information - my %detrends_to_use = (); - { - my $chipData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine chip component information.", - $chip_id,$PS_EXIT_PROG_ERROR); - my $chipData2= parse_md_list($chipData); - my $chipEntry = ${ $chipData2 }[0]; - - $exp_id = $chipEntry->{exp_id}; - my $filter = $chipEntry->{filter}; - my $altfilt= $filter; - $altfilt =~ s/.00000//; - my $dateobs= $chipEntry->{dateobs}; - my ($date, $time) = split /T/, $dateobs; - my ($year,$month,$day) = split /-/, $date; - my ($hour,$minute,$second) = split /:/, $time; #/; - my $dateobs_obj = DateTime->new(year => $year, month => $month, day => $day, hour => $hour, minute => $minute, second => $second); - - foreach my $det_type (%detrends) { - if (($filter !~ /y/)&&($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y - - foreach my $det_obj (@{ $detrends{$det_type} }) { # Iterate over the available options - if (defined($det_obj->{filter})) { - if (($det_obj->{filter} ne $filter)&&($det_obj->{filter} ne $altfilt)) { - next; # Skip detrends with a filter that we can't use. - } - } - if (defined($det_obj->{time_begin})) { - if (DateTime->compare($dateobs_obj,$det_obj->{time_begin}) == -1) { - next; # Skip detrends that come into effect after this exposure (d1 < d2) - } - } - if (defined($det_obj->{time_end})) { - if (DateTime->compare($dateobs_obj,$det_obj->{time_end}) == 1) { - next; # Skip detrends that stop working before this exposure (d1 > d2) - } - } - $detrends_to_use{$det_type} = $det_obj; - last; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool to determine stage parameters.", $remote_id, $chip_id, $error_code, $fail_state); + } + + # Parse chipRun level data to determine exposure information + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine chip component information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + my $metadata = parse_md_list($MDlist); + my $chipEntry = $metadata->[0]; + + $exp_id = $chipEntry->{exp_id}; + $filter = $chipEntry->{filter}; + $altfilt = $filter; + $altfilt =~ s/.00000//; + $dateobs = $chipEntry->{dateobs}; +} + +# STEP 2 : select the det_ids for each desired det_type (appropriate to this exposure) +my %det_ids = (); +my %det_iters = (); + +foreach my $det_type (keys (%det_types)) { + if (($filter !~ /y/) && ($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y + + my $command = "detselect -search -det_type $det_type -time $dateobs"; + $command .= " -dbname $dbname " if defined($dbname); + + if (($det_type eq 'FLAT') || ($det_type eq 'FRINGE')) { + $command .= " -filter $altfilt"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $dbverbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("No valid detrend available for this image: $det_type", $remote_id, $chip_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Could not parse detrend information for this image: $det_type", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + my $metadata = parse_md_list($MDlist); + my $detEntry = $metadata->[0]; + + if ($verbose) { + print STDERR "det_id: $detEntry->{det_id}\n"; + print STDERR "det_iter: $detEntry->{iteration}\n"; + } + + # save det_id and iteration + $det_ids{$det_type} = $detEntry->{det_id}; + $det_iters{$det_type} = $detEntry->{iteration}; +} + +# Step 3: Iterate over the sub-components +{ + # select chipProcessedImfile entries + my $command = "$chiptool -pendingimfile -chip_id $chip_id"; + $command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $dbverbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool -pendingimfile", $remote_id, $chip_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse chiptool -pendingimfile information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + my $metadata = parse_md_list($MDlist); + + # Iterate over the chipProcessedImfile level data. + foreach my $chipEntry (@$metadata) { + # Get information we need to pass to ppImage + my $uri = $chipEntry->{uri}; + my $class_id = $chipEntry->{class_id}; + my $video_cells = $chipEntry->{video_cells}; + my $reduction = $chipEntry->{reduction}; + my $exp_tag = $chipEntry->{exp_tag}; + my $workdir = $chipEntry->{workdir}; + my $chip_imfile_id = $chipEntry->{chip_imfile_id}; + + # Process the image and burntool table + # IMAGE CHECK: Look up the image location, prove it exists, and if not, abort the prep for this object. + my ($ipp_uri, $remote_uri) = uri_convert($uri); + unless (-e $ipp_uri) { + # Try to fix it. + system("neb-repair $uri"); + ($ipp_uri, $remote_uri) = uri_convert($uri); + unless (-e $ipp_uri) { + # This image file is missing, so it will require manual intervention. + &my_die("Required image file $ipp_uri ($chip_id, $class_id) not found. Stopping this prep.", + $remote_id,$chip_id,$PS_EXIT_CONFIG_ERROR,$hard_fail_state); } } - } - -# -# Step 3: Iterate over the sub-components -# Iterate over the chipProcessedImfile level data. - my $reg_command = "$chiptool -pendingimfile "; - $reg_command .= " -chip_id $chip_id " if defined($chip_id); - $reg_command .= " -dbname $dbname " if defined($dbname); - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $reg_command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - - &my_die("Unable to run chiptool -pendingimfile", - $chip_id,$error_code); - } - - my $chipData = $mdcParser->parse(join "", @$stdout_buf) or - next; -# &my_die("Unable to parse chiptool -pendingimfile information.", -# $chip_id,$error_code); - my $chipData2= parse_md_list($chipData); - - - foreach my $chipEntry (@$chipData2) { - # Get information we need to pass to ppImage - my $uri = $chipEntry->{uri}; - my $class_id = $chipEntry->{class_id}; - my $video_cells = $chipEntry->{video_cells}; - my $reduction = $chipEntry->{reduction}; - my $exp_tag = $chipEntry->{exp_tag}; - my $workdir = $chipEntry->{workdir}; - my $chip_imfile_id = $chipEntry->{chip_imfile_id}; - - # Process the image and burntool table - my ($ipp_uri,$remote_uri) = uri_to_outputs_raw($uri); - my $btt = $uri; - $btt =~ s/fits$/burn.tbl/; - my ($ipp_btt,$remote_btt) = uri_to_outputs_raw($btt); - - # Initialize the ppI command - my $ppImage_command = "ppImage -file $remote_uri"; - $ppImage_command .= " -burntool $remote_btt "; - - # Add detrend information to the command line - foreach my $det (keys %detrends_to_use) { - if ((($video_cells)&&($det eq 'DARK'))|| - ((!$video_cells)&&($det eq 'VIDEODARK'))) { - next; + + # The image exists, so continue. This resets the variables, but also outputs the file locations. + ($ipp_uri, $remote_uri) = uri_to_outputs_raw($uri); + my $btt = $uri; + $btt =~ s/fits$/burn.tbl/; + + # Check burntool table for existance, and if it doesn't, regenerate it. + my ($ipp_btt, $remote_btt) = uri_convert($btt); + unless (-e $ipp_btt) { + my $exp_name = $chipEntry->{exp_name}; + my $burntool_command = "$ipp_burntool_fix --exp_name $exp_name --class_id $class_id"; + $burntool_command .= " -dbname $dbname" if defined($dbname); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $burntool_command, verbose => $dbverbose); + # That command repairs the tables, but the instance we learned about may not be correct. + # Look up the new one. + ($ipp_btt, $remote_btt) = uri_convert($btt); + + unless (-e $ipp_btt) { + &my_die("Attempted regeneration of burntool table ($chip_id, $class_id) has failed. Stopping this prep.", + $remote_id,$chip_id, $PS_EXIT_CONFIG_ERROR, $hard_fail_state); } - my $det_code = $det_types{$det}; - my $duri = $detrends_to_use{$det}->{$class_id}; - my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri); - $ppImage_command .= " $det_code $remote_det_uri "; } - - - # Add output root - my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}"; - my $remote_outroot = uri_local_to_remote($ipp_outroot); - $ppImage_command .= " $remote_outroot "; - print STDERR "$remote_outroot $ipp_outroot $class_id\n"; - # Complete reduction information. - $reduction = 'DEFAULT' unless defined $reduction; - my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage - my $recipe_psphot = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot - - $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage "; - $ppImage_command .= " -recipe PSPHOT $recipe_psphot "; - $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats "; - $ppImage_command .= " -threads 4 "; - $ppImage_command .= " -image_id $chip_imfile_id "; - $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace "; - $ppImage_command .= " -log ${remote_outroot}.${class_id}.log "; - -# push @main_commands, $ppImage_command; - - # Calculate pre and post commands - my $remote_outroot_dir = dirname($remote_outroot); - my $pre_command = "mkdir -p $remote_outroot_dir"; - - my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; - $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; - $post_commandA .= " -magicked 0 -hostname REMOTE -dtime_script 0 "; - $post_commandA .= " -dbname $dbname " if defined $dbname; - - my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo "; - my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo"; - -# push @post_commands, "$post_commandB $post_commandC"; - - $job_index++; - - print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}"; - - - # Determine which output files need to be returned. - foreach my $component (@return_component_list) { -# uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) ); - my $filename = $ipprc->filename($component, $ipp_outroot, $class_id); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename ); - my $remote_outroot_dir = dirname($ipp_disk); - print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; -# print " $filename $ipp_disk $remote_disk\n"; -# die; - } - - print CONFIG "\n"; - } -} - -# Actually put the commands into the output file. -# my %unique_cmds = (); -# foreach my $cmd (@pre_commands) { -# unless(exists($unique_cmds{$cmd})) { -# print CONFIG $cmd . "\n"; -# $unique_cmds{$cmd} = 1; -# } -# } -# foreach my $cmd (@main_commands) { -# unless(exists($unique_cmds{$cmd})) { -# print CONFIG $cmd . "\n"; -# $unique_cmds{$cmd} = 1; -# } -# } -# foreach my $cmd (@post_commands) { -# unless(exists($unique_cmds{$cmd})) { -# print CONFIG $cmd . "\n"; -# $unique_cmds{$cmd} = 1; -# } -# } + ($ipp_btt, $remote_btt) = uri_to_outputs_raw($btt); + + # Initialize the ppI command + my $ppImage_command = "ppImage -file $remote_uri"; + $ppImage_command .= " -burntool $remote_btt "; + + foreach my $det_type (keys (%det_types)) { + if (( $video_cells) && ($det_type eq 'DARK')) { next; } + if ((!$video_cells) && ($det_type eq 'VIDEODARK')) { next; } + + my $det_id = $det_ids{$det_type}; + my $det_iter = $det_iters{$det_type}; + + if (not defined $det_id) { next; } + + if ($verbose) { print STDERR "det_type, det_id, det_iter: $det_type, $det_id, $det_iter\n"; } + + # Add detrend information to the command line + my $detselect_command = "detselect -select "; + $detselect_command .= " -det_id $det_id"; + $detselect_command .= " -iteration $det_iter"; + $detselect_command .= " -class_id $class_id"; + $detselect_command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $detselect_command, verbose => $dbverbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $error_code, $fail_state); + } + + my $detMDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + my $detImfileMD = parse_md_list($detMDlist); + my $detImfile = $detImfileMD->[0]; + my $duri = $detImfile->{uri}; + + my $det_code = $det_types{$det_type}; + my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri); + $ppImage_command .= " $det_code $remote_det_uri "; + } + + # Add output root + my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}"; + my $remote_outroot = uri_local_to_remote($ipp_outroot); + $ppImage_command .= " $remote_outroot "; + print STDERR "$remote_outroot $ipp_outroot $class_id\n"; + # Complete reduction information. + $reduction = 'DEFAULT' unless defined $reduction; + my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage + my $recipe_psphot = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot + + $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage "; + $ppImage_command .= " -recipe PSPHOT $recipe_psphot "; + $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats "; + $ppImage_command .= " -threads $threads_req "; + $ppImage_command .= " -image_id $chip_imfile_id "; + $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace "; + $ppImage_command .= " -log ${remote_outroot}.${class_id}.log "; + + # Calculate pre and post commands + my $remote_outroot_dir = dirname($remote_outroot); + my $pre_command = "mkdir -p $remote_outroot_dir"; + + my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; + $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; + $post_commandA .= " -magicked 0 -hostname $remote_hostname -dtime_script 0 "; + $post_commandA .= " -dbname $dbname " if defined $dbname; + + my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo "; + my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo"; + + $job_index++; + + print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}"; + + # Determine which output files need to be returned. + foreach my $component (@return_component_list) { + my $filename = $ipprc->filename($component, $ipp_outroot, $class_id); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename ); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + } + print CONFIG "\n"; + } +} close(CONFIG); @@ -386,42 +326,4 @@ close(GENERATE); -# -# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. -my $proc_need = $job_index * $threads; # how many total processors do we need? -my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? -my $time_need = $job_index * $job_cost; # How many seconds will this take? - -my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill -my ($time_req,$node_req); -if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { - $time_req = $min_time; - $node_req = $min_nodes; -} -elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { - $time_req = $max_time; - $node_req = $max_nodes; - print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; -} -else { - $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; - $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; -} -$time_req += 1; # Safety addition. - -open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$chip_id,$PS_EXIT_SYS_ERROR); -print COMMAND "#!/bin/tcsh\n"; -print COMMAND "##### Moab controll lines\n"; -print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES -print COMMAND "#MSUB -j oe\n"; -print COMMAND "#MSUB -V\n"; -print COMMAND "#MSUB -o ${remote_root}chip.${remote_id}.out\n"; -print COMMAND "date\n"; -print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; -print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; -#print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n"; -print COMMAND "date\n"; -close(COMMAND); - - ## We're done here. The execution and handling are done elsewhere. # Quick review: @@ -429,15 +331,13 @@ # auth unless($no_update) { - my $command = "remotetool -updaterun -remote_id $remote_id "; - $command .= " -set_state pending "; + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $chip_id "; + $command .= " -set_jobs $job_index"; + $command .= " -set_path_base $path_base"; + $command .= " -set_state prep_done"; $command .= " -dbname $dbname " if defined $dbname; system($command); } - - - - - +exit (0); sub uri_convert { @@ -445,7 +345,7 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri ); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { - my_die(); + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); } @@ -461,7 +361,7 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { - my_die(); + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); } @@ -476,10 +376,7 @@ my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print TRANSFER "$ipp_disk\n"; - print CHECK "$remote_disk\n"; - } + + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; return($ipp_disk,$remote_disk); } @@ -489,9 +386,7 @@ my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); $remote_disk = $remote_raw . $ipp_disk; - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print TRANSFER "$ipp_disk\n"; - print CHECK "$remote_disk\n"; - } + + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; return($ipp_disk,$remote_disk); } @@ -500,9 +395,6 @@ my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print RETURN "$ipp_disk\n"; - print GENERATE "$remote_disk\n"; - } + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; return($ipp_disk,$remote_disk); } @@ -518,5 +410,5 @@ return($remote_uri); } - + sub uri_remote_to_local { # This needs to replace the remote root directory with the nebulous tag. @@ -524,10 +416,12 @@ $remote_uri =~ s%${remote_root}%%; my $local_uri = "neb:///" . $remote_uri; - + return($local_uri); } + sub my_die { my $msg = shift; - my $id = shift; + my $remote_id = shift; + my $stage_id = shift; my $exit_code = shift; my $exit_state = shift; @@ -536,12 +430,11 @@ carp($msg); - - if (defined $id and not $no_update) { - my $command = "remotetool -updaterun -remote_id $id"; - $command .= " -fault $exit_code " if defined $exit_code; - $command .= " -set_state $exit_state " if defined $exit_state; - $command .= " -dbname $dbname " if defined $dbname; - - system($command); + + if (defined $remote_id and defined $stage_id and not $no_update) { + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); } Index: /trunk/ippScripts/scripts/sc_prepare_run.pl =================================================================== --- /trunk/ippScripts/scripts/sc_prepare_run.pl (revision 37833) +++ /trunk/ippScripts/scripts/sc_prepare_run.pl (revision 37833) @@ -0,0 +1,317 @@ +#! /usr/bin/env perl + +# this script takes a list of run components ready to be run +# (remoteComponent.state=prep_done) and generates the master run +# scripts for that set. Components which have +# (remoteComponent.state=prep_fail) are skipped + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use DateTime; +use Data::Dumper; +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Hard coded values +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = '/scratch3/watersc1/'; +my $remote_raw = "${remote_root}/tmp/"; # Directory to find raw data in. + +my $threads = 2; # How many threads are we going to use? + +my $fail_state = "prep_fail"; + +# Estimate of how long a job runs, in hours (depends on stage) +my %job_cost = (); +$job_cost{"camera"} = 1700 / 60 / 60; +$job_cost{"chip"} = 150 / 60 / 60; +$job_cost{"warp"} = 110 / 60 / 60; +$job_cost{"stack"} = 1500 / 60 / 60; +$job_cost{"staticsky"} = 2; # really? avg time of 5720s? + +# This object holds how "expensive" a given job is in terms of cores on a node. +# Passed to stask to limit how many jobs run simultaneously +my %job_subscription = (); +$job_subscription{"camera"} = 1; +$job_subscription{"chip"} = 1; +$job_subscription{"warp"} = 1; +$job_subscription{"stack"} = 3; +$job_subscription{"staticsky"} = 3; + +my $proc_per_node = 24; # processors per node +my $min_nodes = 1; # smallest allocation to ask for +my $max_nodes = 1000; # largest allocation to ask for +my $min_time = 1; # shortest allocation to ask for +my $max_time = 8; # longest allocation to ask for + +# We need to ensure we only ever try to transfer a file once. +my %file_filter = (); + +# Look for programs we need +my $missing_tools; +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Options +my ($remote_id,$stage,$camera,$path_base,$dbname,$verbose,$no_update); +GetOptions( + 'remote_id=s' => \$remote_id, + 'stage=s' => \$stage, + 'camera|c=s' => \$camera, + 'path_base=s' => \$path_base, + 'dbname|d=s' => \$dbname, + 'verbose' => \$verbose, + 'no_update' => \$no_update, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --stage --camera --dbname --path_base", -exitval => 3) unless + defined($remote_id) and + defined($stage) and + defined($camera) and + defined($path_base) and + defined($dbname); + +my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# STEP 1: Get a list of the components that make up this remoteRun +my $compData; +{ + my $command = "$remotetool -listcomponent -remote_id $remote_id -state prep_done"; + $command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run remotetool to determine stage parameters.", $remote_id, $error_code, $fail_state); + } + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine component information.", $remote_id, $PS_EXIT_PROG_ERROR, $fail_state); + $compData = parse_md_list($MDlist); +} + +# STEP 2: Open master output files +my $uri_command = $path_base . ".cmd"; +my $uri_transfer= $path_base . ".transfer"; +my $uri_check = $path_base . ".check"; +my $uri_config = $path_base . ".config"; +my $uri_generate= $path_base . ".generate"; +my $uri_return = $path_base . ".return"; + +my $disk_command = $ipprc->file_resolve($uri_command,1); +my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); +my $disk_check = $ipprc->file_resolve($uri_check,1); +my $disk_config = $ipprc->file_resolve($uri_config,1); +my $disk_generate= $ipprc->file_resolve($uri_generate,1); +my $disk_return = $ipprc->file_resolve($uri_return,1); + +my %have_transfer = (); +my %have_check = (); +my %have_config = (); +my %have_generate = (); +my %have_return = (); + +my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check", $remote_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config", $remote_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $PS_EXIT_SYS_ERROR, $fail_state); + +# Step 2: Iterate over all componenets in this remote run. +my $job_index = 0; +foreach my $compEntry (@$compData) { + my $line; + + my $stage_id = $compEntry->{stage_id}; + + my $in_path_base = $compEntry->{path_base}; + + my $state = $compEntry->{state}; + if ($state eq 'fail') { next; } + + my $uri_transfer = $in_path_base . ".transfer"; + my $disk_transfer= $ipprc->file_resolve($uri_transfer); + open(INPUT, "$disk_transfer"); + while ($line = ) { + unless ($have_transfer{$line}) { + $have_transfer{$line} = 1; + print TRANSFER $line; + } + } + close (INPUT); + + my $uri_check = $in_path_base . ".check"; + my $disk_check= $ipprc->file_resolve($uri_check); + open(INPUT, "$disk_check"); + while ($line = ) { + unless ($have_check{$line}) { + $have_check{$line} = 1; + print CHECK $line; + } + } + close (INPUT); + + my $uri_config = $in_path_base . ".config"; + my $disk_config= $ipprc->file_resolve($uri_config); + open(INPUT, "$disk_config"); + while ($line = ) { + unless ($have_config{$line}) { + $have_config{$line} = 1; + print CONFIG $line; + } + } + close (INPUT); + + my $uri_generate = $in_path_base . ".generate"; + my $disk_generate= $ipprc->file_resolve($uri_generate); + open(INPUT, "$disk_generate"); + while ($line = ) { + unless ($have_generate{$line}) { + $have_generate{$line} = 1; + print GENERATE $line; + } + } + close (INPUT); + + my $uri_return = $in_path_base . ".return"; + my $disk_return= $ipprc->file_resolve($uri_return); + open(INPUT, "$disk_return"); + while ($line = ) { + unless ($have_return{$line}) { + $have_return{$line} = 1; + print RETURN $line; + } + } + close (INPUT); + + if (($job_index % 100) == 0) { + print CONFIG "top -b -n 1\n"; + } + + $job_index += $compEntry->{jobs}; +} + +close(CONFIG); +close(TRANSFER); +close(CHECK); +close(RETURN); +close(GENERATE); + +# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. +my $proc_need = $job_index * $threads; # how many total processors do we need? +my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? +my $time_need = $job_index * $job_cost{$stage}; # How many seconds will this take? + +my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill +my ($time_req,$node_req); +if ($node_need * $job_cost{$stage} < $fill_factor * $min_nodes * $min_time) { + $time_req = $min_time; + $node_req = $min_nodes; +} +elsif ($node_need * $job_cost{$stage} > $fill_factor * $max_nodes * $max_time) { + $time_req = $max_time; + $node_req = $max_nodes; + print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost{$stage} time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; +} +else { + $time_req = int(($node_need * $job_cost{$stage}) / ($fill_factor * $max_nodes)) + 1; + $node_req = int(($node_need * $job_cost{$stage}) / ($fill_factor * $time_req)) + 1; +} + +# Most jobs only take 1 core, but ensure that we split them appropriately +$node_req *= $job_subscription{$stage}; + +if (($stage eq "stack")||($stage eq "staticsky")) { + $time_req += 2; +} else { + $time_req += 1; # Safety addition. +} + +open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command", $remote_id, $PS_EXIT_SYS_ERROR, $fail_state); +print COMMAND "#!/bin/tcsh\n"; +print COMMAND "##### Moab controll lines\n"; +print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES +print COMMAND "#MSUB -j oe\n"; +print COMMAND "#MSUB -V\n"; +print COMMAND "#MSUB -o ${remote_root}/stask_logs/${stage}.${remote_id}.out\n"; +print COMMAND "date\n"; +print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; +if (($stage eq "stack")||($stage eq "staticsky")) { + print COMMAND "${remote_root}/stask_stack.py $remote_config " . 'hosts.${SLURM_JOB_ID} ' . $job_subscription{$stage} . "\n"; +} +else { + print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} ' . $job_subscription{$stage} . "\n"; +} +print COMMAND "date\n"; +close(COMMAND); + +## We're done here. The execution and handling are done elsewhere. +# Quick review: +# new -> pending -> run -> full +# auth +unless($no_update) { + my $command = "remotetool -updaterun -remote_id $remote_id "; + $command .= " -set_state pending "; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); +} + +exit (0); + +sub uri_convert { + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $PS_EXIT_SYS_ERROR, $fail_state); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub my_die { + my $msg = shift; + my $remote_id = shift; + my $exit_code = shift; + my $exit_state = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $remote_id and not $no_update) { + my $command = "remotetool -updaterun -remote_id $remote_id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} + Index: /trunk/ippScripts/scripts/sc_prepare_stack.pl =================================================================== --- /trunk/ippScripts/scripts/sc_prepare_stack.pl (revision 37832) +++ /trunk/ippScripts/scripts/sc_prepare_stack.pl (revision 37833) @@ -1,3 +1,5 @@ #! /usr/bin/env perl + +# generate the input & output files lists and commands for a single stackRun use Carp; @@ -16,27 +18,16 @@ use Pod::Usage qw( pod2usage ); - # Hard coded values -#my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -$remote_root = '/scratch3/watersc1/'; -my $threads = 2; # How many threads are we going to use? -my $job_cost = 1500 / 60 / 60; # Estimate of how long a job runs, in hours. -my $proc_per_node = 24; # processors per node -my $min_nodes = 1; # smallest allocation to ask for -my $max_nodes = 1000; # largest allocation to ask for -my $min_time = 1; # shortest allocation to ask for -my $max_time = 8; # longest allocation to ask for -my $remote_hostname = "LANL/Mustang"; # Name of the remote node. -my @return_component_list = ("DBINFO.EXP", "PPSTACK.UNCONV","PPSTACK.UNCONV.MASK","PPSTACK.UNCONV.VARIANCE", - "PPSTACK.UNCONV.EXP","PPSTACK.UNCONV.EXPWT","PPSTACK.UNCONV.EXPNUM","PSPHOT.OUTPUT", - "LOG.EXP", "PSASTRO.STATS", - "PPSTACK.CONFIG","PPSTACK.TARGET.PSF","PSPHOT.STACK.PSF.SAVE"); -#,"PPSTACK.CONV.KERNEL"); +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $remote_hostname = "LANL/Mustang"; # Name of the remote node. +my $threads_req = 4; # How many threads are we going to use? + +my $fail_state = "prep_fail"; + # Look for programs we need my $missing_tools; - my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); -my $stacktool = can_run('stacktool') or (warn "Can't find warptool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); if ($missing_tools) { @@ -48,4 +39,5 @@ GetOptions( 'remote_id=s' => \$remote_id, + 'stack_id=s' => \$stack_id, 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, @@ -56,35 +48,23 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id --stack_id --camera --dbname --path_base", -exitval => 3) unless defined($remote_id) and + defined($stack_id) and defined($camera) and defined($path_base) and defined($dbname); -my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id); + +my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $stack_id, $PS_EXIT_CONFIG_ERROR, $fail_state); my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -# -# Step 1: Get a list of the components that make up this remoteRun - -# SHould this call listrun to ensure we're in state new? -my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; -$rt_cmd .= " -dbname $dbname " if defined($dbname); - -my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $rt_cmd, verbose => $verbose); -unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run remotetool to determine stage parameters.", - $remote_id,$error_code); -} -my $compData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine component information.", - $remote_id,$PS_EXIT_PROG_ERROR); -my $compData2= parse_md_list($compData); - -# -# Step 1b: Open output files -my $uri_command = $path_base . ".cmd"; +my @return_component_list = ("DBINFO.EXP", "PPSTACK.UNCONV","PPSTACK.UNCONV.MASK","PPSTACK.UNCONV.VARIANCE", + "PPSTACK.UNCONV.EXP","PPSTACK.UNCONV.EXPWT","PPSTACK.UNCONV.EXPNUM", + "PSPHOT.OUT.CMF.MEF", # Use this output product to get the one that exists. + "LOG.EXP", "PSASTRO.STATS", + "PPSTACK.CONFIG","PPSTACK.TARGET.PSF","PSPHOT.STACK.PSF.SAVE"); +#,"PPSTACK.CONV.KERNEL"); + +# STEP 0: Open output files my $uri_transfer= $path_base . ".transfer"; my $uri_check = $path_base . ".check"; @@ -93,5 +73,4 @@ my $uri_return = $path_base . ".return"; -my $disk_command = $ipprc->file_resolve($uri_command,1); my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); my $disk_check = $ipprc->file_resolve($uri_check,1); @@ -100,84 +79,102 @@ my $disk_return = $ipprc->file_resolve($uri_return,1); -my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. - -open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id,$PS_EXIT_SYS_ERROR); -open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$remote_id,$PS_EXIT_SYS_ERROR); -open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$remote_id,$PS_EXIT_SYS_ERROR); -open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id,$PS_EXIT_SYS_ERROR); -open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $PS_EXIT_SYS_ERROR); - - -# -# Step 2: Iterate over all componenets in this remote run. +my (undef, $remote_config) = &uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); + my $job_index = 0; -my @pre_commands = (); -my @main_commands = (); -my @post_commands = (); -my %file_filter = (); - - -foreach my $compEntry (@$compData2) { - my $stack_id = $compEntry->{stage_id}; - -# Get exposure level information from the stackRun we're working from. - my $run_command = "$stacktool -tosum -stack_id $stack_id"; - $run_command .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $run_command, verbose => 0); + +# STEP 1: Get exposure level information from the stackRun we're working from. +my ($reduction, $workdir, $tess_id, $skycell_id); +{ + my $command = "$stacktool -tosum -stack_id $stack_id"; + $command .= " -dbname $dbname " if defined($dbname); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run stacktool -tosum ", - $stack_id,$error_code); - } - - my $stackData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine stack component information.", - $stack_id,$PS_EXIT_PROG_ERROR); - my $stackData2= parse_md_list($stackData); - - my $stack = ${ $stackData2 }[0]; - my $reduction = $stack->{reduction}; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run stacktool -tosum ", $remote_id, $stack_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine stack component information.", $remote_id, $stack_id, $PS_EXIT_PROG_ERROR, $fail_state); + my $metadata = parse_md_list($MDlist); + + my $stack = $metadata->[0]; + + $reduction = $stack->{reduction}; $reduction = 'DEFAULT' unless defined($reduction); - my $workdir = $stack->{workdir}; - my $tess_id = $stack->{tess_id}; - my $skycell_id= $stack->{skycell_id}; - - my $ipp_outroot = "${workdir}/${tess_id}/${skycell_id}/${tess_id}.${skycell_id}.stk.${stack_id}"; - my $remote_outroot = uri_local_to_remote($ipp_outroot); - my $remote_outdir = uri_local_to_remote("${workdir}/${tess_id}/${skycell_id}"); + $workdir = $stack->{workdir}; + $tess_id = $stack->{tess_id}; + $skycell_id= $stack->{skycell_id}; +} +my $ipp_outroot = "${workdir}/${tess_id}/${skycell_id}/${tess_id}.${skycell_id}.stk.${stack_id}"; +my $remote_outroot = &uri_local_to_remote($ipp_outroot); +my $remote_outdir = &uri_local_to_remote("${workdir}/${tess_id}/${skycell_id}"); + +# STEP 2: Get the list of input warps +my ($warpData); +{ # This actually returns all the individual stack/skyfiles that comprise this run. Because consistency. my $command = "$stacktool -inputskyfile -stack_id $stack_id "; $command .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run stacktool -inputskyfile ", - $stack_id,$error_code); - } - - my $warpData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine stack component information.", - $stack_id,$PS_EXIT_PROG_ERROR); - my $warpData2= parse_md_list($warpData); - -# -# Step 3: Iterate over the sub-components + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run stacktool -inputskyfile ", $remote_id, $stack_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine stack component information.", $remote_id, $stack_id, $PS_EXIT_PROG_ERROR, $fail_state); + $warpData = parse_md_list($MDlist); +} + +# STEP 3: Iterate over the sub-components +{ + # Generate a file mapping the symbolic and "disk" representations of each needed file. + my $uri_compmap = "${ipp_outroot}.compmap"; + my $component_map = $ipprc->file_resolve($uri_compmap,1); + my ($ipp_compmap, $remote_compmap) = &uri_to_outputs($component_map); + open(COMPMAP,">$ipp_compmap") || &my_die("Couldn't open file? $ipp_compmap", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); + + # Loop over all needed components for this stack. my $warp_path_base_string = ""; - foreach my $warpEntry ( @{ $warpData2 } ) { - my $warp_path_base = $warpEntry->{path_base}; - my $remote_path_base = uri_local_to_remote($warp_path_base); - $warp_path_base_string .= " $remote_path_base "; - - # Append file names to transfer lists so we can recover if warps are deleted. - uri_to_outputs($warp_path_base . ".fits"); - uri_to_outputs($warp_path_base . ".wt.fits"); - uri_to_outputs($warp_path_base . ".mask.fits"); - uri_to_outputs($warp_path_base . ".cmf"); -# uri_to_outputs($warp_path_base . ".psf"); - } - my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl $warp_path_base_string > ${remote_outroot}.in.mdc"; - my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use + foreach my $warpEntry ( @{ $warpData } ) { + my $warp_path_base = $warpEntry->{path_base}; + my $remote_path_base = &uri_local_to_remote($warp_path_base); + $warp_path_base_string .= " $remote_path_base "; + + if ($verbose) { print STDERR "$warp_path_base_string\n"; } + + # Append file names to transfer lists so we can recover if warps are deleted. + my $remote_file; + my $ipp_disk; + # ${remote_root}/tmp/${ipp_disk} + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".fits"); + ($ipp_disk,undef) = &uri_convert($warp_path_base . ".fits"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".wt.fits"); + ($ipp_disk,undef) = &uri_convert($warp_path_base . ".wt.fits"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".mask.fits"); + ($ipp_disk,undef) = &uri_convert($warp_path_base . ".mask.fits"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".cmf"); + ($ipp_disk,undef) = &uri_convert($warp_path_base . ".cmf"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".psf"); + ($ipp_disk,undef) = &uri_convert($warp_path_base . ".psf"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + } + + close(COMPMAP); + + my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl --compmap ${remote_root}/tmp/${component_map} $warp_path_base_string > ${remote_outroot}.in.mdc"; + my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use my $ppstack_command = " ppStack -input ${remote_outroot}.in.mdc "; $ppstack_command .= " ${remote_outroot} -stats ${remote_outroot}.stats "; @@ -191,5 +188,5 @@ $ppstack_command .= " -R PPSTACK.UNCONV.VARIANCE FITS.TYPE COMP_STACK"; $ppstack_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; - $ppstack_command .= " -threads 4 "; + $ppstack_command .= " -threads $threads_req "; $ppstack_command .= " -dumpconfig ${remote_outroot}.mdc "; $ppstack_command .= " -stack_id $stack_id -skycell_id $skycell_id -tess_id $tess_id "; @@ -199,5 +196,5 @@ $post_cmd_echo .= " -dbname $dbname " if defined $dbname; $post_cmd_echo .= " -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; - + my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - STACK_SKYCELL >> ${remote_outroot}.dbinfo "; @@ -207,13 +204,9 @@ # Determine which output files need to be returned foreach my $component(@return_component_list) { - my $filename = $ipprc->filename($component,$ipp_outroot); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); - my $remote_outroot_dir = dirname($ipp_disk); - print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; - } - -# my $cleanup_cmd = " (rm -f /scratch3/watersc1/sys_temp/${tess_id}.${skycell_id}.stk.${stack_id}.*conv.*.fits || true) "; -# print CONFIG " && ${cleanup_cmd} \n"; - + my $filename = $ipprc->filename($component,$ipp_outroot); + my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + } print CONFIG "\n"; } @@ -224,44 +217,9 @@ close(GENERATE); -# -# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. -my $proc_need = $job_index * $threads; # how many total processors do we need? -my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? -my $time_need = $job_index * $job_cost; # How many seconds will this take? - -my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill -my ($time_req,$node_req); -if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { - $time_req = $min_time; - $node_req = $min_nodes; -} -elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { - $time_req = $max_time; - $node_req = $max_nodes; - print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; -} -else { - $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; - $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; -} - -$time_req += 2; -$node_req *= 3; -open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$remote_id,$PS_EXIT_SYS_ERROR); -print COMMAND "#!/bin/tcsh\n"; -print COMMAND "##### Moab controll lines\n"; -print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES -print COMMAND "#MSUB -j oe\n"; -print COMMAND "#MSUB -V\n"; -print COMMAND "#MSUB -o ${remote_root}stack.${remote_id}.out\n"; -print COMMAND "date\n"; -print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; -print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 3' . "\n"; -print COMMAND "date\n"; -close(COMMAND); - unless($no_update) { - my $command = "remotetool -updaterun -remote_id $remote_id "; - $command .= " -set_state pending "; + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stack_id "; + $command .= " -set_jobs $job_index"; + $command .= " -set_path_base $path_base"; + $command .= " -set_state prep_done"; $command .= " -dbname $dbname " if defined $dbname; @@ -269,14 +227,14 @@ } - ## Common SC routines +# KEEP sub uri_convert { # (ipp_disk,remote_disk) = uri_convert(neb_uri); my $neb_uri = shift; my $ipp_disk= $ipprc->file_resolve( $neb_uri ); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { - my_die(); + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); } @@ -288,11 +246,12 @@ } +# KEEP sub uri_convert_and_create { # (ipp_disk,remote_disk) = uri_convert_and_create(neb_uri); ipp_disk is created if it doesn't exist my $neb_uri = shift; my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { - my_die(); + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); } @@ -304,27 +263,25 @@ } +# KEEP sub uri_to_outputs { # (ipp_disk,remote_disk) = uri_to_output(neb_uri); Appends to TRANSFER and CHECK filehandles my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print TRANSFER "$ipp_disk\n"; - print CHECK "$remote_disk\n"; - } + + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; return($ipp_disk,$remote_disk); } +# KEEP sub uri_to_outputs_for_return { # (ipp_disk,remote_disk) = uri_to_outputs_for_return(neb_uri); create ipp_disk, append to RETURN and GENERATE my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print RETURN "$ipp_disk\n"; - print GENERATE "$remote_disk\n"; - } + + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; return($ipp_disk,$remote_disk); } +# KEEP sub uri_local_to_remote { #(remote_uri) = uri_local_to_remote(local_neb_uri); # This needs to replace the nebulous tag with the remote root. @@ -337,18 +294,9 @@ return($remote_uri); } - -sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); - # This needs to replace the remote root directory with the nebulous tag. - my $remote_uri = shift; - $remote_uri =~ s%${remote_root}%%; - my $local_uri = "neb:///" . $remote_uri; - - return($local_uri); -} - sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status); my $msg = shift; - my $id = shift; + my $remote_id = shift; + my $stage_id = shift; my $exit_code = shift; my $exit_state = shift; @@ -357,8 +305,7 @@ carp($msg); - - if (defined $id and not $no_update) { - my $command = "remotetool -updaterun -remote_id $id"; - $command .= " -fault $exit_code " if defined $exit_code; + + if (defined $remote_id and defined $stage_id and not $no_update) { + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; $command .= " -set_state $exit_state " if defined $exit_state; $command .= " -dbname $dbname " if defined $dbname; Index: /trunk/ippScripts/scripts/sc_prepare_staticsky.pl =================================================================== --- /trunk/ippScripts/scripts/sc_prepare_staticsky.pl (revision 37833) +++ /trunk/ippScripts/scripts/sc_prepare_staticsky.pl (revision 37833) @@ -0,0 +1,338 @@ +#! /usr/bin/env perl + +# generate the input & output files lists and commands for a single staticskyRun. + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use DateTime; +use Data::Dumper; +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Hard coded values +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $remote_hostname = "LANL/Mustang"; # Name of the remote node. +my $threads_req = 4; # How many threads are we going to use? + +my $fail_state = "prep_fail"; + +# Look for programs we need +my $missing_tools; +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $staticskytool= can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($remote_id,$sky_id,$camera,$dbname,$verbose,$path_base,$no_update); +GetOptions( + 'remote_id=s' => \$remote_id, + 'sky_id=s' => \$sky_id, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, + 'path_base=s' => \$path_base, + 'no_update' => \$no_update, + 'verbose' => \$verbose, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --sky_id --camera --dbname --path_base", -exitval => 3) unless + defined($remote_id) and + defined($sky_id) and + defined($camera) and + defined($path_base) and + defined($dbname); + +my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $sky_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my @return_component_list = ("LOG.EXP","PSPHOT.STACK.CONFIG","DBINFO.EXP"); + + # "PSPHOT.CHISQ.IMAGE","PSPHOT.CHISQ.MASK","PSPHOT.CHISQ.VARIANCE"); + +my @return_component_list_per_input = (#"PSPHOT.STACK.OUTPUT.IMAGE","PSPHOT.STACK.OUTPUT.MASK","PSPHOT.STACK.OUTPUT.VARIANCE","PSPHOT.STACK.OUTPUT"); + "PSPHOT.STACK.OUTPUT", + "PSPHOT.STACK.PSF.SAVE", + "PSPHOT.STACK.BACKMDL"); + +# STEP 0: Open output files +my $uri_transfer= $path_base . ".transfer"; +my $uri_check = $path_base . ".check"; +my $uri_config = $path_base . ".config"; +my $uri_generate= $path_base . ".generate"; +my $uri_return = $path_base . ".return"; + +my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); +my $disk_check = $ipprc->file_resolve($uri_check,1); +my $disk_config = $ipprc->file_resolve($uri_config,1); +my $disk_generate= $ipprc->file_resolve($uri_generate,1); +my $disk_return = $ipprc->file_resolve($uri_return,1); + +my (undef, $remote_config) = &uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id, $sky_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check", $remote_id, $sky_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config", $remote_id, $sky_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id, $sky_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $sky_id, $PS_EXIT_SYS_ERROR, $fail_state); + +my $job_index = 0; + +# STEP 1: Get exposure level information from the stackRun we're working from. +my ($reduction, $workdir, $tess_id, $skycell_id); +{ + my $command = "$staticskytool -todo -sky_id $sky_id"; + $command .= " -dbname $dbname " if defined($dbname); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run staticskytool -inputs ", $remote_id, $sky_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine stack component information.", $remote_id, $sky_id, $PS_EXIT_PROG_ERROR, $fail_state); + my $metadata = parse_md_list($MDlist); + + my $stack = $metadata->[0]; + + $reduction = $stack->{reduction}; + $reduction = 'DEFAULT' unless defined($reduction); + $workdir = $stack->{workdir}; + $tess_id = $stack->{tess_id}; + $skycell_id= $stack->{skycell_id}; +} +my $ipp_outroot = "${workdir}/${tess_id}/${skycell_id}/${tess_id}.${skycell_id}.sky.${sky_id}"; +my $remote_outroot = &uri_local_to_remote($ipp_outroot); +my $remote_outdir = &uri_local_to_remote("${workdir}/${tess_id}/${skycell_id}"); + +# STEP 2: Get the list of inputs +my ($inputData); +{ + # This actually returns all the individual stack/skyfiles that comprise this run. Because consistency. + my $command = "$staticskytool -inputs -sky_id $sky_id "; + $command .= " -dbname $dbname " if defined($dbname); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run stacktool -inputskyfile ", $remote_id, $sky_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine stack component information.", $remote_id, $sky_id, $PS_EXIT_PROG_ERROR, $fail_state); + $inputData = parse_md_list($MDlist); +} + +# STEP 3: Iterate over the sub-components +{ + # Generate a file mapping the symbolic and "disk" representations of each needed file. + my $uri_compmap = "${ipp_outroot}.compmap"; + my $component_map = $ipprc->file_resolve($uri_compmap,1); + my ($ipp_compmap, $remote_compmap) = &uri_to_outputs($component_map); + open(COMPMAP,">$ipp_compmap") || &my_die("Couldn't open file? $ipp_compmap", $remote_id, $sky_id, $PS_EXIT_SYS_ERROR, $fail_state); + + # Loop over all needed components for this stack. + my $nInputs = 0; + my $input_path_base_string = ""; + foreach my $inputEntry ( @{ $inputData } ) { + my $input_stack_id = $inputEntry->{stack_id}; + + my $input_path_base = $inputEntry->{path_base}; + my $remote_path_base = &uri_local_to_remote($input_path_base); + $input_path_base_string .= " $remote_path_base "; + $input_path_base_string .= " $input_stack_id "; + if ($verbose) { print STDERR "$input_path_base_string\n"; } + + # Append file names to transfer lists so we can recover if inputs are deleted. + my $remote_file; + my $ipp_disk; + # ${remote_root}/tmp/${ipp_disk} + (undef,$remote_file) = &uri_to_outputs($input_path_base . ".unconv.fits"); + ($ipp_disk,undef) = &uri_convert($input_path_base . ".unconv.fits"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + (undef,$remote_file) = &uri_to_outputs($input_path_base . ".unconv.wt.fits"); + ($ipp_disk,undef) = &uri_convert($input_path_base . ".unconv.wt.fits"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + (undef,$remote_file) = &uri_to_outputs($input_path_base . ".unconv.mask.fits"); + ($ipp_disk,undef) = &uri_convert($input_path_base . ".unconv.mask.fits"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + (undef,$remote_file) = &uri_to_outputs($input_path_base . ".unconv.num.fits"); + ($ipp_disk,undef) = &uri_convert($input_path_base . ".unconv.num.fits"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; +# (undef,$remote_file) = &uri_to_outputs($input_path_base . ".cmf"); +# ($ipp_disk,undef) = &uri_convert($input_path_base . ".cmf"); +# print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; +# (undef,$remote_file) = &uri_to_outputs($input_path_base . ".psf"); +# ($ipp_disk,undef) = &uri_convert($input_path_base . ".psf"); +# print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + + $nInputs++; + } + + close(COMPMAP); + + my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_staticsky_mdc.pl --compmap ${remote_root}/tmp/${component_map} $input_path_base_string > ${remote_outroot}.in.mdc"; + my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_PSPHOT'); + my $recipe_ppsub = $ipprc->reduction($reduction, 'STACKPHOT_PPSUB'); + my $recipe_ppstack= $ipprc->reduction($reduction, 'STACKPHOT_PPSTACK'); # Recipe to use + + my $pps_command = " psphotStack -input ${remote_outroot}.in.mdc "; + $pps_command .= " ${remote_outroot} "; + $pps_command .= " -recipe PSPHOT $recipe_psphot "; + $pps_command .= " -recipe PPSUB $recipe_ppsub "; + $pps_command .= " -recipe PPSTACK $recipe_ppstack "; + $pps_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; + $pps_command .= " -threads $threads_req "; + $pps_command .= " -dumpconfig ${remote_outroot}.psphotStack.mdc "; + + my $post_cmd_echo = " echo -n \"staticskytool -addresult -sky_id $sky_id "; + $post_cmd_echo .= " -path_base $ipp_outroot -num_inputs $nInputs "; + $post_cmd_echo .= " -dbname $dbname " if defined $dbname; + $post_cmd_echo .= " -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; + +# CZW: 2014-09-26 We currently do not save any information into the database here, so I'm leaving the dummy from stack. +# my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - STACK_SKYCELL >> ${remote_outroot}.dbinfo "; +# print CONFIG "${mk_mdc_command} && ${pps_command} && ${post_cmd_echo} && ${post_cmd_SfM} "; + + print CONFIG "${mk_mdc_command} && ${pps_command} && ${post_cmd_echo} "; + $job_index++; + + # Determine which output files need to be returned + foreach my $component(@return_component_list) { + my $filename = $ipprc->filename($component,$ipp_outroot); + my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + } + # But this creates outputs for each input stack, so iterate over those as well. + foreach my $inputEntry (@{ $inputData }) { + my $stack_id = $inputEntry->{stack_id}; + foreach my $component (@return_component_list_per_input) { + my $filename = $ipprc->filename($component,$ipp_outroot,$stack_id); + my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename ); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + } + } + + print CONFIG "\n"; +} +close(CONFIG); +close(TRANSFER); +close(CHECK); +close(RETURN); +close(GENERATE); + +unless($no_update) { + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $sky_id "; + $command .= " -set_jobs $job_index"; + $command .= " -set_path_base $path_base"; + $command .= " -set_state prep_done"; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); +} + +## Common SC routines + +# KEEP +sub uri_convert { # (ipp_disk,remote_disk) = uri_convert(neb_uri); + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $sky_id, $PS_EXIT_SYS_ERROR, $fail_state); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +# KEEP +sub uri_convert_and_create { # (ipp_disk,remote_disk) = uri_convert_and_create(neb_uri); ipp_disk is created if it doesn't exist + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $sky_id, $PS_EXIT_SYS_ERROR, $fail_state); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +# KEEP +sub uri_to_outputs { # (ipp_disk,remote_disk) = uri_to_output(neb_uri); Appends to TRANSFER and CHECK filehandles + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + return($ipp_disk,$remote_disk); +} + +# KEEP +sub uri_to_outputs_for_return { # (ipp_disk,remote_disk) = uri_to_outputs_for_return(neb_uri); create ipp_disk, append to RETURN and GENERATE + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); + + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; + return($ipp_disk,$remote_disk); +} + +# KEEP +sub uri_local_to_remote { #(remote_uri) = uri_local_to_remote(local_neb_uri); + # This needs to replace the nebulous tag with the remote root. + my $local_uri = shift; + $local_uri =~ s%^.*?/%%; # neb:/ + $local_uri =~ s%^.*?/%%; # / + $local_uri =~ s%^.*?/%%; # @HOST@.0/ + my $remote_uri = "${remote_root}/" . $local_uri; + + return($remote_uri); +} + +sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status); + my $msg = shift; + my $remote_id = shift; + my $stage_id = shift; + my $exit_code = shift; + my $exit_state = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $remote_id and defined $stage_id and not $no_update) { + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} Index: /trunk/ippScripts/scripts/sc_prepare_warp.pl =================================================================== --- /trunk/ippScripts/scripts/sc_prepare_warp.pl (revision 37832) +++ /trunk/ippScripts/scripts/sc_prepare_warp.pl (revision 37833) @@ -1,3 +1,5 @@ #! /usr/bin/env perl + +# generate the input & output files lists and commands for a single warpRun use Carp; @@ -16,21 +18,14 @@ use Pod::Usage qw( pod2usage ); - # Hard coded values -# my $remote_root = '/scratch3/watersc1/'; -my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -$remote_root = '/scratch3/watersc1/'; -my $threads = 2; # How many threads are we going to use? -my $job_cost = 110 / 60 / 60; # Estimate of how long a job runs, in hours. -my $proc_per_node = 24; # processors per node -my $min_nodes = 1; # smallest allocation to ask for -my $max_nodes = 1000; # largest allocation to ask for -my $min_time = 1; # shortest allocation to ask for -my $max_time = 8; # longest allocation to ask for -my $remote_hostname= "LANL/Mustang"; # Name of the remote node. -my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SKY.SAVE","LOG.EXP"); +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = '/scratch3/watersc1/'; +my $remote_hostname = "LANL/Mustang"; # Name of the remote node. +my $threads_req = 4; # How many threads are we going to use? + +my $fail_state = "prep_fail"; + # Look for programs we need my $missing_tools; - my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); @@ -41,7 +36,8 @@ } -my ($remote_id,$warp_id,,$camera,$dbname,$verbose,$path_base,$no_update); +my ($remote_id,$warp_id,$camera,$dbname,$verbose,$path_base,$no_update); GetOptions( 'remote_id=s' => \$remote_id, + 'warp_id=s' => \$warp_id, 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, @@ -52,35 +48,18 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id -warp_id --camera --dbname --path_base", -exitval => 3) unless defined($remote_id) and + defined($warp_id) and defined($camera) and defined($path_base) and defined($dbname); -my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id); + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id, $warp_id, $PS_EXIT_CONFIG_ERROR, $fail_state); my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -# -# Step 1: Get a list of the components that make up this remoteRun - -# SHould this call listrun to ensure we're in state new? -my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; -$rt_cmd .= " -dbname $dbname " if defined($dbname); - -my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $rt_cmd, verbose => $verbose); -unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run remotetool to determine stage parameters.", - $remote_id,$error_code); -} -my $compData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine component information.", - $remote_id,$PS_EXIT_PROG_ERROR); -my $compData2= parse_md_list($compData); - -# -# Step 1b: Open output files -my $uri_command = $path_base . ".cmd"; +my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SKY.SAVE","LOG.EXP"); + +# STEP 0: Open output files my $uri_transfer= $path_base . ".transfer"; my $uri_check = $path_base . ".check"; @@ -89,5 +68,4 @@ my $uri_return = $path_base . ".return"; -my $disk_command = $ipprc->file_resolve($uri_command,1); my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); my $disk_check = $ipprc->file_resolve($uri_check,1); @@ -96,136 +74,132 @@ my $disk_return = $ipprc->file_resolve($uri_return,1); -my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. - -open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id,$PS_EXIT_SYS_ERROR); -open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$remote_id,$PS_EXIT_SYS_ERROR); -open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$remote_id,$PS_EXIT_SYS_ERROR); -open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id,$PS_EXIT_SYS_ERROR); -open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $PS_EXIT_SYS_ERROR); - - -# -# Step 2: Iterate over all componenets in this remote run. +my (undef, $remote_config) = &uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); + my $job_index = 0; -my @pre_commands = (); -my @main_commands = (); -my @post_commands = (); - -my %file_filter = (); - -foreach my $compEntry (@$compData2) { - my $warp_id = $compEntry->{stage_id}; - -# Get exposure level information from the warpRun we're working from. + +# STEP 1: Get exposure level information from the warpRun we're working from. +my ($warpData); +{ # This actually returns all the individual warp/skyfiles that comprise this run. Because consistency. my $command = "$warptool -towarped -warp_id $warp_id "; $command .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run warptool -pendingimfile ", - $warp_id,$error_code); - } - my $warpData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine warp component information.", - $warp_id,$PS_EXIT_PROG_ERROR); - my $warpData2= parse_md_list($warpData); - -# -# Step 3: Iterate over the sub-components - foreach my $warpEntry ( @{ $warpData2 } ) { - my $workdir = $warpEntry->{workdir}; - my $exp_tag = $warpEntry->{exp_tag}; - my $skycell_id = $warpEntry->{skycell_id}; - my $tess_id = $warpEntry->{tess_id}; - my $reduction = 'DEFAULT' unless defined $warpEntry->{reduction}; - my $recipe_pswarp = $ipprc->reduction($reduction, 'WARP_PSWARP'); # Recipe to use - my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.wrp.${warp_id}.${skycell_id}"; - print "$ipp_outroot\n"; - my $remote_outroot = uri_local_to_remote($ipp_outroot); - - my $reg_command = "$warptool -scmap -warp_id $warp_id -skycell_id ${skycell_id}"; - $reg_command .= " -dbname $dbname " if defined($dbname); - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $reg_command, verbose => 0); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run warptool -pendingimfile: $reg_command ", - $warp_id,$error_code); - } - - my $inpData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine warp component information.", - $warp_id, $PS_EXIT_PROG_ERROR); - my $inpData2=parse_md_list($inpData); - - my $pre_cmd_ims = "ls -1 "; - my $pre_cmd_masks= "ls -1 "; - my $pre_cmd_vars = "ls -1 "; - my $pre_cmd_astrom = ""; - - foreach my $imfile (@{ $inpData2 }) { - my $image = $ipprc->filename("PPIMAGE.CHIP", $imfile->{chip_path_base}, $imfile->{class_id}); - my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); - my $var = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); - - my $astrom= $ipprc->filename("PSASTRO.OUTPUT", $imfile->{cam_path_base}); - - my $remote_image = uri_local_to_remote($image); - my $remote_mask = uri_local_to_remote($mask); - my $remote_var = uri_local_to_remote($var); - my $remote_astrom= uri_local_to_remote($astrom); - - $pre_cmd_ims .= " $remote_image "; - $pre_cmd_masks .= " $remote_mask "; - $pre_cmd_vars .= " $remote_var "; - $pre_cmd_astrom = "echo $remote_astrom > ${remote_outroot}.astrom "; - } - $pre_cmd_ims .= " > ${remote_outroot}.imlist "; - $pre_cmd_masks .= " > ${remote_outroot}.masklist "; - $pre_cmd_vars .= " > ${remote_outroot}.varlist "; - - my $skycell_command = " dvoImageExtract -D CATDIR /turquoise/usr/projects/ps1/watersc1/tess/${tess_id} $skycell_id -o ${remote_outroot}.skyfile "; - - my $pswarp_command = " pswarp -list ${remote_outroot}.imlist "; - $pswarp_command .= " -masklist ${remote_outroot}.masklist -variancelist ${remote_outroot}.varlist "; - $pswarp_command .= " -astromlist ${remote_outroot}.astrom "; - $pswarp_command .= " ${remote_outroot} ${remote_outroot}.skyfile "; - $pswarp_command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE "; - $pswarp_command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF "; - $pswarp_command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF "; - $pswarp_command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS "; - $pswarp_command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL "; - $pswarp_command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID "; - $pswarp_command .= " -recipe PSWARP $recipe_pswarp "; - $pswarp_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; - $pswarp_command .= " -threads 4 "; # -image_id ${image_id} -source_id ${source_id} "; - $pswarp_command .= " -recipe PPSTATS WARPSTATS "; - $pswarp_command .= " -dumpconfig ${remote_outroot}.pswarp.mdc -stats ${remote_outroot}.stats "; - -# print "$pswarp_command \n"; - - my $post_cmd_echo = " echo -n \"warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id "; - $post_cmd_echo .= " -path_base $remote_outroot -uri UNKNOWN "; - $post_cmd_echo .= " -dbname $dbname " if defined $dbname; - $post_cmd_echo .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; - - my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - WARP_SKYCELL >> ${remote_outroot}.dbinfo "; - - print CONFIG "${pre_cmd_ims} && ${pre_cmd_masks} && ${pre_cmd_vars} && ${pre_cmd_astrom} && ${skycell_command} && ${pswarp_command} && ${post_cmd_echo} && ${post_cmd_SfM} "; - $job_index++; - - # Determine which output files need to be returned - foreach my $component(@return_component_list) { - my $filename = $ipprc->filename($component,$ipp_outroot,$skycell_id); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); - my $remote_outroot_dir = dirname($ipp_disk); - print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; - } - - print CONFIG "\n"; -# die(); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run warptool -pendingimfile ", $remote_id, $warp_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine warp component information.", $remote_id, $warp_id, $PS_EXIT_PROG_ERROR, $fail_state); + $warpData = parse_md_list($MDlist); +} + +# STEP 2: Iterate over the sub-components +{ + foreach my $warpEntry ( @{ $warpData } ) { + my $workdir = $warpEntry->{workdir}; + my $exp_tag = $warpEntry->{exp_tag}; + my $skycell_id = $warpEntry->{skycell_id}; + my $tess_id = $warpEntry->{tess_id}; + + my $reduction = $warpEntry->{reduction}; + $reduction = 'DEFAULT' unless defined $warpEntry->{reduction}; + + my $recipe_pswarp = $ipprc->reduction($reduction, 'WARP_PSWARP'); # Recipe to use + if ($recipe_pswarp eq "") { + &my_die("Unable to determine PSWARP recipe for $reduction", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); + } + + my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.wrp.${warp_id}.${skycell_id}"; + print "$ipp_outroot\n"; + my $remote_outroot = &uri_local_to_remote($ipp_outroot); + + my $command = "$warptool -scmap -warp_id $warp_id -skycell_id ${skycell_id}"; + $command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run warptool -pendingimfile: $command ", $remote_id, $warp_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine warp component information.", $remote_id, $warp_id, $PS_EXIT_PROG_ERROR, $fail_state); + my $inData = parse_md_list($MDlist); + + my $pre_cmd_ims = "ls -1 "; + my $pre_cmd_masks= "ls -1 "; + my $pre_cmd_vars = "ls -1 "; + my $pre_cmd_astrom = ""; + + foreach my $imfile (@{ $inData }) { + my $image = $ipprc->filename("PPIMAGE.CHIP", $imfile->{chip_path_base}, $imfile->{class_id}); + my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); + my $var = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); + + my $astrom= $ipprc->filename("PSASTRO.OUTPUT", $imfile->{cam_path_base}); + + my $remote_image = &uri_local_to_remote($image); + my $remote_mask = &uri_local_to_remote($mask); + my $remote_var = &uri_local_to_remote($var); + my $remote_astrom= &uri_local_to_remote($astrom); + + $pre_cmd_ims .= " $remote_image "; + $pre_cmd_masks .= " $remote_mask "; + $pre_cmd_vars .= " $remote_var "; + $pre_cmd_astrom = "echo $remote_astrom > ${remote_outroot}.astrom "; + } + $pre_cmd_ims .= " > ${remote_outroot}.imlist "; + $pre_cmd_masks .= " > ${remote_outroot}.masklist "; + $pre_cmd_vars .= " > ${remote_outroot}.varlist "; + + my $skycell_command = " dvoImageExtract -D CATDIR /turquoise/usr/projects/ps1/watersc1/tess/${tess_id} $skycell_id -o ${remote_outroot}.skyfile "; + + my $pswarp_command = " pswarp -list ${remote_outroot}.imlist "; + $pswarp_command .= " -masklist ${remote_outroot}.masklist -variancelist ${remote_outroot}.varlist "; + $pswarp_command .= " -astromlist ${remote_outroot}.astrom "; + $pswarp_command .= " ${remote_outroot} ${remote_outroot}.skyfile "; + $pswarp_command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE "; + $pswarp_command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF "; + $pswarp_command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF "; + $pswarp_command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS "; + $pswarp_command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL "; + $pswarp_command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID "; + $pswarp_command .= " -recipe PSWARP $recipe_pswarp "; + $pswarp_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; + $pswarp_command .= " -threads $threads_req "; # -image_id ${image_id} -source_id ${source_id} "; + $pswarp_command .= " -recipe PPSTATS WARPSTATS "; + $pswarp_command .= " -dumpconfig ${remote_outroot}.pswarp.mdc -stats ${remote_outroot}.stats "; + +# print "$pswarp_command \n"; + + my $post_cmd_echo = " echo -n \"warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id "; + $post_cmd_echo .= " -dbname $dbname " if defined $dbname; + $post_cmd_echo .= " -uri ${ipp_outroot}.fits "; + $post_cmd_echo .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; + + my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - WARP_SKYCELL >> ${remote_outroot}.dbinfo "; + + print CONFIG "${pre_cmd_ims} && ${pre_cmd_masks} && ${pre_cmd_vars} && ${pre_cmd_astrom} && ${skycell_command} && ${pswarp_command} && ${post_cmd_echo} && ${post_cmd_SfM} "; + $job_index++; + + # Determine which output files need to be returned + foreach my $component(@return_component_list) { + my $filename = $ipprc->filename($component,$ipp_outroot,$skycell_id); + my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + } + print CONFIG "\n"; +# die(); } } @@ -236,48 +210,14 @@ close(GENERATE); -# -# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. -my $proc_need = $job_index * $threads; # how many total processors do we need? -my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? -my $time_need = $job_index * $job_cost; # How many seconds will this take? - -my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill -my ($time_req,$node_req); -if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { - $time_req = $min_time; - $node_req = $min_nodes; -} -elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { - $time_req = $max_time; - $node_req = $max_nodes; - print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; -} -else { - $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; - $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; -} -$time_req += 1; # Safety addition. - -open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$remote_id,$PS_EXIT_SYS_ERROR); -print COMMAND "#!/bin/tcsh\n"; -print COMMAND "##### Moab controll lines\n"; -print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES -print COMMAND "#MSUB -j oe\n"; -print COMMAND "#MSUB -V\n"; -print COMMAND "#MSUB -o ${remote_root}warp.${remote_id}.out\n"; -print COMMAND "date\n"; -print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; -print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; -print COMMAND "date\n"; -close(COMMAND); - unless($no_update) { - my $command = "remotetool -updaterun -remote_id $remote_id "; - $command .= " -set_state pending "; + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $warp_id "; + $command .= " -set_jobs $job_index"; + $command .= " -set_path_base $path_base"; + $command .= " -set_state prep_done"; $command .= " -dbname $dbname " if defined $dbname; system($command); } - +exit (0); ## Common SC routines @@ -287,7 +227,7 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri ); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { - my_die(); + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); } @@ -303,7 +243,7 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { - my_die(); + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); } @@ -317,11 +257,8 @@ sub uri_to_outputs { # (ipp_disk,remote_disk) = uri_to_output(neb_uri); Appends to TRANSFER and CHECK filehandles my $neb_uri = shift; - my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print TRANSFER "$ipp_disk\n"; - print CHECK "$remote_disk\n"; - } + my ($ipp_disk, $remote_disk) = &uri_convert( $neb_uri ); + + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; return($ipp_disk,$remote_disk); } @@ -329,10 +266,8 @@ sub uri_to_outputs_for_return { # (ipp_disk,remote_disk) = uri_to_outputs_for_return(neb_uri); create ipp_disk, append to RETURN and GENERATE my $neb_uri = shift; - my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print RETURN "$ipp_disk\n"; - print GENERATE "$remote_disk\n"; - } + my ($ipp_disk, $remote_disk) = &uri_convert_and_create( $neb_uri ); + + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; return($ipp_disk,$remote_disk); } @@ -348,5 +283,5 @@ return($remote_uri); } - + sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); # This needs to replace the remote root directory with the nebulous tag. @@ -354,5 +289,5 @@ $remote_uri =~ s%${remote_root}%%; my $local_uri = "neb:///" . $remote_uri; - + return($local_uri); } @@ -361,5 +296,6 @@ sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status); my $msg = shift; - my $id = shift; + my $remote_id = shift; + my $stage_id = shift; my $exit_code = shift; my $exit_state = shift; @@ -368,8 +304,7 @@ carp($msg); - - if (defined $id and not $no_update) { - my $command = "remotetool -updaterun -remote_id $id"; - $command .= " -fault $exit_code " if defined $exit_code; + + if (defined $remote_id and defined $stage_id and not $no_update) { + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; $command .= " -set_state $exit_state " if defined $exit_state; $command .= " -dbname $dbname " if defined $dbname; Index: /trunk/ippScripts/scripts/sc_remote_exec.pl =================================================================== --- /trunk/ippScripts/scripts/sc_remote_exec.pl (revision 37832) +++ /trunk/ippScripts/scripts/sc_remote_exec.pl (revision 37833) @@ -1,3 +1,3 @@ -#!/usr/bin/env perl +#!/usr/bin/env perl use Carp; @@ -16,5 +16,8 @@ # Hard coded values my $DMZ_HOST = 'wtrw'; -my $SEC_HOST = 'mu-fe'; +my @SEC_HOSTS= ('mu-fe1','mu-fe2','mu-fe3' + # ,'mu-fe4' + ); +my $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; # 'mu-fe'; my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/'; my $remote_root = '/scratch3/watersc1/'; @@ -32,14 +35,9 @@ # Options -my ($remote_id,$path_base,$policy,$poll,$force,$job_id,$dbname,$verbose,$no_update,$camera); -$force = 0; +my ($remote_id,$job_id,$path_base,$dbname,$verbose,$no_update,$camera); $verbose = 0; GetOptions( 'remote_id=s' => \$remote_id, - 'job_id=s' => \$job_id, 'path_base=s' => \$path_base, - 'policy=s' => \$policy, - 'poll' => \$poll, - 'force-poll' => \$force, 'camera=s' => \$camera, 'dbname=s' => \$dbname, @@ -54,5 +52,5 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id); -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files # Phase 1: See if we can actually do anything. @@ -69,202 +67,63 @@ unless ($rt_success) { $rt_error_code = (($rt_error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("UNable to run remotetool to determine remote run status", - $remote_id,$rt_error_code); -} -my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or - &my_die("UNable to run remotetool to determine remote run status", - $remote_id,$rt_error_code); + &my_die("Unable to run remotetool to determine remote run status", $remote_id, $rt_error_code); +} +my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or + &my_die("Unable to run remotetool to determine remote run status", $remote_id, $rt_error_code); my $rrData2 = parse_md_list($rrData); my $runData = ${ $rrData2 }[0]; # There should be only one -# These are operations we only need to do on the first call, not on subsequent poll operations. -unless ($poll) { +my ($remote_path) = &uri_local_to_remote($path_base); +print "REMOTE PATH: $remote_path\n"; + # Phase 2: Ensure files are in place: # Copy command files - my @files = (); - - my $uri_command = $path_base . ".cmd"; - my $uri_transfer= $path_base . ".transfer"; - my $uri_check = $path_base . ".check"; - my $uri_config = $path_base . ".config"; - my $uri_generate= $path_base . ".generate"; - my $uri_return = $path_base . ".return"; - - my $disk_command = $ipprc->file_resolve($uri_command); - my $disk_transfer= $ipprc->file_resolve($uri_transfer); - my $disk_check = $ipprc->file_resolve($uri_check); - my $disk_config = $ipprc->file_resolve($uri_config); - my $disk_generate= $ipprc->file_resolve($uri_generate); - my $disk_return = $ipprc->file_resolve($uri_return); - - scp_put($disk_command,uri_local_to_remote($uri_command)); - scp_put($disk_transfer,uri_local_to_remote($uri_transfer)); - scp_put($disk_check,uri_local_to_remote($uri_check)); - scp_put($disk_config,uri_local_to_remote($uri_config)); - scp_put($disk_generate,uri_local_to_remote($uri_generate)); - scp_put($disk_return, uri_local_to_remote($uri_return)); - -# Run check command - my (undef,$remote_check) = uri_convert($uri_check); - my (undef,$remote_transfer)= uri_convert($uri_transfer); - - my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer"); - # We no longer need to parse this output, as it retrieves files itself. - foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) { - print "$l\n"; - } +my @files = (); + +my $uri_command = $path_base . ".cmd"; +my $uri_transfer= $path_base . ".transfer"; +my $uri_check = $path_base . ".check"; +my $uri_config = $path_base . ".config"; +my $uri_generate= $path_base . ".generate"; +my $uri_return = $path_base . ".return"; + +my $disk_command = $ipprc->file_resolve($uri_command); +my $disk_transfer= $ipprc->file_resolve($uri_transfer); +my $disk_check = $ipprc->file_resolve($uri_check); +my $disk_config = $ipprc->file_resolve($uri_config); +my $disk_generate= $ipprc->file_resolve($uri_generate); +my $disk_return = $ipprc->file_resolve($uri_return); + +&scp_put($disk_command, &uri_local_to_remote($uri_command)); +&scp_put($disk_transfer, &uri_local_to_remote($uri_transfer)); +&scp_put($disk_check, &uri_local_to_remote($uri_check)); +&scp_put($disk_config, &uri_local_to_remote($uri_config)); +&scp_put($disk_generate, &uri_local_to_remote($uri_generate)); +&scp_put($disk_return, &uri_local_to_remote($uri_return)); + +my $ssh_check_stdout = &ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_path --fetch"); + +# We no longer need to parse this output, as it retrieves files itself. +foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) { + print "$l\n"; +} # Run real command - my (undef,$remote_command) = uri_convert($uri_command); - my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command"); - if ($#{ $ssh_exec_stdout } != -1) { # Parse the output - $job_id = ${ $ssh_exec_stdout }[0]; - chomp($job_id); - $job_id =~ s/\s+//g; - } - - unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. - &my_die("No job_id returned. Sorry.",$remote_id,$PS_EXIT_PROG_ERROR); - } - - # Notify the database that this entry is currently running. - my $command = "$remotetool -updaterun -remote_id $remote_id "; - $command .= " -set_state run "; - $command .= " -job_id $job_id "; - $command .= " -dbname $dbname " if defined $dbname; - - system($command); - -} # End of exec phase - -# Check that we have a valid job_id, either from the command line, or from the msub command. +my (undef,$remote_command) = &uri_convert($uri_command); +my $ssh_exec_stdout = &ssh_exec_command("msub -V $remote_command"); +if ($#{ $ssh_exec_stdout } != -1) { # Parse the output + $job_id = ${ $ssh_exec_stdout }[0]; + chomp($job_id); + $job_id =~ s/\s+//g; +} + unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. - &my_die("No job_id returned. Sorry.",$remote_id,$PS_EXIT_PROG_ERROR); -} - - -# Poll the job status? -my $poll_count = 0; -my $poll_max = 1; # If it isn't finished now, it's unlikely to magically finish in the next five minutes. -my $poll_sleep = 60; -my $poll_response = 0; -my $poll_word = "nothing"; - -if ($force) { - $poll_response = 1; - $poll_word = "Completed"; -} - -while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)&&($force != 1)) { - if ($verbose) { - print "Polling code. Sleeping. $poll_count of $poll_max $poll_response\n"; - } - unless ($poll_count == 0) { - sleep($poll_sleep); - } - ($poll_response,$poll_word) = poll_job($job_id); - - if ($poll_response == 1) { # This job has completed - last; - } - elsif ($poll_response == -1) { # This job has an error - warn("The job exited incorrectly: $remote_id $job_id $poll_word. Attempting to recover what we can."); - $poll_response = 1; # To fool the check later into proceding - } - $poll_count++; -} -if ($verbose) { - print "Stopped Polling code. $poll_count of $poll_max $poll_response\n"; -} -if ($poll_response != 1) { - &my_die("Job hasn't completed. Ending polling.", $remote_id, 0, "run", $job_id); -} - - -# Initialize the remote side to sync things - -# Once sync completes, execute all the database commands -# Grab the return list -my $uri_return = $path_base . ".return"; -my ($ipp_return,$remote_return) = uri_convert($uri_return); -# We shouldn't need to get this file. We made that file. -#scp_get($remote_return,$ipp_return); - -# Construct list of dbinfo files we'll need to execute. -my @returned_files = (); -my @dbinfo_files = (); -my @dbindex = (); -my $iter = 0; -open(RETURN, "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id,$PS_EXIT_SYS_ERROR); -while () { - chomp; - push @returned_files, $_; - if ($_ =~ /dbinfo/) { - push @dbinfo_files, $_; - push @dbindex, $iter; - } - $iter++; - -} -close(RETURN); -print STDERR "Expect $#dbinfo_files dbinfo files\n"; - -my $first = 0; -my $last = $#dbinfo_files; -my $mid = 0; -my $range = 20; # How wide of a chunk to look through to catch threading drops -my $offset = 0; # Offset to start from. - -# Feed the return list into the transfer tool -# If this fails, we should go to an auth wait point. -if ($poll_word eq 'Completed') { - # We're complete. All files should exist on disk - my $iter = 0; - my $transfer_success = 1; - do { - $transfer_success = 1; - $iter++; - check_ssh_connection(); - my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset"); - my $return_push_output_print = join "\n", @{ $return_push_output }; - if ($return_push_output_print =~ /exited with value 255/) { $transfer_success = 0; } - print "$return_push_output_print\n"; - } while (($transfer_success != 1) && ($iter < 10)); # Try harder; -} -else { - # Just try and fetch things, because we didn't complete correctly, it seems. - my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset"); - my $return_push_output_print = join "\n", @{ $return_push_output }; - print "$return_push_output_print\n"; -} - -foreach my $file (@dbinfo_files) { - open(DBF,"$file") || warn "Missing file $file\n"; - my $cmd = ; - close(DBF); - chomp($cmd); -# print STDERR "$file $cmd\n"; - $cmd =~ s/-/ -/g; # This is just a safety check for missing space. - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $cmd, verbose => $verbose); - unless ($success) { - - # This shouldn't fail, but we also can't suddenly abort if something does fail. - # Now we're going to drop the component that owns this dbinfo, which should force a retry. - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code"); - if ($poll_word ne 'Completed') { # If the job claims it completed, then we're fighting the network. - drop_component($remote_id,$cmd,$file); - } - } - -} - -# We're done, so set the state and exit. -&my_die("Finished",$remote_id,0,"full"); - + &my_die("No job_id returned. Sorry.", $remote_id, $PS_EXIT_PROG_ERROR); +} + +# Notify the database that this entry is currently running. +&my_die("Finished", $remote_id, 0, "run", $job_id); # END PROGRAM - sub check_ssh_connection { @@ -272,8 +131,8 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Authorization check failed.",$remote_id,0,'auth'); + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Authorization check failed.", $remote_id, 0, 'pending'); } } @@ -285,11 +144,11 @@ my $directory = dirname($destination); - ssh_exec_command("mkdir -p $directory"); - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die(); + &ssh_exec_command("mkdir -p $directory"); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("failed to send file $file to LANL", $remote_id, $error_code); } } @@ -302,8 +161,8 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die(); + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("failed to get file $file from LANL", $remote_id, $error_code); } } @@ -311,47 +170,25 @@ sub ssh_exec_command { my $cmd = shift; - $cmd = "$ssh $DMZ_HOST ssh ${SEC_HOST} $cmd"; + $cmd = "$ssh -n $DMZ_HOST ssh ${SEC_HOST} $cmd"; print "EXEC: $cmd\n"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - # If we failed, see if we failed due to authorization. - check_ssh_connection(); - &my_die("Failed to execute command: >>$cmd<<"); + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + # If we failed, see if we failed due to authorization. + check_ssh_connection(); + &my_die("Failed to execute remote command: >>$cmd<<", $remote_id, $error_code); } return ($stdout_buf); -} - -sub poll_job { - my $job_id = shift; - my $response_array = ssh_exec_command("checkjob -v $job_id"); - my $response = join "\n", @$response_array; - - if ($response =~ /State: Running/) { - return(0,"Running"); - } - elsif ($response =~ /State: Completed/) { - return(1,"Completed"); - } - elsif ($response =~ /State: Idle/) { - return(-2,"Idle"); - } - else { - my $response_word = $response; - $response_word =~ s/.*State: //; - $response_word =~ s/\n.*//; - return(-1,$response_word); - } -} - - +} + +# neb URI to (ipp_disk,remote_disk) sub uri_convert { my $neb_uri = shift; my $ipp_disk= $ipprc->file_resolve( $neb_uri ); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { - my_die(); + &my_die("Failed to generate or find uri $neb_uri", $remote_id, $PS_EXIT_SYS_ERROR); } @@ -363,13 +200,5 @@ } -sub uri_to_outputs { - my $neb_uri = shift; - my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - -# print TRANSFER "$ipp_disk\n"; -# print CHECK "$remote_disk\n"; - return($ipp_disk,$remote_disk); -} - +# neb URI to remote URI-on-disk sub uri_local_to_remote { # This needs to replace the nebulous tag with the remote root. @@ -382,46 +211,4 @@ return($remote_uri); } - -sub uri_remote_to_local { - # This needs to replace the remote root directory with the nebulous tag. - my $remote_uri = shift; - $remote_uri =~ s%${remote_root}%%; - my $local_uri = "neb:///" . $remote_uri; - - return($local_uri); -} - -sub drop_component { - my $remote_id = shift; - my $cmd = shift; - my $file = shift; - - if ($cmd eq '') { - # Zero byte file. This dbinfo was never constructed properly. - - # Steal the stage_id from the filename. This isn't something I'm happy about. - my $stage = $runData->{stage}; - my $stage_id = $file; - if ($stage eq 'chip') { - $stage_id =~ s/^.*ch\.//; - } - elsif ($stage eq 'camera') { - $stage_id =~ s/^.*cm\.//; - } - elsif ($stage eq 'warp') { - $stage_id =~ s/^.*wrp\.//; - } - elsif ($stage eq 'stack') { - $stage_id =~ s/^.*stk\.//; - } - $stage_id =~ s/\..*$//; - - # Drop this from the database - my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id "; - $drop_cmd .= " -dbname $dbname " if defined($dbname); - system($drop_cmd); - } - -} sub my_die { @@ -434,13 +221,13 @@ carp($msg); - + if (defined $id and not $no_update) { - my $command = "$remotetool -updaterun -remote_id $id"; - $command .= " -fault $exit_code " if defined $exit_code; - $command .= " -job_id $job_id " if defined $jobid; - $command .= " -set_state $exit_state " if defined $exit_state; - $command .= " -dbname $dbname " if defined $dbname; - - system($command); + my $command = "$remotetool -updaterun -remote_id $id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -job_id $job_id " if defined $jobid; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); } Index: /trunk/ippScripts/scripts/sc_remote_poll.pl =================================================================== --- /trunk/ippScripts/scripts/sc_remote_poll.pl (revision 37833) +++ /trunk/ippScripts/scripts/sc_remote_poll.pl (revision 37833) @@ -0,0 +1,634 @@ +#!/usr/bin/env perl + +# we have a remoteRun with an associated job_id in state of run +# 1) poll until MOAB says the processing has completed +# 2) check the status of the jobs by looking for dbinfo files on the remote server +# 3) return a list of the dbinfo files + status (exists, missing) +# 4) generate a subset transfer file with only the items that have dbinfo files? +# 5) rsync back the valid files + +# alternative method: +# 1) poll for completion as above +# 2) check for the existence of each file, generate a return table with only existing files +# 3) rsync back the existing files +# 4) run the retrieved dbinfo files + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use File::Basename; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Hard coded values +my $DMZ_HOST = 'wtrw'; +my @SEC_HOSTS= ('mu-fe1','mu-fe2','mu-fe3' + # ,'mu-fe4' + ); +my $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; # 'mu-fe'; +my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/'; +my $remote_root = '/scratch3/watersc1/'; + +# tools +my $missing_tools; +my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); +my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Options +my ($remote_id,$path_base,$skip_poll,$skip_transfer,$job_id,$dbname,$retry,$verbose,$no_update,$camera); +$verbose = 0; +GetOptions( + 'remote_id=s' => \$remote_id, + 'job_id=s' => \$job_id, + 'path_base=s' => \$path_base, + 'skip_poll' => \$skip_poll, + 'skip_transfer' => \$skip_transfer, + 'camera=s' => \$camera, + 'dbname=s' => \$dbname, + 'retry' => \$retry, + 'verbose' => \$verbose, + 'no_update' => \$no_update, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --job_id --path_base", -exitval => 3) unless + defined($path_base) and + defined($remote_id) and + defined($job_id); + +my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id); +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# STEP 1: See if we can actually do anything. +# If the link is down, there's no benefit in trying to do anything else. +&check_ssh_connection(); + +print "passed authentication challenge.\n"; + +# STEP 2: Grab the information about this run; +my $runData; +{ + my $command = "$remotetool -listrun -remote_id $remote_id "; + $command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run remotetool to determine remote run status", $remote_id, $error_code); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to run remotetool to determine remote run status", $remote_id, $error_code); + + my $metadata = parse_md_list($MDlist); + $runData = $metadata->[0]; # There should be only one +} +my $stage = $runData->{stage}; + +# STEP 3: Poll the job status +my $poll_response = 0; +my $poll_word = "nothing"; + +if ($skip_poll) { + $poll_word = "Completed"; +} else { + + my $poll_file = "$remote_root/stask_logs/$runData->{stage}.$remote_id.out"; + ($poll_response,$poll_word) = poll_job_alt($poll_file); + + # ($poll_response,$poll_word) = poll_job($job_id); + + if ($poll_response == -1) { # This job has an error + print "The job exited incorrectly: $remote_id $job_id $poll_word $poll_response.\n"; + # warn("The job exited incorrectly: $remote_id $job_id $poll_word."); + &my_die("problem with MOAB processing?", $remote_id, $PS_EXIT_SYS_ERROR); + exit (2); + } + + if ($poll_response != 1) { # This job has NOT completed + print "The job has not exited at LANL: $remote_id $job_id $poll_word $poll_response.\n"; + # NOTE: this should not be a failure condition + exit (0); + } +} + +my $remote_path = uri_local_to_remote($path_base); + +# STEP 4: transfer back the files. Transfer tool should check the +# existence of files on the remote end and only return those that +# exist +unless ($skip_transfer) { + my $offset = 0; # Offset to start from. + my $iter = 0; + my $transfer_success = 1; + do { + $transfer_success = 1; + $iter++; + check_ssh_connection(); + my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_path --offset $offset"); + my $return_push_output_print = join "\n", @{ $return_push_output }; + if ($return_push_output_print =~ /exited with value 255/) { $transfer_success = 0; } + print "$return_push_output_print\n"; + } while (($transfer_success != 1) && ($iter < 10)); # Try harder; + + if (!$transfer_success) { &my_die ("failed to get all return files", $remote_id, 23); } +} +print STDERR "rsync of all files is complete, checking the results\n"; + +# STEP 5: grab the .stat file from LANL (this file is first constructed at LANL) + +my $uri_stats = $path_base . ".stat"; +my $disk_stats = $ipprc->file_resolve($uri_stats, 1); +my $remote_stats = uri_local_to_remote($uri_stats); + +scp_get($remote_stats,$disk_stats); + +# STEP 6: check dbinfo files for PASS / PART / FAIL + +my %stage_pass = (); +my %stage_fail = (); +my %stage_part = (); + +my $Npass = 0; +my $Npart = 0; +my $Nfail = 0; +my @dbinfo_files = (); +open (STATFILE, "$disk_stats") || &my_die("Couldn't open file? $disk_stats", $remote_id, $PS_EXIT_SYS_ERROR); +while (my $line = ) { + chomp $line; +# unless ($line =~ /dbinfo/) { next; } + + my @words = split (" ", $line); + + my ($stage_id, $class_id, $component) = get_stage_id_from_filename ($words[0],$stage); + unless(defined($stage_id)) { + print STDERR "Unable to convert file: $words[0]\n"; + next; + } +# print STDERR "stage_id : $stage_id, class_id: $class_id, \n"; + + if ($words[1] eq "PASS") { + if ($line =~ /dbinfo/) { + push @dbinfo_files, $words[0]; + } + $stage_pass{$stage_id} = 1; + $Npass ++; + } elsif ($words[1] eq "FAIL") { + $stage_fail{$stage_id} = 1; + $Nfail ++; + } elsif ($words[1] eq "PART") { + $stage_part{$stage_id} = 1; + $Npart ++; + } +} +close (STATFILE); +print STDERR "dbinfo files: $Npass PASS, $Nfail FAIL, $Npart PART\n"; + +# STEP 7: if dbinfo is PASS, execute it (fail ones that are not PASS?). +if (1) { + foreach my $file (@dbinfo_files) { + my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); + if (defined($stage_part{$stage_id})|| + defined($stage_fail{$stage_id})) { + print STDERR "Skipping $file as it is not complete: " . $stage_part{$stage_id} . " " . $stage_fail{$stage_id} . "\n"; + next; + + } + open(DBF,"$file") || warn "Missing file $file\n"; + my $cmd = ; + close(DBF); + chomp($cmd); + + $cmd =~ s/-/ -/g; # This is just a safety check for missing space. + + if ($cmd eq "") { + print STDERR "problem: empty dbinfo file\n"; + exit (1); + next; + } + + print "dbinfo: $file\n"; + + if ($retry) { + # use the dbinfo filename to get stage id, check if we have already updated this entry: + my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); + + # we just need to check for the existence of an entry: + if ($runData->{stage} eq "chip") { + my $cmd1 = "$chiptool -simple -processedimfile -chip_id $stage_id -class_id $class_id"; + $cmd1 .= " -dbname $dbname " if defined($dbname); + + my ( $success1, $error_code1, $full_buf1, $stdout_buf1, $stderr_buf1 ) = + run(command => $cmd1, verbose => 0); + + unless ($success1) { + $error_code1 = (($error_code1 >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run chiptool to check on results", $remote_id, $error_code1); + } + + my $Nrows = @$stdout_buf1; + if ($Nrows > 0) { + print STDERR "already updated: $stage_id, $class_id\n"; + next; + } + } elsif ($runData->{stage} eq "camera") { + my $cmd1 = "$camtool -simple -processedexp -cam_id $stage_id"; + $cmd1 .= " -dbname $dbname " if defined($dbname); + + my ( $success1, $error_code1, $full_buf1, $stdout_buf1, $stderr_buf1 ) = + run(command => $cmd1, verbose => 0); + + unless ($success1) { + $error_code1 = (($error_code1 >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run chiptool to check on results", $remote_id, $error_code1); + } + + my $Nrows = @$stdout_buf1; + if ($Nrows > 0) { + print STDERR "already updated: cam_id: $stage_id\n"; + next; + } + } elsif ($runData->{stage} eq "warp") { + my $cmd1 = "$warptool -simple -warped -warp_id $stage_id -skycell_id $class_id"; + $cmd1 .= " -dbname $dbname " if defined($dbname); + + my ( $success1, $error_code1, $full_buf1, $stdout_buf1, $stderr_buf1 ) = + run(command => $cmd1, verbose => 0); + + unless ($success1) { + $error_code1 = (($error_code1 >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run chiptool to check on results", $remote_id, $error_code1); + } + + my $Nrows = @$stdout_buf1; + if ($Nrows > 0) { + print STDERR "already updated: warp_id: $stage_id, $class_id\n"; + next; + } + } else { + print STDERR "do not know how to handle stage $runData->{stage}\n"; + exit (3); + } + } + + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + # This shouldn't fail, but we also can't suddenly abort if something does fail. + # Now we're going to drop the component that owns this dbinfo, which should force a retry. + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code"); + my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); + $stage_fail{$stage_id} = 1; + } + } + } + print STDERR "dbinfo files all run (or skipped)\n"; +} + +# STEP 8: unify the stage states and update the remoteComponent entries +foreach my $entry (keys (%stage_fail)) { + $stage_pass{$entry} = 0; +} +foreach my $entry (keys (%stage_part)) { + $stage_pass{$entry} = 0; +} +foreach my $entry (keys (%stage_pass)) { + my $cmd = "remotetool -updatecomponent -remote_id $remote_id -stage_id $entry"; + $cmd .= " -dbname $dbname " if defined($dbname); + if ($stage_pass{$entry}) { + $cmd .= " -set_state pass"; + } else { + $cmd .= " -set_state fail"; + } + + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + &my_die("Unable to run remotetool to determine remote run status", $remote_id); + } + } else { + print STDERR "no_update: $cmd\n"; + } +} + +# We're done, so set the state and exit. +&my_die("Finished", $remote_id, 0, "full"); + +# END PROGRAM + +sub check_ssh_connection { + my $cmd = "$ssh -O check $DMZ_HOST"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Authorization check failed.",$remote_id,0,'auth'); + } +} + +sub scp_put { + my $file = shift; + my $destination = shift; + my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}"; + + my $directory = dirname($destination); + ssh_exec_command("mkdir -p $directory"); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Failed to send file $file to LANL", $remote_id, $PS_EXIT_SYS_ERROR); + } +} + +sub scp_get { + my $destination = shift; + my $file = shift; + + my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file "; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Failed to get file $file from LANL", $remote_id, $PS_EXIT_SYS_ERROR); + } +} + +sub ssh_exec_command { + my $cmd = shift; + $cmd = "$ssh -n $DMZ_HOST ssh ${SEC_HOST} $cmd"; + print "EXEC: $cmd\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + # If we failed, see if we failed due to authorization. + check_ssh_connection(); + &my_die("Failed to execute command: >>$cmd<<", $remote_id, $PS_EXIT_SYS_ERROR); + } + return ($stdout_buf); +} + +sub poll_job_alt { + my $poll_file = shift; + + my $response_array = ssh_exec_command("$remote_root/sc_checkjob.pl --poll_file $poll_file"); + my $response = join "\n", @$response_array; + + print STDERR "response: $response\n"; + + if ($response =~ /STATUS: RUNNING/) { + return(0,"Running"); + } + elsif ($response =~ /STATUS: COMPLETED/) { + return(1,"Completed"); + } + elsif ($response =~ /STATUS: PENDING/) { + return(-2,"Idle"); + } + else { + my $response_word = $response; + $response_word =~ s/.*State: //; + $response_word =~ s/\n.*//; + return(-1,$response_word); + } +} + + +sub poll_job { + my $job_id = shift; + my $response_array = ssh_exec_command("checkjob -v $job_id"); + my $response = join "\n", @$response_array; + + if ($response =~ /State: Running/) { + return(0,"Running"); + } + elsif ($response =~ /State: Completed/) { + return(1,"Completed"); + } + elsif ($response =~ /State: Idle/) { + return(-2,"Idle"); + } + else { + my $response_word = $response; + $response_word =~ s/.*State: //; + $response_word =~ s/\n.*//; + return(-1,$response_word); + } +} + + +sub uri_convert { + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + &my_die("Failed to generate or find uri $neb_uri", $remote_id, $PS_EXIT_SYS_ERROR); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs { + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + +# print TRANSFER "$ipp_disk\n"; +# print CHECK "$remote_disk\n"; + return($ipp_disk,$remote_disk); +} + +sub uri_local_to_remote { + # This needs to replace the nebulous tag with the remote root. + my $local_uri = shift; + $local_uri =~ s%^.*?/%%; # neb:/ + $local_uri =~ s%^.*?/%%; # / + $local_uri =~ s%^.*?/%%; # @HOST@.0/ + my $remote_uri = "${remote_root}/" . $local_uri; + + return($remote_uri); +} + +sub uri_remote_to_local { + # This needs to replace the remote root directory with the nebulous tag. + my $remote_uri = shift; + $remote_uri =~ s%${remote_root}%%; + my $local_uri = "neb:///" . $remote_uri; + + return($local_uri); +} + +# assumes $runData->{stage} exists +sub get_stage_id_from_dbinfo { + my $file = shift; + + my $stage_id = 0; + my $class_id = 0; + + # Steal the stage_id from the filename. This isn't something I'm happy about. + my $stage = $runData->{stage}; + if ($stage eq 'chip') { + ($stage_id, $class_id) = $file =~ /ch\.(\d*).(XY\d\d).dbinfo/; + } + elsif ($stage eq 'camera') { + ($stage_id) = $file =~ /cm\.(\d*).dbinfo/; + $class_id = 0; + } + elsif ($stage eq 'warp') { + ($stage_id, $class_id) = $file =~ /wrp\.(\d*)\.(skycell.\d\d\d\d.\d\d\d).dbinfo/; + } + elsif ($stage eq 'stack') { + print STDERR "dbinfo: $file\n"; + ($class_id, $stage_id) = $file =~ /(skycell.\d\d\d\d.\d\d\d).stk.(\d*).dbinfo/; + } + + return ($stage_id, $class_id); +} + +sub get_stage_id_from_filename { + my $file = shift; + my $stage = shift; + my $stage_id = -1; + my $class_id = ''; + my $component = ''; + + if ($stage eq 'chip') { + ($stage_id, $class_id, $component) = $file =~ /ch\.(\d*).(XY\d\d).([\w\.]+)$/; + } + elsif ($stage eq 'camera') { + if ($file =~ /XY/) { + ($stage_id, $class_id, $component) = $file =~ /cm\.(\d*).(XY\d\d)\.([\w\.]+)$/; + } + else { + ($stage_id, $component) = $file =~ /cm\.(\d*)\.([\w\.]+)$/; + $class_id = 0; + } + } + elsif ($stage eq 'warp') { + ($stage_id, $class_id, $component) = $file =~ /wrp\.(\d*)\.(skycell.\d\d\d\d.\d\d\d).([\w\.]+)$/; + } + elsif ($stage eq 'stack') { + ($stage_id, $component) = $file =~ /stk\.(\d*)\.([\w\.]+)$/; + $class_id = 0; + } + return($stage_id,$class_id,$component); +} + + + +sub fail_component { + my $remote_id = shift; + my $cmd = shift; + my $file = shift; + + # Steal the stage_id from the filename. This isn't something I'm happy about. + my $stage = $runData->{stage}; + my $stage_id = $file; + if ($stage eq 'chip') { + $stage_id =~ s/^.*ch\.//; + } + elsif ($stage eq 'camera') { + $stage_id =~ s/^.*cm\.//; + } + elsif ($stage eq 'warp') { + $stage_id =~ s/^.*wrp\.//; + } + elsif ($stage eq 'stack') { + $stage_id =~ s/^.*stk\.//; + } + $stage_id =~ s/\..*$//; + + print STDERR "failed component: $remote_id, $stage_id\n"; + + # Drop this from the database + ## my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id "; + ## $drop_cmd .= " -dbname $dbname " if defined($dbname); + ## system($drop_cmd); +} + +sub drop_component { + my $remote_id = shift; + my $cmd = shift; + my $file = shift; + + if ($cmd eq '') { + # Zero byte file. This dbinfo was never constructed properly. + + # Steal the stage_id from the filename. This isn't something I'm happy about. + my $stage = $runData->{stage}; + my $stage_id = $file; + if ($stage eq 'chip') { + $stage_id =~ s/^.*ch\.//; + } + elsif ($stage eq 'camera') { + $stage_id =~ s/^.*cm\.//; + } + elsif ($stage eq 'warp') { + $stage_id =~ s/^.*wrp\.//; + } + elsif ($stage eq 'stack') { + $stage_id =~ s/^.*stk\.//; + } + $stage_id =~ s/\..*$//; + + # Drop this from the database + my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id "; + $drop_cmd .= " -dbname $dbname " if defined($dbname); + system($drop_cmd); + } + +} + +sub my_die { + my $msg = shift; + my $id = shift; + my $exit_code = shift; + my $exit_state = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $id and not $no_update) { + my $command = "$remotetool -updaterun -remote_id $id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} + +# Quick review: +# new -> pending -> run -> full +# auth Index: /trunk/ippScripts/scripts/sc_retry_return.pl =================================================================== --- /trunk/ippScripts/scripts/sc_retry_return.pl (revision 37833) +++ /trunk/ippScripts/scripts/sc_retry_return.pl (revision 37833) @@ -0,0 +1,346 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use File::Basename; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Hard coded values +my $DMZ_HOST = 'wtrw'; +my $SEC_HOST = 'mu-fe'; +my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/'; +my $remote_root = '/scratch3/watersc1/'; + +# tools +my $missing_tools; +my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); +my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Options +my ($remote_id,$path_base,$dbname,$verbose,$no_update,$camera); +$verbose = 0; +GetOptions( + 'remote_id=s' => \$remote_id, + 'path_base=s' => \$path_base, + 'camera=s' => \$camera, + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'no_update' => \$no_update, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --path_base", -exitval => 3) unless + defined($path_base) and + defined($remote_id); + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id); +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Phase 1: See if we can actually do anything. +# If the link is down, there's no benefit in trying to do anything else. +&check_ssh_connection(); + +print "passed authentication challenge.\n"; + +# Phase 1.5: Grab the information about this run; +my $rt_cmd = "$remotetool -listrun -remote_id $remote_id "; +$rt_cmd .= " -dbname $dbname " if defined($dbname); + +my ( $rt_success, $rt_error_code, $rt_full_buf, $rt_stdout_buf, $rt_stderr_buf ) = + run(command => $rt_cmd, verbose => $verbose); +unless ($rt_success) { + $rt_error_code = (($rt_error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run remotetool to determine remote run status", + $remote_id,$rt_error_code); +} + +my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or + &my_die("Unable to run remotetool to determine remote run status", + $remote_id,$rt_error_code); +my $rrData2 = parse_md_list($rrData); +my $runData = ${ $rrData2 }[0]; # There should be only one + +# my job ID : +my $job_id = $runData->{job_id}; +print "remote job ID: $job_id\n"; + +# Poll the job status? (check status at LANL) +my $poll_response = 0; +my $poll_word = "nothing"; +($poll_response, $poll_word) = poll_job($job_id); + +if ($poll_response == -1) { # This job has an error + print "The job exited incorrectly: $remote_id $job_id $poll_word $poll_response.\n"; + # warn("The job exited incorrectly: $remote_id $job_id $poll_word."); + exit (2); +} + +if ($poll_response != 1) { # This job has NOT completed + print "The job has not exited at LANL: $remote_id $job_id $poll_word $poll_response.\n"; + exit (2); +} + +# Initialize the remote side to sync things +my $uri_return = $path_base . ".return"; +my ($ipp_return, $remote_return) = uri_convert($uri_return); + +# Construct list of dbinfo files we'll need to execute. +my @returned_files = (); +my @dbinfo_files = (); + +# get a list of the dbinfo files +open(RETURN, "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id, $PS_EXIT_SYS_ERROR); +while () { + chomp; + if ($_ =~ /dbinfo/) { + push @dbinfo_files, $_; + } +} +close(RETURN); +print STDERR "Expect $#dbinfo_files dbinfo files\n"; + +# Feed the return list into the transfer tool +# If this fails, we should exit : do not update gpc1 database until all files are back +{ + my $offset = 0; # Offset to start from. + my $iter = 0; + my $transfer_success = 1; + do { + $transfer_success = 1; + $iter++; + check_ssh_connection(); + my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset"); + my $return_push_output_print = join "\n", @{ $return_push_output }; + if ($return_push_output_print =~ /exited with value 255/) { $transfer_success = 0; } + print "$return_push_output_print\n"; + } while (($transfer_success != 1) && ($iter < 10)); # Try harder; + + if (!$transfer_success) { &my_die ("failed to get all return files"); } +} +print STDERR "rsync of all files is complete, updating gpc1 database\n"; + +foreach my $file (@dbinfo_files) { + + # use the dbinfo filename to get stage id, check if we have already updated this entry: + my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); + + # chiptool -dbname -chip_id $stage_id -class_id $class_id + + # we just need to check for the existence of an entry: + if ($runData->{stage} == "chip") { + my $cmd1 = "$chiptool -simple -processedimfile -chip_id $stage_id -class_id $class_id"; + $cmd1 .= " -dbname $dbname " if defined($dbname); + + my ( $success1, $error_code1, $full_buf1, $stdout_buf1, $stderr_buf1 ) = + run(command => $cmd1, verbose => $verbose); + + unless ($success1) { + $error_code1 = (($error_code1 >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run chiptool to check on results", $remote_id, $error_code1); + } + + my $Nrows = @$stdout_buf1; + if ($Nrows > 0) { + print STDERR "skip this entry: $stage_id, $class_id\n"; + next; + } + } else { + print STDERR "do not know how to handle stage $runData->{stage}\n"; + exit (3); + } + + open(DBF,"$file") || warn "Missing file $file\n"; + my $cmd = ; + close(DBF); + + chomp($cmd); + + $cmd =~ s/-/ -/g; # This is just a safety check for missing space. + if ($cmd == "") { + print STDERR "empty dbinfo file for $stage_id, $class_id, skipping\n"; + next; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + # This shouldn't fail, but we also can't suddenly abort if something does fail. + # Now we're going to drop the component that owns this dbinfo, which should force a retry. + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code"); + if ($poll_word ne 'Completed') { # If the job claims it completed, then we're fighting the network. + # drop_component($remote_id,$cmd,$file); + print STDERR "we should drop this component: $cmd\n"; + } + } +} + +# We're done, so set the state and exit. +&my_die("Finished",$remote_id,0,"full"); + +# END PROGRAM + +# yes +sub check_ssh_connection { + my $cmd = "$ssh -O check $DMZ_HOST"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Authorization check failed", $remote_id, 0, 'auth'); + } +} + +sub ssh_exec_command { + my $cmd = shift; + $cmd = "$ssh $DMZ_HOST ssh ${SEC_HOST} $cmd"; + print "EXEC: $cmd\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + # If we failed, see if we failed due to authorization. + check_ssh_connection(); + &my_die("Failed to execute command: >>$cmd<<"); + } + return ($stdout_buf); +} + +sub uri_convert { + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk, $remote_disk); +} + +sub poll_job { + my $job_id = shift; + my $response_array = ssh_exec_command("checkjob -v $job_id"); + my $response = join "\n", @$response_array; + + if ($response =~ /State: Running/) { + return(0,"Running"); + } + elsif ($response =~ /State: Completed/) { + return(1,"Completed"); + } + elsif ($response =~ /State: Idle/) { + return(-2,"Idle"); + } + else { + my $response_word = $response; + $response_word =~ s/.*State: //; + $response_word =~ s/\n.*//; + return(-1,$response_word); + } +} + + +# assumes $runData->{stage} exists +sub get_stage_id_from_dbinfo { + my $file = shift; + + my $stage_id = 0; + my $class_id = 0; + + # Steal the stage_id from the filename. This isn't something I'm happy about. + my $stage = $runData->{stage}; + if ($stage eq 'chip') { + ($stage_id, $class_id) = $file =~ /ch\.(\d*).(XY\d\d).dbinfo/; + } + elsif ($stage eq 'camera') { + $stage_id =~ s/^.*cm\.//; + $class_id = 0; + } + elsif ($stage eq 'warp') { + $stage_id =~ s/^.*wrp\.//; + $class_id = 0; + } + elsif ($stage eq 'stack') { + $stage_id =~ s/^.*stk\.//; + $class_id = 0; + } + + return ($stage_id, $class_id); +} + +sub drop_component { + my $remote_id = shift; + my $cmd = shift; + my $file = shift; + + if ($cmd eq '') { + # Zero byte file. This dbinfo was never constructed properly. + + # Steal the stage_id from the filename. This isn't something I'm happy about. + my $stage = $runData->{stage}; + my $stage_id = $file; + if ($stage eq 'chip') { + $stage_id =~ s/^.*ch\.//; + } + elsif ($stage eq 'camera') { + $stage_id =~ s/^.*cm\.//; + } + elsif ($stage eq 'warp') { + $stage_id =~ s/^.*wrp\.//; + } + elsif ($stage eq 'stack') { + $stage_id =~ s/^.*stk\.//; + } + $stage_id =~ s/\..*$//; + + # Drop this from the database + my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id "; + $drop_cmd .= " -dbname $dbname " if defined($dbname); + system($drop_cmd); + } + +} + +sub my_die { + my $msg = shift; + my $id = shift; + my $exit_code = shift; + my $exit_state = shift; + my $jobid = shift; + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $id and not $no_update) { + my $command = "$remotetool -updaterun -remote_id $id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -job_id $job_id " if defined $jobid; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} Index: /trunk/ippScripts/scripts/sc_transfer_tool.pl =================================================================== --- /trunk/ippScripts/scripts/sc_transfer_tool.pl (revision 37832) +++ /trunk/ippScripts/scripts/sc_transfer_tool.pl (revision 37833) @@ -10,29 +10,67 @@ use Sys::Hostname; +my $scp_cmd = "/usr/projects/cosmo/amd6100/bin/scp"; +my $ssh_cmd = "/usr/projects/cosmo/amd6100/bin/ssh"; + my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; -$remote_root = '/scratch3/watersc1/'; -my $local_raw = "${remote_root}/raw/"; + $remote_root = '/scratch3/watersc1/'; + +#my $local_raw = "${remote_root}/tmp/"; my $local_tmp = "${remote_root}/tmp/"; -my $threads = 10; -my @hosts = ('ippc20.ipp.ifa.hawaii.edu','ippc24.ipp.ifa.hawaii.edu','ippc28.ipp.ifa.hawaii.edu', - 'ippc21.ipp.ifa.hawaii.edu','ippc25.ipp.ifa.hawaii.edu','ippc29.ipp.ifa.hawaii.edu', - 'ippc22.ipp.ifa.hawaii.edu','ippc26.ipp.ifa.hawaii.edu', - 'ippc23.ipp.ifa.hawaii.edu','ippc27.ipp.ifa.hawaii.edu' ); +my $threads = 11; +my @hosts = ( + 'ippc20.ipp.ifa.hawaii.edu', + 'ippc21.ipp.ifa.hawaii.edu', + 'ippc22.ipp.ifa.hawaii.edu', + 'ippc23.ipp.ifa.hawaii.edu', + 'ippc24.ipp.ifa.hawaii.edu', + 'ippc25.ipp.ifa.hawaii.edu', + 'ippc26.ipp.ifa.hawaii.edu', + 'ippc27.ipp.ifa.hawaii.edu', + 'ippc28.ipp.ifa.hawaii.edu', +#'ippc29.ipp.ifa.hawaii.edu', (mark's hi-mem test machine) +#'ippc30.ipp.ifa.hawaii.edu', (postage stamp server) + 'ippc31.ipp.ifa.hawaii.edu', + 'ippc32.ipp.ifa.hawaii.edu' + ); @hosts = (@hosts, @hosts, @hosts, @hosts, @hosts); -my $input_file; +my %server_options = ( + 'ippc20.ipp.ifa.hawaii.edu' => '', + 'ippc21.ipp.ifa.hawaii.edu' => '', + 'ippc22.ipp.ifa.hawaii.edu' => '', + 'ippc23.ipp.ifa.hawaii.edu' => '', + 'ippc24.ipp.ifa.hawaii.edu' => '', + 'ippc25.ipp.ifa.hawaii.edu' => '', + 'ippc26.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes', + 'ippc27.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes', + 'ippc28.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes', + 'ippc29.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes', + 'ippc30.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes', + 'ippc31.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes', + 'ippc32.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes' + ); + +my $input_path; +my $local_file; +my $remote_file; + my $verbose = 0; my $fetch = 0; my $offset = 0; +my $retry = 0; +my $quickfetch = 0; GetOptions( 'threads=s' => \$threads, - 'input=s' => \$input_file, - 'fetch=s' => \$fetch, + 'input=s' => \$input_path, 'offset=s' => \$offset, + 'fetch' => \$fetch, 'verbose' => \$verbose, + 'retry' => \$retry, + 'quickfetch' => \$quickfetch, ) or pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV; pod2usage( -msg => "Required options: --input", -exitval => 3) unless - defined($input_file); + defined($input_path); my $hostname = hostname; @@ -41,45 +79,90 @@ # split in input file list unless(-d $local_tmp) { system("mkdir -p $local_tmp"); } -unless(-d $local_raw) { system("mkdir -p $local_raw"); } - -open(I,$input_file) || die "Couldn't find input file specified\n"; +#unless(-d $local_raw) { system("mkdir -p $local_raw"); } + +# we have two modes: fetch (-> LANL) and return (-> IPP) + if ($fetch) { - open(F,$fetch) || die "Couldn't find fetch file specified\n"; -} -my $input_base = basename($input_file); -my @filehandles; + $local_file = "$input_path.check"; + $remote_file = "$input_path.transfer"; +} else { + $local_file = "$input_path.generate"; + $remote_file = "$input_path.return"; +} +my $input_base = basename($local_file); + +unless ($retry) { + # open the input file list + open (LOCAL,$local_file) || die "Couldn't find input file specified $local_file\n"; + open (REMOTE,$remote_file) || die "Couldn't find input file specified $remote_file\n"; + + # generate N output files (to be fed to the N rsync / tar threads) + my @filehandles; + my $i; + my $line = 0; + for ($i = 0; $i < $threads; $i++) { + open($filehandles[$i], ">${local_tmp}/${input_base}.${i}"); + } + + unless ($fetch) { + my $stat_file = "$input_path.stat"; + print "STATFILE: $stat_file\n"; + open (STATFILE, ">$stat_file"); + } + + $i = 0; + while (my $Lname = ) { + my $Rname = ; + + chomp $Lname; + chomp $Rname; + + $line++; + if ($line < $offset) { next; } # I think this is off-by-one in a safe direction + + if ($line % 250 == 0) { print STDERR "line $line : $Lname\n"; } + + $i = int(rand($#filehandles + 1)); + if ($i >= $threads) { + print STDERR "HUH: impossible file handle?\n"; + die; + } + + # if we are fetching, and 'quickfetch' is requested, skip file if it exists + # NOTE: this does not check that the transfer was complete or the file is current + + if ($fetch) { + # We are fetching, and do not already have this file. + if ($quickfetch && (-e $Lname)) { next; } + print { $filehandles[$i] } "$Rname\n"; + } else { + # We are pushing, but only send back files we actually have + # we cannot have the remote file unless the local file exists + # (it is a link to the local file and is generated after the local file) + my $haveRemote = -e "$local_tmp/$Rname"; + if ($haveRemote) { + print { $filehandles[$i] } "$Rname\n"; + print STATFILE "$Rname PASS\n"; + } else { + my $haveLocal = -e $Lname; + if ($haveLocal) { + print STATFILE "$Rname PART\n"; + } else { + print STATFILE "$Rname FAIL\n"; + } + } + } + } + close(LOCAL); + close(REMOTE); + unless ($fetch) { close(STATFILE); } + + for ($i = 0; $i < $threads; $i++) { + close($filehandles[$i]); + } +} + my $i; -my $line = 0; -for ($i = 0; $i < $threads; $i++) { - open($filehandles[$i], ">${local_tmp}/${input_base}.${i}"); -} -$i = 0; -while() { - chomp; - $line++; - if ($line < $offset) { next; } # I think this is off-by-one in a safe direction - - $i = int(rand($#filehandles + 1)); - - my $fline; - if ($fetch) { - $fline = ; - } - if (($fetch)&&(!(-e $_))) { # We are fetching, and do not already have this file. - print { $filehandles[$i] } $fline; - } - elsif (!($fetch)) { # We are pushing - # The rsync call expects to find files of a given name in the directory specified. -# print { $filehandles[$i] } "${local_tmp}/$_" . "\n"; - print { $filehandles[$i] } "$_" . "\n"; - } -} -close(I); - -for ($i = 0; $i < $threads; $i++) { - close($filehandles[$i]); -} - -# fork the tars +# fork the rsync my @pids = (); for ($i = 0; $i < $threads; $i++) { @@ -89,14 +172,19 @@ my $code = 0; if ($fetch) { - $code = fetch_task($host,"${input_base}.${i}", 0); - } - else { + $code = fetch_task($host,"${local_tmp}/${input_base}.${i}", 0); + } else { $code = transfer_task($host,"${local_tmp}/${input_base}.${i}", 0); } + print STDERR "$host $input_base $i $code\n"; exit($code); } } + +my $global_status = 0; for ($i = 0; $i < $threads; $i++) { waitpid($pids[$i],0); + my $this_status = $?; + print "exit status $hosts[$i] : $this_status\n"; + if ($this_status) { $global_status = ($this_status >> 8); } } @@ -105,4 +193,11 @@ } +# my $scp_command = "$scp_cmd $option ${local_tmp}/${input_base}.stat $hosts[0]:/tmp/"; +# print "$scp_command\n"; +# system("$scp_command"); +# XXX check return status + +print "global status: $global_status\n"; +exit ($global_status); # distribute bundles to nodes @@ -115,38 +210,33 @@ my $error = shift; - # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date? -# my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; - my $command = "rsync -Lpt -e '/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/"; + my $option = $server_options{$destination_host}; + + my $command = "rsync -L --size-only --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/"; print STDERR "$command\n"; - + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + + my $return_value = 0; unless ($success) { - print STDERR "ERROR: $error_code\n"; -# my $std = join "\n", @{ $stdout_buf }; -# print "STDOUT: $std\n"; -# $std = join "\n", @{ $stderr_buf }; -# print "STDERR: $std\n"; - -# foreach my $line (@{ $stderr_buf }) { # This is a hack. A messy ugly hack. -# if ($line =~ /No such file or directory/) { -# my $file = (split /\s+/, $line)[1]; -# $file =~ s/:$//; -# if ($file !~ /nebulous/) { next; } -# print STDERR "Touching $file so tar can continue.\n"; -# system("touch $file"); -# } -# } - - warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code."); - $error++; - if ($error < 4) { - $error_code = transfer_task($destination_host,$transfer_filelist,$error); - } - else { - die("Failed too many times $error $destination_host $transfer_filelist"); + if ($error_code =~ /value 23/) { + print STDERR "Some files failed to transfer. Partial run?\n"; + $error = 23; + $return_value = 23; + } else { + print STDERR "ERROR: $error_code\n"; + warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code."); + sleep(5); + $error++; + if ($error < 4) { + $error_code = transfer_task($destination_host,$transfer_filelist,$error); + } + else { + die("Failed too many times $error $destination_host $transfer_filelist"); + } + $return_value = 1; } } - return($error_code); + return($return_value); } @@ -156,19 +246,33 @@ my $error = shift; - system("/usr/projects/cosmo/amd6100/bin/scp -o NoneSwitch=yes -o NoneEnabled=yes ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/"); - my $command = "/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar cf - --ignore-failed-read --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference "; + my $option = $server_options{$destination_host}; + + my $command = "rsync --size-only -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${destination_host}:/ ${local_tmp}"; print STDERR "$command\n"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + my $error_code = 0; + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or 4); - warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code."); - $error++; - if ($error < 4) { - $error_code = fetch_task($destination_host,$transfer_filelist,$error); - } - else { - die("Failed too many times $error $destination_host $transfer_filelist"); + print "raw error_msg: $error_msg...\n"; + ($error_code) = $error_msg =~ m/exited with value (\d+)/; + warn("Transfer of files in $transfer_filelist from $destination_host failed with error ($error_code) $error_msg."); + print "*** stdout: *** \n"; + foreach my $line (@$stdout_buf) { + print STDERR "stdout: $line\n"; + } + print "*** stderr: *** \n"; + foreach my $line (@$stderr_buf) { + print STDERR "stderr: $line\n"; + } + if ($error_code == 23) { + print STDERR "Some files failed to transfer\n"; + } else { + $error++; + if ($error < 4) { + $error_code = fetch_task($destination_host,$transfer_filelist,$error); + } else { + print STDERR "Failed too many times $error $destination_host $transfer_filelist\n"; + } } } Index: /trunk/ippScripts/scripts/stack_skycell.pl =================================================================== --- /trunk/ippScripts/scripts/stack_skycell.pl (revision 37832) +++ /trunk/ippScripts/scripts/stack_skycell.pl (revision 37833) @@ -225,5 +225,6 @@ $num++; - my $image = $file->{uri}; # Image name +# my $image = $file->{uri}; # Image name + my $image = $ipprc->filename("PSWARP.OUTPUT", $file->{path_base} ); # Image name my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name my $weight = $ipprc->filename( "PSWARP.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 37832) +++ /trunk/ippTools/share/Makefile.am (revision 37833) @@ -524,4 +524,7 @@ remotetool_definebyquery_warp.sql \ remotetool_definebyquery_stack.sql \ + remotetool_definebyquery_staticsky.sql \ remotetool_dropcomponent.sql \ + remotetool_listcomponent.sql \ + remotetool_doneprep.sql \ remotetool_updatepoll.sql Index: /trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_create_tables.sql (revision 37832) +++ /trunk/ippTools/share/pxadmin_create_tables.sql (revision 37833) @@ -2385,4 +2385,7 @@ remote_id BIGINT, stage_id BIGINT, + jobs INT + state VARCHAR(64) NOT NULL, + path_base VARCHAR(255), KEY (stage_id), FOREIGN KEY (remote_id) REFERENCES remoteRun(remote_id) Index: /trunk/ippTools/share/remotetool_definebyquery_cam.sql =================================================================== --- /trunk/ippTools/share/remotetool_definebyquery_cam.sql (revision 37832) +++ /trunk/ippTools/share/remotetool_definebyquery_cam.sql (revision 37833) @@ -11,9 +11,9 @@ LEFT JOIN -(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state - FROM remoteRun +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun JOIN remoteComponent USING (remote_id) WHERE -remoteRun.stage = 'camera' +remoteRun.stage = 'camera' and remoteComponent.state != 'retry' -- where hook %s ) AS R Index: /trunk/ippTools/share/remotetool_definebyquery_chip.sql =================================================================== --- /trunk/ippTools/share/remotetool_definebyquery_chip.sql (revision 37832) +++ /trunk/ippTools/share/remotetool_definebyquery_chip.sql (revision 37833) @@ -11,9 +11,9 @@ LEFT JOIN -(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state - FROM remoteRun +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun JOIN remoteComponent USING (remote_id) WHERE -remoteRun.stage = 'chip' +remoteRun.stage = 'chip' and remoteComponent.state != 'retry' -- where hook %s ) AS R @@ -24,3 +24,3 @@ - + Index: /trunk/ippTools/share/remotetool_definebyquery_stack.sql =================================================================== --- /trunk/ippTools/share/remotetool_definebyquery_stack.sql (revision 37832) +++ /trunk/ippTools/share/remotetool_definebyquery_stack.sql (revision 37833) @@ -15,5 +15,5 @@ JOIN remoteComponent USING (remote_id) WHERE -remoteRun.stage = 'stack' +remoteRun.stage = 'stack' and remoteComponent.state != 'retry' -- where hook %s ) AS R Index: /trunk/ippTools/share/remotetool_definebyquery_staticsky.sql =================================================================== --- /trunk/ippTools/share/remotetool_definebyquery_staticsky.sql (revision 37833) +++ /trunk/ippTools/share/remotetool_definebyquery_staticsky.sql (revision 37833) @@ -0,0 +1,21 @@ +SELECT stage_id FROM + +( SELECT sky_id AS stage_id + FROM staticskyRun +WHERE +staticskyRun.state = 'new' +-- where hook %s +GROUP BY sky_id +) AS W + +LEFT JOIN + +(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state + FROM remoteRun + JOIN remoteComponent USING(remote_id) +WHERE +remoteRun.stage = 'staticsky' AND remoteComponent.state != 'retry' +-- where hook %s +) AS R +USING(stage_id) +WHERE R.remote_id IS NULL Index: /trunk/ippTools/share/remotetool_definebyquery_warp.sql =================================================================== --- /trunk/ippTools/share/remotetool_definebyquery_warp.sql (revision 37832) +++ /trunk/ippTools/share/remotetool_definebyquery_warp.sql (revision 37833) @@ -17,5 +17,5 @@ JOIN remoteComponent USING (remote_id) WHERE -remoteRun.stage = 'warp' +remoteRun.stage = 'warp' and remoteComponent.state != 'retry' -- where hook %s ) AS R Index: /trunk/ippTools/share/remotetool_doneprep.sql =================================================================== --- /trunk/ippTools/share/remotetool_doneprep.sql (revision 37833) +++ /trunk/ippTools/share/remotetool_doneprep.sql (revision 37833) @@ -0,0 +1,30 @@ +SELECT * FROM + (SELECT + remote_id, + remoteRun.path_base, + stage, + count(stage_id) as Ndone +FROM remoteRun +JOIN remoteComponent +USING (remote_id) +WHERE +((remoteComponent.state = 'prep_done') OR (remoteComponent.state = 'prep_fail') OR (remoteComponent.state = 'fail')) +-- and hook %s +GROUP BY remote_id) +AS DONE + +JOIN + +(SELECT +remote_id, +count(stage_id) as Njobs +FROM remoteRun +JOIN remoteComponent +USING (remote_id) +-- where hook %s +GROUP BY remote_id) +AS JOBS + +using (remote_id) + +where Ndone = Njobs Index: /trunk/ippTools/share/remotetool_listcomponent.sql =================================================================== --- /trunk/ippTools/share/remotetool_listcomponent.sql (revision 37833) +++ /trunk/ippTools/share/remotetool_listcomponent.sql (revision 37833) @@ -0,0 +1,11 @@ +SELECT DISTINCT + remoteComponent.remote_id, + remoteComponent.stage_id, + remoteComponent.jobs, + remoteComponent.state, + remoteComponent.path_base, + remoteRun.stage, + remoteRun.path_base as run_path_base +FROM remoteRun +JOIN remoteComponent + USING(remote_id) Index: /trunk/ippTools/src/remotetool.c =================================================================== --- /trunk/ippTools/src/remotetool.c (revision 37832) +++ /trunk/ippTools/src/remotetool.c (revision 37833) @@ -21,4 +21,7 @@ static bool listrunMode(pxConfig *config); static bool listcomponentMode(pxConfig *config); + +// list remoteRuns with all remoteComponents in state 'prep_done' or 'prep_fail' +static bool doneprepMode(pxConfig *config); // Update @@ -26,6 +29,8 @@ static bool updaterunMode(pxConfig *config); static bool updatepollMode(pxConfig *config); +static bool updatecomponentMode(pxConfig *config); static bool revertrunMode(pxConfig *config); static bool revertauthMode(pxConfig *config); +static bool revertcomponentMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -47,12 +52,15 @@ switch (config->mode) { - MODECASE(REMOTETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); - MODECASE(REMOTETOOL_MODE_LISTRUN, listrunMode); - MODECASE(REMOTETOOL_MODE_LISTCOMPONENT, listcomponentMode); - MODECASE(REMOTETOOL_MODE_DROPCOMPONENT, dropcomponentMode); - MODECASE(REMOTETOOL_MODE_UPDATERUN, updaterunMode); - MODECASE(REMOTETOOL_MODE_UPDATEPOLL, updatepollMode); - MODECASE(REMOTETOOL_MODE_REVERTRUN, revertrunMode); - MODECASE(REMOTETOOL_MODE_REVERTAUTH, revertauthMode); + MODECASE(REMOTETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(REMOTETOOL_MODE_LISTRUN, listrunMode); + MODECASE(REMOTETOOL_MODE_DONEPREP, doneprepMode); + MODECASE(REMOTETOOL_MODE_LISTCOMPONENT, listcomponentMode); + MODECASE(REMOTETOOL_MODE_DROPCOMPONENT, dropcomponentMode); + MODECASE(REMOTETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(REMOTETOOL_MODE_UPDATEPOLL, updatepollMode); + MODECASE(REMOTETOOL_MODE_UPDATECOMPONENT, updatecomponentMode); + MODECASE(REMOTETOOL_MODE_REVERTRUN, revertrunMode); + MODECASE(REMOTETOOL_MODE_REVERTAUTH, revertauthMode); + MODECASE(REMOTETOOL_MODE_REVERTCOMPONENT, revertcomponentMode); default: @@ -92,5 +100,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); - + set_label = set_label ? set_label : label; @@ -105,5 +113,5 @@ return(false); } - + if (label) { psStringAppend(&whereOption, "\n AND (label = '%s')", label); @@ -116,9 +124,9 @@ return(false); } - + if (label) { psStringAppend(&whereOption, "\n AND (label = '%s')", label); } - } + } else if (!strcmp(stage,"warp")) { query = pxDataGet("remotetool_definebyquery_warp.sql"); @@ -127,5 +135,5 @@ return(false); } - + if (label) { // Because warp has to check to see if something other than the state is set correctly, I have to use this wonky way to pass the label in. @@ -133,5 +141,5 @@ psStringAppend(&whereOption, "\n AND (label = '%s')", label); } - } + } else if (!strcmp(stage,"stack")) { query = pxDataGet("remotetool_definebyquery_stack.sql"); @@ -140,9 +148,20 @@ return(false); } - + if (label) { psStringAppend(&whereOption, "\n AND (label = '%s')", label); } - } + } + else if (!strcmp(stage,"staticsky")) { + query = pxDataGet("remotetool_definebyquery_staticsky.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + if (label) { + psStringAppend(&whereOption, "\n AND (label = '%s')", label); + } + } else { psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage); @@ -191,17 +210,17 @@ // Insert the top level run object remoteRunRow *run = remoteRunRowAlloc(0, // remote_id - "new", // state - stage, - set_label, - path_base, - -1, // job_id - NULL, // last_poll - 0 // fault - ); + "new", // state + stage, + set_label, + path_base, + -1, // job_id + NULL, // last_poll + 0 // fault + ); if (!run) { - psError(PS_ERR_UNKNOWN, false, "failed to alloc lapRun object"); + psError(PS_ERR_UNKNOWN, false, "failed to alloc remoteRun object"); return(true); } - + if (!remoteRunInsertObject(config->dbh, run)) { if (!psDBRollback(config->dbh)) { @@ -213,5 +232,5 @@ } psS64 remote_id = psDBLastInsertID(config->dbh); - + psArray *list = psArrayAllocEmpty(limit); for (long i=0; i < psArrayLength(output); i++) { @@ -219,14 +238,13 @@ psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id"); - remoteComponentRow *comp = remoteComponentRowAlloc(remote_id, - stage_id); + remoteComponentRow *comp = remoteComponentRowAlloc(remote_id, stage_id, 0, "new", NULL); if (!comp) { - psError(PS_ERR_UNKNOWN, false, "failed to alloc lapRun object"); + psError(PS_ERR_UNKNOWN, false, "failed to alloc remoteComponent object"); return(true); } - + if (!remoteComponentInsertObject(config->dbh, comp)) { if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error"); } psError(PS_ERR_UNKNOWN, false, "database error"); @@ -237,5 +255,5 @@ psArrayAdd(list, list->n, comp); psFree(comp); - + } psFree(run); @@ -245,5 +263,5 @@ return false; } - + if (!remoteComponentPrintObjects(stdout, list, !simple)) { if (!psDBRollback(config->dbh)) { @@ -254,8 +272,8 @@ return false; } - + psFree(list); psFree(output); - + return true; } @@ -265,5 +283,5 @@ { psMetadata *where = psMetadataAlloc(); - + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); @@ -274,14 +292,13 @@ PXOPT_COPY_TIME(config->args, where, "-poll_begin", "last_poll", ">="); PXOPT_COPY_TIME(config->args, where, "-poll_end", "last_poll", "<="); - + PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); PXOPT_LOOKUP_S16(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psArray *remRuns = NULL; remRuns = psDBSelectRows(config->dbh, "remoteRun", where, limit); - + if (!ippdbPrintMetadatas(stdout, remRuns, "remoteRun", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); @@ -299,23 +316,139 @@ PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_S64(config->args, where, "-jobs", "jobs", "=="); + + PXOPT_COPY_STR(config->args, where, "-state", "remoteComponent.state", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "remoteRun.label", "=="); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psArray *remComps = NULL; - remComps = psDBSelectRows(config->dbh, "remoteComponent", where, 0); - - if (!ippdbPrintMetadatas(stdout, remComps, "remoteComponent", !simple)) { + PXOPT_LOOKUP_S16(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("remotetool_listcomponent.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("remotetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "remoteComponent", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(remComps); - return false; - } - psFree(remComps); + psFree(output); + return false; + } + psFree(output); return true; } +// List +static bool doneprepMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "remoteRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_LOOKUP_S16(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("remotetool_doneprep.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + psString whereDone = NULL; + psString whereJobs = NULL; + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereDone, " \n AND %s", whereClause); + psStringAppend(&whereJobs, " \n WHERE %s", whereClause); + psFree(whereClause); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + // fprintf (stderr, "query: %s\n", query); + // fprintf (stderr, "whereDone: %s\n", whereDone); + // fprintf (stderr, "whereJobs: %s\n", whereJobs); + + if (!p_psDBRunQueryF(config->dbh, query, whereDone, whereJobs)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereDone); + psFree(whereJobs); + psFree(query); + return false; + } + psFree(whereDone); + psFree(whereJobs); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("remotetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "remoteRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + // Update static bool dropcomponentMode(pxConfig *config) { + fprintf (stderr, "do we really want to use -dropcomponent?\n"); + exit (1); + // Assert that we have a unique component to operate on. PXOPT_LOOKUP_S64(remote_id,config->args, "-remote_id",true, false); @@ -329,5 +462,5 @@ return false; } - + if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -341,6 +474,4 @@ } psFree(query); - - // point of no return @@ -349,5 +480,5 @@ return false; } - + return true; } @@ -364,5 +495,5 @@ // Wheres PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); - + // Values to set PXOPT_COPY_STR(config->args, values, "-set_label", "label", "=="); @@ -374,9 +505,5 @@ psFree(values); - if (rows < 1) { - psError(PS_ERR_UNKNOWN,false, "no rows affected"); - return(false); - } - + if (rows < 1) fprintf (stderr, "no rows changed (run may already be in desired state)\n"); return(true); } @@ -393,5 +520,5 @@ psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psFree(where); - + psString query = NULL; query = pxDataGet("remotetool_updatepoll.sql"); @@ -412,7 +539,30 @@ return(true); } - - - + +static bool updatecomponentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); + psAssert(remote_id,"This should have failed before this point."); + + psMetadata *where = psMetadataAlloc(); + psMetadata *values = psMetadataAlloc(); + + // Wheres + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + + // Values to set + PXOPT_COPY_S32(config->args, values, "-set_jobs", "jobs", "=="); + PXOPT_COPY_STR(config->args, values, "-set_path_base", "path_base", "=="); + PXOPT_COPY_STR(config->args, values, "-set_state", "state", "=="); + + long rows = psDBUpdateRows(config->dbh, "remoteComponent", where, values); + psFree(values); + + if (rows < 1) fprintf (stderr, "no rows changed (component may already be in desired state)\n"); + return true; +} static bool revertrunMode(pxConfig *config) @@ -426,33 +576,7 @@ PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - + PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); // Value to set - PXOPT_COPY_S16(config->args, values, "-fault", "fault", "=="); - long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values); - psFree(values); - psFree(where); - if (rows < 1) { - psError(PS_ERR_UNKNOWN,false, "no rows affected"); - return(false); - } - - return(true); -} - -static bool revertauthMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - psMetadata *values = psMetadataAlloc(); - - // required - PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - - psMetadataAddStr(where, PS_LIST_TAIL, "state", 0, "==", "auth"); - - // Value to set - psMetadataAddStr(values, PS_LIST_TAIL, "state", 0, "==", "run"); + psMetadataAddS16(values,PS_LIST_TAIL,"fault",0, "==", 0); long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values); @@ -466,3 +590,58 @@ return(true); } - + +static bool revertcomponentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config,false); + psMetadata *where = psMetadataAlloc(); + psMetadata *values = psMetadataAlloc(); + + // required + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + psMetadataAddStr(where,PS_LIST_TAIL,"state",0, "LIKE", "%fail"); + + // value to set + psMetadataAddStr(values, PS_LIST_TAIL, "state",0, "==", "retry"); + + + long rows = psDBUpdateRows(config->dbh, "remoteComponent", where, values); + psFree(values); + psFree(where); + if (rows < 1) { + psError(PS_ERR_UNKNOWN,false, "no rows affected"); + return(false); + } + + return(true); +} + + + +static bool revertauthMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + psMetadata *values = psMetadataAlloc(); + + // required + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psMetadataAddStr(where, PS_LIST_TAIL, "state", 0, "==", "auth"); + + // Value to set + psMetadataAddStr(values, PS_LIST_TAIL, "state", 0, "==", "run"); + + long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values); + psFree(values); + psFree(where); + if (rows < 1) { + psError(PS_ERR_UNKNOWN,false, "no rows affected"); + return(false); + } + + return(true); +} + Index: /trunk/ippTools/src/remotetool.h =================================================================== --- /trunk/ippTools/src/remotetool.h (revision 37832) +++ /trunk/ippTools/src/remotetool.h (revision 37833) @@ -12,10 +12,13 @@ REMOTETOOL_MODE_DEFINEBYQUERY, REMOTETOOL_MODE_LISTRUN, + REMOTETOOL_MODE_DONEPREP, REMOTETOOL_MODE_LISTCOMPONENT, REMOTETOOL_MODE_DROPCOMPONENT, REMOTETOOL_MODE_UPDATERUN, REMOTETOOL_MODE_UPDATEPOLL, - REMOTETOOL_MODE_REVERTRUN, - REMOTETOOL_MODE_REVERTAUTH, + REMOTETOOL_MODE_UPDATECOMPONENT, + REMOTETOOL_MODE_REVERTRUN, + REMOTETOOL_MODE_REVERTAUTH, + REMOTETOOL_MODE_REVERTCOMPONENT, } remotetoolMode; Index: /trunk/ippTools/src/remotetoolConfig.c =================================================================== --- /trunk/ippTools/src/remotetoolConfig.c (revision 37832) +++ /trunk/ippTools/src/remotetoolConfig.c (revision 37833) @@ -40,5 +40,5 @@ ADD_OPT(Bool, definebyqueryArgs, "-pretend", "pretend to do this.", 0); ADD_OPT(Bool, definebyqueryArgs,"-simple", "simple print format",0); - + // -listrun psMetadata *listrunArgs = psMetadataAlloc(); @@ -59,9 +59,30 @@ ADD_OPT(S16, listrunArgs, "-limit", "limit to number of runs to display", 0); + // -doneprep + psMetadata *doneprepArgs = psMetadataAlloc(); + ADD_OPT(S64, doneprepArgs, "-remote_id", "run to list", 0); + ADD_OPT(S64, doneprepArgs, "-job_id", "job_id", 0); + ADD_OPT(Str, doneprepArgs, "-stage", "stage to return", NULL); + ADD_OPT(Str, doneprepArgs, "-state", "state to return", NULL); + ADD_OPT(Str, doneprepArgs, "-label", "remRun label to use", NULL); + + ADD_OPT(Bool, doneprepArgs, "-simple", "simple print format", false); + ADD_OPT(S16, doneprepArgs, "-limit", "limit to number of runs to display", 0); + + // -revertcomponent + psMetadata *revertcomponentArgs = psMetadataAlloc(); + ADD_OPT(S64, revertcomponentArgs, "-remote_id", "run to revert components from", 0); + ADD_OPT(S64, revertcomponentArgs, "-stage_id", "component stage_id to revert", 0); + + // -listcomponents - psMetadata *listcomponentArgs = psMetadataAlloc(); - ADD_OPT(S64, listcomponentArgs, "-remote_id", "run to list", 0); - ADD_OPT(S16, listcomponentArgs, "-limit", "limit to number of runs to display", 0); + psMetadata *listcomponentArgs = psMetadataAlloc(); + ADD_OPT(S64, listcomponentArgs, "-remote_id", "run to list", 0); + ADD_OPT(S64, listcomponentArgs, "-stage_id", "run to list", 0); + ADD_OPT(S32, listcomponentArgs, "-jobs", "run to list", 0); + ADD_OPT(Str, listcomponentArgs, "-state", "run to list", NULL); + ADD_OPT(S16, listcomponentArgs, "-limit", "limit to number of runs to display", 0); ADD_OPT(Bool, listcomponentArgs, "-simple", "simple print format", false); + ADD_OPT(Str, listcomponentArgs, "-label", "label to use", NULL); // -dropcomponent @@ -69,5 +90,5 @@ ADD_OPT(S64, dropcomponentArgs, "-remote_id", "run to update", 0); ADD_OPT(S64, dropcomponentArgs, "-stage_id", "stage_id to remote", 0); - + // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); @@ -83,5 +104,14 @@ psMetadata *updatepollArgs = psMetadataAlloc(); ADD_OPT(S64, updatepollArgs, "-remote_id", "run to update", 0); - + + // -updatecomponent + psMetadata *updatecomponentArgs = psMetadataAlloc(); + ADD_OPT(S64, updatecomponentArgs, "-remote_id", "remote_id to update", 0); + ADD_OPT(S64, updatecomponentArgs, "-stage_id", "stage_id to update", 0); + ADD_OPT(Str, updatecomponentArgs, "-state", "limit to this remoteComponent state", NULL); + ADD_OPT(Str, updatecomponentArgs, "-set_state", "remoteComponent state to assign", NULL); + ADD_OPT(Str, updatecomponentArgs, "-set_path_base", "remoteRun path_base to assign", NULL); + ADD_OPT(S32, updatecomponentArgs, "-set_jobs", "number of jobs for this stage_id", 0); + // -revertrun psMetadata *revertrunArgs = psMetadataAlloc(); @@ -89,5 +119,5 @@ ADD_OPT(Str, revertrunArgs, "-label", "remRun label to use", NULL); ADD_OPT(S16, revertrunArgs, "-fault", "fault code", 0); - + // -revertauth psMetadata *revertauthArgs = psMetadataAlloc(); @@ -98,12 +128,15 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "", REMOTETOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-listrun", "", REMOTETOOL_MODE_LISTRUN, listrunArgs); - PXOPT_ADD_MODE("-listcomponent", "", REMOTETOOL_MODE_LISTCOMPONENT, listcomponentArgs); - PXOPT_ADD_MODE("-dropcomponent", "", REMOTETOOL_MODE_DROPCOMPONENT, dropcomponentArgs); - PXOPT_ADD_MODE("-updaterun", "", REMOTETOOL_MODE_UPDATERUN, updaterunArgs); - PXOPT_ADD_MODE("-updatepoll", "", REMOTETOOL_MODE_UPDATEPOLL, updatepollArgs); - PXOPT_ADD_MODE("-revertrun", "", REMOTETOOL_MODE_REVERTRUN, revertrunArgs); - PXOPT_ADD_MODE("-revertauth", "", REMOTETOOL_MODE_REVERTAUTH, revertauthArgs); + PXOPT_ADD_MODE("-definebyquery", "", REMOTETOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-listrun", "", REMOTETOOL_MODE_LISTRUN, listrunArgs); + PXOPT_ADD_MODE("-doneprep", "", REMOTETOOL_MODE_DONEPREP, doneprepArgs); + PXOPT_ADD_MODE("-listcomponent", "", REMOTETOOL_MODE_LISTCOMPONENT, listcomponentArgs); + PXOPT_ADD_MODE("-revertcomponent", "", REMOTETOOL_MODE_REVERTCOMPONENT, revertcomponentArgs); + PXOPT_ADD_MODE("-dropcomponent", "", REMOTETOOL_MODE_DROPCOMPONENT, dropcomponentArgs); + PXOPT_ADD_MODE("-updaterun", "", REMOTETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-updatepoll", "", REMOTETOOL_MODE_UPDATEPOLL, updatepollArgs); + PXOPT_ADD_MODE("-updatecomponent", "", REMOTETOOL_MODE_UPDATECOMPONENT, updatecomponentArgs); + PXOPT_ADD_MODE("-revertrun", "", REMOTETOOL_MODE_REVERTRUN, revertrunArgs); + PXOPT_ADD_MODE("-revertauth", "", REMOTETOOL_MODE_REVERTAUTH, revertauthArgs);