Index: /tags/ipp-pv3-20140717/ippScripts/Build.PL =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/Build.PL (revision 37297) +++ /tags/ipp-pv3-20140717/ippScripts/Build.PL (revision 37298) @@ -143,4 +143,5 @@ scripts/sc_prepare_run.pl scripts/sc_remote_exec.pl + scripts/sc_remote_poll.pl scripts/sc_retry_return.pl scripts/sc_transfer_tool.pl Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_camera.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_camera.pl (revision 37297) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_camera.pl (revision 37298) @@ -20,6 +20,7 @@ # Hard coded values # my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $threads_req = 0; # How many threads are we going to use? +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $remote_hostname = "LANL/Mustang"; # Name of the remote node. +my $threads_req = 0; # How many threads are we going to use? my $fail_state = "prep_fail"; @@ -38,5 +39,5 @@ GetOptions( 'remote_id=s' => \$remote_id, - 'cam_d=s' => \$cam_id, + 'cam_id=s' => \$cam_id, 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, @@ -85,30 +86,32 @@ # STEP 1: Get exposure level information for this camRun -my ($workdir,$exp_tag); -{ +my ($workdir,$exp_tag,$reduction); +{ my $command = "$camtool -pendingexp -cam_id $cam_id "; $command .= " -dbname $dbname " if defined($dbname); - + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => 0); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - + &my_die("Unable to run camtool to determine stage parameters.", $remote_id, $cam_id, $error_code, $fail_state); } - + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine cam component information.", $remote_id, $cam_id, $PS_EXIT_PROG_ERROR, $fail_state); + &my_die("Unable to determine cam component information.", $remote_id, $cam_id, $PS_EXIT_PROG_ERROR, $fail_state); my $metadata = parse_md_list($MDlist); my $camEntry = $metadata->[0]; - - $workdir = $camEntry->{workdir}; - $exp_tag = $camEntry->{exp_tag}; + + $workdir = $camEntry->{workdir}; + $exp_tag = $camEntry->{exp_tag}; + $reduction = $camEntry->{reduction}; + $reduction = 'DEFAULT' unless defined $reduction; unless (defined($workdir)) { - while( my ($k, $v) = each %$camEntry ) { - print "key: $k, value: $v.\n"; - } - print "%{ $camEntry }\n"; - die; + while( my ($k, $v) = each %$camEntry ) { + print "key: $k, value: $v.\n"; + } + print "%{ $camEntry }\n"; + die; } } @@ -122,8 +125,8 @@ $command .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to run camtool -pendingimfile ", $remote_id, $cam_id, $error_code, $fail_state); } @@ -131,5 +134,5 @@ # We don't actually care about the input cam data other than to know which mask files to instantiate. my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine cam component information.", $remote_id, $cam_id, $PS_EXIT_PROG_ERROR, $fail_state); + &my_die("Unable to determine cam component information.", $remote_id, $cam_id, $PS_EXIT_PROG_ERROR, $fail_state); my $metadata = parse_md_list($MDlist); @@ -140,14 +143,14 @@ my $pre_cmd_masks= "ls -1 ${remote_chip_path}*.mk.fits > ${remote_outroot}.masklist"; - my $reduction = 'DEFAULT' unless defined $camEntry->{reduction}; - my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use + my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use + if ($recipe_psastro eq "") { + &my_die("Unable to determine PSASTRO recipe", $cam_id, $PS_EXIT_PROG_ERROR); + } my $psastro_command = " psastro -list ${remote_outroot}.cmflist "; $psastro_command .= " -masklist ${remote_outroot}.masklist ${remote_outroot} "; -# $psastro_command .= " -refmasklist ${remote_outroot}.masklist ${remote_outroot} "; -# This is used to do edge calculations. It should probably be smart enough to know that it can just use the masks I just specified, but it's not. - # Uncomment when remote psastro supports this +# $psastro_command .= " -refmasklist ${remote_outroot}.masklist ${remote_outroot} "; + $psastro_command .= " -refmasklist /turquoise/usr/projects/ps1/watersc1/references/gpc1.refmask.list "; $psastro_command .= " -kh-correct /turquoise/usr/projects/ps1/watersc1/references/khcorrect.20140606.v0.fits "; - $psastro_command .= " -refmasklist /turquoise/usr/projects/ps1/watersc1/references/gpc1.refmask.list "; # $psastro_command .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20080909.asm "; $psastro_command .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20140505.asm "; @@ -156,7 +159,7 @@ $psastro_command .= " -dumpconfig ${remote_outroot}.psastro.mdc -stats ${remote_outroot}.stats "; $psastro_command .= " -recipe PPSTATS CAMSTATS "; - + my $camtool_post_cmd = "camtool -cam_id $cam_id -addprocessedexp -uri UNKNOWN "; - $camtool_post_cmd .= " -dbname $dbname " if defined $dbname; + $camtool_post_cmd .= " -dbname $dbname " if defined $dbname; $camtool_post_cmd .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 "; my $post_cmd_echo = " echo -n \"$camtool_post_cmd\" > ${remote_outroot}.dbinfo "; @@ -168,14 +171,14 @@ # Determine which output files need to be returned foreach my $component(@return_component_list) { - my $filename = $ipprc->filename($component,$ipp_outroot); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); - my $remote_outroot_dir = dirname($ipp_disk); - print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + my $filename = $ipprc->filename($component,$ipp_outroot); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; } foreach my $chipInfo (@{ $metadata }) { - my $filename = $ipprc->filename("PSASTRO.OUTPUT.MASK",$ipp_outroot,$chipInfo->{class_id}); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); - my $remote_outroot_dir = dirname($ipp_disk); - print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + my $filename = $ipprc->filename("PSASTRO.OUTPUT.MASK",$ipp_outroot,$chipInfo->{class_id}); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; } print CONFIG "\n"; @@ -204,5 +207,5 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri ); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { &my_die( "Unable to generate file for $neb_uri ", $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); @@ -220,5 +223,5 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { &my_die( "Unable to generate file for $neb_uri ", $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); @@ -235,5 +238,5 @@ my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - + unless (exists($file_filter{$neb_uri})) { $file_filter{$neb_uri} = 1; @@ -265,5 +268,5 @@ return($remote_uri); } - + sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); # This needs to replace the remote root directory with the nebulous tag. @@ -271,5 +274,5 @@ $remote_uri =~ s%${remote_root}%%; my $local_uri = "neb:///" . $remote_uri; - + return($local_uri); } @@ -286,7 +289,7 @@ carp($msg); - + if (defined $remote_id and defined $stage_id and not $no_update) { - my $command = "remotetool -updatecomponent -remote_id $id -stage_id $stage_id"; + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; $command .= " -set_state $exit_state " if defined $exit_state; $command .= " -dbname $dbname " if defined $dbname; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl (revision 37297) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl (revision 37298) @@ -20,7 +20,8 @@ # Hard coded values # my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $remote_raw = "${remote_root}/raw/"; # Directory to find raw data in. -my $threads_req = 4; +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $remote_hostname = "LANL/Mustang"; # Name of the remote node. +my $remote_raw = "${remote_root}/raw/"; # Directory to find raw data in. +my $threads_req = 4; my $fail_state = "prep_fail"; @@ -264,5 +265,5 @@ my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; - $post_commandA .= " -magicked 0 -hostname REMOTE -dtime_script 0 "; + $post_commandA .= " -magicked 0 -hostname $remote_hostname -dtime_script 0 "; $post_commandA .= " -dbname $dbname " if defined $dbname; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl (revision 37297) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl (revision 37298) @@ -32,5 +32,5 @@ # Estimate of how long a job runs, in hours (depends on stage) my %job_cost = (); -$job_cost{"camera"} = 1700 / 60 / 60; +$job_cost{"camera"} = 1700 / 60 / 60; $job_cost{"chip"} = 150 / 60 / 60; $job_cost{"warp"} = 110 / 60 / 60; @@ -42,5 +42,4 @@ my $min_time = 1; # shortest allocation to ask for my $max_time = 8; # longest allocation to ask for -my $remote_hostname= "LANL/Mustang"; # Name of the remote node. # We need to ensure we only ever try to transfer a file once. Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_stack.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_stack.pl (revision 37297) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_stack.pl (revision 37298) @@ -20,6 +20,7 @@ # Hard coded values # my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $threads_req = 4; # How many threads are we going to use? +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $remote_hostname = "LANL/Mustang"; # Name of the remote node. +my $threads_req = 4; # How many threads are we going to use? my $fail_state = "prep_fail"; @@ -59,7 +60,7 @@ my @return_component_list = ("DBINFO.EXP", "PPSTACK.UNCONV","PPSTACK.UNCONV.MASK","PPSTACK.UNCONV.VARIANCE", - "PPSTACK.UNCONV.EXP","PPSTACK.UNCONV.EXPWT","PPSTACK.UNCONV.EXPNUM","PSPHOT.OUTPUT", - "LOG.EXP", "PSASTRO.STATS", - "PPSTACK.CONFIG","PPSTACK.TARGET.PSF","PSPHOT.STACK.PSF.SAVE"); + "PPSTACK.UNCONV.EXP","PPSTACK.UNCONV.EXPWT","PPSTACK.UNCONV.EXPNUM","PSPHOT.OUTPUT", + "LOG.EXP", "PSASTRO.STATS", + "PPSTACK.CONFIG","PPSTACK.TARGET.PSF","PSPHOT.STACK.PSF.SAVE"); #,"PPSTACK.CONV.KERNEL"); @@ -93,12 +94,12 @@ $command .= " -dbname $dbname " if defined($dbname); ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + run(command => $command, verbose => 0); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run stacktool -tosum ", $remote_id, $stack_id, $error_code, $fail_state); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run stacktool -tosum ", $remote_id, $stack_id, $error_code, $fail_state); } my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine stack component information.", $remote_id, $stack_id, $PS_EXIT_PROG_ERROR, $fail_state); + &my_die("Unable to determine stack component information.", $remote_id, $stack_id, $PS_EXIT_PROG_ERROR, $fail_state); my $metadata = parse_md_list($MDlist); @@ -111,6 +112,6 @@ $skycell_id= $stack->{skycell_id}; } -my $ipp_outroot = "${workdir}/${tess_id}/${skycell_id}/${tess_id}.${skycell_id}.stk.${stack_id}"; -my $remote_outroot = uri_local_to_remote($ipp_outroot); +my $ipp_outroot = "${workdir}/${tess_id}/${skycell_id}/${tess_id}.${skycell_id}.stk.${stack_id}"; +my $remote_outroot = uri_local_to_remote($ipp_outroot); my $remote_outdir = uri_local_to_remote("${workdir}/${tess_id}/${skycell_id}"); @@ -122,12 +123,12 @@ $command .= " -dbname $dbname " if defined($dbname); ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + run(command => $command, verbose => 0); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run stacktool -inputskyfile ", $remote_id, $stack_id, $error_code, $fail_state); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run stacktool -inputskyfile ", $remote_id, $stack_id, $error_code, $fail_state); } my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine stack component information.", $remote_id, $stack_id, $PS_EXIT_PROG_ERROR, $fail_state); + &my_die("Unable to determine stack component information.", $remote_id, $stack_id, $PS_EXIT_PROG_ERROR, $fail_state); my $warpData = parse_md_list($MDlist); } @@ -137,17 +138,17 @@ my $warp_path_base_string = ""; foreach my $warpEntry ( @{ $warpData } ) { - my $warp_path_base = $warpEntry->{path_base}; - my $remote_path_base = uri_local_to_remote($warp_path_base); - $warp_path_base_string .= " $remote_path_base "; - - # Append file names to transfer lists so we can recover if warps are deleted. - uri_to_outputs($warp_path_base . ".fits"); - uri_to_outputs($warp_path_base . ".wt.fits"); - uri_to_outputs($warp_path_base . ".mask.fits"); - uri_to_outputs($warp_path_base . ".cmf"); -# uri_to_outputs($warp_path_base . ".psf"); + my $warp_path_base = $warpEntry->{path_base}; + my $remote_path_base = uri_local_to_remote($warp_path_base); + $warp_path_base_string .= " $remote_path_base "; + + # Append file names to transfer lists so we can recover if warps are deleted. + uri_to_outputs($warp_path_base . ".fits"); + uri_to_outputs($warp_path_base . ".wt.fits"); + uri_to_outputs($warp_path_base . ".mask.fits"); + uri_to_outputs($warp_path_base . ".cmf"); +# uri_to_outputs($warp_path_base . ".psf"); } my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl $warp_path_base_string > ${remote_outroot}.in.mdc"; - my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use + my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use my $ppstack_command = " ppStack -input ${remote_outroot}.in.mdc "; $ppstack_command .= " ${remote_outroot} -stats ${remote_outroot}.stats "; @@ -169,5 +170,5 @@ $post_cmd_echo .= " -dbname $dbname " if defined $dbname; $post_cmd_echo .= " -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; - + my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - STACK_SKYCELL >> ${remote_outroot}.dbinfo "; @@ -177,8 +178,8 @@ # Determine which output files need to be returned foreach my $component(@return_component_list) { - my $filename = $ipprc->filename($component,$ipp_outroot); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); - my $remote_outroot_dir = dirname($ipp_disk); - print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + my $filename = $ipprc->filename($component,$ipp_outroot); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; } print CONFIG "\n"; @@ -206,5 +207,5 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri ); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { &my_die( "Unable to generate file for $neb_uri ", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); @@ -222,5 +223,5 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { &my_die( "Unable to generate file for $neb_uri ", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); @@ -237,5 +238,5 @@ my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - + unless (exists($file_filter{$neb_uri})) { $file_filter{$neb_uri} = 1; @@ -267,5 +268,5 @@ return($remote_uri); } - + sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); # This needs to replace the remote root directory with the nebulous tag. @@ -273,5 +274,5 @@ $remote_uri =~ s%${remote_root}%%; my $local_uri = "neb:///" . $remote_uri; - + return($local_uri); } @@ -288,5 +289,5 @@ carp($msg); - + if (defined $remote_id and defined $stage_id and not $no_update) { my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl (revision 37297) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl (revision 37298) @@ -20,6 +20,7 @@ # Hard coded values # my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -my $remote_root = '/scratch3/watersc1/'; -my $threads_req = 4; # How many threads are we going to use? +my $remote_root = '/scratch3/watersc1/'; +my $remote_hostname = "LANL/Mustang"; # Name of the remote node. +my $threads_req = 4; # How many threads are we going to use? my $fail_state = "prep_fail"; @@ -90,11 +91,11 @@ $command .= " -dbname $dbname " if defined($dbname); ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + run(command => $command, verbose => 0); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run warptool -pendingimfile ", $remote_id, $warp_id, $error_code, $fail_state); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run warptool -pendingimfile ", $remote_id, $warp_id, $error_code, $fail_state); } my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine warp component information.", $remote_id, $warp_id, $PS_EXIT_PROG_ERROR, $fail_state); + &my_die("Unable to determine warp component information.", $remote_id, $warp_id, $PS_EXIT_PROG_ERROR, $fail_state); $warpData = parse_md_list($MDlist); } @@ -103,93 +104,93 @@ { foreach my $warpEntry ( @{ $warpData } ) { - my $workdir = $warpEntry->{workdir}; - my $exp_tag = $warpEntry->{exp_tag}; - my $skycell_id = $warpEntry->{skycell_id}; - my $tess_id = $warpEntry->{tess_id}; - my $reduction = 'DEFAULT' unless defined $warpEntry->{reduction}; - my $recipe_pswarp = $ipprc->reduction($reduction, 'WARP_PSWARP'); # Recipe to use - my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.wrp.${warp_id}.${skycell_id}"; - print "$ipp_outroot\n"; - my $remote_outroot = uri_local_to_remote($ipp_outroot); - - my $command = "$warptool -scmap -warp_id $warp_id -skycell_id ${skycell_id}"; - $command .= " -dbname $dbname " if defined($dbname); - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run warptool -pendingimfile: $command ", $remote_id, $warp_id, $error_code, $fail_state); - } - - my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine warp component information.", $remote_id, $warp_id, $PS_EXIT_PROG_ERROR, $fail_state); - my $inData = parse_md_list($MDlist); - - my $pre_cmd_ims = "ls -1 "; - my $pre_cmd_masks= "ls -1 "; - my $pre_cmd_vars = "ls -1 "; - my $pre_cmd_astrom = ""; - - foreach my $imfile (@{ $inData2 }) { - my $image = $ipprc->filename("PPIMAGE.CHIP", $imfile->{chip_path_base}, $imfile->{class_id}); - my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); - my $var = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); - - my $astrom= $ipprc->filename("PSASTRO.OUTPUT", $imfile->{cam_path_base}); - - my $remote_image = uri_local_to_remote($image); - my $remote_mask = uri_local_to_remote($mask); - my $remote_var = uri_local_to_remote($var); - my $remote_astrom= uri_local_to_remote($astrom); - - $pre_cmd_ims .= " $remote_image "; - $pre_cmd_masks .= " $remote_mask "; - $pre_cmd_vars .= " $remote_var "; - $pre_cmd_astrom = "echo $remote_astrom > ${remote_outroot}.astrom "; - } - $pre_cmd_ims .= " > ${remote_outroot}.imlist "; - $pre_cmd_masks .= " > ${remote_outroot}.masklist "; - $pre_cmd_vars .= " > ${remote_outroot}.varlist "; - - my $skycell_command = " dvoImageExtract -D CATDIR /turquoise/usr/projects/ps1/watersc1/tess/${tess_id} $skycell_id -o ${remote_outroot}.skyfile "; - - my $pswarp_command = " pswarp -list ${remote_outroot}.imlist "; - $pswarp_command .= " -masklist ${remote_outroot}.masklist -variancelist ${remote_outroot}.varlist "; - $pswarp_command .= " -astromlist ${remote_outroot}.astrom "; - $pswarp_command .= " ${remote_outroot} ${remote_outroot}.skyfile "; - $pswarp_command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE "; - $pswarp_command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF "; - $pswarp_command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF "; - $pswarp_command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS "; - $pswarp_command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL "; - $pswarp_command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID "; - $pswarp_command .= " -recipe PSWARP $recipe_pswarp "; - $pswarp_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; - $pswarp_command .= " -threads $threads_req "; # -image_id ${image_id} -source_id ${source_id} "; - $pswarp_command .= " -recipe PPSTATS WARPSTATS "; - $pswarp_command .= " -dumpconfig ${remote_outroot}.pswarp.mdc -stats ${remote_outroot}.stats "; - -# print "$pswarp_command \n"; - - my $post_cmd_echo = " echo -n \"warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id "; - $post_cmd_echo .= " -path_base $remote_outroot -uri UNKNOWN "; - $post_cmd_echo .= " -dbname $dbname " if defined $dbname; - $post_cmd_echo .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; - - my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - WARP_SKYCELL >> ${remote_outroot}.dbinfo "; - - print CONFIG "${pre_cmd_ims} && ${pre_cmd_masks} && ${pre_cmd_vars} && ${pre_cmd_astrom} && ${skycell_command} && ${pswarp_command} && ${post_cmd_echo} && ${post_cmd_SfM} "; - $job_index++; - - # Determine which output files need to be returned - foreach my $component(@return_component_list) { - my $filename = $ipprc->filename($component,$ipp_outroot,$skycell_id); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); - my $remote_outroot_dir = dirname($ipp_disk); - print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; - } - print CONFIG "\n"; -# die(); + my $workdir = $warpEntry->{workdir}; + my $exp_tag = $warpEntry->{exp_tag}; + my $skycell_id = $warpEntry->{skycell_id}; + my $tess_id = $warpEntry->{tess_id}; + my $reduction = 'DEFAULT' unless defined $warpEntry->{reduction}; + my $recipe_pswarp = $ipprc->reduction($reduction, 'WARP_PSWARP'); # Recipe to use + my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.wrp.${warp_id}.${skycell_id}"; + print "$ipp_outroot\n"; + my $remote_outroot = uri_local_to_remote($ipp_outroot); + + my $command = "$warptool -scmap -warp_id $warp_id -skycell_id ${skycell_id}"; + $command .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run warptool -pendingimfile: $command ", $remote_id, $warp_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine warp component information.", $remote_id, $warp_id, $PS_EXIT_PROG_ERROR, $fail_state); + my $inData = parse_md_list($MDlist); + + my $pre_cmd_ims = "ls -1 "; + my $pre_cmd_masks= "ls -1 "; + my $pre_cmd_vars = "ls -1 "; + my $pre_cmd_astrom = ""; + + foreach my $imfile (@{ $inData2 }) { + my $image = $ipprc->filename("PPIMAGE.CHIP", $imfile->{chip_path_base}, $imfile->{class_id}); + my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); + my $var = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); + + my $astrom= $ipprc->filename("PSASTRO.OUTPUT", $imfile->{cam_path_base}); + + my $remote_image = uri_local_to_remote($image); + my $remote_mask = uri_local_to_remote($mask); + my $remote_var = uri_local_to_remote($var); + my $remote_astrom= uri_local_to_remote($astrom); + + $pre_cmd_ims .= " $remote_image "; + $pre_cmd_masks .= " $remote_mask "; + $pre_cmd_vars .= " $remote_var "; + $pre_cmd_astrom = "echo $remote_astrom > ${remote_outroot}.astrom "; + } + $pre_cmd_ims .= " > ${remote_outroot}.imlist "; + $pre_cmd_masks .= " > ${remote_outroot}.masklist "; + $pre_cmd_vars .= " > ${remote_outroot}.varlist "; + + my $skycell_command = " dvoImageExtract -D CATDIR /turquoise/usr/projects/ps1/watersc1/tess/${tess_id} $skycell_id -o ${remote_outroot}.skyfile "; + + my $pswarp_command = " pswarp -list ${remote_outroot}.imlist "; + $pswarp_command .= " -masklist ${remote_outroot}.masklist -variancelist ${remote_outroot}.varlist "; + $pswarp_command .= " -astromlist ${remote_outroot}.astrom "; + $pswarp_command .= " ${remote_outroot} ${remote_outroot}.skyfile "; + $pswarp_command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE "; + $pswarp_command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF "; + $pswarp_command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF "; + $pswarp_command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS "; + $pswarp_command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL "; + $pswarp_command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID "; + $pswarp_command .= " -recipe PSWARP $recipe_pswarp "; + $pswarp_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; + $pswarp_command .= " -threads $threads_req "; # -image_id ${image_id} -source_id ${source_id} "; + $pswarp_command .= " -recipe PPSTATS WARPSTATS "; + $pswarp_command .= " -dumpconfig ${remote_outroot}.pswarp.mdc -stats ${remote_outroot}.stats "; + +# print "$pswarp_command \n"; + + my $post_cmd_echo = " echo -n \"warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id "; + $post_cmd_echo .= " -path_base $remote_outroot -uri UNKNOWN "; + $post_cmd_echo .= " -dbname $dbname " if defined $dbname; + $post_cmd_echo .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; + + my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - WARP_SKYCELL >> ${remote_outroot}.dbinfo "; + + print CONFIG "${pre_cmd_ims} && ${pre_cmd_masks} && ${pre_cmd_vars} && ${pre_cmd_astrom} && ${skycell_command} && ${pswarp_command} && ${post_cmd_echo} && ${post_cmd_SfM} "; + $job_index++; + + # Determine which output files need to be returned + foreach my $component(@return_component_list) { + my $filename = $ipprc->filename($component,$ipp_outroot,$skycell_id); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + } + print CONFIG "\n"; +# die(); } } @@ -217,5 +218,5 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri ); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { &my_die( "Unable to generate file for $neb_uri ", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); @@ -233,5 +234,5 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { &my_die( "Unable to generate file for $neb_uri ", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); @@ -248,5 +249,5 @@ my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - + unless (exists($file_filter{$neb_uri})) { $file_filter{$neb_uri} = 1; @@ -278,5 +279,5 @@ return($remote_uri); } - + sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); # This needs to replace the remote root directory with the nebulous tag. @@ -284,5 +285,5 @@ $remote_uri =~ s%${remote_root}%%; my $local_uri = "neb:///" . $remote_uri; - + return($local_uri); } @@ -299,5 +300,5 @@ carp($msg); - + if (defined $remote_id and defined $stage_id and not $no_update) { my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_exec.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_exec.pl (revision 37297) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_exec.pl (revision 37298) @@ -32,5 +32,5 @@ # Options -my ($remote_id,$path_base,$dbname,$verbose,$no_update,$camera); +my ($remote_id,$job_id,$path_base,$dbname,$verbose,$no_update,$camera); $verbose = 0; GetOptions( @@ -64,14 +64,12 @@ unless ($rt_success) { $rt_error_code = (($rt_error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run remotetool to determine remote run status", - $remote_id,$rt_error_code); + &my_die("Unable to run remotetool to determine remote run status", $remote_id, $rt_error_code); } my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or - &my_die("Unable to run remotetool to determine remote run status", - $remote_id,$rt_error_code); + &my_die("Unable to run remotetool to determine remote run status", $remote_id, $rt_error_code); my $rrData2 = parse_md_list($rrData); my $runData = ${ $rrData2 }[0]; # There should be only one -my ($remote_path) = uri_convert($path_base); +my ($remote_path) = &uri_local_to_remote($path_base); print "REMOTE PATH: $remote_path\n"; @@ -94,16 +92,12 @@ my $disk_return = $ipprc->file_resolve($uri_return); -scp_put($disk_command,uri_local_to_remote($uri_command)); -scp_put($disk_transfer,uri_local_to_remote($uri_transfer)); -scp_put($disk_check,uri_local_to_remote($uri_check)); -scp_put($disk_config,uri_local_to_remote($uri_config)); -scp_put($disk_generate,uri_local_to_remote($uri_generate)); -scp_put($disk_return, uri_local_to_remote($uri_return)); - -# Run check command -# my (undef,$remote_check) = uri_convert($uri_check); -# my (undef,$remote_transfer)= uri_convert($uri_transfer); - -my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_path --fetch"); +&scp_put($disk_command, &uri_local_to_remote($uri_command)); +&scp_put($disk_transfer, &uri_local_to_remote($uri_transfer)); +&scp_put($disk_check, &uri_local_to_remote($uri_check)); +&scp_put($disk_config, &uri_local_to_remote($uri_config)); +&scp_put($disk_generate, &uri_local_to_remote($uri_generate)); +&scp_put($disk_return, &uri_local_to_remote($uri_return)); + +my $ssh_check_stdout = &ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_path --fetch"); # We no longer need to parse this output, as it retrieves files itself. @@ -113,6 +107,6 @@ # Run real command -my (undef,$remote_command) = uri_convert($uri_command); -my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command"); +my (undef,$remote_command) = &uri_convert($uri_command); +my $ssh_exec_stdout = &ssh_exec_command("msub -V $remote_command"); if ($#{ $ssh_exec_stdout } != -1) { # Parse the output $job_id = ${ $ssh_exec_stdout }[0]; @@ -122,9 +116,9 @@ unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. - &my_die("No job_id returned. Sorry.",$remote_id,$PS_EXIT_PROG_ERROR); + &my_die("No job_id returned. Sorry.", $remote_id, $PS_EXIT_PROG_ERROR); } # Notify the database that this entry is currently running. -&my_die("Finished",$remote_id,0,"run",$job_id); +&my_die("Finished", $remote_id, 0, "run", $job_id); # END PROGRAM @@ -137,5 +131,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Authorization check failed.",$remote_id,0,'auth'); + &my_die("Authorization check failed.", $remote_id, 0, 'auth'); } } @@ -147,11 +141,11 @@ my $directory = dirname($destination); - ssh_exec_command("mkdir -p $directory"); - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die(); + &ssh_exec_command("mkdir -p $directory"); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("failed to send file $file to LANL", $remote_id, $error_code); } } @@ -167,5 +161,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die(); + &my_die("failed to get file $file from LANL", $remote_id, $error_code); } } @@ -173,5 +167,5 @@ sub ssh_exec_command { my $cmd = shift; - $cmd = "$ssh $DMZ_HOST ssh ${SEC_HOST} $cmd"; + $cmd = "$ssh -n $DMZ_HOST ssh ${SEC_HOST} $cmd"; print "EXEC: $cmd\n"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -181,32 +175,10 @@ # If we failed, see if we failed due to authorization. check_ssh_connection(); - &my_die("Failed to execute command: >>$cmd<<"); + &my_die("Failed to execute remote command: >>$cmd<<", $remote_id, $error_code); } return ($stdout_buf); } -sub poll_job { - my $job_id = shift; - my $response_array = ssh_exec_command("checkjob -v $job_id"); - my $response = join "\n", @$response_array; - - if ($response =~ /State: Running/) { - return(0,"Running"); - } - elsif ($response =~ /State: Completed/) { - return(1,"Completed"); - } - elsif ($response =~ /State: Idle/) { - return(-2,"Idle"); - } - else { - my $response_word = $response; - $response_word =~ s/.*State: //; - $response_word =~ s/\n.*//; - return(-1,$response_word); - } -} - - +# neb URI to (ipp_disk,remote_disk) sub uri_convert { my $neb_uri = shift; @@ -215,5 +187,5 @@ unless(defined($ipp_disk)) { - my_die(); + &my_die("Failed to generate or find uri $neb_uri", $remote_id, $PS_EXIT_SYS_ERROR); } @@ -225,13 +197,5 @@ } -sub uri_to_outputs { - my $neb_uri = shift; - my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - -# print TRANSFER "$ipp_disk\n"; -# print CHECK "$remote_disk\n"; - return($ipp_disk,$remote_disk); -} - +# neb URI to remote URI-on-disk sub uri_local_to_remote { # This needs to replace the nebulous tag with the remote root. @@ -245,46 +209,4 @@ } -sub uri_remote_to_local { - # This needs to replace the remote root directory with the nebulous tag. - my $remote_uri = shift; - $remote_uri =~ s%${remote_root}%%; - my $local_uri = "neb:///" . $remote_uri; - - return($local_uri); -} - -sub drop_component { - my $remote_id = shift; - my $cmd = shift; - my $file = shift; - - if ($cmd eq '') { - # Zero byte file. This dbinfo was never constructed properly. - - # Steal the stage_id from the filename. This isn't something I'm happy about. - my $stage = $runData->{stage}; - my $stage_id = $file; - if ($stage eq 'chip') { - $stage_id =~ s/^.*ch\.//; - } - elsif ($stage eq 'camera') { - $stage_id =~ s/^.*cm\.//; - } - elsif ($stage eq 'warp') { - $stage_id =~ s/^.*wrp\.//; - } - elsif ($stage eq 'stack') { - $stage_id =~ s/^.*stk\.//; - } - $stage_id =~ s/\..*$//; - - # Drop this from the database - my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id "; - $drop_cmd .= " -dbname $dbname " if defined($dbname); - system($drop_cmd); - } - -} - sub my_die { my $msg = shift; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_poll.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_poll.pl (revision 37297) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_poll.pl (revision 37298) @@ -35,7 +35,8 @@ # tools my $missing_tools; -my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); -my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); +my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); +my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); if ($missing_tools) { @@ -45,6 +46,5 @@ # Options -my ($remote_id,$path_base,$policy,$poll,$force,$job_id,$dbname,$verbose,$no_update,$camera); -$force = 0; +my ($remote_id,$path_base,$skip_poll,$skip_transfer,$job_id,$dbname,$retry,$verbose,$no_update,$camera); $verbose = 0; GetOptions( @@ -52,9 +52,9 @@ 'job_id=s' => \$job_id, 'path_base=s' => \$path_base, - 'policy=s' => \$policy, - 'poll' => \$poll, - 'force-poll' => \$force, + 'skip_poll' => \$skip_poll, + 'skip_transfer' => \$skip_transfer, 'camera=s' => \$camera, 'dbname=s' => \$dbname, + 'retry' => \$retry, 'verbose' => \$verbose, 'no_update' => \$no_update, @@ -78,14 +78,14 @@ # STEP 2: Grab the information about this run; my $runData; -{ +{ my $command = "$remotetool -listrun -remote_id $remote_id "; $command .= " -dbname $dbname " if defined($dbname); my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to run remotetool to determine remote run status", $remote_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run remotetool to determine remote run status", $remote_id, $error_code); } @@ -101,22 +101,27 @@ my $poll_word = "nothing"; -($poll_response,$poll_word) = poll_job($job_id); - -if ($poll_response == -1) { # This job has an error - print "The job exited incorrectly: $remote_id $job_id $poll_word $poll_response.\n"; - # warn("The job exited incorrectly: $remote_id $job_id $poll_word."); - exit (2); -} - -if ($poll_response != 1) { # This job has NOT completed - print "The job has not exited at LANL: $remote_id $job_id $poll_word $poll_response.\n"; - exit (2); +if ($skip_poll) { + $poll_word = "Completed"; +} else { + ($poll_response,$poll_word) = poll_job($job_id); + + if ($poll_response == -1) { # This job has an error + print "The job exited incorrectly: $remote_id $job_id $poll_word $poll_response.\n"; + # warn("The job exited incorrectly: $remote_id $job_id $poll_word."); + exit (2); + } + + if ($poll_response != 1) { # This job has NOT completed + print "The job has not exited at LANL: $remote_id $job_id $poll_word $poll_response.\n"; + exit (2); + } } my $remote_path = uri_local_to_remote($path_base); -# transfer tool should check the existence of files on the remote end -# and only return those that exist -{ +# STEP 4: transfer back the files. Transfer tool should check the +# existence of files on the remote end and only return those that +# exist +unless ($skip_transfer) { my $offset = 0; # Offset to start from. my $iter = 0; @@ -136,13 +141,17 @@ print STDERR "rsync of all files is complete, checking the results\n"; -# grab the .stats file - -my $uri_stats = $path_base . ".stats"; -my $disk_stats = $ipprc->file_resolve($uri_stats); +# STEP 5: grab the .stat file from LANL (this file is first constructed at LANL) + +my $uri_stats = $path_base . ".stat"; +my $disk_stats = $ipprc->file_resolve($uri_stats, 1); my $remote_stats = uri_local_to_remote($uri_stats); scp_get($remote_stats,$disk_stats); -# check for PASS / PART / FAIL : if dbinfo is PASS, go ahead and execute that. +# STEP 6: check dbinfo files for PASS / PART / FAIL + +my %stage_pass = (); +my %stage_fail = (); +my %stage_part = (); my $Npass = 0; @@ -151,21 +160,28 @@ my @dbinfo_files = (); open (STATFILE, "$disk_stats") || &my_die("Couldn't open file? $disk_stats", $remote_id, $PS_EXIT_SYS_ERROR); -while (my $line = ) { +while (my $line = ) { chomp $line; - unless ($_ =~ /dbinfo/) { next; } + unless ($line =~ /dbinfo/) { next; } my @words = split (" ", $line); + + my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($words[0]); + if ($words[1] eq "PASS") { push @dbinfo_files, $words[0]; - $Npass ++; + $stage_pass{$stage_id} = 1; + $Npass ++; } elsif ($words[1] eq "FAIL") { - $Nfail ++; + $stage_fail{$stage_id} = 1; + $Nfail ++; } elsif ($words[1] eq "PART") { - $Npart ++; - } -} -close(RETURN); + $stage_part{$stage_id} = 1; + $Npart ++; + } +} +close (STATFILE); print STDERR "dbinfo files: $Npass PASS, $Nfail FAIL, $Npart PART\n"; +# STEP 7: if dbinfo is PASS, execute it (fail ones that are not PASS?). foreach my $file (@dbinfo_files) { open(DBF,"$file") || warn "Missing file $file\n"; @@ -176,21 +192,80 @@ $cmd =~ s/-/ -/g; # This is just a safety check for missing space. - if ($cmd == "") { - print STDERR "problem: empty dbinfo file\n"; - next; - } - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $cmd, verbose => $verbose); - unless ($success) { - # This shouldn't fail, but we also can't suddenly abort if something does fail. - # Now we're going to drop the component that owns this dbinfo, which should force a retry. - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code"); - if ($poll_word ne 'Completed') { # If the job claims it completed, then we're fighting the network. - fail_component($remote_id, $cmd, $file); - } - } - + if ($cmd eq "") { + print STDERR "problem: empty dbinfo file\n"; + exit (1); + next; + } + + print "dbinfo: $file\n"; + + if ($retry) { + # use the dbinfo filename to get stage id, check if we have already updated this entry: + my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); + + # we just need to check for the existence of an entry: + if ($runData->{stage} eq "chip") { + my $cmd1 = "$chiptool -simple -processedimfile -chip_id $stage_id -class_id $class_id"; + $cmd1 .= " -dbname $dbname " if defined($dbname); + + my ( $success1, $error_code1, $full_buf1, $stdout_buf1, $stderr_buf1 ) = + run(command => $cmd1, verbose => 0); + + unless ($success1) { + $error_code1 = (($error_code1 >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run chiptool to check on results", $remote_id, $error_code1); + } + + my $Nrows = @$stdout_buf1; + if ($Nrows > 0) { + print STDERR "already updated: $stage_id, $class_id\n"; + next; + } + } else { + print STDERR "do not know how to handle stage $runData->{stage}\n"; + exit (3); + } + } + + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + # This shouldn't fail, but we also can't suddenly abort if something does fail. + # Now we're going to drop the component that owns this dbinfo, which should force a retry. + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code"); + my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); + $stage_fail{$stage_id} = 1; + } + } +} +print STDERR "dbinfo files all run (or skipped)\n"; + +# STEP 8: unify the stage states and update the remoteComponent entries +foreach my $entry (keys (%stage_fail)) { + $stage_pass{$entry} = 0; +} +foreach my $entry (keys (%stage_part)) { + $stage_pass{$entry} = 0; +} +foreach my $entry (keys (%stage_pass)) { + my $cmd = "remotetool -updatecomponent -remote_id $remote_id -stage_id $entry"; + $cmd .= " -dbname $dbname " if defined($dbname); + if ($stage_pass{$entry}) { + $cmd .= " -set_state pass"; + } else { + $cmd .= " -set_state fail"; + } + + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + &my_die("Unable to run remotetool to determine remote run status", $remote_id); + } + } else { + print STDERR "no_update: $cmd\n"; + } } @@ -223,5 +298,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die(); + &my_die("Failed to send file $file to LANL", $remote_id, $PS_EXIT_SYS_ERROR); } } @@ -237,5 +312,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die(); + &my_die("Failed to get file $file from LANL", $remote_id, $PS_EXIT_SYS_ERROR); } } @@ -243,5 +318,5 @@ sub ssh_exec_command { my $cmd = shift; - $cmd = "$ssh $DMZ_HOST ssh ${SEC_HOST} $cmd"; + $cmd = "$ssh -n $DMZ_HOST ssh ${SEC_HOST} $cmd"; print "EXEC: $cmd\n"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -251,5 +326,5 @@ # If we failed, see if we failed due to authorization. check_ssh_connection(); - &my_die("Failed to execute command: >>$cmd<<"); + &my_die("Failed to execute command: >>$cmd<<", $remote_id, $PS_EXIT_SYS_ERROR); } return ($stdout_buf); @@ -285,5 +360,5 @@ unless(defined($ipp_disk)) { - my_die(); + &my_die("Failed to generate or find uri $neb_uri", $remote_id, $PS_EXIT_SYS_ERROR); } @@ -324,4 +399,32 @@ } +# assumes $runData->{stage} exists +sub get_stage_id_from_dbinfo { + my $file = shift; + + my $stage_id = 0; + my $class_id = 0; + + # Steal the stage_id from the filename. This isn't something I'm happy about. + my $stage = $runData->{stage}; + if ($stage eq 'chip') { + ($stage_id, $class_id) = $file =~ /ch\.(\d*).(XY\d\d).dbinfo/; + } + elsif ($stage eq 'camera') { + $stage_id =~ s/^.*cm\.//; + $class_id = 0; + } + elsif ($stage eq 'warp') { + $stage_id =~ s/^.*wrp\.//; + $class_id = 0; + } + elsif ($stage eq 'stack') { + $stage_id =~ s/^.*stk\.//; + $class_id = 0; + } + + return ($stage_id, $class_id); +} + sub fail_component { my $remote_id = shift; @@ -333,19 +436,19 @@ my $stage_id = $file; if ($stage eq 'chip') { - $stage_id =~ s/^.*ch\.//; + $stage_id =~ s/^.*ch\.//; } elsif ($stage eq 'camera') { - $stage_id =~ s/^.*cm\.//; + $stage_id =~ s/^.*cm\.//; } elsif ($stage eq 'warp') { - $stage_id =~ s/^.*wrp\.//; + $stage_id =~ s/^.*wrp\.//; } elsif ($stage eq 'stack') { - $stage_id =~ s/^.*stk\.//; + $stage_id =~ s/^.*stk\.//; } $stage_id =~ s/\..*$//; - + print STDERR "failed component: $remote_id, $stage_id\n"; - + # Drop this from the database ## my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id "; @@ -400,5 +503,4 @@ my $command = "$remotetool -updaterun -remote_id $id"; $command .= " -fault $exit_code " if defined $exit_code; - $command .= " -job_id $job_id " if defined $jobid; $command .= " -set_state $exit_state " if defined $exit_state; $command .= " -dbname $dbname " if defined $dbname; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl (revision 37297) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl (revision 37298) @@ -102,7 +102,9 @@ } -my $stat_file = "$input_path.stat"; -print "STATFILE: $stat_file\n"; -open (STATFILE, ">$stat_file"); +unless ($fetch) { + my $stat_file = "$input_path.stat"; + print "STATFILE: $stat_file\n"; + open (STATFILE, ">$stat_file"); +} $i = 0; @@ -116,8 +118,12 @@ if ($line < $offset) { next; } # I think this is off-by-one in a safe direction - if ($line % 250 == 0) { print "line $line : $Lname\n"; } + if ($line % 250 == 0) { print STDERR "line $line : $Lname\n"; } $i = int(rand($#filehandles + 1)); - + if ($i >= $threads) { + print STDERR "HUH: impossible file handle?\n"; + die; + } + # if we are fetching, and 'quickfetch' is requested, skip file if it exists # NOTE: this does not check that the transfer was complete or the file is current @@ -129,9 +135,17 @@ } else { # We are pushing, but only send back files we actually have - if (-e $Lname) { + # we cannot have the remote file unless the local file exists + # (it is a link to the local file and is generated after the local file) + my $haveRemote = -e "$local_tmp/$Rname"; + if ($haveRemote) { print { $filehandles[$i] } "$Rname\n"; print STATFILE "$Rname PASS\n"; } else { - print STATFILE "$Rname FAIL\n"; + my $haveLocal = -e $Lname; + if ($haveLocal) { + print STATFILE "$Rname PART\n"; + } else { + print STATFILE "$Rname FAIL\n"; + } } } @@ -139,5 +153,5 @@ close(LOCAL); close(REMOTE); -close(STATFILE); +unless ($fetch) { close(STATFILE); } for ($i = 0; $i < $threads; $i++) {