Changeset 37298 for tags/ipp-pv3-20140717
- Timestamp:
- Aug 24, 2014, 1:15:51 PM (12 years ago)
- Location:
- tags/ipp-pv3-20140717/ippScripts
- Files:
-
- 9 edited
-
Build.PL (modified) (1 diff)
-
scripts/sc_prepare_camera.pl (modified) (14 diffs)
-
scripts/sc_prepare_chip.pl (modified) (2 diffs)
-
scripts/sc_prepare_run.pl (modified) (2 diffs)
-
scripts/sc_prepare_stack.pl (modified) (14 diffs)
-
scripts/sc_prepare_warp.pl (modified) (9 diffs)
-
scripts/sc_remote_exec.pl (modified) (13 diffs)
-
scripts/sc_remote_poll.pl (modified) (16 diffs)
-
scripts/sc_transfer_tool.pl (modified) (4 diffs)
Legend:
- Unmodified
- Added
- Removed
-
tags/ipp-pv3-20140717/ippScripts/Build.PL
r37284 r37298 143 143 scripts/sc_prepare_run.pl 144 144 scripts/sc_remote_exec.pl 145 scripts/sc_remote_poll.pl 145 146 scripts/sc_retry_return.pl 146 147 scripts/sc_transfer_tool.pl -
tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_camera.pl
r37291 r37298 20 20 # Hard coded values 21 21 # my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location 22 my $remote_root = '/scratch3/watersc1/'; # Far side destination base location 23 my $threads_req = 0; # How many threads are we going to use? 22 my $remote_root = '/scratch3/watersc1/'; # Far side destination base location 23 my $remote_hostname = "LANL/Mustang"; # Name of the remote node. 24 my $threads_req = 0; # How many threads are we going to use? 24 25 25 26 my $fail_state = "prep_fail"; … … 38 39 GetOptions( 39 40 'remote_id=s' => \$remote_id, 40 'cam_ d=s'=> \$cam_id,41 'cam_id=s' => \$cam_id, 41 42 'camera|c=s' => \$camera, 42 43 'dbname|d=s' => \$dbname, … … 85 86 86 87 # STEP 1: Get exposure level information for this camRun 87 my ($workdir,$exp_tag );88 { 88 my ($workdir,$exp_tag,$reduction); 89 { 89 90 my $command = "$camtool -pendingexp -cam_id $cam_id "; 90 91 $command .= " -dbname $dbname " if defined($dbname); 91 92 92 93 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 93 94 run(command => $command, verbose => 0); 94 95 unless ($success) { 95 96 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 96 97 97 98 &my_die("Unable to run camtool to determine stage parameters.", $remote_id, $cam_id, $error_code, $fail_state); 98 99 } 99 100 100 101 my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or 101 102 &my_die("Unable to determine cam component information.", $remote_id, $cam_id, $PS_EXIT_PROG_ERROR, $fail_state); 102 103 my $metadata = parse_md_list($MDlist); 103 104 my $camEntry = $metadata->[0]; 104 105 $workdir = $camEntry->{workdir}; 106 $exp_tag = $camEntry->{exp_tag}; 105 106 $workdir = $camEntry->{workdir}; 107 $exp_tag = $camEntry->{exp_tag}; 108 $reduction = $camEntry->{reduction}; 109 $reduction = 'DEFAULT' unless defined $reduction; 107 110 unless (defined($workdir)) { 108 109 110 111 112 111 while( my ($k, $v) = each %$camEntry ) { 112 print "key: $k, value: $v.\n"; 113 } 114 print "%{ $camEntry }\n"; 115 die; 113 116 } 114 117 } … … 122 125 $command .= " -dbname $dbname " if defined($dbname); 123 126 124 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =125 127 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 128 run(command => $command, verbose => 0); 126 129 unless ($success) { 127 130 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 128 131 &my_die("Unable to run camtool -pendingimfile ", $remote_id, $cam_id, $error_code, $fail_state); 129 132 } … … 131 134 # We don't actually care about the input cam data other than to know which mask files to instantiate. 132 135 my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or 133 136 &my_die("Unable to determine cam component information.", $remote_id, $cam_id, $PS_EXIT_PROG_ERROR, $fail_state); 134 137 135 138 my $metadata = parse_md_list($MDlist); … … 140 143 my $pre_cmd_masks= "ls -1 ${remote_chip_path}*.mk.fits > ${remote_outroot}.masklist"; 141 144 142 my $reduction = 'DEFAULT' unless defined $camEntry->{reduction}; 143 my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use 145 my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use 146 if ($recipe_psastro eq "") { 147 &my_die("Unable to determine PSASTRO recipe", $cam_id, $PS_EXIT_PROG_ERROR); 148 } 144 149 145 150 my $psastro_command = " psastro -list ${remote_outroot}.cmflist "; 146 151 $psastro_command .= " -masklist ${remote_outroot}.masklist ${remote_outroot} "; 147 # $psastro_command .= " -refmasklist ${remote_outroot}.masklist ${remote_outroot} "; 148 # This is used to do edge calculations. It should probably be smart enough to know that it can just use the masks I just specified, but it's not. 149 # Uncomment when remote psastro supports this 152 # $psastro_command .= " -refmasklist ${remote_outroot}.masklist ${remote_outroot} "; 153 $psastro_command .= " -refmasklist /turquoise/usr/projects/ps1/watersc1/references/gpc1.refmask.list "; 150 154 $psastro_command .= " -kh-correct /turquoise/usr/projects/ps1/watersc1/references/khcorrect.20140606.v0.fits "; 151 $psastro_command .= " -refmasklist /turquoise/usr/projects/ps1/watersc1/references/gpc1.refmask.list ";152 155 # $psastro_command .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20080909.asm "; 153 156 $psastro_command .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20140505.asm "; … … 156 159 $psastro_command .= " -dumpconfig ${remote_outroot}.psastro.mdc -stats ${remote_outroot}.stats "; 157 160 $psastro_command .= " -recipe PPSTATS CAMSTATS "; 158 161 159 162 my $camtool_post_cmd = "camtool -cam_id $cam_id -addprocessedexp -uri UNKNOWN "; 160 $camtool_post_cmd .= " -dbname $dbname " if defined $dbname; 163 $camtool_post_cmd .= " -dbname $dbname " if defined $dbname; 161 164 $camtool_post_cmd .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 "; 162 165 my $post_cmd_echo = " echo -n \"$camtool_post_cmd\" > ${remote_outroot}.dbinfo "; … … 168 171 # Determine which output files need to be returned 169 172 foreach my $component(@return_component_list) { 170 171 172 173 173 my $filename = $ipprc->filename($component,$ipp_outroot); 174 my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); 175 my $remote_outroot_dir = dirname($ipp_disk); 176 print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; 174 177 } 175 178 foreach my $chipInfo (@{ $metadata }) { 176 177 178 179 179 my $filename = $ipprc->filename("PSASTRO.OUTPUT.MASK",$ipp_outroot,$chipInfo->{class_id}); 180 my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); 181 my $remote_outroot_dir = dirname($ipp_disk); 182 print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; 180 183 } 181 184 print CONFIG "\n"; … … 204 207 my $ipp_disk= $ipprc->file_resolve( $neb_uri ); 205 208 my $remote_disk = $ipp_disk; 206 209 207 210 unless(defined($ipp_disk)) { 208 211 &my_die( "Unable to generate file for $neb_uri ", $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); … … 220 223 my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); 221 224 my $remote_disk = $ipp_disk; 222 225 223 226 unless(defined($ipp_disk)) { 224 227 &my_die( "Unable to generate file for $neb_uri ", $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state); … … 235 238 my $neb_uri = shift; 236 239 my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); 237 240 238 241 unless (exists($file_filter{$neb_uri})) { 239 242 $file_filter{$neb_uri} = 1; … … 265 268 return($remote_uri); 266 269 } 267 270 268 271 sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); 269 272 # This needs to replace the remote root directory with the nebulous tag. … … 271 274 $remote_uri =~ s%${remote_root}%%; 272 275 my $local_uri = "neb:///" . $remote_uri; 273 276 274 277 return($local_uri); 275 278 } … … 286 289 287 290 carp($msg); 288 291 289 292 if (defined $remote_id and defined $stage_id and not $no_update) { 290 my $command = "remotetool -updatecomponent -remote_id $ id -stage_id $stage_id";293 my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; 291 294 $command .= " -set_state $exit_state " if defined $exit_state; 292 295 $command .= " -dbname $dbname " if defined $dbname; -
tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl
r37291 r37298 20 20 # Hard coded values 21 21 # my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location 22 my $remote_root = '/scratch3/watersc1/'; # Far side destination base location 23 my $remote_raw = "${remote_root}/raw/"; # Directory to find raw data in. 24 my $threads_req = 4; 22 my $remote_root = '/scratch3/watersc1/'; # Far side destination base location 23 my $remote_hostname = "LANL/Mustang"; # Name of the remote node. 24 my $remote_raw = "${remote_root}/raw/"; # Directory to find raw data in. 25 my $threads_req = 4; 25 26 26 27 my $fail_state = "prep_fail"; … … 264 265 my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; 265 266 $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; 266 $post_commandA .= " -magicked 0 -hostname REMOTE-dtime_script 0 ";267 $post_commandA .= " -magicked 0 -hostname $remote_hostname -dtime_script 0 "; 267 268 $post_commandA .= " -dbname $dbname " if defined $dbname; 268 269 -
tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl
r37291 r37298 32 32 # Estimate of how long a job runs, in hours (depends on stage) 33 33 my %job_cost = (); 34 $job_cost{"camera"} = 1700 / 60 / 60; 34 $job_cost{"camera"} = 1700 / 60 / 60; 35 35 $job_cost{"chip"} = 150 / 60 / 60; 36 36 $job_cost{"warp"} = 110 / 60 / 60; … … 42 42 my $min_time = 1; # shortest allocation to ask for 43 43 my $max_time = 8; # longest allocation to ask for 44 my $remote_hostname= "LANL/Mustang"; # Name of the remote node.45 44 46 45 # We need to ensure we only ever try to transfer a file once. -
tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_stack.pl
r37291 r37298 20 20 # Hard coded values 21 21 # my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location 22 my $remote_root = '/scratch3/watersc1/'; # Far side destination base location 23 my $threads_req = 4; # How many threads are we going to use? 22 my $remote_root = '/scratch3/watersc1/'; # Far side destination base location 23 my $remote_hostname = "LANL/Mustang"; # Name of the remote node. 24 my $threads_req = 4; # How many threads are we going to use? 24 25 25 26 my $fail_state = "prep_fail"; … … 59 60 60 61 my @return_component_list = ("DBINFO.EXP", "PPSTACK.UNCONV","PPSTACK.UNCONV.MASK","PPSTACK.UNCONV.VARIANCE", 61 62 63 62 "PPSTACK.UNCONV.EXP","PPSTACK.UNCONV.EXPWT","PPSTACK.UNCONV.EXPNUM","PSPHOT.OUTPUT", 63 "LOG.EXP", "PSASTRO.STATS", 64 "PPSTACK.CONFIG","PPSTACK.TARGET.PSF","PSPHOT.STACK.PSF.SAVE"); 64 65 #,"PPSTACK.CONV.KERNEL"); 65 66 … … 93 94 $command .= " -dbname $dbname " if defined($dbname); 94 95 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 95 96 run(command => $command, verbose => 0); 96 97 unless ($success) { 97 98 98 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 99 &my_die("Unable to run stacktool -tosum ", $remote_id, $stack_id, $error_code, $fail_state); 99 100 } 100 101 101 102 my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or 102 103 &my_die("Unable to determine stack component information.", $remote_id, $stack_id, $PS_EXIT_PROG_ERROR, $fail_state); 103 104 my $metadata = parse_md_list($MDlist); 104 105 … … 111 112 $skycell_id= $stack->{skycell_id}; 112 113 } 113 my $ipp_outroot = "${workdir}/${tess_id}/${skycell_id}/${tess_id}.${skycell_id}.stk.${stack_id}"; 114 my $remote_outroot = uri_local_to_remote($ipp_outroot); 114 my $ipp_outroot = "${workdir}/${tess_id}/${skycell_id}/${tess_id}.${skycell_id}.stk.${stack_id}"; 115 my $remote_outroot = uri_local_to_remote($ipp_outroot); 115 116 my $remote_outdir = uri_local_to_remote("${workdir}/${tess_id}/${skycell_id}"); 116 117 … … 122 123 $command .= " -dbname $dbname " if defined($dbname); 123 124 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 124 125 run(command => $command, verbose => 0); 125 126 unless ($success) { 126 127 127 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 128 &my_die("Unable to run stacktool -inputskyfile ", $remote_id, $stack_id, $error_code, $fail_state); 128 129 } 129 130 130 131 my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or 131 132 &my_die("Unable to determine stack component information.", $remote_id, $stack_id, $PS_EXIT_PROG_ERROR, $fail_state); 132 133 my $warpData = parse_md_list($MDlist); 133 134 } … … 137 138 my $warp_path_base_string = ""; 138 139 foreach my $warpEntry ( @{ $warpData } ) { 139 140 141 142 143 144 145 146 147 148 # 140 my $warp_path_base = $warpEntry->{path_base}; 141 my $remote_path_base = uri_local_to_remote($warp_path_base); 142 $warp_path_base_string .= " $remote_path_base "; 143 144 # Append file names to transfer lists so we can recover if warps are deleted. 145 uri_to_outputs($warp_path_base . ".fits"); 146 uri_to_outputs($warp_path_base . ".wt.fits"); 147 uri_to_outputs($warp_path_base . ".mask.fits"); 148 uri_to_outputs($warp_path_base . ".cmf"); 149 # uri_to_outputs($warp_path_base . ".psf"); 149 150 } 150 151 my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl $warp_path_base_string > ${remote_outroot}.in.mdc"; 151 my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use 152 my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use 152 153 my $ppstack_command = " ppStack -input ${remote_outroot}.in.mdc "; 153 154 $ppstack_command .= " ${remote_outroot} -stats ${remote_outroot}.stats "; … … 169 170 $post_cmd_echo .= " -dbname $dbname " if defined $dbname; 170 171 $post_cmd_echo .= " -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; 171 172 172 173 my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - STACK_SKYCELL >> ${remote_outroot}.dbinfo "; 173 174 … … 177 178 # Determine which output files need to be returned 178 179 foreach my $component(@return_component_list) { 179 180 181 182 180 my $filename = $ipprc->filename($component,$ipp_outroot); 181 my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); 182 my $remote_outroot_dir = dirname($ipp_disk); 183 print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; 183 184 } 184 185 print CONFIG "\n"; … … 206 207 my $ipp_disk= $ipprc->file_resolve( $neb_uri ); 207 208 my $remote_disk = $ipp_disk; 208 209 209 210 unless(defined($ipp_disk)) { 210 211 &my_die( "Unable to generate file for $neb_uri ", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); … … 222 223 my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); 223 224 my $remote_disk = $ipp_disk; 224 225 225 226 unless(defined($ipp_disk)) { 226 227 &my_die( "Unable to generate file for $neb_uri ", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state); … … 237 238 my $neb_uri = shift; 238 239 my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); 239 240 240 241 unless (exists($file_filter{$neb_uri})) { 241 242 $file_filter{$neb_uri} = 1; … … 267 268 return($remote_uri); 268 269 } 269 270 270 271 sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); 271 272 # This needs to replace the remote root directory with the nebulous tag. … … 273 274 $remote_uri =~ s%${remote_root}%%; 274 275 my $local_uri = "neb:///" . $remote_uri; 275 276 276 277 return($local_uri); 277 278 } … … 288 289 289 290 carp($msg); 290 291 291 292 if (defined $remote_id and defined $stage_id and not $no_update) { 292 293 my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; -
tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl
r37291 r37298 20 20 # Hard coded values 21 21 # my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location 22 my $remote_root = '/scratch3/watersc1/'; 23 my $threads_req = 4; # How many threads are we going to use? 22 my $remote_root = '/scratch3/watersc1/'; 23 my $remote_hostname = "LANL/Mustang"; # Name of the remote node. 24 my $threads_req = 4; # How many threads are we going to use? 24 25 25 26 my $fail_state = "prep_fail"; … … 90 91 $command .= " -dbname $dbname " if defined($dbname); 91 92 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 92 93 run(command => $command, verbose => 0); 93 94 unless ($success) { 94 95 95 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 96 &my_die("Unable to run warptool -pendingimfile ", $remote_id, $warp_id, $error_code, $fail_state); 96 97 } 97 98 my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or 98 99 &my_die("Unable to determine warp component information.", $remote_id, $warp_id, $PS_EXIT_PROG_ERROR, $fail_state); 99 100 $warpData = parse_md_list($MDlist); 100 101 } … … 103 104 { 104 105 foreach my $warpEntry ( @{ $warpData } ) { 105 106 107 108 109 110 my $recipe_pswarp = $ipprc->reduction($reduction, 'WARP_PSWARP'); # Recipe to use 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 $pre_cmd_masks .= " $remote_mask "; 148 $pre_cmd_vars .= " $remote_var "; 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 # 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 # 106 my $workdir = $warpEntry->{workdir}; 107 my $exp_tag = $warpEntry->{exp_tag}; 108 my $skycell_id = $warpEntry->{skycell_id}; 109 my $tess_id = $warpEntry->{tess_id}; 110 my $reduction = 'DEFAULT' unless defined $warpEntry->{reduction}; 111 my $recipe_pswarp = $ipprc->reduction($reduction, 'WARP_PSWARP'); # Recipe to use 112 my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.wrp.${warp_id}.${skycell_id}"; 113 print "$ipp_outroot\n"; 114 my $remote_outroot = uri_local_to_remote($ipp_outroot); 115 116 my $command = "$warptool -scmap -warp_id $warp_id -skycell_id ${skycell_id}"; 117 $command .= " -dbname $dbname " if defined($dbname); 118 119 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 120 run(command => $command, verbose => 0); 121 unless ($success) { 122 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 123 &my_die("Unable to run warptool -pendingimfile: $command ", $remote_id, $warp_id, $error_code, $fail_state); 124 } 125 126 my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or 127 &my_die("Unable to determine warp component information.", $remote_id, $warp_id, $PS_EXIT_PROG_ERROR, $fail_state); 128 my $inData = parse_md_list($MDlist); 129 130 my $pre_cmd_ims = "ls -1 "; 131 my $pre_cmd_masks= "ls -1 "; 132 my $pre_cmd_vars = "ls -1 "; 133 my $pre_cmd_astrom = ""; 134 135 foreach my $imfile (@{ $inData2 }) { 136 my $image = $ipprc->filename("PPIMAGE.CHIP", $imfile->{chip_path_base}, $imfile->{class_id}); 137 my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); 138 my $var = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); 139 140 my $astrom= $ipprc->filename("PSASTRO.OUTPUT", $imfile->{cam_path_base}); 141 142 my $remote_image = uri_local_to_remote($image); 143 my $remote_mask = uri_local_to_remote($mask); 144 my $remote_var = uri_local_to_remote($var); 145 my $remote_astrom= uri_local_to_remote($astrom); 146 147 $pre_cmd_ims .= " $remote_image "; 148 $pre_cmd_masks .= " $remote_mask "; 149 $pre_cmd_vars .= " $remote_var "; 150 $pre_cmd_astrom = "echo $remote_astrom > ${remote_outroot}.astrom "; 151 } 152 $pre_cmd_ims .= " > ${remote_outroot}.imlist "; 153 $pre_cmd_masks .= " > ${remote_outroot}.masklist "; 154 $pre_cmd_vars .= " > ${remote_outroot}.varlist "; 155 156 my $skycell_command = " dvoImageExtract -D CATDIR /turquoise/usr/projects/ps1/watersc1/tess/${tess_id} $skycell_id -o ${remote_outroot}.skyfile "; 157 158 my $pswarp_command = " pswarp -list ${remote_outroot}.imlist "; 159 $pswarp_command .= " -masklist ${remote_outroot}.masklist -variancelist ${remote_outroot}.varlist "; 160 $pswarp_command .= " -astromlist ${remote_outroot}.astrom "; 161 $pswarp_command .= " ${remote_outroot} ${remote_outroot}.skyfile "; 162 $pswarp_command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE "; 163 $pswarp_command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF "; 164 $pswarp_command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF "; 165 $pswarp_command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS "; 166 $pswarp_command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL "; 167 $pswarp_command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID "; 168 $pswarp_command .= " -recipe PSWARP $recipe_pswarp "; 169 $pswarp_command .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log "; 170 $pswarp_command .= " -threads $threads_req "; # -image_id ${image_id} -source_id ${source_id} "; 171 $pswarp_command .= " -recipe PPSTATS WARPSTATS "; 172 $pswarp_command .= " -dumpconfig ${remote_outroot}.pswarp.mdc -stats ${remote_outroot}.stats "; 173 174 # print "$pswarp_command \n"; 175 176 my $post_cmd_echo = " echo -n \"warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id "; 177 $post_cmd_echo .= " -path_base $remote_outroot -uri UNKNOWN "; 178 $post_cmd_echo .= " -dbname $dbname " if defined $dbname; 179 $post_cmd_echo .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo "; 180 181 my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - WARP_SKYCELL >> ${remote_outroot}.dbinfo "; 182 183 print CONFIG "${pre_cmd_ims} && ${pre_cmd_masks} && ${pre_cmd_vars} && ${pre_cmd_astrom} && ${skycell_command} && ${pswarp_command} && ${post_cmd_echo} && ${post_cmd_SfM} "; 184 $job_index++; 185 186 # Determine which output files need to be returned 187 foreach my $component(@return_component_list) { 188 my $filename = $ipprc->filename($component,$ipp_outroot,$skycell_id); 189 my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename); 190 my $remote_outroot_dir = dirname($ipp_disk); 191 print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; 192 } 193 print CONFIG "\n"; 194 # die(); 194 195 } 195 196 } … … 217 218 my $ipp_disk= $ipprc->file_resolve( $neb_uri ); 218 219 my $remote_disk = $ipp_disk; 219 220 220 221 unless(defined($ipp_disk)) { 221 222 &my_die( "Unable to generate file for $neb_uri ", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); … … 233 234 my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); 234 235 my $remote_disk = $ipp_disk; 235 236 236 237 unless(defined($ipp_disk)) { 237 238 &my_die( "Unable to generate file for $neb_uri ", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state); … … 248 249 my $neb_uri = shift; 249 250 my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); 250 251 251 252 unless (exists($file_filter{$neb_uri})) { 252 253 $file_filter{$neb_uri} = 1; … … 278 279 return($remote_uri); 279 280 } 280 281 281 282 sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri); 282 283 # This needs to replace the remote root directory with the nebulous tag. … … 284 285 $remote_uri =~ s%${remote_root}%%; 285 286 my $local_uri = "neb:///" . $remote_uri; 286 287 287 288 return($local_uri); 288 289 } … … 299 300 300 301 carp($msg); 301 302 302 303 if (defined $remote_id and defined $stage_id and not $no_update) { 303 304 my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; -
tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_exec.pl
r37289 r37298 32 32 33 33 # Options 34 my ($remote_id,$ path_base,$dbname,$verbose,$no_update,$camera);34 my ($remote_id,$job_id,$path_base,$dbname,$verbose,$no_update,$camera); 35 35 $verbose = 0; 36 36 GetOptions( … … 64 64 unless ($rt_success) { 65 65 $rt_error_code = (($rt_error_code >> 8) or $PS_EXIT_PROG_ERROR); 66 &my_die("Unable to run remotetool to determine remote run status", 67 $remote_id,$rt_error_code); 66 &my_die("Unable to run remotetool to determine remote run status", $remote_id, $rt_error_code); 68 67 } 69 68 my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or 70 &my_die("Unable to run remotetool to determine remote run status", 71 $remote_id,$rt_error_code); 69 &my_die("Unable to run remotetool to determine remote run status", $remote_id, $rt_error_code); 72 70 my $rrData2 = parse_md_list($rrData); 73 71 my $runData = ${ $rrData2 }[0]; # There should be only one 74 72 75 my ($remote_path) = uri_convert($path_base);73 my ($remote_path) = &uri_local_to_remote($path_base); 76 74 print "REMOTE PATH: $remote_path\n"; 77 75 … … 94 92 my $disk_return = $ipprc->file_resolve($uri_return); 95 93 96 scp_put($disk_command,uri_local_to_remote($uri_command)); 97 scp_put($disk_transfer,uri_local_to_remote($uri_transfer)); 98 scp_put($disk_check,uri_local_to_remote($uri_check)); 99 scp_put($disk_config,uri_local_to_remote($uri_config)); 100 scp_put($disk_generate,uri_local_to_remote($uri_generate)); 101 scp_put($disk_return, uri_local_to_remote($uri_return)); 102 103 # Run check command 104 # my (undef,$remote_check) = uri_convert($uri_check); 105 # my (undef,$remote_transfer)= uri_convert($uri_transfer); 106 107 my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_path --fetch"); 94 &scp_put($disk_command, &uri_local_to_remote($uri_command)); 95 &scp_put($disk_transfer, &uri_local_to_remote($uri_transfer)); 96 &scp_put($disk_check, &uri_local_to_remote($uri_check)); 97 &scp_put($disk_config, &uri_local_to_remote($uri_config)); 98 &scp_put($disk_generate, &uri_local_to_remote($uri_generate)); 99 &scp_put($disk_return, &uri_local_to_remote($uri_return)); 100 101 my $ssh_check_stdout = &ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_path --fetch"); 108 102 109 103 # We no longer need to parse this output, as it retrieves files itself. … … 113 107 114 108 # Run real command 115 my (undef,$remote_command) = uri_convert($uri_command);116 my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command");109 my (undef,$remote_command) = &uri_convert($uri_command); 110 my $ssh_exec_stdout = &ssh_exec_command("msub -V $remote_command"); 117 111 if ($#{ $ssh_exec_stdout } != -1) { # Parse the output 118 112 $job_id = ${ $ssh_exec_stdout }[0]; … … 122 116 123 117 unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. 124 &my_die("No job_id returned. Sorry.", $remote_id,$PS_EXIT_PROG_ERROR);118 &my_die("No job_id returned. Sorry.", $remote_id, $PS_EXIT_PROG_ERROR); 125 119 } 126 120 127 121 # Notify the database that this entry is currently running. 128 &my_die("Finished", $remote_id,0,"run",$job_id);122 &my_die("Finished", $remote_id, 0, "run", $job_id); 129 123 130 124 # END PROGRAM … … 137 131 unless ($success) { 138 132 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 139 &my_die("Authorization check failed.", $remote_id,0,'auth');133 &my_die("Authorization check failed.", $remote_id, 0, 'auth'); 140 134 } 141 135 } … … 147 141 148 142 my $directory = dirname($destination); 149 ssh_exec_command("mkdir -p $directory");150 151 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 152 run(command => $cmd, verbose => $verbose); 153 unless ($success) { 154 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 155 &my_die( );143 &ssh_exec_command("mkdir -p $directory"); 144 145 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 146 run(command => $cmd, verbose => $verbose); 147 unless ($success) { 148 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 149 &my_die("failed to send file $file to LANL", $remote_id, $error_code); 156 150 } 157 151 } … … 167 161 unless ($success) { 168 162 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 169 &my_die( );163 &my_die("failed to get file $file from LANL", $remote_id, $error_code); 170 164 } 171 165 } … … 173 167 sub ssh_exec_command { 174 168 my $cmd = shift; 175 $cmd = "$ssh $DMZ_HOST ssh ${SEC_HOST} $cmd";169 $cmd = "$ssh -n $DMZ_HOST ssh ${SEC_HOST} $cmd"; 176 170 print "EXEC: $cmd\n"; 177 171 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 181 175 # If we failed, see if we failed due to authorization. 182 176 check_ssh_connection(); 183 &my_die("Failed to execute command: >>$cmd<<");177 &my_die("Failed to execute remote command: >>$cmd<<", $remote_id, $error_code); 184 178 } 185 179 return ($stdout_buf); 186 180 } 187 181 188 sub poll_job { 189 my $job_id = shift; 190 my $response_array = ssh_exec_command("checkjob -v $job_id"); 191 my $response = join "\n", @$response_array; 192 193 if ($response =~ /State: Running/) { 194 return(0,"Running"); 195 } 196 elsif ($response =~ /State: Completed/) { 197 return(1,"Completed"); 198 } 199 elsif ($response =~ /State: Idle/) { 200 return(-2,"Idle"); 201 } 202 else { 203 my $response_word = $response; 204 $response_word =~ s/.*State: //; 205 $response_word =~ s/\n.*//; 206 return(-1,$response_word); 207 } 208 } 209 210 182 # neb URI to (ipp_disk,remote_disk) 211 183 sub uri_convert { 212 184 my $neb_uri = shift; … … 215 187 216 188 unless(defined($ipp_disk)) { 217 my_die();189 &my_die("Failed to generate or find uri $neb_uri", $remote_id, $PS_EXIT_SYS_ERROR); 218 190 } 219 191 … … 225 197 } 226 198 227 sub uri_to_outputs { 228 my $neb_uri = shift; 229 my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); 230 231 # print TRANSFER "$ipp_disk\n"; 232 # print CHECK "$remote_disk\n"; 233 return($ipp_disk,$remote_disk); 234 } 235 199 # neb URI to remote URI-on-disk 236 200 sub uri_local_to_remote { 237 201 # This needs to replace the nebulous tag with the remote root. … … 245 209 } 246 210 247 sub uri_remote_to_local {248 # This needs to replace the remote root directory with the nebulous tag.249 my $remote_uri = shift;250 $remote_uri =~ s%${remote_root}%%;251 my $local_uri = "neb:///" . $remote_uri;252 253 return($local_uri);254 }255 256 sub drop_component {257 my $remote_id = shift;258 my $cmd = shift;259 my $file = shift;260 261 if ($cmd eq '') {262 # Zero byte file. This dbinfo was never constructed properly.263 264 # Steal the stage_id from the filename. This isn't something I'm happy about.265 my $stage = $runData->{stage};266 my $stage_id = $file;267 if ($stage eq 'chip') {268 $stage_id =~ s/^.*ch\.//;269 }270 elsif ($stage eq 'camera') {271 $stage_id =~ s/^.*cm\.//;272 }273 elsif ($stage eq 'warp') {274 $stage_id =~ s/^.*wrp\.//;275 }276 elsif ($stage eq 'stack') {277 $stage_id =~ s/^.*stk\.//;278 }279 $stage_id =~ s/\..*$//;280 281 # Drop this from the database282 my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id ";283 $drop_cmd .= " -dbname $dbname " if defined($dbname);284 system($drop_cmd);285 }286 287 }288 289 211 sub my_die { 290 212 my $msg = shift; -
tags/ipp-pv3-20140717/ippScripts/scripts/sc_remote_poll.pl
r37288 r37298 35 35 # tools 36 36 my $missing_tools; 37 my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1);38 my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1);37 my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); 38 my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); 39 39 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); 40 my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); 40 41 41 42 if ($missing_tools) { … … 45 46 46 47 # Options 47 my ($remote_id,$path_base,$policy,$poll,$force,$job_id,$dbname,$verbose,$no_update,$camera); 48 $force = 0; 48 my ($remote_id,$path_base,$skip_poll,$skip_transfer,$job_id,$dbname,$retry,$verbose,$no_update,$camera); 49 49 $verbose = 0; 50 50 GetOptions( … … 52 52 'job_id=s' => \$job_id, 53 53 'path_base=s' => \$path_base, 54 'policy=s' => \$policy, 55 'poll' => \$poll, 56 'force-poll' => \$force, 54 'skip_poll' => \$skip_poll, 55 'skip_transfer' => \$skip_transfer, 57 56 'camera=s' => \$camera, 58 57 'dbname=s' => \$dbname, 58 'retry' => \$retry, 59 59 'verbose' => \$verbose, 60 60 'no_update' => \$no_update, … … 78 78 # STEP 2: Grab the information about this run; 79 79 my $runData; 80 { 80 { 81 81 my $command = "$remotetool -listrun -remote_id $remote_id "; 82 82 $command .= " -dbname $dbname " if defined($dbname); 83 83 84 84 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 85 85 run(command => $command, verbose => $verbose); 86 86 87 87 unless ($success) { 88 89 88 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 89 &my_die("Unable to run remotetool to determine remote run status", $remote_id, $error_code); 90 90 } 91 91 … … 101 101 my $poll_word = "nothing"; 102 102 103 ($poll_response,$poll_word) = poll_job($job_id); 104 105 if ($poll_response == -1) { # This job has an error 106 print "The job exited incorrectly: $remote_id $job_id $poll_word $poll_response.\n"; 107 # warn("The job exited incorrectly: $remote_id $job_id $poll_word."); 108 exit (2); 109 } 110 111 if ($poll_response != 1) { # This job has NOT completed 112 print "The job has not exited at LANL: $remote_id $job_id $poll_word $poll_response.\n"; 113 exit (2); 103 if ($skip_poll) { 104 $poll_word = "Completed"; 105 } else { 106 ($poll_response,$poll_word) = poll_job($job_id); 107 108 if ($poll_response == -1) { # This job has an error 109 print "The job exited incorrectly: $remote_id $job_id $poll_word $poll_response.\n"; 110 # warn("The job exited incorrectly: $remote_id $job_id $poll_word."); 111 exit (2); 112 } 113 114 if ($poll_response != 1) { # This job has NOT completed 115 print "The job has not exited at LANL: $remote_id $job_id $poll_word $poll_response.\n"; 116 exit (2); 117 } 114 118 } 115 119 116 120 my $remote_path = uri_local_to_remote($path_base); 117 121 118 # transfer tool should check the existence of files on the remote end 119 # and only return those that exist 120 { 122 # STEP 4: transfer back the files. Transfer tool should check the 123 # existence of files on the remote end and only return those that 124 # exist 125 unless ($skip_transfer) { 121 126 my $offset = 0; # Offset to start from. 122 127 my $iter = 0; … … 136 141 print STDERR "rsync of all files is complete, checking the results\n"; 137 142 138 # grab the .stats file139 140 my $uri_stats = $path_base . ".stat s";141 my $disk_stats = $ipprc->file_resolve($uri_stats );143 # STEP 5: grab the .stat file from LANL (this file is first constructed at LANL) 144 145 my $uri_stats = $path_base . ".stat"; 146 my $disk_stats = $ipprc->file_resolve($uri_stats, 1); 142 147 my $remote_stats = uri_local_to_remote($uri_stats); 143 148 144 149 scp_get($remote_stats,$disk_stats); 145 150 146 # check for PASS / PART / FAIL : if dbinfo is PASS, go ahead and execute that. 151 # STEP 6: check dbinfo files for PASS / PART / FAIL 152 153 my %stage_pass = (); 154 my %stage_fail = (); 155 my %stage_part = (); 147 156 148 157 my $Npass = 0; … … 151 160 my @dbinfo_files = (); 152 161 open (STATFILE, "$disk_stats") || &my_die("Couldn't open file? $disk_stats", $remote_id, $PS_EXIT_SYS_ERROR); 153 while (my $line = < RETURN>) {162 while (my $line = <STATFILE>) { 154 163 chomp $line; 155 unless ($ _=~ /dbinfo/) { next; }164 unless ($line =~ /dbinfo/) { next; } 156 165 157 166 my @words = split (" ", $line); 167 168 my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($words[0]); 169 158 170 if ($words[1] eq "PASS") { 159 171 push @dbinfo_files, $words[0]; 160 $Npass ++; 172 $stage_pass{$stage_id} = 1; 173 $Npass ++; 161 174 } elsif ($words[1] eq "FAIL") { 162 $Nfail ++; 175 $stage_fail{$stage_id} = 1; 176 $Nfail ++; 163 177 } elsif ($words[1] eq "PART") { 164 $Npart ++; 165 } 166 } 167 close(RETURN); 178 $stage_part{$stage_id} = 1; 179 $Npart ++; 180 } 181 } 182 close (STATFILE); 168 183 print STDERR "dbinfo files: $Npass PASS, $Nfail FAIL, $Npart PART\n"; 169 184 185 # STEP 7: if dbinfo is PASS, execute it (fail ones that are not PASS?). 170 186 foreach my $file (@dbinfo_files) { 171 187 open(DBF,"$file") || warn "Missing file $file\n"; … … 176 192 $cmd =~ s/-/ -/g; # This is just a safety check for missing space. 177 193 178 if ($cmd == "") { 179 print STDERR "problem: empty dbinfo file\n"; 180 next; 181 } 182 183 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 184 run(command => $cmd, verbose => $verbose); 185 unless ($success) { 186 # This shouldn't fail, but we also can't suddenly abort if something does fail. 187 # Now we're going to drop the component that owns this dbinfo, which should force a retry. 188 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 189 warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code"); 190 if ($poll_word ne 'Completed') { # If the job claims it completed, then we're fighting the network. 191 fail_component($remote_id, $cmd, $file); 192 } 193 } 194 194 if ($cmd eq "") { 195 print STDERR "problem: empty dbinfo file\n"; 196 exit (1); 197 next; 198 } 199 200 print "dbinfo: $file\n"; 201 202 if ($retry) { 203 # use the dbinfo filename to get stage id, check if we have already updated this entry: 204 my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); 205 206 # we just need to check for the existence of an entry: 207 if ($runData->{stage} eq "chip") { 208 my $cmd1 = "$chiptool -simple -processedimfile -chip_id $stage_id -class_id $class_id"; 209 $cmd1 .= " -dbname $dbname " if defined($dbname); 210 211 my ( $success1, $error_code1, $full_buf1, $stdout_buf1, $stderr_buf1 ) = 212 run(command => $cmd1, verbose => 0); 213 214 unless ($success1) { 215 $error_code1 = (($error_code1 >> 8) or $PS_EXIT_PROG_ERROR); 216 &my_die("Unable to run chiptool to check on results", $remote_id, $error_code1); 217 } 218 219 my $Nrows = @$stdout_buf1; 220 if ($Nrows > 0) { 221 print STDERR "already updated: $stage_id, $class_id\n"; 222 next; 223 } 224 } else { 225 print STDERR "do not know how to handle stage $runData->{stage}\n"; 226 exit (3); 227 } 228 } 229 230 unless ($no_update) { 231 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 232 run(command => $cmd, verbose => $verbose); 233 unless ($success) { 234 # This shouldn't fail, but we also can't suddenly abort if something does fail. 235 # Now we're going to drop the component that owns this dbinfo, which should force a retry. 236 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 237 warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code"); 238 my ($stage_id, $class_id) = get_stage_id_from_dbinfo ($file); 239 $stage_fail{$stage_id} = 1; 240 } 241 } 242 } 243 print STDERR "dbinfo files all run (or skipped)\n"; 244 245 # STEP 8: unify the stage states and update the remoteComponent entries 246 foreach my $entry (keys (%stage_fail)) { 247 $stage_pass{$entry} = 0; 248 } 249 foreach my $entry (keys (%stage_part)) { 250 $stage_pass{$entry} = 0; 251 } 252 foreach my $entry (keys (%stage_pass)) { 253 my $cmd = "remotetool -updatecomponent -remote_id $remote_id -stage_id $entry"; 254 $cmd .= " -dbname $dbname " if defined($dbname); 255 if ($stage_pass{$entry}) { 256 $cmd .= " -set_state pass"; 257 } else { 258 $cmd .= " -set_state fail"; 259 } 260 261 unless ($no_update) { 262 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 263 run(command => $cmd, verbose => $verbose); 264 unless ($success) { 265 &my_die("Unable to run remotetool to determine remote run status", $remote_id); 266 } 267 } else { 268 print STDERR "no_update: $cmd\n"; 269 } 195 270 } 196 271 … … 223 298 unless ($success) { 224 299 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 225 &my_die( );300 &my_die("Failed to send file $file to LANL", $remote_id, $PS_EXIT_SYS_ERROR); 226 301 } 227 302 } … … 237 312 unless ($success) { 238 313 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 239 &my_die( );314 &my_die("Failed to get file $file from LANL", $remote_id, $PS_EXIT_SYS_ERROR); 240 315 } 241 316 } … … 243 318 sub ssh_exec_command { 244 319 my $cmd = shift; 245 $cmd = "$ssh $DMZ_HOST ssh ${SEC_HOST} $cmd";320 $cmd = "$ssh -n $DMZ_HOST ssh ${SEC_HOST} $cmd"; 246 321 print "EXEC: $cmd\n"; 247 322 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 251 326 # If we failed, see if we failed due to authorization. 252 327 check_ssh_connection(); 253 &my_die("Failed to execute command: >>$cmd<<" );328 &my_die("Failed to execute command: >>$cmd<<", $remote_id, $PS_EXIT_SYS_ERROR); 254 329 } 255 330 return ($stdout_buf); … … 285 360 286 361 unless(defined($ipp_disk)) { 287 my_die();362 &my_die("Failed to generate or find uri $neb_uri", $remote_id, $PS_EXIT_SYS_ERROR); 288 363 } 289 364 … … 324 399 } 325 400 401 # assumes $runData->{stage} exists 402 sub get_stage_id_from_dbinfo { 403 my $file = shift; 404 405 my $stage_id = 0; 406 my $class_id = 0; 407 408 # Steal the stage_id from the filename. This isn't something I'm happy about. 409 my $stage = $runData->{stage}; 410 if ($stage eq 'chip') { 411 ($stage_id, $class_id) = $file =~ /ch\.(\d*).(XY\d\d).dbinfo/; 412 } 413 elsif ($stage eq 'camera') { 414 $stage_id =~ s/^.*cm\.//; 415 $class_id = 0; 416 } 417 elsif ($stage eq 'warp') { 418 $stage_id =~ s/^.*wrp\.//; 419 $class_id = 0; 420 } 421 elsif ($stage eq 'stack') { 422 $stage_id =~ s/^.*stk\.//; 423 $class_id = 0; 424 } 425 426 return ($stage_id, $class_id); 427 } 428 326 429 sub fail_component { 327 430 my $remote_id = shift; … … 333 436 my $stage_id = $file; 334 437 if ($stage eq 'chip') { 335 438 $stage_id =~ s/^.*ch\.//; 336 439 } 337 440 elsif ($stage eq 'camera') { 338 441 $stage_id =~ s/^.*cm\.//; 339 442 } 340 443 elsif ($stage eq 'warp') { 341 444 $stage_id =~ s/^.*wrp\.//; 342 445 } 343 446 elsif ($stage eq 'stack') { 344 447 $stage_id =~ s/^.*stk\.//; 345 448 } 346 449 $stage_id =~ s/\..*$//; 347 450 348 451 print STDERR "failed component: $remote_id, $stage_id\n"; 349 452 350 453 # Drop this from the database 351 454 ## my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id "; … … 400 503 my $command = "$remotetool -updaterun -remote_id $id"; 401 504 $command .= " -fault $exit_code " if defined $exit_code; 402 $command .= " -job_id $job_id " if defined $jobid;403 505 $command .= " -set_state $exit_state " if defined $exit_state; 404 506 $command .= " -dbname $dbname " if defined $dbname; -
tags/ipp-pv3-20140717/ippScripts/scripts/sc_transfer_tool.pl
r37287 r37298 102 102 } 103 103 104 my $stat_file = "$input_path.stat"; 105 print "STATFILE: $stat_file\n"; 106 open (STATFILE, ">$stat_file"); 104 unless ($fetch) { 105 my $stat_file = "$input_path.stat"; 106 print "STATFILE: $stat_file\n"; 107 open (STATFILE, ">$stat_file"); 108 } 107 109 108 110 $i = 0; … … 116 118 if ($line < $offset) { next; } # I think this is off-by-one in a safe direction 117 119 118 if ($line % 250 == 0) { print "line $line : $Lname\n"; }120 if ($line % 250 == 0) { print STDERR "line $line : $Lname\n"; } 119 121 120 122 $i = int(rand($#filehandles + 1)); 121 123 if ($i >= $threads) { 124 print STDERR "HUH: impossible file handle?\n"; 125 die; 126 } 127 122 128 # if we are fetching, and 'quickfetch' is requested, skip file if it exists 123 129 # NOTE: this does not check that the transfer was complete or the file is current … … 129 135 } else { 130 136 # We are pushing, but only send back files we actually have 131 if (-e $Lname) { 137 # we cannot have the remote file unless the local file exists 138 # (it is a link to the local file and is generated after the local file) 139 my $haveRemote = -e "$local_tmp/$Rname"; 140 if ($haveRemote) { 132 141 print { $filehandles[$i] } "$Rname\n"; 133 142 print STATFILE "$Rname PASS\n"; 134 143 } else { 135 print STATFILE "$Rname FAIL\n"; 144 my $haveLocal = -e $Lname; 145 if ($haveLocal) { 146 print STATFILE "$Rname PART\n"; 147 } else { 148 print STATFILE "$Rname FAIL\n"; 149 } 136 150 } 137 151 } … … 139 153 close(LOCAL); 140 154 close(REMOTE); 141 close(STATFILE); 155 unless ($fetch) { close(STATFILE); } 142 156 143 157 for ($i = 0; $i < $threads; $i++) {
Note:
See TracChangeset
for help on using the changeset viewer.
