Changeset 36596
- Timestamp:
- Mar 18, 2014, 12:36:53 PM (12 years ago)
- File:
-
- 1 edited
-
trunk/ippScripts/scripts/sc_prepare_chip.pl (modified) (10 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/sc_prepare_chip.pl
r36583 r36596 11 11 use DateTime; 12 12 use Data::Dumper; 13 use File::Basename; 13 14 14 15 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); … … 126 127 $detselect_command2 .= " -iteration $det_iter "; 127 128 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 128 run(command => $detselect_command2, verbose => $verbose);129 run(command => $detselect_command2, verbose => 0); 129 130 unless ($success) { 130 131 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); … … 171 172 # Step 2: Iterate over all componenets in this remote run. 172 173 my $job_index = 0; 174 my @pre_commands = (); 175 my @main_commands = (); 176 my @post_commands = (); 177 173 178 foreach my $compEntry (@$compData2) { 174 179 175 180 my $chip_id = $compEntry->{stage_id}; 176 181 my $exp_id ; 177 182 # Get exposure level information from the chipRun we're working from. 178 183 my $command = "$chiptool -listrun -chip_id $chip_id "; … … 197 202 my $chipEntry = ${ $chipData2 }[0]; 198 203 199 my $exp_id = $chipEntry->{exp_id};204 $exp_id = $chipEntry->{exp_id}; 200 205 my $filter = $chipEntry->{filter}; 201 206 my $altfilt= $filter; … … 230 235 } 231 236 } 232 233 234 # if ($filter eq 'y.00000') { 235 # $det_types{'FRINGE'} = '-fringe'; 236 # } 237 # $filter =~ s/.00000//; # The database doesn't have the trailing zeros, and the query isn't a like. 238 # foreach my $det (keys %det_types) { 239 # my $detselect_command = "detselect -search "; 240 # $detselect_command .= " -det_type $det "; 241 # $detselect_command .= " -inst $camera " if defined($camera); 242 # $detselect_command .= " -filter $filter " if ($det eq 'FLAT'); 243 # $detselect_command .= " -time $dateobs "; 244 245 # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 246 # run(command => $detselect_command, verbose => $verbose); 247 # unless ($success) { 248 # $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 249 250 # &my_die("Unable to run dettool to obtain detrend information: $det $camera $filter", 251 # $chip_id,$error_code); 252 # } 253 254 # my $detExp = $mdcParser->parse(join "", @$stdout_buf) or 255 # &my_die("Unable to parse detrend information: $det $camera $filter", 256 # $chip_id,$error_code); 257 # my $detExp2= parse_md_list($detExp); 258 259 # foreach $detExp (@$detExp2) { 260 # my $det_id = $detExp->{det_id}; 261 # my $iteration = $detExp->{iteration}; 262 # $detrends{$det}{ID} = $det_id; 263 # $detrends{$det}{ITER}= $iteration; 264 # } 265 # } 266 } 237 } 238 267 239 # 268 240 # Step 3: Iterate over the sub-components … … 306 278 my $ppImage_command = "ppImage -file $remote_uri"; 307 279 $ppImage_command .= " -burntool $remote_btt "; 308 309 # Add detrend information to the command line310 # foreach my $det (keys %detrends) {311 # my $det_id = $detrends{$det}{ID};312 # my $det_iter = $detrends{$det}{ITER};313 # my $det_code = $det_types{$det};314 315 # if ((($video_cells)&&($det eq 'DARK'))||316 # ((!$video_cells)&&($det eq 'VIDEODARK'))) {317 # next;318 # }319 320 # my $detselect_command2 = "detselect -select ";321 # $detselect_command2 .= " -det_id $det_id ";322 # $detselect_command2 .= " -iteration $det_iter ";323 # $detselect_command2 .= " -class_id $class_id ";324 # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =325 # run(command => $detselect_command2, verbose => $verbose);326 # unless ($success) {327 # $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);328 329 # &my_die("No valid detrend available for this image: $det $det_id $det_iter $class_id",330 # $chip_id,$error_code);331 # }332 333 # my $detImfile = $mdcParser->parse(join "", @$stdout_buf) or334 # &my_die("Could not parse detrend information for this image: $det $det_id $det_iter $class_id",335 # $chip_id,$error_code);336 # my $detImfile2= parse_md_list($detImfile);337 # foreach $detImfile (@$detImfile2) {338 # my $duri = $detImfile->{uri};339 # my (undef, $remote_det_uri) = uri_to_outputs($duri);340 # $ppImage_command .= " $det_code $remote_det_uri ";341 # }342 # }343 280 344 281 # Add detrend information to the command line … … 359 296 my $remote_outroot = uri_local_to_remote($ipp_outroot); 360 297 $ppImage_command .= " $remote_outroot "; 361 print STDERR "$remote_outroot $ipp_outroot \n";298 print STDERR "$remote_outroot $ipp_outroot $class_id\n"; 362 299 # Complete reduction information. 363 300 $reduction = 'DEFAULT' unless defined $reduction; … … 369 306 $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats "; 370 307 $ppImage_command .= " -threads $threads "; 371 $ppImage_command .= " - chip_imfile_id $chip_imfile_id ";308 $ppImage_command .= " -image_id $chip_imfile_id "; 372 309 $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace "; 373 310 $ppImage_command .= " -log ${remote_outroot}.${class_id}.log "; 374 375 print CONFIG "$ppImage_command\n"; 311 312 push @main_commands, $ppImage_command; 313 314 # Calculate pre and post commands 315 my $remote_outroot_dir = dirname($remote_outroot); 316 push @pre_commands, "mkdir -p $remote_outroot_dir"; 317 318 my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; 319 $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; 320 $post_commandA .= " -magicked 0 -hostname REMOTE -dtime_script 0 "; 321 $post_commandA .= " -dbname $dbname " if defined $dbname; 322 323 my $post_commandB = "echo -n $post_commandA > ${remote_outroot}.${class_id}.dbinfo && "; 324 my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo"; 325 326 push @post_commands, "$post_commandB $post_commandC"; 327 376 328 $job_index++; 377 329 } 378 330 } 331 332 # Actually put the commands into the output file. 333 my %unique_cmds = (); 334 foreach my $cmd (@pre_commands) { 335 unless(exists($unique_cmds{$cmd})) { 336 print CONFIG $cmd . "\n"; 337 $unique_cmds{$cmd} = 1; 338 } 339 } 340 foreach my $cmd (@main_commands) { 341 unless(exists($unique_cmds{$cmd})) { 342 print CONFIG $cmd . "\n"; 343 $unique_cmds{$cmd} = 1; 344 } 345 } 346 foreach my $cmd (@post_commands) { 347 unless(exists($unique_cmds{$cmd})) { 348 print CONFIG $cmd . "\n"; 349 $unique_cmds{$cmd} = 1; 350 } 351 } 352 379 353 close(CONFIG); 380 354 close(TRANSFER); … … 406 380 print COMMAND "#!/bin/tcsh\n"; 407 381 print COMMAND "##### Moab controll lines\n"; 408 #print COMMAND "#MSUB -l nodes=60:ppn=2,walltime=1:00:00\n"; ## CHECK RESOURCES409 382 print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES 410 383 print COMMAND "#MSUB -j oe\n"; … … 412 385 print COMMAND "#MSUB -o ${remote_root}chip.${remote_id}.out\n"; 413 386 print COMMAND "date\n"; 414 #print COMMAND 'id=$SLURM_JOB_ID' . "\n";415 #print COMMAND 'SLURM_NNODES=$SLURM_JOB_NUM_NODES' . "\n";416 387 print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; 417 388 print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; 418 #print COMMAND "srun -n2 -l --multi-prog $remote_config\n"; 419 print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n"; 389 #print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n"; 420 390 print COMMAND "date\n"; 421 391 close(COMMAND);
Note:
See TracChangeset
for help on using the changeset viewer.
