Changeset 37280
- Timestamp:
- Aug 21, 2014, 2:37:38 PM (12 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl
r37278 r37280 24 24 my $threads_req = 4; 25 25 26 my $fail_state = "prep_fail"; 27 26 28 # Look for programs we need 27 29 my $missing_tools; … … 36 38 37 39 # Options 38 my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose );40 my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose,$dbverbose); 39 41 GetOptions( 40 42 'remote_id=s' => \$remote_id, … … 45 47 'no_update' => \$no_update, 46 48 'verbose' => \$verbose, 49 'dbverbose' => \$dbverbose, 47 50 ) or pod2usage( 2 ); 48 51 … … 55 58 defined($dbname); 56 59 57 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $ exp_id);60 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); 58 61 59 62 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files … … 61 64 # Detrend concept holders (name, ppImage option) 62 65 my %det_types = ( 63 'MASK' => '-mask',64 'FLAT' => '-flat',65 'DARK' => '-dark',66 'MASK' => '-mask', 67 'FLAT' => '-flat', 68 'DARK' => '-dark', 66 69 'VIDEODARK' => '-dark', 67 70 'LINEARITY' => '-linearity', … … 73 76 74 77 # STEP 0: Open output files 75 my $uri_command = $path_base . ".cmd";76 78 my $uri_transfer= $path_base . ".transfer"; 77 79 my $uri_check = $path_base . ".check"; … … 80 82 my $uri_return = $path_base . ".return"; 81 83 82 my $disk_command = $ipprc->file_resolve($uri_command,1);83 84 my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); 84 85 my $disk_check = $ipprc->file_resolve($uri_check,1); … … 89 90 my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. 90 91 91 open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer", $chip_id, $PS_EXIT_SYS_ERROR);92 open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check", $chip_id, $PS_EXIT_SYS_ERROR);93 open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config", $chip_id, $PS_EXIT_SYS_ERROR);94 open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate", $chip_id, $PS_EXIT_SYS_ERROR);95 open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR);92 open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); 93 open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); 94 open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); 95 open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); 96 open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); 96 97 97 98 my $job_index = 0; 98 my @pre_commands = ();99 my @main_commands = ();100 my @post_commands = ();101 99 102 100 # STEP 1 : Get exposure level information for this chipRun 103 my $exp_id, $filter, $altfilt, $dateobs;101 my ($exp_id, $filter, $altfilt, $dateobs); 104 102 { 105 103 my $command = "$chiptool -listrun -chip_id $chip_id "; 106 104 $command .= " -dbname $dbname " if defined($dbname); 107 105 108 106 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 109 run(command => $command, verbose => 0);107 run(command => $command, verbose => $dbverbose); 110 108 unless ($success) { 111 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 112 113 &my_die("Unable to run chiptool to determine stage parameters.", 114 $chip_id,$error_code); 115 } 116 117 # Parse chipRun level data to determine exposure information 109 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 110 111 &my_die("Unable to run chiptool to determine stage parameters.", $remote_id, $chip_id, $error_code, $fail_state); 112 } 113 114 # Parse chipRun level data to determine exposure information 118 115 my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or 119 &my_die("Unable to determine chip component information.", $chip_id,$PS_EXIT_PROG_ERROR);116 &my_die("Unable to determine chip component information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); 120 117 my $metadata = parse_md_list($MDlist); 121 my $chipEntry = $metadata [0];122 123 $exp_id = $chipEntry->{exp_id}; 118 my $chipEntry = $metadata->[0]; 119 120 $exp_id = $chipEntry->{exp_id}; 124 121 $filter = $chipEntry->{filter}; 125 122 $altfilt = $filter; … … 134 131 foreach my $det_type (keys (%det_types)) { 135 132 if (($filter !~ /y/) && ($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y 136 133 137 134 my $command = "detselect -search -det_type $det_type -time $dateobs"; 138 135 $command .= " -dbname $dbname " if defined($dbname); 139 140 if (($det_type eq 'FLAT') || ($det_type eq 'FRINGE')) { 141 $command .= " -filter $altfilt";142 } 143 136 137 if (($det_type eq 'FLAT') || ($det_type eq 'FRINGE')) { 138 $command .= " -filter $altfilt"; 139 } 140 144 141 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 145 run(command => $command, verbose => 0);142 run(command => $command, verbose => $dbverbose); 146 143 unless ($success) { 147 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);148 149 &my_die("No valid detrend available for this image: $det_type", $chip_id, $error_code);150 } 151 152 my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or 153 &my_die("Could not parse detrend information for this image: $det_type", $chip_id,$error_code);144 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 145 146 &my_die("No valid detrend available for this image: $det_type", $remote_id, $chip_id, $error_code, $fail_state); 147 } 148 149 my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or 150 &my_die("Could not parse detrend information for this image: $det_type", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); 154 151 my $metadata = parse_md_list($MDlist); 155 my $detEntry = $metadata[0]; 152 my $detEntry = $metadata->[0]; 153 154 if ($verbose) { 155 print STDERR "det_id: $detEntry->{det_id}\n"; 156 print STDERR "det_iter: $detEntry->{iteration}\n"; 157 } 156 158 157 159 # save det_id and iteration … … 165 167 my $command = "$chiptool -pendingimfile -chip_id $chip_id"; 166 168 $command .= " -dbname $dbname " if defined($dbname); 167 169 168 170 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 169 run(command => $command, verbose => $verbose);171 run(command => $command, verbose => $dbverbose); 170 172 171 173 unless ($success) { 172 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 173 174 &my_die("Unable to run chiptool -pendingimfile", 175 $chip_id,$error_code); 174 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 175 176 &my_die("Unable to run chiptool -pendingimfile", $remote_id, $chip_id, $error_code, $fail_state); 176 177 } 177 178 178 179 my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or 179 &my_die("Unable to parse chiptool -pendingimfile information.", $chip_id,$error_code);180 &my_die("Unable to parse chiptool -pendingimfile information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); 180 181 my $metadata = parse_md_list($MDlist); 181 182 182 183 # Iterate over the chipProcessedImfile level data. 183 184 foreach my $chipEntry (@$metadata) { 184 # Get information we need to pass to ppImage 185 my $uri = $chipEntry->{uri}; 186 my $class_id = $chipEntry->{class_id}; 187 my $video_cells = $chipEntry->{video_cells}; 188 my $reduction = $chipEntry->{reduction}; 189 my $exp_tag = $chipEntry->{exp_tag}; 190 my $workdir = $chipEntry->{workdir}; 191 my $chip_imfile_id = $chipEntry->{chip_imfile_id}; 192 193 # Process the image and burntool table 194 my ($ipp_uri, $remote_uri) = uri_to_outputs_raw($uri); 195 my $btt = $uri; 196 $btt =~ s/fits$/burn.tbl/; 197 my ($ipp_btt, $remote_btt) = uri_to_outputs_raw($btt); 198 199 # Initialize the ppI command 200 my $ppImage_command = "ppImage -file $remote_uri"; 201 $ppImage_command .= " -burntool $remote_btt "; 202 203 foreach my $det_type (keys (%det_types)) { 204 if (( $video_cells) && ($det_type eq 'DARK')) next; 205 if ((!$video_cells) && ($det_type eq 'VIDEODARK')) next; 206 207 my $det_id = $det_ids{$det_type}; 208 my $det_iter = $det_iters{$det_type}; 209 210 # Add detrend information to the command line 211 my $detselect_command = "detselect -select "; 212 $detselect_command .= " -det_id $det_id"; 213 $detselect_command .= " -iteration $det_iter"; 214 $detselect_command .= " -class_id $class_id"; 215 $detselect_command .= " -dbname $dbname " if defined($dbname); 216 217 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 218 run(command => $detselect_command, verbose => 0); 219 220 unless ($success) { 221 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 222 223 &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", 224 $chip_id,$error_code); 225 } 226 227 my $detMDlist = $mdcParser->parse(join "", @$stdout_buf) or 228 &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", 229 $chip_id,$error_code); 230 my $detImfileMD = parse_md_list($detMDlist); 231 my $duri = $detImfileMD[0]->{uri}; 232 233 my $det_code = $det_types{$det_type}; 234 my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri); 235 $ppImage_command .= " $det_code $remote_det_uri "; 236 } 237 238 # Add output root 239 my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}"; 240 my $remote_outroot = uri_local_to_remote($ipp_outroot); 241 $ppImage_command .= " $remote_outroot "; 242 print STDERR "$remote_outroot $ipp_outroot $class_id\n"; 243 # Complete reduction information. 244 $reduction = 'DEFAULT' unless defined $reduction; 245 my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage 246 my $recipe_psphot = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot 247 248 $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage "; 249 $ppImage_command .= " -recipe PSPHOT $recipe_psphot "; 250 $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats "; 251 $ppImage_command .= " -threads $threads_req "; 252 $ppImage_command .= " -image_id $chip_imfile_id "; 253 $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace "; 254 $ppImage_command .= " -log ${remote_outroot}.${class_id}.log "; 255 256 # push @main_commands, $ppImage_command; 257 258 # Calculate pre and post commands 259 my $remote_outroot_dir = dirname($remote_outroot); 260 my $pre_command = "mkdir -p $remote_outroot_dir"; 261 262 my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; 263 $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; 264 $post_commandA .= " -magicked 0 -hostname REMOTE -dtime_script 0 "; 265 $post_commandA .= " -dbname $dbname " if defined $dbname; 266 267 my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo "; 268 my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo"; 269 270 # push @post_commands, "$post_commandB $post_commandC"; 271 272 $job_index++; 273 274 print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}"; 275 276 # Determine which output files need to be returned. 277 foreach my $component (@return_component_list) { 278 # uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) ); 279 my $filename = $ipprc->filename($component, $ipp_outroot, $class_id); 280 my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename ); 281 my $remote_outroot_dir = dirname($ipp_disk); 282 print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; 283 # print " $filename $ipp_disk $remote_disk\n"; 284 # die; 285 } 286 print CONFIG "\n"; 185 # Get information we need to pass to ppImage 186 my $uri = $chipEntry->{uri}; 187 my $class_id = $chipEntry->{class_id}; 188 my $video_cells = $chipEntry->{video_cells}; 189 my $reduction = $chipEntry->{reduction}; 190 my $exp_tag = $chipEntry->{exp_tag}; 191 my $workdir = $chipEntry->{workdir}; 192 my $chip_imfile_id = $chipEntry->{chip_imfile_id}; 193 194 # Process the image and burntool table 195 my ($ipp_uri, $remote_uri) = uri_to_outputs_raw($uri); 196 my $btt = $uri; 197 $btt =~ s/fits$/burn.tbl/; 198 my ($ipp_btt, $remote_btt) = uri_to_outputs_raw($btt); 199 200 # Initialize the ppI command 201 my $ppImage_command = "ppImage -file $remote_uri"; 202 $ppImage_command .= " -burntool $remote_btt "; 203 204 foreach my $det_type (keys (%det_types)) { 205 if (( $video_cells) && ($det_type eq 'DARK')) { next; } 206 if ((!$video_cells) && ($det_type eq 'VIDEODARK')) { next; } 207 208 my $det_id = $det_ids{$det_type}; 209 my $det_iter = $det_iters{$det_type}; 210 211 if (not defined $det_id) { next; } 212 213 if ($verbose) { print STDERR "det_type, det_id, det_iter: $det_type, $det_id, $det_iter\n"; } 214 215 # Add detrend information to the command line 216 my $detselect_command = "detselect -select "; 217 $detselect_command .= " -det_id $det_id"; 218 $detselect_command .= " -iteration $det_iter"; 219 $detselect_command .= " -class_id $class_id"; 220 $detselect_command .= " -dbname $dbname " if defined($dbname); 221 222 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 223 run(command => $detselect_command, verbose => $dbverbose); 224 225 unless ($success) { 226 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 227 228 &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $error_code, $fail_state); 229 } 230 231 my $detMDlist = $mdcParser->parse(join "", @$stdout_buf) or 232 &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); 233 my $detImfileMD = parse_md_list($detMDlist); 234 my $detImfile = $detImfileMD->[0]; 235 my $duri = $detImfile->{uri}; 236 237 my $det_code = $det_types{$det_type}; 238 my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri); 239 $ppImage_command .= " $det_code $remote_det_uri "; 240 } 241 242 # Add output root 243 my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}"; 244 my $remote_outroot = uri_local_to_remote($ipp_outroot); 245 $ppImage_command .= " $remote_outroot "; 246 print STDERR "$remote_outroot $ipp_outroot $class_id\n"; 247 # Complete reduction information. 248 $reduction = 'DEFAULT' unless defined $reduction; 249 my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage 250 my $recipe_psphot = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot 251 252 $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage "; 253 $ppImage_command .= " -recipe PSPHOT $recipe_psphot "; 254 $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats "; 255 $ppImage_command .= " -threads $threads_req "; 256 $ppImage_command .= " -image_id $chip_imfile_id "; 257 $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace "; 258 $ppImage_command .= " -log ${remote_outroot}.${class_id}.log "; 259 260 # Calculate pre and post commands 261 my $remote_outroot_dir = dirname($remote_outroot); 262 my $pre_command = "mkdir -p $remote_outroot_dir"; 263 264 my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; 265 $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; 266 $post_commandA .= " -magicked 0 -hostname REMOTE -dtime_script 0 "; 267 $post_commandA .= " -dbname $dbname " if defined $dbname; 268 269 my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo "; 270 my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo"; 271 272 $job_index++; 273 274 print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}"; 275 276 # Determine which output files need to be returned. 277 foreach my $component (@return_component_list) { 278 my $filename = $ipprc->filename($component, $ipp_outroot, $class_id); 279 my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename ); 280 my $remote_outroot_dir = dirname($ipp_disk); 281 print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; 282 } 283 print CONFIG "\n"; 287 284 } 288 285 } … … 314 311 my $ipp_disk= $ipprc->file_resolve( $neb_uri ); 315 312 my $remote_disk = $ipp_disk; 316 313 317 314 unless(defined($ipp_disk)) { 318 my_die();315 my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); 319 316 } 320 317 … … 330 327 my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); 331 328 my $remote_disk = $ipp_disk; 332 329 333 330 unless(defined($ipp_disk)) { 334 my_die();331 my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); 335 332 } 336 333 … … 345 342 my $neb_uri = shift; 346 343 my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); 347 348 unless (exists($file_filter{$neb_uri})) { 349 $file_filter{$neb_uri} = 1; 350 print TRANSFER "$ipp_disk\n"; 351 print CHECK "$remote_disk\n"; 352 } 344 345 print TRANSFER "$ipp_disk\n"; 346 print CHECK "$remote_disk\n"; 353 347 return($ipp_disk,$remote_disk); 354 348 } … … 358 352 my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); 359 353 $remote_disk = $remote_raw . $ipp_disk; 360 unless (exists($file_filter{$neb_uri})) { 361 $file_filter{$neb_uri} = 1; 362 print TRANSFER "$ipp_disk\n"; 363 print CHECK "$remote_disk\n"; 364 } 354 355 print TRANSFER "$ipp_disk\n"; 356 print CHECK "$remote_disk\n"; 365 357 return($ipp_disk,$remote_disk); 366 358 } … … 369 361 my $neb_uri = shift; 370 362 my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); 371 unless (exists($file_filter{$neb_uri})) { 372 $file_filter{$neb_uri} = 1; 373 print RETURN "$ipp_disk\n"; 374 print GENERATE "$remote_disk\n"; 375 } 363 print RETURN "$ipp_disk\n"; 364 print GENERATE "$remote_disk\n"; 376 365 return($ipp_disk,$remote_disk); 377 366 } … … 387 376 return($remote_uri); 388 377 } 389 378 390 379 sub uri_remote_to_local { 391 380 # This needs to replace the remote root directory with the nebulous tag. … … 393 382 $remote_uri =~ s%${remote_root}%%; 394 383 my $local_uri = "neb:///" . $remote_uri; 395 384 396 385 return($local_uri); 397 386 } 387 398 388 sub my_die { 399 389 my $msg = shift; 400 my $id = shift; 390 my $remote_id = shift; 391 my $stage_id = shift; 401 392 my $exit_code = shift; 402 393 my $exit_state = shift; … … 405 396 406 397 carp($msg); 407 408 I need stage id here 409 410 411 if (defined $id and not $no_update) { 412 my $command = "remotetool -updatecomponent -remote_id $id"; 413 $command .= " -fault $exit_code " if defined $exit_code; 414 $command .= " -set_state $exit_state " if defined $exit_state; 415 $command .= " -dbname $dbname " if defined $dbname; 416 417 system($command); 398 399 if (defined $remote_id and defined $stage_id and not $no_update) { 400 my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; 401 $command .= " -set_state $exit_state " if defined $exit_state; 402 $command .= " -dbname $dbname " if defined $dbname; 403 404 system($command); 418 405 } 419 406
Note:
See TracChangeset
for help on using the changeset viewer.
