- Timestamp:
- Mar 5, 2012, 5:19:48 PM (14 years ago)
- Location:
- branches/meh_branches/ppstack_test
- Files:
-
- 3 edited
-
. (modified) (1 prop)
-
pstamp/scripts (modified) (1 prop)
-
pstamp/scripts/detectability_respond.pl (modified) (15 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/meh_branches/ppstack_test
- Property svn:mergeinfo changed
-
branches/meh_branches/ppstack_test/pstamp/scripts
- Property svn:mergeinfo set to
-
branches/meh_branches/ppstack_test/pstamp/scripts/detectability_respond.pl
r29616 r33415 20 20 use PS::IPP::PStamp::RequestFile qw( :standard ); 21 21 use PS::IPP::PStamp::Job qw( :standard ); 22 use PS::IPP::Config qw($PS_EXIT_SUCCESS 23 $PS_EXIT_UNKNOWN_ERROR 24 $PS_EXIT_SYS_ERROR 25 $PS_EXIT_CONFIG_ERROR 26 $PS_EXIT_PROG_ERROR 27 $PS_EXIT_DATA_ERROR 28 $PS_EXIT_TIMEOUT_ERROR 29 metadataLookupStr 30 metadataLookupBool 31 caturi 32 ); 22 use PS::IPP::Config qw( :standard ); 23 use PS::IPP::Metadata::List qw( parse_md_list ); 24 33 25 use Astro::FITS::CFITSIO qw( :constants ); 34 26 Astro::FITS::CFITSIO::PerlyUnpacking(1); … … 41 33 my $EXTVER = 1.0; 42 34 my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE'; 43 my ($req_id,$ req_name,$product,$need_magic,$missing_tools,$project);35 my ($req_id,$job_id,$req_name,$product,$need_magic,$missing_tools,$project); 44 36 my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps,$ignore_wisdom); 45 37 GetOptions( … … 47 39 'output=s' => \$output, 48 40 'workdir=s' => \$workdir, 41 'job_id=s' => \$job_id, 49 42 'dbname=s' => \$dbname, 50 43 'dbserver=s' => \$dbserver, … … 74 67 75 68 my $ipprc = PS::IPP::Config->new(); 69 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files 70 76 71 77 72 if (!$dbserver) { … … 108 103 } # End reading wisdom. 109 104 else { 110 #111 # Parse input request file using detect_query_read (as it's already written).112 #105 # 106 # Parse input request file using detect_query_read (as it's already written). 107 # 113 108 my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file"; 114 109 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 123 118 { 124 119 my @column_names = (); 120 # split output into lines skip until the line which lists the column names is found 121 # Parse subsequent lines 125 122 foreach my $entry (split /\n/, (join "", @$stdout_buf)) { 126 123 if ($entry =~ /^#/) { … … 128 125 shift(@column_names); # Dump the hash sign. 129 126 } 130 els e{127 elsif (scalar @column_names) { 131 128 # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE 132 129 my %row_data; 133 130 @row_data{@column_names} = (split /\s+/, $entry); 134 131 for (my $i = 0; $i <= $#column_names; $i++) { 135 136 137 #print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n";138 139 } 140 } 141 } 142 #143 # Identify target images. This should properly collate targets on a single imfile.144 #132 push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]}; 133 $Nrows = scalar(keys(%query)); 134 # print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n"; 135 } 136 } 137 } 138 } 139 # 140 # Identify target images. This should properly collate targets on a single imfile. 141 # 145 142 foreach my $fpa_id (keys %query) { 146 143 my %temp_hash; … … 212 209 } 213 210 elsif ($fpa_id eq 'Not_Set') { 211 # no matching file was found skip 214 212 next; 215 213 } … … 227 225 $need_magic = 0; 228 226 } 229 my $mjd_min = $mjd;227 my $mjd_min = int $mjd; 230 228 my $mjd_max = $mjd + 1; 231 229 230 my $req_filter; 231 if ($filter ne 'Not_Set') { 232 $req_filter = $filter . '%'; 233 } 232 234 233 235 # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter. … … 237 239 $fpa_id,undef,undef, 238 240 $option_mask,$need_magic, 239 # $ra,$dec, 240 $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose); 241 $mjd_min,$mjd_max,$req_filter,undef,$verbose); 241 242 242 243 foreach my $this_image_ref (@{ $pstamp_images_ref }) { 243 foreach my $key (sort (keys %{ $this_image_ref } )) { 244 my $value = ${ $this_image_ref }{$key}; 245 if ($key eq 'row_index') { 246 $value = join ' ', @{ $this_image_ref->{$key} }; 247 } 248 249 # print "$this_image_ref $key $value\n"; 250 foreach my $valid_index (@{ $this_image_ref->{row_index} }) { 251 $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image}; 252 $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask}; 253 $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight}; 254 $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf}; 255 $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id}; 256 $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb}; 257 $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic; 258 $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0; 259 260 if (exists($this_image_ref->{astrom})) { 261 $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom}; 262 } 263 else { 264 $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf}; 265 } 266 if (exists($this_image_ref->{class_id})) { 267 $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id}; 268 $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id}; 269 270 } 271 else { 272 $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id}; 273 $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa'; 274 } 275 276 $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state}; 277 if (exists($this_image_ref->{data_state})) { 278 $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state}; 279 } 280 else { 281 $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state}; 282 } 283 $query{$fpa_id}{FAULT}[$valid_index] = 0; 244 245 if (0) { 246 foreach my $key (sort (keys %{ $this_image_ref } )) { 247 my $value = ${ $this_image_ref }{$key}; 248 if ($key eq 'row_index') { 249 $value = join ' ', @{ $this_image_ref->{$key} }; 250 } 251 print "$this_image_ref $key $value\n"; 252 } 253 } 254 255 256 foreach my $valid_index (@{ $this_image_ref->{row_index} }) { 257 $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image}; 258 $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask}; 259 $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight}; 260 $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf}; 261 $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id}; 262 $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb}; 263 $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic; 264 $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0; 265 266 if (exists($this_image_ref->{astrom})) { 267 $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom}; 268 } 269 else { 270 $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf}; 271 } 272 if (exists($this_image_ref->{class_id})) { 273 $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id}; 274 $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id}; 275 276 } 277 else { 278 $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id}; 279 $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa'; 280 } 281 282 $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state}; 283 if (exists($this_image_ref->{data_state})) { 284 $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state}; 285 } 286 else { 287 $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state}; 288 } 289 $query{$fpa_id}{FAULT}[$valid_index] = 0; 284 290 $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked}; 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] =0)) {301 302 303 304 305 306 307 308 309 310 311 312 313 314 291 if ($stage eq 'chip') { 292 $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state}; 293 } 294 295 # Determine if the data exists. 296 if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or 297 ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or 298 ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or 299 ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or 300 ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or 301 ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) { 302 303 # image is gone and it's not coming back 304 $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE; 305 } 306 elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) { 307 $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED; 308 } 309 elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) { 310 $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE; 311 312 # updating stacks isn't implemented 313 if (($stage eq 'stack')) { 314 $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; 315 } 316 # updating old burntool data isn't implemented 317 elsif ($stage eq 'chip') { 318 if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and 319 (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) { 320 $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; 315 321 } 316 } 317 } # End determining error faults. 318 } 319 } 320 } 322 } 323 } # End determining error faults. 324 } 325 } 321 326 } 322 327 } # End calculating wisdom … … 324 329 my %processing_request; 325 330 331 if ($job_id) { 332 $wisdom_file = "$wisdom_file.$job_id"; 333 } 326 334 open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file"); 327 335 foreach my $fpa_id (keys %query) { 328 336 for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) { 329 print WISDOM "$fpa_id\t"; 330 foreach my $key (keys %{ $query{$fpa_id} }) { 331 print WISDOM "$key $query{$fpa_id}{$key}[$i]\t"; 332 } 333 print WISDOM "\n"; 334 if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) { 335 @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = 336 ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i], 337 $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]); 338 push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i; 339 } 337 print WISDOM "$fpa_id\t"; 338 foreach my $key (keys %{ $query{$fpa_id} }) { 339 print WISDOM "$key $query{$fpa_id}{$key}[$i]\t"; 340 } 341 print WISDOM "\n"; 342 my $data_state = $query{$fpa_id}{DATA_STATE}[$i]; 343 if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) { 344 if ($data_state ne 'full') { 345 346 @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = 347 ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i], 348 $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]); 349 } 350 push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i; 351 } 340 352 } 341 353 } 342 354 close(WISDOM); 343 355 344 # If there is anything that needs to be updated, create the update request list, and then exit the program. 345 my $exit_code = 0; 346 my $update_request_file = "${workdir}/update_request.dat"; 347 open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file"); 348 foreach my $images (keys %update_request) { 349 foreach my $fault (keys %{ $update_request{$images} }) { 350 if ($fault == 25) { 351 $exit_code = 25; 352 } 353 elsif ($fault != 0) { 354 $exit_code = 21; 355 } 356 my $update_request = join ' ', @{ $update_request{$images}{$fault} }; 357 print UPDATE_REQUEST "$update_request\n"; 358 } 359 } 360 close(UPDATE_REQUEST); 361 if ($exit_code != 0) { 362 exit($exit_code); 356 if (! defined $job_id) { 357 # If there is anything that needs to be updated, create the update request list, and then exit the program. 358 my $exit_code = 0; 359 my $update_request_file = "${workdir}/update_request.dat"; 360 open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file"); 361 foreach my $images (keys %update_request) { 362 foreach my $fault (keys %{ $update_request{$images} }) { 363 if ($fault == 25) { 364 $exit_code = 25; 365 } 366 elsif ($fault != 0) { 367 $exit_code = 21; 368 } 369 my $update_request = join ' ', @{ $update_request{$images}{$fault} }; 370 print UPDATE_REQUEST "$update_request\n"; 371 } 372 } 373 close(UPDATE_REQUEST); 374 if ($exit_code != 0) { 375 exit($exit_code); 376 } 363 377 } 364 378 … … 388 402 my $weight= $query{$fpa_id}{WEIGHT}[$index]; 389 403 my $stage = $query{$fpa_id}{STAGE}[$index]; 390 # if there's a fault, then we can't process this image. 391 if (($fault != 0)||($query{$fpa_id}{BAD_COMPONENT}[$index] == 1)) { 392 $query{$fpa_id}{PROC_ERROR}[$index] = 23; 393 394 $query{$fpa_id}{NPIX}[$index] = 0; 395 $query{$fpa_id}{QFACTOR}[$index] = 0.0; 396 $query{$fpa_id}{FLUX}[$index] = 0.0; 397 $query{$fpa_id}{FLUX_SIG}[$index] = 0.0; 404 my $stage_id = $query{$fpa_id}{STAGE_ID}[$index]; 405 my $component = $query{$fpa_id}{COMPONENT_ID}[$index]; 406 # print "Input is from $stage $stage_id $component\n"; 407 408 # if there's a fault or quality problem, then we can't process this image. 409 # if (($fault != 0)||($query{$fpa_id}{BAD_COMPONENT}[$index] == 1)) { 410 if ($fault = check_component($stage, $stage_id, $component, $imagedb)) { 411 foreach my $i (@{ $processing_request{$fpa_id}{$image} }) { 412 $query{$fpa_id}{PROC_ERROR}[$i] = $fault; 413 414 $query{$fpa_id}{NPIX}[$i] = 0; 415 $query{$fpa_id}{QFACTOR}[$i] = 0.0; 416 $query{$fpa_id}{FLUX}[$i] = 0.0; 417 $query{$fpa_id}{FLUX_SIG}[$i] = 0.0; 418 } 398 419 399 420 next; … … 695 716 $inHeader->{FPA_ID}->{value} = $fpa_id; 696 717 } 718 719 my @keysa = keys %{$query{$fpa_id}}; 720 721 print "$fpa_id: @keysa\n"; 697 722 698 723 push @{$colData{'ROWNUM'}}, @{ $query{$fpa_id}{ROWNUM} }; … … 710 735 $status = 0; 711 736 # print "$output\n"; 737 if (-e $output) { 738 unlink $output or die "failed to unlink existing response file $output\n"; 739 } 712 740 my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status ); 713 741 check_fitsio( $status ); … … 779 807 } 780 808 809 sub check_component { 810 my ($stage, $stage_id, $component, $imagedb) = @_; 811 812 print "Checking status of component $stage $stage_id $component\n"; 813 814 my $command; 815 if ($stage eq 'diff') { 816 $command = "difftool -diffskyfile -diff_id $stage_id -skycell_id $component"; 817 } elsif ($stage eq 'stack') { 818 $command = "stacktool -sumskyfile -stack_id $stage_id"; 819 } elsif ($stage eq 'warp') { 820 $command = "warptool -warped -warp_id $stage_id -skycell_id $component"; 821 } elsif ($stage eq 'chip') { 822 $command = "chiptool -processedimfile -chip_id $stage_id -class_id $component"; 823 } else { 824 die("check_component not implemented for stage $stage yet"); 825 } 826 827 $command .= " -dbname $imagedb"; 828 829 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 830 run(command => $command, verbose => $verbose); 831 unless ($success) { 832 my $rc = $error_code >> 8; 833 carp "$command failed $error_code $rc"; 834 exit $PS_EXIT_UNKNOWN_ERROR; 835 } 836 my $output = join "", @$stdout_buf; 837 my $metadata = $mdcParser->parse($output); 838 my $results = parse_md_list($metadata); 839 if (scalar @$results != 1) { 840 carp "$command returned too many components: " . scalar @$results; 841 exit $PS_EXIT_UNKNOWN_ERROR; 842 } 843 my $it = $results->[0]; 844 845 if ($stage eq 'stack') { 846 $it->{data_state} = $it->{state}; 847 } 848 849 my $return_status = 0; 850 if ($it->{quality}) { 851 print " Bad quality: $it->{quality}\n"; 852 $return_status = $PSTAMP_GONE; 853 } elsif ($it->{fault}) { 854 print " Faulted: $it->{fault}\n"; 855 $return_status = $PSTAMP_GONE; 856 } elsif ($it->{data_state} ne 'full') { 857 # XXX does this work for stack? 858 carp " Faulted: $it->{data_state}\n"; 859 $return_status = $PSTAMP_GONE; 860 } else { 861 print " Component ok.\n"; 862 } 863 864 return $return_status; 865 } 866 781 867 sub my_die { 782 868 my $message = shift;
Note:
See TracChangeset
for help on using the changeset viewer.
