Changeset 28003 for branches/pap/pstamp/scripts/detectability_respond.pl
- Timestamp:
- May 18, 2010, 12:49:05 PM (16 years ago)
- Location:
- branches/pap
- Files:
-
- 2 edited
-
. (modified) (1 prop)
-
pstamp/scripts/detectability_respond.pl (modified) (20 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/pap
- Property svn:mergeinfo changed
-
branches/pap/pstamp/scripts/detectability_respond.pl
r27643 r28003 41 41 my $EXTVER = 1.0; 42 42 my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE'; 43 my ($req_id,$req_name,$product,$need_magic,$missing_tools );44 my ($request_file,$output,$ dbname,$verbose,$save_temps);43 my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project); 44 my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps); 45 45 GetOptions( 46 'input=s' =>\$request_file,46 'input=s' => \$request_file, 47 47 'output=s' => \$output, 48 'workdir=s' => \$workdir, 48 49 'dbname=s' => \$dbname, 50 'dbserver=s' => \$dbserver, 49 51 'verbose' => \$verbose, 50 52 'save-temps' => \$save_temps, … … 55 57 -exitval => 3, 56 58 ) unless 57 defined $request_file and defined $output and defined $ dbname;59 defined $request_file and defined $output and defined $workdir and defined $dbname; 58 60 59 61 my $detect_query_read = can_run('detect_query_read') or (warn "Can't find detect_query_read" and $missing_tools = 1); … … 71 73 72 74 my $ipprc = PS::IPP::Config->new(); 73 #my $tmp_dir = "/data/${host}.0/tmp/"; 75 76 if (!$dbserver) { 77 $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); 78 } 79 # This is hardcoded in for the moment. 80 $project = resolve_project($ipprc,"gpc1",$dbname,$dbserver); 81 my $imagedb = $project->{dbname}; 82 if (!$imagedb) { 83 carp("failed to find imagedb for project: $project"); 84 } 74 85 75 86 # 76 87 # Parse input request file using detect_query_read (as it's already written). 77 88 # 78 79 89 my $dqr_command = "$detect_query_read --input $request_file"; 80 90 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 81 91 run(command => $dqr_command, verbose => $verbose); 82 92 unless ($success) { 83 my_die("Unable to perform $dqr_command error code: $error_code",-1,-1,-1,-1,-1,-1); 93 # This is a problem, because I'm not sure how we handle a failure to read something. 94 # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 95 # we can't send that response back. 96 die("Unable to perform $dqr_command error code: $error_code"); 84 97 } 85 98 my %query = (); … … 119 132 my %image_list_hash; 120 133 for (my $i = 1; $i < $Nrows; $i++) { 121 # print "$i $Nrows $query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]\n"; 122 134 # This could use the fact that locate_images now accepts position arrays, but 135 # I'll save that for after I get the majority of things working. 123 136 my $image_set_tmp = find_image_set($query{HEADER}{FPA_ID}[0],$query{HEADER}{STAGE}[0], 124 137 $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0], … … 126 139 $query{CONTENT}{ROWNUM}[$i],$verbose); 127 140 unless (%$image_set_tmp) { 128 my_die("No images were returned.",$query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0], 129 $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0], 130 $query{HEADER}{STAGE}[0]); 131 } 132 print "=== $image_set_tmp->{IMAGE}\n $image_set_tmp->{PSF}\n $image_set_tmp->{MASK}\n $image_set_tmp->{WEIGHT}\n $image_set_tmp->{SKY_COORDINATES}\n $image_set_tmp->{ROWNUM}\n"; 133 # This appends, assuming that if we get an image, we also get the identical psf/mask/weight/etc. 141 # No images were returned, so create a dummy entry that 142 $image_list_hash{'no_image'}{IMAGE} = 'no_image'; 143 $image_list_hash{'no_image'}{PSF} = 'no_psf'; 144 $image_list_hash{'no_image'}{MASK} = 'no_mask'; 145 $image_list_hash{'no_image'}{WEIGHT} = 'no_weight'; 146 $image_list_hash{'no_image'}{CATALOG} = 'no_catalog'; 147 $image_list_hash{'no_image'}{CLASS_ID} = 'no_class'; 148 $image_list_hash{'no_image'}{ERROR} = $PSTAMP_NO_IMAGE_MATCH; 149 push @{ $image_list_hash{'no_image'}{SKY_COORDINATES} }, "$query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]"; 150 push @{ $image_list_hash{'no_image'}{ROWNUM} }, $query{CONTENT}{ROWNUM}[$i]; 151 next; 152 } 153 # print "=== $image_set_tmp->{IMAGE}\n $image_set_tmp->{PSF}\n"; 154 # print " $image_set_tmp->{MASK}\n $image_set_tmp->{WEIGHT}\n"; 155 # print " $image_set_tmp->{SKY_COORDINATES}\n $image_set_tmp->{ROWNUM}\n"; 156 157 # This indexes the results for identical images into the same hash. 134 158 $image_list_hash{$image_set_tmp->{IMAGE}}{IMAGE} = $image_set_tmp->{IMAGE}; 135 159 $image_list_hash{$image_set_tmp->{IMAGE}}{PSF} = $image_set_tmp->{PSF}; … … 138 162 $image_list_hash{$image_set_tmp->{IMAGE}}{CATALOG} = $image_set_tmp->{CATALOG}; 139 163 $image_list_hash{$image_set_tmp->{IMAGE}}{CLASS_ID} = $image_set_tmp->{CLASS_ID}; 164 $image_list_hash{$image_set_tmp->{IMAGE}}{ERROR} = $image_set_tmp->{ERROR}; 140 165 push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{SKY_COORDINATES} }, $image_set_tmp->{SKY_COORDINATES}; 141 166 push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{ROWNUM} }, $image_set_tmp->{ROWNUM}; 142 167 } 168 143 169 my $i = 0; 144 145 170 foreach my $k (keys %image_list_hash) { 171 # If we errored out on finding an image, we need to not try to run psphot here. 172 if ($image_list_hash{$k}{ERROR} != 0) { 173 next; 174 } 146 175 # Write coordinates of the requested targets to a file. 147 my ($coordfile,$coordname) = tempfile(" /tmp/detect.coords.$i.XXXX",176 my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$i.XXXX", 148 177 UNLINK => !$save_temps); 149 my ($targetfile,$targetname) = tempfile(" /tmp/detect.targets.$i.XXXX",178 my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$i.XXXX", 150 179 UNLINK => !$save_temps); 151 180 … … 153 182 print $coordfile "$image_list_hash{$k}{SKY_COORDINATES}[$j]\n"; 154 183 } 155 184 # print "$k\n"; 156 185 # Convert the sky coordinates to image coordinates with ppCoord. 157 186 my $command = "ppCoord -astrom $image_list_hash{$k}{CATALOG} -radec $coordname"; … … 171 200 } 172 201 173 202 # print "psphot $image_list_hash{$k}{PSF}\n"; 174 203 # Run psphotForced on the target list. 175 my $tmpdir = tempdir("detect.$i.XXXX", DIR => " /tmp/", CLEANUP => !$save_temps);204 my $tmpdir = tempdir("detect.$i.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps); 176 205 $image_list_hash{$k}{OUTROOT} = "$tmpdir/detectability.$query{HEADER}{STAGE}[0].$query{HEADER}{FPA_ID}[0]"; 177 206 … … 185 214 run(command => $psphot_cmd, verbose => $verbose); 186 215 unless ($success) { 187 my_die("Unable to perform $psphot_cmd. Error_code: $error_code", 188 $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0], 189 $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0], 190 $query{HEADER}{STAGE}[0],$error_code); 216 $image_list_hash{$k}{ERROR} = $PSTAMP_SYSTEM_ERROR; 191 217 } 192 218 } … … 196 222 # 197 223 my @rownums = (); 224 my @out_errors = (); 198 225 my @psphot_Npix = (); 199 226 my @psphot_Qfact= (); 200 227 my @psphot_flux = (); 228 201 229 foreach my $k (keys %image_list_hash) { 202 my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf"; 203 204 my ($tmp_Npix,$tmp_Qfact,$tmp_flux) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE}); 205 206 push @rownums, @{ $image_list_hash{$k}{ROWNUM} }; 207 push @psphot_Npix, @{ $tmp_Npix }; 208 push @psphot_Qfact, @{ $tmp_Qfact }; 209 push @psphot_flux, @{ $tmp_flux }; 230 if ($image_list_hash{$k}{ERROR} == 0) { 231 my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf"; 232 233 my ($tmp_Npix,$tmp_Qfact,$tmp_flux) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE}); 234 235 push @rownums, @{ $image_list_hash{$k}{ROWNUM} }; 236 push @out_errors, (map { $image_list_hash{$k}{ERROR} } @{ $image_list_hash{$k}{ROWNUM} }); 237 push @psphot_Npix, @{ $tmp_Npix }; 238 push @psphot_Qfact, @{ $tmp_Qfact }; 239 push @psphot_flux, @{ $tmp_flux }; 240 } 241 else { 242 push @rownums, @{ $image_list_hash{$k}{ROWNUM} }; 243 push @out_errors, (map { $image_list_hash{$k}{ERROR} } @{ $image_list_hash{$k}{ROWNUM} }); 244 push @psphot_Npix, (map { 0 } @{ $image_list_hash{$k}{ROWNUM} }); 245 push @psphot_Qfact, (map { 0.0 } @{ $image_list_hash{$k}{ROWNUM} }); 246 push @psphot_flux, (map { 0.0 } @{ $image_list_hash{$k}{ROWNUM} }); 247 } 210 248 } 211 249 … … 214 252 $query{HEADER}{MJD_OBS}[0],$query{HEADER}{filter}[0], 215 253 $query{HEADER}{obscode}[0], 216 \@rownums, \@ psphot_Npix, \@psphot_Qfact, \@psphot_flux);217 254 \@rownums, \@out_errors, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux); 255 # print "Wrote response file $output\n"; 218 256 # 219 257 # Add to datastore 220 258 # 221 # my $finish_command = "dquery_finish.pl --req_id $req_id --req_name $req_name --product $product"; 222 # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 223 # run(command => $finish_command, verbose => $verbose); 224 # unless ($success) { 225 # warn ("Unable to perform $finish_command error code: $error_code"); 226 # } 227 259 # Files are added to the datastore by dquery_finish.pl 228 260 # 229 261 # Cleanup 230 262 # 231 263 # Since everything is written to temporary files, there should be nothing to cleanup. 264 232 265 # 233 266 # Utilities … … 242 275 my $index = shift; 243 276 my $verbose = shift; 244 277 278 # This is the set of things that we need in order to run psphotForced 245 279 my $option_mask |= 1; 246 280 $option_mask |= $PSTAMP_SELECT_IMAGE; … … 252 286 my $mjd_max = $mjd + 1; 253 287 288 # Construct a row list. 289 my @rowList; 290 $rowList[0]->{CENTER_X} = $ra; 291 $rowList[0]->{CENTER_Y} = $dec; 292 $rowList[0]->{ID} = 1; 293 $rowList[0]->{STAGE} = $stage; 294 $rowList[0]->{COORD_MASK} = 0; 295 296 # print "$stage\n"; 254 297 # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter. 255 my @images = locate_images($ipprc,$dbname,"bycoord",$stage, 256 undef,undef,undef,$option_mask,$need_magic, 257 $ra,$dec,$mjd_min,$mjd_max,$filter . ".00000",undef,$verbose); 298 my @images = locate_images($ipprc,$imagedb, 299 \@rowList, 300 "bycoord",$stage, 301 undef,undef,undef, 302 $option_mask,$need_magic, 303 # $ra,$dec, 304 $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose); 258 305 259 306 my %image_info = (); … … 270 317 } 271 318 elsif ($stage eq 'stack') { 272 # Stacks hide the exposure name very well, so I'm 273 # choosing the stack_id as the FPA_ID to match. This 274 # probably needs to be approved somehow. 319 # Stacks hide the exposure name very well, so we can only match against stage_id 275 320 if (${ $j }{stage_id} ne $FPA_ID) { 276 321 next; … … 285 330 } 286 331 # Debug prints of all the components of this image 287 foreach my $k (keys %{ $j }) { 288 print "$i $j $k ${ $j }{$k}\n"; 289 } 290 332 # foreach my $k (keys %{ $j }) { 333 # if ($k eq 'row_index') { 334 # print "$i $j $k @{${ $j }{$k} }\n"; 335 # } 336 # print "$i $j $k ${ $j }{$k}\n"; 337 # } 338 339 # Check for existance of the images. Drawn mostly from pstampparse.pl 340 my $run_state = ${ $j }{state}; 341 my $data_state = ${ $j }{data_state}; 342 $data_state = $run_state if $stage eq 'stack'; 343 my $fault = 0; 344 if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or 345 ($run_state eq 'drop') or 346 ($run_state eq 'error_cleaned') or 347 ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { 348 # image is gone and it's not coming back 349 $fault = $PSTAMP_GONE; 350 } 351 elsif (($data_state ne 'full') or ($need_magic and (${ $j }{magicked} < 0))) { 352 if (($stage eq 'stack')||($stage eq 'diff')) { 353 # updating stacks and diffs isn't implemented 354 $fault = $PSTAMP_NOT_IMPLEMENTED; 355 } 356 if ($stage eq 'chip') { 357 my $burntool_state = ${ $j }{burntool_state}; 358 if ($burntool_state and (abs($burntool_state) < 14)) { 359 $fault = $PSTAMP_NOT_AVAILABLE; 360 } 361 } 362 363 if ($fault == 0) { 364 # This bombs us out to dqueryparse, which will then flag a job for this run to be updated. 365 my_die_for_update($data_state,$query{HEADER}{STAGE}[0], 366 ${ $j }{stage_id},${ $j }{class_id} || ${ $j }{skycell_id}, 367 $need_magic,$imagedb,$PSTAMP_NOT_AVAILABLE); 368 } 369 } 370 291 371 # This image matches, so we want to save the information into our output structure 292 372 $image_info{ROWNUM} = $index; … … 295 375 $image_info{MASK} = ${ $j }{mask}; 296 376 $image_info{WEIGHT} = ${ $j }{weight}; 377 $image_info{ERROR} = $fault; 297 378 $image_info{SKY_COORDINATES} = "$ra $dec"; 298 379 # To do sky->image coordinate transformations, we need to use the cmf/smf file. If … … 405 486 my $obscode = shift; 406 487 my $rownum_ref = shift; 488 my $out_err_ref = shift; 407 489 my $psphot_Npix_ref = shift; 408 490 my $psphot_Qfact_ref = shift; … … 413 495 my $columns = [ 414 496 # matching rownum from detectability original request 415 { name => 'ROWNUM', type => '20A', writetype => TSTRING }, 497 { name => 'ROWNUM', type => 'V', writetype => TULONG }, 498 # any errors that occurred during processing 499 { name => 'ERROR_CODE', type => 'V', writetype => TULONG }, 416 500 # number of pixels used in hypothetical PSF for the query detection 417 501 { name => 'DETECT_N', type => 'V', writetype => TULONG }, … … 455 539 for (my $i = 0; $i < $numRows; $i++) { 456 540 push @{$colData{'ROWNUM'}}, ${ $rownum_ref }[$i]; 541 push @{$colData{'ERROR_CODE'}}, ${ $out_err_ref }[$i]; 457 542 push @{$colData{'DETECT_N'}}, ${ $psphot_Npix_ref }[$i]; 458 543 push @{$colData{'DETECT_F'}}, ${ $psphot_Qfact_ref }[$i]; … … 529 614 530 615 } 616 617 sub my_die_for_update { 618 my $state = shift; 619 my $stage = shift; 620 my $stage_id = shift; 621 my $component = shift; 622 my $need_magic = shift; 623 my $imagedb = shift; 624 my $exit_code = shift; 625 626 print "$state $stage $stage_id $component $need_magic $imagedb\n"; 627 print STDERR "$state $stage $stage_id $component $need_magic $imagedb\n"; 628 exit($exit_code); 629 }
Note:
See TracChangeset
for help on using the changeset viewer.
