- Timestamp:
- Jul 30, 2010, 9:31:50 AM (16 years ago)
- Location:
- branches/eam_branches/ipp-20100621/pstamp
- Files:
-
- 2 edited
-
. (modified) (1 prop)
-
scripts/detectability_respond.pl (modified) (13 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20100621/pstamp
- Property svn:ignore
-
old new 16 16 ltmain.sh 17 17 libtool 18 ppstamp.pc
-
- Property svn:ignore
-
branches/eam_branches/ipp-20100621/pstamp/scripts/detectability_respond.pl
r28262 r28794 42 42 my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE'; 43 43 my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project); 44 my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps );44 my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps,$ignore_wisdom); 45 45 GetOptions( 46 46 'input=s' => \$request_file, … … 51 51 'verbose' => \$verbose, 52 52 'save-temps' => \$save_temps, 53 'ignore-wisdom' => \$ignore_wisdom, 53 54 ) or pod2usage(2); 54 55 … … 84 85 } 85 86 87 my %query = (); 88 my %image_list_hash = (); 89 my $wisdom_file = "${workdir}/wisdom.dat"; 90 if ((-e $wisdom_file)&&!($ignore_wisdom)) { 91 print "Reading wisdom file $wisdom_file instead of parsing...\n"; 92 open(WISDOM,"$wisdom_file") or my_die("failed to open wisdom file $wisdom_file"); 93 my $i = 0; 94 while(<WISDOM>) { 95 chomp; 96 my ($fpa_id,@key_values) = split /\s+/; 97 while ($#key_values > -1) { 98 my $key = shift(@key_values); 99 my $val = shift(@key_values); 100 $query{$fpa_id}{$key}[$i] = $val; 101 } 102 $i++; 103 } 104 close(WISDOM); 105 } # End reading wisdom. 106 else { 86 107 # 87 108 # Parse input request file using detect_query_read (as it's already written). 88 109 # 89 my $dqr_command = "$detect_query_read --input $request_file"; 90 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 91 run(command => $dqr_command, verbose => $verbose); 92 unless ($success) { 93 # This is a problem, because I'm not sure how we handle a failure to read something. 94 # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 95 # we can't send that response back. 96 die("Unable to perform $dqr_command error code: $error_code"); 97 } 98 my %query = (); 99 my $Nrows = 0; 100 { 101 my @column_names = (); 102 my $section = ''; 103 foreach my $entry (split /\n/, (join "", @$stdout_buf)) { 104 if ($entry =~ /^#/) { 105 @column_names = split /\s+/, $entry; 106 shift(@column_names); # Dump the hash sign. 107 if ($section eq 'HEADER') { 108 $section = 'CONTENT'; 110 my $dqr_command = "$detect_query_read --input $request_file"; 111 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 112 run(command => $dqr_command, verbose => $verbose); 113 unless ($success) { 114 # This is a problem, because I'm not sure how we handle a failure to read something. 115 # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 116 # we can't send that response back. 117 die("Unable to perform $dqr_command error code: $error_code"); 118 } 119 my $Nrows = 0; 120 { 121 my @column_names = (); 122 foreach my $entry (split /\n/, (join "", @$stdout_buf)) { 123 if ($entry =~ /^#/) { 124 @column_names = split /\s+/, $entry; 125 shift(@column_names); # Dump the hash sign. 109 126 } 110 127 else { 111 $section = 'HEADER'; 128 # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE 129 my %row_data; 130 @row_data{@column_names} = (split /\s+/, $entry); 131 for (my $i = 0; $i <= $#column_names; $i++) { 132 push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]}; 133 $Nrows = scalar(keys(%query)); 134 # print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n"; 112 135 } 113 } 136 } 137 } 138 } 139 # 140 # Identify target images. This should properly collate targets on a single imfile. 141 # 142 foreach my $fpa_id (keys %query) { 143 my %temp_hash; 144 my $query_style = 'byexp'; 145 my $stage; 146 my $filter; 147 my $mjd; 148 # Determine the query style for this fpa_id 149 if ($fpa_id =~ /o.*g.*o/) { 150 $query_style = 'byexp'; 151 } 152 elsif ($fpa_id =~ /\d+/) { 153 $query_style = 'byid'; 154 } 114 155 else { 115 # HEADER: 116 # QUERY_ID FPA_ID MJD_OBS FILTER OBSCODE STAGE 117 # CONTENT: 118 # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG 119 my @columns = split /\s+/, $entry; 120 for (my $i = 0; $i <= $#columns; $i++) { 121 $query{$section}{$column_names[$i]}[$Nrows] = $columns[$i]; 122 # print "$section $column_names[$i] $Nrows $columns[$i]\n"; 123 } 124 $Nrows++; 125 } 126 } 127 } 128 129 # 130 # Identify target images. This should properly collate targets on a single imfile. 131 # 132 my %image_list_hash; 133 for (my $i = 1; $i < $Nrows; $i++) { 134 # This could use the fact that locate_images now accepts position arrays, but 135 # I'll save that for after I get the majority of things working. 136 my $image_set_tmp = find_image_set($query{HEADER}{FPA_ID}[0],$query{HEADER}{STAGE}[0], 137 $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0], 138 $query{CONTENT}{RA1_DEG}[$i],$query{CONTENT}{DEC1_DEG}[$i], 139 $query{CONTENT}{ROWNUM}[$i],$verbose); 140 unless (%$image_set_tmp) { 141 # No images were returned, so create a dummy entry that 142 $image_list_hash{'no_image'}{IMAGE} = 'no_image'; 143 $image_list_hash{'no_image'}{PSF} = 'no_psf'; 144 $image_list_hash{'no_image'}{MASK} = 'no_mask'; 145 $image_list_hash{'no_image'}{WEIGHT} = 'no_weight'; 146 $image_list_hash{'no_image'}{CATALOG} = 'no_catalog'; 147 $image_list_hash{'no_image'}{CLASS_ID} = 'no_class'; 148 $image_list_hash{'no_image'}{ERROR} = $PSTAMP_NO_IMAGE_MATCH; 149 push @{ $image_list_hash{'no_image'}{SKY_COORDINATES} }, "$query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]"; 150 push @{ $image_list_hash{'no_image'}{ROWNUM} }, $query{CONTENT}{ROWNUM}[$i]; 151 next; 152 } 153 # print "=== $image_set_tmp->{IMAGE}\n $image_set_tmp->{PSF}\n"; 154 # print " $image_set_tmp->{MASK}\n $image_set_tmp->{WEIGHT}\n"; 155 # print " $image_set_tmp->{SKY_COORDINATES}\n $image_set_tmp->{ROWNUM}\n"; 156 157 # This indexes the results for identical images into the same hash. 158 $image_list_hash{$image_set_tmp->{IMAGE}}{IMAGE} = $image_set_tmp->{IMAGE}; 159 $image_list_hash{$image_set_tmp->{IMAGE}}{PSF} = $image_set_tmp->{PSF}; 160 $image_list_hash{$image_set_tmp->{IMAGE}}{MASK} = $image_set_tmp->{MASK}; 161 $image_list_hash{$image_set_tmp->{IMAGE}}{WEIGHT} = $image_set_tmp->{WEIGHT}; 162 $image_list_hash{$image_set_tmp->{IMAGE}}{CATALOG} = $image_set_tmp->{CATALOG}; 163 $image_list_hash{$image_set_tmp->{IMAGE}}{CLASS_ID} = $image_set_tmp->{CLASS_ID}; 164 $image_list_hash{$image_set_tmp->{IMAGE}}{ERROR} = $image_set_tmp->{ERROR}; 165 push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{SKY_COORDINATES} }, $image_set_tmp->{SKY_COORDINATES}; 166 push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{ROWNUM} }, $image_set_tmp->{ROWNUM}; 167 } 168 169 my $i = 0; 170 foreach my $k (keys %image_list_hash) { 171 # If we errored out on finding an image, we need to not try to run psphot here. 172 if ($image_list_hash{$k}{ERROR} != 0) { 173 next; 174 } 175 # Write coordinates of the requested targets to a file. 176 my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$i.XXXX", 177 UNLINK => !$save_temps); 178 my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$i.XXXX", 179 UNLINK => !$save_temps); 180 181 for (my $j = 0; $j <= $#{ $image_list_hash{$k}{SKY_COORDINATES} }; $j++) { 182 print $coordfile "$image_list_hash{$k}{SKY_COORDINATES}[$j]\n"; 183 } 184 # print "$k\n"; 185 # Convert the sky coordinates to image coordinates with ppCoord. 186 my $command = "ppCoord -astrom $image_list_hash{$k}{CATALOG} -radec $coordname"; 187 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 188 run(command => $command, verbose => $verbose); 189 unless ($success) { 190 my_die("Unable to perform $command. Error_code: $error_code", 191 $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0], 192 $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0], 193 $query{HEADER}{STAGE}[0],$error_code); 194 } 195 my @response = split /\n/, (join "", @$stdout_buf); 196 foreach my $line (@response) { 197 my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line; 198 print $targetfile "$r_x $r_y\n"; 199 $image_list_hash{$k}{EXTENSION_BASE} = $r_chip; 200 } 201 202 # print "psphot $image_list_hash{$k}{PSF}\n"; 203 # Run psphotForced on the target list. 204 my $tmpdir = tempdir("detect.$i.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps); 205 $image_list_hash{$k}{OUTROOT} = "$tmpdir/detectability.$query{HEADER}{STAGE}[0].$query{HEADER}{FPA_ID}[0]"; 206 207 my $psphot_cmd = "$psphotForced -psf $image_list_hash{$k}{PSF} "; 208 $psphot_cmd .= "-file $image_list_hash{$k}{IMAGE} "; 209 $psphot_cmd .= "-mask $image_list_hash{$k}{MASK} "; 210 $psphot_cmd .= "-variance $image_list_hash{$k}{WEIGHT} "; 211 $psphot_cmd .= "-srctext $targetname $image_list_hash{$k}{OUTROOT}"; 212 213 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 214 run(command => $psphot_cmd, verbose => $verbose); 215 unless ($success) { 216 $image_list_hash{$k}{ERROR} = $PSTAMP_SYSTEM_ERROR; 217 } 218 } 219 220 # 221 # Convert psphot output to response 222 # 223 my @rownums = (); 224 my @out_errors = (); 225 my @psphot_Npix = (); 226 my @psphot_Qfact= (); 227 my @psphot_flux = (); 228 my @psphot_error = (); 229 230 foreach my $k (keys %image_list_hash) { 231 if ($image_list_hash{$k}{ERROR} == 0) { 232 my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf"; 233 234 my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE}); 235 236 push @rownums, @{ $image_list_hash{$k}{ROWNUM} }; 237 push @out_errors, (map { $image_list_hash{$k}{ERROR} } @{ $image_list_hash{$k}{ROWNUM} }); 238 push @psphot_Npix, @{ $tmp_Npix }; 239 push @psphot_Qfact, @{ $tmp_Qfact }; 240 push @psphot_flux, @{ $tmp_flux }; 241 push @psphot_error, @{ $tmp_flux_error }; 242 } 243 else { 244 push @rownums, @{ $image_list_hash{$k}{ROWNUM} }; 245 push @out_errors, (map { $image_list_hash{$k}{ERROR} } @{ $image_list_hash{$k}{ROWNUM} }); 246 push @psphot_Npix, (map { 0 } @{ $image_list_hash{$k}{ROWNUM} }); 247 push @psphot_Qfact, (map { 0.0 } @{ $image_list_hash{$k}{ROWNUM} }); 248 push @psphot_flux, (map { 0.0 } @{ $image_list_hash{$k}{ROWNUM} }); 249 push @psphot_error, (map { 0.0 } @{ $image_list_hash{$k}{ROWNUM} }); 250 } 251 } 252 253 write_response_file($output, 254 $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0], 255 $query{HEADER}{MJD_OBS}[0],$query{HEADER}{filter}[0], 256 $query{HEADER}{obscode}[0], 257 \@rownums, \@out_errors, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux, \@psphot_error); 258 # print "Wrote response file $output\n"; 259 # 260 # Add to datastore 261 # 262 # Files are added to the datastore by dquery_finish.pl 263 # 264 # Cleanup 265 # 266 # Since everything is written to temporary files, there should be nothing to cleanup. 267 268 # 269 # Utilities 270 # 271 sub find_image_set { 272 my $FPA_ID = shift; 273 my $stage = lc(shift); 274 my $mjd = shift; 275 my $filter = shift; 276 my $ra = shift; 277 my $dec = shift; 278 my $index = shift; 279 my $verbose = shift; 280 281 # This is the set of things that we need in order to run psphotForced 282 my $option_mask |= 1; 283 $option_mask |= $PSTAMP_SELECT_IMAGE; 284 $option_mask |= $PSTAMP_SELECT_MASK; 285 $option_mask |= $PSTAMP_SELECT_VARIANCE; 286 $option_mask |= $PSTAMP_SELECT_PSF; 287 my $need_magic = 1; 288 my $mjd_min = $mjd; 289 my $mjd_max = $mjd + 1; 290 291 # Construct a row list. 292 my @rowList; 293 $rowList[0]->{CENTER_X} = $ra; 294 $rowList[0]->{CENTER_Y} = $dec; 295 $rowList[0]->{ID} = 1; 296 $rowList[0]->{STAGE} = $stage; 297 $rowList[0]->{COORD_MASK} = 0; 298 299 # print "$stage\n"; 300 # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter. 301 my @images = locate_images($ipprc,$imagedb, 302 \@rowList, 303 "bycoord",$stage, 304 undef,undef,undef, 305 $option_mask,$need_magic, 306 # $ra,$dec, 307 $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose); 308 309 my %image_info = (); 310 foreach my $i (@images) { # Scan over each result 311 foreach my $j (@{ $i }) { # Scan over each image in the result 312 # We only care about an image if it matches the FPA_ID in the request 313 if ($stage eq 'diff') { 314 # Diffs match if either exposure name is defined and matches the FPA_ID 315 unless ((defined(${ $j }{exp_name_1}) && (${ $j }{exp_name_1} eq $FPA_ID))|| 316 (defined(${ $j }{exp_name_2}) && (${ $j }{exp_name_2} eq $FPA_ID))|| 317 (${ $j }{exp_id_1} eq $FPA_ID)||(${ $j }{exp_id_2} eq $FPA_ID)) { 318 next; 156 exit_with_failure(21,"Parse error in request file"); 157 } 158 # Confirm that we only have one stage/filter/mjd 159 for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) { 160 $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1; 161 $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1; 162 $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1; 163 } 164 if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) { 165 $stage = (keys(%{ $temp_hash{STAGE} }))[0]; 166 } 167 else { 168 exit_with_failure(21,"Too many STAGEs specified"); 169 } 170 if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) { 171 $filter = (keys(%{ $temp_hash{FILTER} }))[0]; 172 } 173 else { 174 exit_with_failure(21,"Too many FILTERs specified"); 175 } 176 if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) { 177 $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0]; 178 } 179 else { 180 exit_with_failure(21,"Too many MJD-OBS specified"); 181 } 182 # Set common request components 183 my $option_mask |= 1; 184 $option_mask |= $PSTAMP_SELECT_IMAGE; 185 $option_mask |= $PSTAMP_SELECT_MASK; 186 $option_mask |= $PSTAMP_SELECT_VARIANCE; 187 $option_mask |= $PSTAMP_SELECT_PSF; 188 my $need_magic = 1; 189 if ($stage eq 'stack') { 190 $need_magic = 0; 191 } 192 my $mjd_min = $mjd; 193 my $mjd_max = $mjd + 1; 194 195 # Construct a row list. 196 my @rowList; 197 for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) { 198 $rowList[$i]->{CENTER_X} = $query{$fpa_id}{RA1_DEG}[$i]; 199 $rowList[$i]->{CENTER_Y} = $query{$fpa_id}{DEC1_DEG}[$i]; 200 $rowList[$i]->{ID} = $query{$fpa_id}{ROWNUM}[$i]; 201 $rowList[$i]->{COORD_MASK} = 0; 202 } 203 204 # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter. 205 my $pstamp_images_ref = locate_images($ipprc,$imagedb, 206 \@rowList, 207 $query_style,$stage, 208 $fpa_id,undef,undef, 209 $option_mask,$need_magic, 210 # $ra,$dec, 211 $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose); 212 213 foreach my $this_image_ref (@{ $pstamp_images_ref }) { 214 foreach my $key (sort (keys %{ $this_image_ref } )) { 215 my $value = ${ $this_image_ref }{$key}; 216 if ($key eq 'row_index') { 217 $value = join ' ', @{ $this_image_ref->{$key} }; 218 } 219 # print "$this_image_ref $key $value\n"; 220 foreach my $valid_index (@{ $this_image_ref->{row_index} }) { 221 $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image}; 222 $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask}; 223 $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight}; 224 $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf}; 225 $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id}; 226 $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb}; 227 $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic; 228 229 if (exists($this_image_ref->{astrom})) { 230 $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom}; 231 } 232 else { 233 $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf}; 234 } 235 if (exists($this_image_ref->{class_id})) { 236 $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id}; 237 $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id}; 238 239 } 240 else { 241 $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id}; 242 $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa'; 243 } 244 245 $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state}; 246 if (exists($this_image_ref->{data_state})) { 247 $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state}; 248 } 249 else { 250 $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state}; 251 } 252 $query{$fpa_id}{FAULT}[$valid_index] = 0; 253 $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked}; 254 if ($stage eq 'chip') { 255 $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state}; 256 } 257 258 # Determine if the data exists. 259 if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or 260 ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or 261 ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or 262 ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or 263 ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or 264 ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) { 265 266 # image is gone and it's not coming back 267 $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE; 268 } 269 elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] = 0)) { 270 $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED; 271 } 272 elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) { 273 $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE; 274 275 # updating stacks isn't implemented 276 if (($stage eq 'stack')) { 277 $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; 278 } 279 # updating old burntool data isn't implemented 280 elsif ($stage eq 'chip') { 281 if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and 282 (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) { 283 $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; 284 } 285 } 286 } # End determining error faults. 319 287 } 320 288 } 321 elsif ($stage eq 'stack') { 322 # Stacks hide the exposure name very well, so we can only match against stage_id 323 if (${ $j }{stage_id} ne $FPA_ID) { 324 next; 325 } 289 } 290 } 291 } # End calculating wisdom 292 my %update_request; 293 my %processing_request; 294 295 open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file"); 296 foreach my $fpa_id (keys %query) { 297 for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) { 298 print WISDOM "$fpa_id\t"; 299 foreach my $key (keys %{ $query{$fpa_id} }) { 300 print WISDOM "$key $query{$fpa_id}{$key}[$i]\t"; 301 } 302 print WISDOM "\n"; 303 @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = 304 ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i], 305 $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]); 306 push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i; 307 } 308 } 309 close(WISDOM); 310 my $exit_code = 0; 311 my $update_request_file = "${workdir}/update_request.dat"; 312 open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file"); 313 foreach my $images (keys %update_request) { 314 foreach my $fault (keys %{ $update_request{$images} }) { 315 if ($fault == 25) { 316 $exit_code = 25; 317 } 318 my $update_request = join ' ', @{ $update_request{$images}{$fault} }; 319 print UPDATE_REQUEST "$update_request\n"; 320 } 321 } 322 close(UPDATE_REQUEST); 323 # if ($exit_code != 0) { 324 # exit($exit_code); 325 # } 326 327 # This duplicates stuff returned by PSTAMP, but my thought is to convert that ---^ into a conditional, in which I read 328 # from the wisdom.dat file. This means only one pass on all that potentially slow stuff. If that's the case, then 329 # I can recalculate the processing request. 330 331 foreach my $fpa_id (keys %processing_request) { 332 foreach my $image (keys %{ $processing_request{$fpa_id} }) { 333 print "$fpa_id $image\t"; 334 foreach my $i (@{ $processing_request{$fpa_id}{$image} }) { 335 print "$i "; 336 } 337 print "\n"; 338 } 339 } 340 #exit(10); 341 342 # run ppCoord and psphotForced to calculate the required data. 343 foreach my $fpa_id (keys %processing_request) { 344 foreach my $image (keys %{ $processing_request{$fpa_id} }) { 345 # Get this image specific data from the first entry. That entry is now the king of this set. 346 my $index = $processing_request{$fpa_id}{$image}[0]; 347 my $fault = $query{$fpa_id}{FAULT}[$index]; 348 my $catalog = $query{$fpa_id}{CATALOG}[$index]; 349 my $psf = $query{$fpa_id}{PSF}[$index]; 350 my $mask = $query{$fpa_id}{MASK}[$index]; 351 my $weight= $query{$fpa_id}{WEIGHT}[$index]; 352 my $stage = $query{$fpa_id}{STAGE}[$index]; 353 # if there's a fault, then we can't process this image. 354 if ($fault != 0) { 355 next; 356 } 357 358 # Create coordinate file to convert to positions. 359 my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$index.XXXX", 360 UNLINK => !$save_temps); 361 my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$index.XXXX", 362 UNLINK => !$save_temps); 363 foreach my $i (@{ $processing_request{$fpa_id}{$image} }) { 364 print $coordfile "$query{$fpa_id}{RA1_DEG}[$i] $query{$fpa_id}{DEC1_DEG}[$i]\n"; 365 } 366 367 # Convert the sky coordinates to image coordinates with ppCoord. 368 my $command = "ppCoord -astrom $catalog -radec $coordname"; 369 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 370 run(command => $command, verbose => $verbose); 371 unless ($success) { 372 my_die("Unable to perform $command. Error_code: $error_code", 373 $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index], 374 $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index], 375 $error_code); 376 } 377 my @response = split /\n/, (join "", @$stdout_buf); 378 my $i = 0; 379 foreach my $line (@response) { 380 my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line; 381 print $targetfile "$r_x $r_y\n"; 382 if (($r_ra == $query{$fpa_id}{RA1_DEG}[$processing_request{$fpa_id}{$image}[$i]])&& 383 ($r_dec == $query{$fpa_id}{DEC1_DEG}[$processing_request{$fpa_id}{$image}[$i]])) { 384 $query{$fpa_id}{X_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_x; 385 $query{$fpa_id}{Y_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_y; 386 $query{$fpa_id}{EXTENSION_BASE}[$processing_request{$fpa_id}{$image}[$i]] = $r_chip; 326 387 } 327 388 else { 328 # For all the other stages (warp and chip are the ones I've tested), we can simply329 # directly match the exposure name to the FPA_ID330 unless ((${ $j }{exp_name} eq $FPA_ID)||(${ $j }{exp_id} eq $FPA_ID)) {331 next;332 }389 $error_code = $PS_EXIT_PROG_ERROR; 390 my_die("Unable to match input RA/DEC with output RA/DEC: ($query{$fpa_id}{RA1_DEG}[$i],$query{$fpa_id}{DEC1_DEG}[$i]) -> ($r_ra,$r_dec) i. $error_code", 391 $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index], 392 $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index], 393 $error_code); 333 394 } 334 # Debug prints of all the components of this image 335 # foreach my $k (keys %{ $j }) { 336 # if ($k eq 'row_index') { 337 # print "$i $j $k @{${ $j }{$k} }\n"; 338 # } 339 # print "$i $j $k ${ $j }{$k}\n"; 340 # } 341 342 # Check for existance of the images. Drawn mostly from pstampparse.pl 343 my $run_state = ${ $j }{state}; 344 my $data_state = ${ $j }{data_state}; 345 $data_state = $run_state if $stage eq 'stack'; 346 my $fault = 0; 347 if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or 348 ($run_state eq 'drop') or 349 ($run_state eq 'error_cleaned') or 350 ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { 351 # image is gone and it's not coming back 352 $fault = $PSTAMP_GONE; 395 $i++; 396 } 397 398 # Run psphotForced on the target list. 399 my $tmpdir = tempdir("detect.$index.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps); 400 my $outroot = "$tmpdir/detectability.${stage}.${fpa_id}.${index}"; 401 $query{$fpa_id}{PROC_ERROR}[$index] = 0; 402 my $psphot_cmd = "$psphotForced -psf $psf "; 403 $psphot_cmd .= "-file $image "; 404 $psphot_cmd .= "-mask $mask "; 405 $psphot_cmd .= "-variance $weight "; 406 $psphot_cmd .= "-srctext $targetname $outroot "; 407 $psphot_cmd .= "-D OUTPUT.FORMAT PS1_DV1 "; 408 409 ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 410 run(command => $psphot_cmd, verbose => $verbose); 411 unless ($success) { 412 $query{$fpa_id}{PROC_ERROR}[$index] = $PSTAMP_SYSTEM_ERROR; 413 } 414 415 # Why not parse out results here? 416 # Convert psphot output to response 417 if ($query{$fpa_id}{PROC_ERROR}[$index] == 0) { 418 my $class_id = $query{$fpa_id}{CLASS_ID}[$index]; 419 my $cmf = "${outroot}.${class_id}.cmf"; 420 my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$query{$fpa_id}{EXTENSION_BASE}[$index]); 421 foreach ($i = 0; $i <= $#{ $processing_request{$fpa_id}{$image} }; $i++) { 422 my $result_index = $processing_request{$fpa_id}{$image}[$i]; 423 $query{$fpa_id}{PROC_ERROR}[$result_index] = $query{$fpa_id}{PROC_ERROR}[$index]; 424 425 $query{$fpa_id}{NPIX}[$result_index] = ${ $tmp_Npix }[$i]; 426 $query{$fpa_id}{QFACTOR}[$result_index] = ${ $tmp_Qfact }[$i]; 427 $query{$fpa_id}{FLUX}[$result_index] = ${ $tmp_flux }[$i]; 428 $query{$fpa_id}{FLUX_SIG}[$result_index] = ${ $tmp_flux_error }[$i]; 429 430 # print "$fpa_id $image $#{ $processing_request{$fpa_id}{$image} } $result_index $i ${ $tmp_Npix }[$i]\n"; 353 431 } 354 elsif (($data_state ne 'full') or ($need_magic and (${ $j }{magicked} < 0))) { 355 if (($stage eq 'stack')||($stage eq 'diff')) { 356 # updating stacks and diffs isn't implemented 357 $fault = $PSTAMP_NOT_IMPLEMENTED; 358 } 359 if ($stage eq 'chip') { 360 my $burntool_state = ${ $j }{burntool_state}; 361 if ($burntool_state and (abs($burntool_state) < 14)) { 362 $fault = $PSTAMP_NOT_AVAILABLE; 363 } 364 } 365 366 if ($fault == 0) { 367 # This bombs us out to dqueryparse, which will then flag a job for this run to be updated. 368 my_die_for_update($data_state,$query{HEADER}{STAGE}[0], 369 ${ $j }{stage_id},${ $j }{class_id} || ${ $j }{skycell_id}, 370 $need_magic,$imagedb,$PSTAMP_NOT_AVAILABLE); 371 } 432 } 433 else { 434 foreach ($i = 0; $i <= $#{ $processing_request{$fpa_id}{$image} }; $i++) { 435 my $result_index = $processing_request{$fpa_id}{$image}[$i]; 436 $query{$fpa_id}{PROC_ERROR}[$result_index] = $query{$fpa_id}{PROC_ERROR}[$index]; 437 438 $query{$fpa_id}{NPIX}[$result_index] = 0; 439 $query{$fpa_id}{QFACTOR}[$result_index] = 0.0; 440 $query{$fpa_id}{FLUX}[$result_index] = 0.0; 441 $query{$fpa_id}{FLUX_SIG}[$result_index] = 0.0; 372 442 } 373 374 # This image matches, so we want to save the information into our output structure 375 $image_info{ROWNUM} = $index; 376 $image_info{IMAGE} = ${ $j }{image}; 377 $image_info{PSF} = ${ $j }{psf}; 378 $image_info{MASK} = ${ $j }{mask}; 379 $image_info{WEIGHT} = ${ $j }{weight}; 380 $image_info{ERROR} = $fault; 381 $image_info{SKY_COORDINATES} = "$ra $dec"; 382 # To do sky->image coordinate transformations, we need to use the cmf/smf file. If 383 # an astrom reference (the camera stage smf file) exists, then use that, as we're dealing with 384 # with the chip stage. Otherwise, use the stage-dependent cmf (and set the class_id to fpa). 385 # The EXTENSION_BASE stores the basename of the extension that will be generated by psphotForced. 386 if (exists(${ $j }{astrom})) { 387 $image_info{CATALOG} = ${ $j }{astrom}; 388 $image_info{CLASS_ID} = ${ $j }{class_id}; 389 390 } 391 else { 392 $image_info{CATALOG} = ${ $j }{cmf}; 393 $image_info{CLASS_ID} = 'fpa'; 394 395 } 396 397 } 398 } 399 return(\%image_info); 400 } 443 } 444 445 446 } 447 } 448 449 write_response_file($output,\%query); 450 451 # 452 # Utilities 453 # 401 454 402 455 # Taken largely from detect_query_read … … 435 488 ]; 436 489 } 437 elsif ($CMFversion eq 'PS1_DV 2') {490 elsif ($CMFversion eq 'PS1_DV1') { 438 491 $column_defs = [ 439 492 { name => 'PSF_QF', type => '1E', writetype => TDOUBLE }, … … 457 510 458 511 my $correct_error = 0; 512 # print STDERR "Ncols:" . $#{ $column_defs } . "\n"; 459 513 foreach my $col (@$column_defs) { 460 514 my ($col_num,$col_type,$col_data); … … 466 520 $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status); 467 521 $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status) and check_fitsio($status); 468 522 # print STDERR "$col\t>>" . $col->{name} . "<<\t>>" . @{ $col_data } . "<<\n"; 469 523 if ($col->{name} eq 'PSF_QF') { 470 524 @tmp_Qfact = @{ $col_data }; … … 477 531 } 478 532 elsif ($col->{name} eq 'PSF_INST_MAG_SIG') { 479 @tmp_flux_err = map { $_ = $_ / (2.5 * log10(exp(1))) } @{ $col_data };533 @tmp_flux_err = map { $_ = $_ / (2.5 * (log(exp(1)) / log(10))) } @{ $col_data }; 480 534 $correct_error = 1; 481 535 } … … 493 547 } 494 548 $inFits->close_file( $status ) and check_fitsio($status); 549 # print STDERR "$CMFversion\n"; 550 # print STDERR "Q: $#tmp_Qfact\t@tmp_Qfact\nN: $#tmp_Npix\t@tmp_Npix\n"; 495 551 return(\@tmp_Npix, \@tmp_Qfact, \@tmp_flux, \@tmp_flux_err); 496 552 } … … 499 555 sub write_response_file { 500 556 my $outfile = shift; 501 my $query_id = shift; 502 my $FPA_ID = shift; 503 my $MJD_OBS = shift; 504 my $filter = shift; 505 my $obscode = shift; 506 my $rownum_ref = shift; 507 my $out_err_ref = shift; 508 my $psphot_Npix_ref = shift; 509 my $psphot_Qfact_ref = shift; 510 my $psphot_flux_ref = shift; 511 my $psphot_error_ref = shift; 512 my $status = 0; 513 514 # Specification of columns to write 515 my $columns = [ 516 # matching rownum from detectability original request 517 { name => 'ROWNUM', type => 'V', writetype => TULONG }, 518 # any errors that occurred during processing 519 { name => 'ERROR_CODE', type => 'V', writetype => TULONG }, 520 # number of pixels used in hypothetical PSF for the query detection 521 { name => 'DETECT_N', type => 'V', writetype => TULONG }, 522 # detectibility, indicating the fraction of PSF pixels detetable by IPP 523 { name => 'DETECT_F', type => 'D', writetype => TDOUBLE }, 524 # flux of the target source 525 { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE }, 526 # error in the flux of the target source 527 { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE }, 528 ]; 529 530 # Header translation table 531 my $headers = { 532 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, 533 comment => 'MOPS Query ID for this batch query' }, 534 'FPA_ID' => { name => 'FPA_ID', type => TSTRING, 535 comment => 'original FPA_ID used at ingest' }, 536 }; 557 my $query_ref = shift; 558 559 my %query = %{ $query_ref }; 560 561 my $columns; 562 my $headers; 563 564 my $EXTVER_IS_1 = (scalar(keys(%query)) == 1); 565 # print "EXTVER: $EXTVER_IS_1\n"; 566 my ($query_id,$FPA_ID,$MJD_OBS,$filter,$obscode,$status); 567 if ($EXTVER_IS_1 == 1) { 568 # Specification of columns to write 569 $columns = [ 570 # matching rownum from detectability original request 571 { name => 'ROWNUM', type => 'V', writetype => TULONG }, 572 # any errors that occurred during processing 573 { name => 'ERROR_CODE', type => 'V', writetype => TULONG }, 574 # number of pixels used in hypothetical PSF for the query detection 575 { name => 'DETECT_N', type => 'V', writetype => TULONG }, 576 # detectibility, indicating the fraction of PSF pixels detetable by IPP 577 { name => 'DETECT_F', type => 'D', writetype => TDOUBLE }, 578 # flux of the target source 579 { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE }, 580 # error in the flux of the target source 581 { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE }, 582 ]; 583 584 # Header translation table 585 $headers = { 586 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, 587 comment => 'MOPS Query ID for this batch query' }, 588 'FPA_ID' => { name => 'FPA_ID', type => TSTRING, 589 comment => 'original FPA_ID used at ingest' }, 590 # 'MJD-OBS' => { name => 'FPA_ID', type => TSTRING, 591 # comment => 'original FPA_ID used at ingest' }, 592 # 'FILTER' => { name => 'FPA_ID', type => TSTRING, 593 # comment => 'original FPA_ID used at ingest' }, 594 # 'OBSCODE' => { name => 'FPA_ID', type => TSTRING, 595 # comment => 'original FPA_ID used at ingest' }, 596 }; 597 } 598 else { 599 # Specification of columns to write 600 $columns = [ 601 # matching rownum from detectability original request 602 { name => 'ROWNUM', type => 'V', writetype => TULONG }, 603 # any errors that occurred during processing 604 { name => 'ERROR_CODE', type => 'V', writetype => TULONG }, 605 # number of pixels used in hypothetical PSF for the query detection 606 { name => 'DETECT_N', type => 'V', writetype => TULONG }, 607 # detectibility, indicating the fraction of PSF pixels detetable by IPP 608 { name => 'DETECT_F', type => 'D', writetype => TDOUBLE }, 609 # flux of the target source 610 { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE }, 611 # error in the flux of the target source 612 { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE }, 613 # The FPA That would be in the header if it were to be there. 614 { name => 'FPA_ID', type => '20A', writetype => TSTRING }, 615 ]; 616 617 # Header translation table 618 $headers = { 619 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, 620 comment => 'MOPS Query ID for this batch query' }, 621 }; 622 } 537 623 538 624 # Parse the list of columns … … 548 634 } 549 635 550 my $numRows = $#{ $rownum_ref } + 1;551 636 my $inHeader = { }; 552 637 553 638 # Hack to force the data to match the detect_response_create formats 639 554 640 $inHeader->{QUERY_ID}->{value} = $query_id; 555 $inHeader->{FPA_ID}->{value} = $FPA_ID;556 $inHeader->{MJD_OBS}->{value} = $MJD_OBS;557 $inHeader->{FILTER}->{value} = $filter;558 $inHeader->{OBSCODE}->{value} = $obscode;641 if ($EXTVER_IS_1 == 1) { 642 my $fpa_id = (keys(%query))[0]; 643 $inHeader->{FPA_ID}->{value} = $fpa_id; 644 } 559 645 560 646 # Fill the table columns with the data, making sure the flux is defined 561 for (my $i = 0; $i < $numRows; $i++) { 562 push @{$colData{'ROWNUM'}}, ${ $rownum_ref }[$i]; 563 push @{$colData{'ERROR_CODE'}}, ${ $out_err_ref }[$i]; 564 push @{$colData{'DETECT_N'}}, ${ $psphot_Npix_ref }[$i]; 565 push @{$colData{'DETECT_F'}}, ${ $psphot_Qfact_ref }[$i]; 566 push @{$colData{'TARGET_FLUX'}}, ${ $psphot_flux_ref }[$i]; 567 push @{$colData{'TARGET_FLUX_SIG'}}, ${ $psphot_error_ref }[$i]; 568 unless (defined(${ $colData{'TARGET_FLUX'}}[-1])) { 569 $colData{'TARGET_FLUX'}[-1] = 0.0; 570 } 571 unless (defined(${ $colData{'TARGET_FLUX_SIG'}}[-1])) { 572 $colData{'TARGET_FLUX_SIG'}[-1] = 0.0; 573 } 574 } 575 647 foreach my $fpa_id (keys %query) { 648 $inHeader->{QUERY_ID}->{value} = $query{$fpa_id}{QUERY_ID}[0]; 649 if ($EXTVER_IS_1 == 1) { 650 my $fpa_id = (keys(%query))[0]; 651 $inHeader->{FPA_ID}->{value} = $fpa_id; 652 } 653 654 push @{$colData{'ROWNUM'}}, @{ $query{$fpa_id}{ROWNUM} }; 655 push @{$colData{'ERROR_CODE'}}, @{ $query{$fpa_id}{PROC_ERROR} }; 656 push @{$colData{'DETECT_N'}}, @{ $query{$fpa_id}{NPIX} }; 657 push @{$colData{'DETECT_F'}}, @{ $query{$fpa_id}{QFACTOR} }; 658 push @{$colData{'TARGET_FLUX'}}, @{ $query{$fpa_id}{FLUX} }; 659 push @{$colData{'TARGET_FLUX_SIG'}}, @{ $query{$fpa_id}{FLUX_SIG} }; 660 if ($EXTVER_IS_1 != 1) { 661 push @{$colData{'FPA_ID'}}, (map {$fpa_id} @{ $query{$fpa_id}{ROWNUM} }); 662 } 663 } 664 my $numRows = 0; 576 665 # Back to detect_response_create: 666 $status = 0; 667 # print "$output\n"; 577 668 my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status ); 578 669 check_fitsio( $status ); … … 580 671 check_fitsio( $status ); 581 672 my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE'; 673 582 674 $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, \@colNames, \@colTypes, undef, $EXTNAME, $status); 583 675 check_fitsio( $status ); … … 601 693 check_fitsio( $status ); 602 694 } 695 603 696 for (my $i = 0; $i < scalar @colNames; $i++) { 604 697 my $colName = $colNames[$i];# Column name 605 698 my $writeType = $colWriteType[$i]; 699 my $numRows = scalar(@{$colData{$colName}}); 700 unless(defined($writeType)) { 701 print "write type undefined for $colName\n"; 702 } 703 unless(defined($numRows)) { 704 print "num Rows undefined for $colName\n"; 705 } 706 unless(defined($status)) { 707 print "status undefined for $colName\n"; 708 } 709 unless(defined($colData{$colName})) { 710 print "col data undefined for $colName\n"; 711 } 712 unless(defined($colName)) { 713 print "column name undefined for $i\n"; 714 } 715 # print STDERR "$writeType $i $numRows $colName $status @{ $colData{$colName} }\n"; 606 716 $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData{$colName}, $status ); 607 717 check_fitsio( $status ); 718 608 719 } 609 720 $outFits->close_file( $status ); … … 641 752 } 642 753 643 sub my_die_for_update { 644 my $state = shift; 645 my $stage = shift; 646 my $stage_id = shift; 647 my $component = shift; 648 my $need_magic = shift; 649 my $imagedb = shift; 650 my $exit_code = shift; 651 652 print "$state $stage $stage_id $component $need_magic $imagedb\n"; 653 print STDERR "$state $stage $stage_id $component $need_magic $imagedb\n"; 654 exit($exit_code); 655 } 754 sub exit_with_failure { 755 my $status = shift; 756 my $message = shift; 757 carp("$message"); 758 exit($status); 759 } 760
Note:
See TracChangeset
for help on using the changeset viewer.
