Index: trunk/pstamp/scripts/detect_query_read =================================================================== --- trunk/pstamp/scripts/detect_query_read (revision 28776) +++ trunk/pstamp/scripts/detect_query_read (revision 28777) @@ -26,5 +26,5 @@ my $no_print_header = 0; # omit the header keywords my $no_print_rows = 0; # omit the rows - +my $version = 0; my ( $input, # Name of input text file @@ -52,57 +52,107 @@ my $status = 0; -# The required keywords -my $header = { - 'QUERY_ID' => { name => 'QUERY_ID', - writetype => TSTRING, - comment => 'MOPS Query ID for this batch query', - value => undef - }, - 'FPA_ID' => { name => 'FPA_ID', - writetype => TSTRING, - comment => 'orginal FPA_ID used at ingest', - value => undef - }, - 'MJD_OBS' => { name => 'MJD-OBS', - writetype => TDOUBLE, - comment => 'starting time of the exposure, MJD', - value => undef - }, - 'FILTER' => { name => 'FILTER', - writetype => TSTRING, - comment => 'effective filter use for the exposure', - value => undef - }, - 'OBSCODE' => { name => 'OBSCODE', - writetype => TSTRING, - comment => 'site identifier (MPC observatory code)', - value => undef - }, - 'STAGE' => { - name => 'STAGE', - writetype => TSTRING, - comment => 'processing stage to examine', - value => undef - } -}; - -# key_array insures that the order that the keywords is printed out is -# the same as the ICD -my @key_array = qw( QUERY_ID FPA_ID MJD_OBS FILTER OBSCODE STAGE); +# Read the input file + +my $inFits = Astro::FITS::CFITSIO::open_file( $input, READONLY, $status ); # FITS file handle +check_fitsio($status); + +$inFits->movnam_hdu(BINARY_TBL, EXTNAME, 0, $status) and check_fitsio($status); + +my $inHeader = $inFits->read_header(); # Header for input + +my $numRows; # Number of rows in table +$inFits->get_num_rows($numRows, $status) and check_fitsio($status); + +# The keywords found in the header +# my $header = { +# 'EXTVER' => { name => 'EXTVER', +# writetype => TSTRING, +# comment => 'Extension version', +# value => undef +# }, +# 'EXTNAME' => { name => 'EXTNAME', +# writetype => TSTRING, +# comment => 'name of this binary table extension', +# value => undef +# }, +# 'QUERY_ID' => { name => 'QUERY_ID', +# writetype => TSTRING, +# comment => 'MOPS Query ID for this batch query', +# value => undef +# }, +# 'FPA_ID' => { name => 'FPA_ID', +# writetype => TSTRING, +# comment => 'orginal FPA_ID used at ingest', +# value => undef +# }, +# 'MJD_OBS' => { name => 'MJD-OBS', +# writetype => TDOUBLE, +# comment => 'starting time of the exposure, MJD', +# value => undef +# }, +# 'FILTER' => { name => 'FILTER', +# writetype => TSTRING, +# comment => 'effective filter use for the exposure', +# value => undef +# }, +# 'OBSCODE' => { name => 'OBSCODE', +# writetype => TSTRING, +# comment => 'site identifier (MPC observatory code)', +# value => undef +# }, +# 'STAGE' => { +# name => 'STAGE', +# writetype => TSTRING, +# comment => 'processing stage to examine', +# value => undef +# } +# }; + +my $parse_error = 0; +# Parse the header to determine what we expect to find. +foreach my $header_key (keys %{ $inHeader }) { + $inHeader->{$header_key} =~ s/\s+//g; + $inHeader->{$header_key} =~ s/\'//g; +} +my ($EXTVER,$headerEXTNAME,$QUERY_ID,$FPA_ID,$MJD_OBS,$FILTER,$OBSCODE,$STAGE) = + ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID},$inHeader->{FPA_ID}, + $inHeader->{'MJD-OBS'},$inHeader->{FILTER},$inHeader->{OBSCODE},$inHeader->{STAGE}); +unless(defined($EXTVER) && defined($headerEXTNAME) && + (($EXTVER == 1)||($EXTVER == 2)) && + ($headerEXTNAME eq EXTNAME)) { + $parse_error = 1; +} +unless(($EXTVER == 2)|| + ((defined($QUERY_ID))&&(defined($FPA_ID))&&(defined($MJD_OBS))&& + (defined($FILTER))&&(defined($OBSCODE)))) { + $parse_error = 2; +} # Specification of columns my $column_defs = [ # matching rownum from detectability original request - { name => 'ROWNUM', type => '20A', writetype => TSTRING }, + { name => 'ROWNUM', type => '20A', writetype => TSTRING, version => 1 }, # coordinate at start of exposure, in degrees - { name => 'RA1_DEG', type => 'D', writetype => TDOUBLE }, + { name => 'RA1_DEG', type => 'D', writetype => TDOUBLE, version => 1 }, # coordinate at start of exposure, in degrees - { name => 'DEC1_DEG', type => 'D', writetype => TDOUBLE }, + { name => 'DEC1_DEG', type => 'D', writetype => TDOUBLE, version => 1 }, # coordinate at end of exposure, in degrees - { name => 'RA2_DEG', type => 'D', writetype => TDOUBLE }, + { name => 'RA2_DEG', type => 'D', writetype => TDOUBLE, version => 1 }, # coordinate at end of exposure, in degrees - { name => 'DEC2_DEG', type => 'D', writetype => TDOUBLE }, + { name => 'DEC2_DEG', type => 'D', writetype => TDOUBLE, version => 1 }, # apparent magnitude - { name => 'MAG', type => 'D', writetype => TDOUBLE }, + { name => 'MAG', type => 'D', writetype => TDOUBLE, version => 1 }, + # v2 query_id: MOPS query ID for this batch query + { name => 'QUERY_ID', type => '20A', writetype => TSTRING, version => 2, default => $QUERY_ID }, + # v2 fpa_id: original FPA_ID used at ingest + { name => 'FPA_ID', type => '20A', writetype => TSTRING, version => 2, default => $FPA_ID }, + # v2 mjd obs: starting time of the exposure, MJD + { name => 'MJD-OBS', type => 'D', writetype => TDOUBLE, version => 2, default => $MJD_OBS }, + # v2 filter: effective filter used for the exposure as a string, allowed values of g, r, i, z, y, B, V, w + { name => 'FILTER', type => '3A', writetype => TSTRING, version => 2, default => $FILTER }, + # v2 obscode: site identifier (MPC observatory code) + { name => 'OBSCODE', type => '3A', writetype => TSTRING, version => 2, default => $OBSCODE }, + # v2 stage: stage name to perform query on, allowed values of 'chip', 'warp', 'stack', and 'diff' + { name => 'STAGE', type => '20A', writetype => TSTRING, version => 2, default => $STAGE }, ]; @@ -118,19 +168,11 @@ } -# Read the input file - -my $inFits = Astro::FITS::CFITSIO::open_file( $input, READONLY, $status ); # FITS file handle -check_fitsio($status); - -$inFits->movnam_hdu(BINARY_TBL, EXTNAME, 0, $status) and check_fitsio($status); - -my $inHeader = $inFits->read_header(); # Header for input - -my $numRows; # Number of rows in table -$inFits->get_num_rows($numRows, $status) and check_fitsio($status); - foreach my $col (@$column_defs) { my ($col_num, $col_type, $col_data); + if ($col->{version} > $EXTVER) { + @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows); + next; + } $inFits->get_colnum(0, $col->{name}, $col_num, $status) and check_fitsio($status); $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status); @@ -138,50 +180,42 @@ and check_fitsio($status); $colData{$col->{name}} = $col_data; -} - -# Now produce the output - -if (!$no_print_header) { - my $label; - my $data; - # I don't do this because I want the keys to be printed in a particular order - #foreach my $key (keys %$header) { - foreach my $key (@key_array) { - my $name = $header->{$key}->{name}; - my $value = $inHeader->{$name}; - if (($key eq 'STAGE')&& !(defined($value))) { - $value = 'DIFF'; - } - # get rid of quotes and whitespace - $value =~ s/\'//g; - if (defined $value) { - #print "$key\t\t\t$value\n"; - $label .= sprintf "%-12s ", $key; - $data .= sprintf "%-12s ", $value; - } else { - die "keyword $key not found in $input\n"; - } - } - print "# " . $label . "\n" unless $no_print_label; - print $data . "\n"; -} - -if (!$no_print_rows) { - if (!$no_print_label) { - print "# "; - foreach my $col (@$column_defs) { - printf "%-12s ", $col->{name}; - } - print "\n"; - } - - for (my $i = 0; $i < $numRows; $i++) { - foreach my $col (@$column_defs) { - printf "%-12s ", $colData{$col->{name}}->[$i]; + if ($col->{name} eq 'MJD-OBS') { + print @{ $col_data } . "\n"; + } +} + +# Verify uniqueness of important columns: +my @unique_fields = ('QUERY_ID','OBSCODE','STAGE'); +foreach my $colName (@unique_fields) { + my %counter = (); + foreach my $row (@{ $colData{$colName} }) { + $counter{$row} = 1; + } + if (scalar(keys(%counter)) != 1) { + $parse_error = 3; + } +} +if ($parse_error) { + die "Unable to parse detectability query: $parse_error " . &error_message($parse_error) . "\n";; +} + +# # Now produce the output + + +if (!$no_print_label) { + print "# "; + foreach my $col (@$column_defs) { + printf "%-12s ", $col->{name}; + } + print "\n"; +} + +for (my $i = 0; $i < $numRows; $i++) { + foreach my $col (@$column_defs) { + printf "%-12s ", $colData{$col->{name}}->[$i]; #foreach my $aref (@col_arrays) { - #printf "%-12s ", $aref->[$i]; - } - print "\n"; - } + #printf "%-12s ", $aref->[$i]; + } + print "\n"; } @@ -202,3 +236,21 @@ } +sub error_message { + my $error = shift; + if ($error == 1) { + return("Unknown EXTVER/EXTNAME"); + } + if ($error == 2) { + return("Required header field not found"); + } + if ($error == 3) { + return("Unique column not uniquely specified"); + } + else { + return("Unknown fault."); + } +} + + + __END__ Index: trunk/pstamp/scripts/detectability_respond.pl =================================================================== --- trunk/pstamp/scripts/detectability_respond.pl (revision 28776) +++ trunk/pstamp/scripts/detectability_respond.pl (revision 28777) @@ -42,5 +42,5 @@ my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE'; my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project); -my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps); +my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps,$ignore_wisdom); GetOptions( 'input=s' => \$request_file, @@ -51,4 +51,5 @@ 'verbose' => \$verbose, 'save-temps' => \$save_temps, + 'ignore-wisdom' => \$ignore_wisdom, ) or pod2usage(2); @@ -84,320 +85,371 @@ } +my %query = (); +my %image_list_hash = (); +my $wisdom_file = "${workdir}/wisdom.dat"; +if ((-e $wisdom_file)&&!($ignore_wisdom)) { + print "Reading wisdom file $wisdom_file instead of parsing...\n"; + open(WISDOM,"$wisdom_file") or my_die("failed to open wisdom file $wisdom_file"); + my $i = 0; + while() { + chomp; + my ($fpa_id,@key_values) = split /\s+/; + while ($#key_values > -1) { + my $key = shift(@key_values); + my $val = shift(@key_values); + $query{$fpa_id}{$key}[$i] = $val; + } + $i++; + } + close(WISDOM); +} # End reading wisdom. +else { # # Parse input request file using detect_query_read (as it's already written). # -my $dqr_command = "$detect_query_read --input $request_file"; -my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $dqr_command, verbose => $verbose); -unless ($success) { - # This is a problem, because I'm not sure how we handle a failure to read something. - # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, - # we can't send that response back. - die("Unable to perform $dqr_command error code: $error_code"); -} -my %query = (); -my $Nrows = 0; -{ - my @column_names = (); - my $section = ''; - foreach my $entry (split /\n/, (join "", @$stdout_buf)) { - if ($entry =~ /^#/) { - @column_names = split /\s+/, $entry; - shift(@column_names); # Dump the hash sign. - if ($section eq 'HEADER') { - $section = 'CONTENT'; + my $dqr_command = "$detect_query_read --input $request_file"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $dqr_command, verbose => $verbose); + unless ($success) { + # This is a problem, because I'm not sure how we handle a failure to read something. + # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, + # we can't send that response back. + die("Unable to perform $dqr_command error code: $error_code"); + } + my $Nrows = 0; + { + my @column_names = (); + foreach my $entry (split /\n/, (join "", @$stdout_buf)) { + if ($entry =~ /^#/) { + @column_names = split /\s+/, $entry; + shift(@column_names); # Dump the hash sign. } else { - $section = 'HEADER'; + # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE + my %row_data; + @row_data{@column_names} = (split /\s+/, $entry); + for (my $i = 0; $i <= $#column_names; $i++) { + push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]}; + $Nrows = scalar(keys(%query)); +# print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n"; } - } + } + } + } +# +# Identify target images. This should properly collate targets on a single imfile. +# + foreach my $fpa_id (keys %query) { + my %temp_hash; + my $query_style = 'byexp'; + my $stage; + my $filter; + my $mjd; + # Determine the query style for this fpa_id + if ($fpa_id =~ /o.*g.*o/) { + $query_style = 'byexp'; + } + elsif ($fpa_id =~ /\d+/) { + $query_style = 'byid'; + } else { - # HEADER: - # QUERY_ID FPA_ID MJD_OBS FILTER OBSCODE STAGE - # CONTENT: - # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG - my @columns = split /\s+/, $entry; - for (my $i = 0; $i <= $#columns; $i++) { - $query{$section}{$column_names[$i]}[$Nrows] = $columns[$i]; -# print "$section $column_names[$i] $Nrows $columns[$i]\n"; - } - $Nrows++; - } - } -} - -# -# Identify target images. This should properly collate targets on a single imfile. -# -my %image_list_hash; -for (my $i = 1; $i < $Nrows; $i++) { - # This could use the fact that locate_images now accepts position arrays, but - # I'll save that for after I get the majority of things working. - my $image_set_tmp = find_image_set($query{HEADER}{FPA_ID}[0],$query{HEADER}{STAGE}[0], - $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0], - $query{CONTENT}{RA1_DEG}[$i],$query{CONTENT}{DEC1_DEG}[$i], - $query{CONTENT}{ROWNUM}[$i],$verbose); - unless (%$image_set_tmp) { - # No images were returned, so create a dummy entry that - $image_list_hash{'no_image'}{IMAGE} = 'no_image'; - $image_list_hash{'no_image'}{PSF} = 'no_psf'; - $image_list_hash{'no_image'}{MASK} = 'no_mask'; - $image_list_hash{'no_image'}{WEIGHT} = 'no_weight'; - $image_list_hash{'no_image'}{CATALOG} = 'no_catalog'; - $image_list_hash{'no_image'}{CLASS_ID} = 'no_class'; - $image_list_hash{'no_image'}{ERROR} = $PSTAMP_NO_IMAGE_MATCH; - push @{ $image_list_hash{'no_image'}{SKY_COORDINATES} }, "$query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]"; - push @{ $image_list_hash{'no_image'}{ROWNUM} }, $query{CONTENT}{ROWNUM}[$i]; - next; - } -# print "=== $image_set_tmp->{IMAGE}\n $image_set_tmp->{PSF}\n"; -# print " $image_set_tmp->{MASK}\n $image_set_tmp->{WEIGHT}\n"; -# print " $image_set_tmp->{SKY_COORDINATES}\n $image_set_tmp->{ROWNUM}\n"; - - # This indexes the results for identical images into the same hash. - $image_list_hash{$image_set_tmp->{IMAGE}}{IMAGE} = $image_set_tmp->{IMAGE}; - $image_list_hash{$image_set_tmp->{IMAGE}}{PSF} = $image_set_tmp->{PSF}; - $image_list_hash{$image_set_tmp->{IMAGE}}{MASK} = $image_set_tmp->{MASK}; - $image_list_hash{$image_set_tmp->{IMAGE}}{WEIGHT} = $image_set_tmp->{WEIGHT}; - $image_list_hash{$image_set_tmp->{IMAGE}}{CATALOG} = $image_set_tmp->{CATALOG}; - $image_list_hash{$image_set_tmp->{IMAGE}}{CLASS_ID} = $image_set_tmp->{CLASS_ID}; - $image_list_hash{$image_set_tmp->{IMAGE}}{ERROR} = $image_set_tmp->{ERROR}; - push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{SKY_COORDINATES} }, $image_set_tmp->{SKY_COORDINATES}; - push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{ROWNUM} }, $image_set_tmp->{ROWNUM}; -} - -my $i = 0; -foreach my $k (keys %image_list_hash) { - # If we errored out on finding an image, we need to not try to run psphot here. - if ($image_list_hash{$k}{ERROR} != 0) { - next; - } - # Write coordinates of the requested targets to a file. - my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$i.XXXX", - UNLINK => !$save_temps); - my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$i.XXXX", - UNLINK => !$save_temps); - - for (my $j = 0; $j <= $#{ $image_list_hash{$k}{SKY_COORDINATES} }; $j++) { - print $coordfile "$image_list_hash{$k}{SKY_COORDINATES}[$j]\n"; - } -# print "$k\n"; - # Convert the sky coordinates to image coordinates with ppCoord. - my $command = "ppCoord -astrom $image_list_hash{$k}{CATALOG} -radec $coordname"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("Unable to perform $command. Error_code: $error_code", - $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0], - $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0], - $query{HEADER}{STAGE}[0],$error_code); - } - my @response = split /\n/, (join "", @$stdout_buf); - foreach my $line (@response) { - my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line; - print $targetfile "$r_x $r_y\n"; - $image_list_hash{$k}{EXTENSION_BASE} = $r_chip; - } - -# print "psphot $image_list_hash{$k}{PSF}\n"; - # Run psphotForced on the target list. - my $tmpdir = tempdir("detect.$i.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps); - $image_list_hash{$k}{OUTROOT} = "$tmpdir/detectability.$query{HEADER}{STAGE}[0].$query{HEADER}{FPA_ID}[0]"; - - my $psphot_cmd = "$psphotForced -psf $image_list_hash{$k}{PSF} "; - $psphot_cmd .= "-file $image_list_hash{$k}{IMAGE} "; - $psphot_cmd .= "-mask $image_list_hash{$k}{MASK} "; - $psphot_cmd .= "-variance $image_list_hash{$k}{WEIGHT} "; - $psphot_cmd .= "-srctext $targetname $image_list_hash{$k}{OUTROOT} "; - $psphot_cmd .= "-D OUTPUT.FORMAT PS1_DV1 "; - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $psphot_cmd, verbose => $verbose); - unless ($success) { - $image_list_hash{$k}{ERROR} = $PSTAMP_SYSTEM_ERROR; - } -} - -# -# Convert psphot output to response -# -my @rownums = (); -my @out_errors = (); -my @psphot_Npix = (); -my @psphot_Qfact= (); -my @psphot_flux = (); -my @psphot_error = (); - -foreach my $k (keys %image_list_hash) { - if ($image_list_hash{$k}{ERROR} == 0) { - my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf"; - - my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE}); - - push @rownums, @{ $image_list_hash{$k}{ROWNUM} }; - push @out_errors, (map { $image_list_hash{$k}{ERROR} } @{ $image_list_hash{$k}{ROWNUM} }); - push @psphot_Npix, @{ $tmp_Npix }; - push @psphot_Qfact, @{ $tmp_Qfact }; - push @psphot_flux, @{ $tmp_flux }; - push @psphot_error, @{ $tmp_flux_error }; - } - else { - push @rownums, @{ $image_list_hash{$k}{ROWNUM} }; - push @out_errors, (map { $image_list_hash{$k}{ERROR} } @{ $image_list_hash{$k}{ROWNUM} }); - push @psphot_Npix, (map { 0 } @{ $image_list_hash{$k}{ROWNUM} }); - push @psphot_Qfact, (map { 0.0 } @{ $image_list_hash{$k}{ROWNUM} }); - push @psphot_flux, (map { 0.0 } @{ $image_list_hash{$k}{ROWNUM} }); - push @psphot_error, (map { 0.0 } @{ $image_list_hash{$k}{ROWNUM} }); - } -} - -write_response_file($output, - $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0], - $query{HEADER}{MJD_OBS}[0],$query{HEADER}{filter}[0], - $query{HEADER}{obscode}[0], - \@rownums, \@out_errors, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux, \@psphot_error); -# print "Wrote response file $output\n"; -# -# Add to datastore -# -# Files are added to the datastore by dquery_finish.pl -# -# Cleanup -# -# Since everything is written to temporary files, there should be nothing to cleanup. - -# -# Utilities -# -sub find_image_set { - my $FPA_ID = shift; - my $stage = lc(shift); - my $mjd = shift; - my $filter = shift; - my $ra = shift; - my $dec = shift; - my $index = shift; - my $verbose = shift; - - # This is the set of things that we need in order to run psphotForced - my $option_mask |= 1; - $option_mask |= $PSTAMP_SELECT_IMAGE; - $option_mask |= $PSTAMP_SELECT_MASK; - $option_mask |= $PSTAMP_SELECT_VARIANCE; - $option_mask |= $PSTAMP_SELECT_PSF; - my $need_magic = 1; - my $mjd_min = $mjd; - my $mjd_max = $mjd + 1; - - # Construct a row list. - my @rowList; - $rowList[0]->{CENTER_X} = $ra; - $rowList[0]->{CENTER_Y} = $dec; - $rowList[0]->{ID} = 1; - $rowList[0]->{STAGE} = $stage; - $rowList[0]->{COORD_MASK} = 0; - - # print "$stage\n"; - # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter. - my @images = locate_images($ipprc,$imagedb, - \@rowList, - "bycoord",$stage, - undef,undef,undef, - $option_mask,$need_magic, - # $ra,$dec, - $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose); - - my %image_info = (); - foreach my $i (@images) { # Scan over each result - foreach my $j (@{ $i }) { # Scan over each image in the result - # We only care about an image if it matches the FPA_ID in the request - if ($stage eq 'diff') { - # Diffs match if either exposure name is defined and matches the FPA_ID - unless ((defined(${ $j }{exp_name_1}) && (${ $j }{exp_name_1} eq $FPA_ID))|| - (defined(${ $j }{exp_name_2}) && (${ $j }{exp_name_2} eq $FPA_ID))|| - (${ $j }{exp_id_1} eq $FPA_ID)||(${ $j }{exp_id_2} eq $FPA_ID)) { - next; + exit_with_failure(21,"Parse error in request file"); + } + # Confirm that we only have one stage/filter/mjd + for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) { + $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1; + $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1; + $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1; + } + if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) { + $stage = (keys(%{ $temp_hash{STAGE} }))[0]; + } + else { + exit_with_failure(21,"Too many STAGEs specified"); + } + if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) { + $filter = (keys(%{ $temp_hash{FILTER} }))[0]; + } + else { + exit_with_failure(21,"Too many FILTERs specified"); + } + if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) { + $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0]; + } + else { + exit_with_failure(21,"Too many MJD-OBS specified"); + } + # Set common request components + my $option_mask |= 1; + $option_mask |= $PSTAMP_SELECT_IMAGE; + $option_mask |= $PSTAMP_SELECT_MASK; + $option_mask |= $PSTAMP_SELECT_VARIANCE; + $option_mask |= $PSTAMP_SELECT_PSF; + my $need_magic = 1; + if ($stage eq 'stack') { + $need_magic = 0; + } + my $mjd_min = $mjd; + my $mjd_max = $mjd + 1; + + # Construct a row list. + my @rowList; + for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) { + $rowList[$i]->{CENTER_X} = $query{$fpa_id}{RA1_DEG}[$i]; + $rowList[$i]->{CENTER_Y} = $query{$fpa_id}{DEC1_DEG}[$i]; + $rowList[$i]->{ID} = $query{$fpa_id}{ROWNUM}[$i]; + $rowList[$i]->{COORD_MASK} = 0; + } + + # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter. + my $pstamp_images_ref = locate_images($ipprc,$imagedb, + \@rowList, + $query_style,$stage, + $fpa_id,undef,undef, + $option_mask,$need_magic, + # $ra,$dec, + $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose); + + foreach my $this_image_ref (@{ $pstamp_images_ref }) { + foreach my $key (sort (keys %{ $this_image_ref } )) { + my $value = ${ $this_image_ref }{$key}; + if ($key eq 'row_index') { + $value = join ' ', @{ $this_image_ref->{$key} }; + } +# print "$this_image_ref $key $value\n"; + foreach my $valid_index (@{ $this_image_ref->{row_index} }) { + $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image}; + $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask}; + $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight}; + $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf}; + $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id}; + $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb}; + $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic; + + if (exists($this_image_ref->{astrom})) { + $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom}; + } + else { + $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf}; + } + if (exists($this_image_ref->{class_id})) { + $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id}; + $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id}; + + } + else { + $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id}; + $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa'; + } + + $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state}; + if (exists($this_image_ref->{data_state})) { + $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state}; + } + else { + $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state}; + } + $query{$fpa_id}{FAULT}[$valid_index] = 0; + $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked}; + if ($stage eq 'chip') { + $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state}; + } + + # Determine if the data exists. + if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or + ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or + ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or + ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) { + + # image is gone and it's not coming back + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE; + } + elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] = 0)) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED; + } + elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE; + + # updating stacks isn't implemented + if (($stage eq 'stack')) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; + } + # updating old burntool data isn't implemented + elsif ($stage eq 'chip') { + if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and + (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) { + $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED; + } + } + } # End determining error faults. } } - elsif ($stage eq 'stack') { - # Stacks hide the exposure name very well, so we can only match against stage_id - if (${ $j }{stage_id} ne $FPA_ID) { - next; - } + } + } +} # End calculating wisdom +my %update_request; +my %processing_request; + +open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file"); +foreach my $fpa_id (keys %query) { + for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) { + print WISDOM "$fpa_id\t"; + foreach my $key (keys %{ $query{$fpa_id} }) { + print WISDOM "$key $query{$fpa_id}{$key}[$i]\t"; + } + print WISDOM "\n"; + @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = + ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i], + $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]); + push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i; + } +} +close(WISDOM); +my $exit_code = 0; +my $update_request_file = "${workdir}/update_request.dat"; +open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file"); +foreach my $images (keys %update_request) { + foreach my $fault (keys %{ $update_request{$images} }) { + if ($fault == 25) { + $exit_code = 25; + } + my $update_request = join ' ', @{ $update_request{$images}{$fault} }; + print UPDATE_REQUEST "$update_request\n"; + } +} +close(UPDATE_REQUEST); +# if ($exit_code != 0) { +# exit($exit_code); +# } + +# This duplicates stuff returned by PSTAMP, but my thought is to convert that ---^ into a conditional, in which I read +# from the wisdom.dat file. This means only one pass on all that potentially slow stuff. If that's the case, then +# I can recalculate the processing request. + +foreach my $fpa_id (keys %processing_request) { + foreach my $image (keys %{ $processing_request{$fpa_id} }) { + print "$fpa_id $image\t"; + foreach my $i (@{ $processing_request{$fpa_id}{$image} }) { + print "$i "; + } + print "\n"; + } +} +#exit(10); + +# run ppCoord and psphotForced to calculate the required data. +foreach my $fpa_id (keys %processing_request) { + foreach my $image (keys %{ $processing_request{$fpa_id} }) { + # Get this image specific data from the first entry. That entry is now the king of this set. + my $index = $processing_request{$fpa_id}{$image}[0]; + my $fault = $query{$fpa_id}{FAULT}[$index]; + my $catalog = $query{$fpa_id}{CATALOG}[$index]; + my $psf = $query{$fpa_id}{PSF}[$index]; + my $mask = $query{$fpa_id}{MASK}[$index]; + my $weight= $query{$fpa_id}{WEIGHT}[$index]; + my $stage = $query{$fpa_id}{STAGE}[$index]; + # if there's a fault, then we can't process this image. + if ($fault != 0) { + next; + } + + # Create coordinate file to convert to positions. + my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$index.XXXX", + UNLINK => !$save_temps); + my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$index.XXXX", + UNLINK => !$save_temps); + foreach my $i (@{ $processing_request{$fpa_id}{$image} }) { + print $coordfile "$query{$fpa_id}{RA1_DEG}[$i] $query{$fpa_id}{DEC1_DEG}[$i]\n"; + } + + # Convert the sky coordinates to image coordinates with ppCoord. + my $command = "ppCoord -astrom $catalog -radec $coordname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("Unable to perform $command. Error_code: $error_code", + $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index], + $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index], + $error_code); + } + my @response = split /\n/, (join "", @$stdout_buf); + my $i = 0; + foreach my $line (@response) { + my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line; + print $targetfile "$r_x $r_y\n"; + if (($r_ra == $query{$fpa_id}{RA1_DEG}[$processing_request{$fpa_id}{$image}[$i]])&& + ($r_dec == $query{$fpa_id}{DEC1_DEG}[$processing_request{$fpa_id}{$image}[$i]])) { + $query{$fpa_id}{X_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_x; + $query{$fpa_id}{Y_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_y; + $query{$fpa_id}{EXTENSION_BASE}[$processing_request{$fpa_id}{$image}[$i]] = $r_chip; } else { - # For all the other stages (warp and chip are the ones I've tested), we can simply - # directly match the exposure name to the FPA_ID - unless ((${ $j }{exp_name} eq $FPA_ID)||(${ $j }{exp_id} eq $FPA_ID)) { - next; - } + $error_code = $PS_EXIT_PROG_ERROR; + my_die("Unable to match input RA/DEC with output RA/DEC: ($query{$fpa_id}{RA1_DEG}[$i],$query{$fpa_id}{DEC1_DEG}[$i]) -> ($r_ra,$r_dec) i. $error_code", + $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index], + $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index], + $error_code); } - # Debug prints of all the components of this image -# foreach my $k (keys %{ $j }) { -# if ($k eq 'row_index') { -# print "$i $j $k @{${ $j }{$k} }\n"; -# } -# print "$i $j $k ${ $j }{$k}\n"; -# } - - # Check for existance of the images. Drawn mostly from pstampparse.pl - my $run_state = ${ $j }{state}; - my $data_state = ${ $j }{data_state}; - $data_state = $run_state if $stage eq 'stack'; - my $fault = 0; - if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or - ($run_state eq 'drop') or - ($run_state eq 'error_cleaned') or - ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { - # image is gone and it's not coming back - $fault = $PSTAMP_GONE; + $i++; + } + + # Run psphotForced on the target list. + my $tmpdir = tempdir("detect.$index.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps); + my $outroot = "$tmpdir/detectability.${stage}.${fpa_id}.${index}"; + $query{$fpa_id}{PROC_ERROR}[$index] = 0; + my $psphot_cmd = "$psphotForced -psf $psf "; + $psphot_cmd .= "-file $image "; + $psphot_cmd .= "-mask $mask "; + $psphot_cmd .= "-variance $weight "; + $psphot_cmd .= "-srctext $targetname $outroot "; + $psphot_cmd .= "-D OUTPUT.FORMAT PS1_DV1 "; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $psphot_cmd, verbose => $verbose); + unless ($success) { + $query{$fpa_id}{PROC_ERROR}[$index] = $PSTAMP_SYSTEM_ERROR; + } + + # Why not parse out results here? +# Convert psphot output to response + if ($query{$fpa_id}{PROC_ERROR}[$index] == 0) { + my $class_id = $query{$fpa_id}{CLASS_ID}[$index]; + my $cmf = "${outroot}.${class_id}.cmf"; + my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$query{$fpa_id}{EXTENSION_BASE}[$index]); + foreach ($i = 0; $i <= $#{ $processing_request{$fpa_id}{$image} }; $i++) { + my $result_index = $processing_request{$fpa_id}{$image}[$i]; + $query{$fpa_id}{PROC_ERROR}[$result_index] = $query{$fpa_id}{PROC_ERROR}[$index]; + + $query{$fpa_id}{NPIX}[$result_index] = ${ $tmp_Npix }[$i]; + $query{$fpa_id}{QFACTOR}[$result_index] = ${ $tmp_Qfact }[$i]; + $query{$fpa_id}{FLUX}[$result_index] = ${ $tmp_flux }[$i]; + $query{$fpa_id}{FLUX_SIG}[$result_index] = ${ $tmp_flux_error }[$i]; + +# print "$fpa_id $image $#{ $processing_request{$fpa_id}{$image} } $result_index $i ${ $tmp_Npix }[$i]\n"; } - elsif (($data_state ne 'full') or ($need_magic and (${ $j }{magicked} < 0))) { - if (($stage eq 'stack')||($stage eq 'diff')) { - # updating stacks and diffs isn't implemented - $fault = $PSTAMP_NOT_IMPLEMENTED; - } - if ($stage eq 'chip') { - my $burntool_state = ${ $j }{burntool_state}; - if ($burntool_state and (abs($burntool_state) < 14)) { - $fault = $PSTAMP_NOT_AVAILABLE; - } - } - - if ($fault == 0) { - # This bombs us out to dqueryparse, which will then flag a job for this run to be updated. - my_die_for_update($data_state,$query{HEADER}{STAGE}[0], - ${ $j }{stage_id},${ $j }{class_id} || ${ $j }{skycell_id}, - $need_magic,$imagedb,$PSTAMP_NOT_AVAILABLE); - } + } + else { + foreach ($i = 0; $i <= $#{ $processing_request{$fpa_id}{$image} }; $i++) { + my $result_index = $processing_request{$fpa_id}{$image}[$i]; + $query{$fpa_id}{PROC_ERROR}[$result_index] = $query{$fpa_id}{PROC_ERROR}[$index]; + + $query{$fpa_id}{NPIX}[$result_index] = 0; + $query{$fpa_id}{QFACTOR}[$result_index] = 0.0; + $query{$fpa_id}{FLUX}[$result_index] = 0.0; + $query{$fpa_id}{FLUX_SIG}[$result_index] = 0.0; } - - # This image matches, so we want to save the information into our output structure - $image_info{ROWNUM} = $index; - $image_info{IMAGE} = ${ $j }{image}; - $image_info{PSF} = ${ $j }{psf}; - $image_info{MASK} = ${ $j }{mask}; - $image_info{WEIGHT} = ${ $j }{weight}; - $image_info{ERROR} = $fault; - $image_info{SKY_COORDINATES} = "$ra $dec"; - # To do sky->image coordinate transformations, we need to use the cmf/smf file. If - # an astrom reference (the camera stage smf file) exists, then use that, as we're dealing with - # with the chip stage. Otherwise, use the stage-dependent cmf (and set the class_id to fpa). - # The EXTENSION_BASE stores the basename of the extension that will be generated by psphotForced. - if (exists(${ $j }{astrom})) { - $image_info{CATALOG} = ${ $j }{astrom}; - $image_info{CLASS_ID} = ${ $j }{class_id}; - - } - else { - $image_info{CATALOG} = ${ $j }{cmf}; - $image_info{CLASS_ID} = 'fpa'; - - } - - } - } - return(\%image_info); -} + } + + + } +} + +write_response_file($output,\%query); + +# +# Utilities +# # Taken largely from detect_query_read @@ -503,40 +555,70 @@ sub write_response_file { my $outfile = shift; - my $query_id = shift; - my $FPA_ID = shift; - my $MJD_OBS = shift; - my $filter = shift; - my $obscode = shift; - my $rownum_ref = shift; - my $out_err_ref = shift; - my $psphot_Npix_ref = shift; - my $psphot_Qfact_ref = shift; - my $psphot_flux_ref = shift; - my $psphot_error_ref = shift; - my $status = 0; - - # Specification of columns to write - my $columns = [ - # matching rownum from detectability original request - { name => 'ROWNUM', type => 'V', writetype => TULONG }, - # any errors that occurred during processing - { name => 'ERROR_CODE', type => 'V', writetype => TULONG }, - # number of pixels used in hypothetical PSF for the query detection - { name => 'DETECT_N', type => 'V', writetype => TULONG }, - # detectibility, indicating the fraction of PSF pixels detetable by IPP - { name => 'DETECT_F', type => 'D', writetype => TDOUBLE }, - # flux of the target source - { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE }, - # error in the flux of the target source - { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE }, - ]; - - # Header translation table - my $headers = { - 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, - comment => 'MOPS Query ID for this batch query' }, - 'FPA_ID' => { name => 'FPA_ID', type => TSTRING, - comment => 'original FPA_ID used at ingest' }, - }; + my $query_ref = shift; + + my %query = %{ $query_ref }; + + my $columns; + my $headers; + + my $EXTVER_IS_1 = (scalar(keys(%query)) == 1); +# print "EXTVER: $EXTVER_IS_1\n"; + my ($query_id,$FPA_ID,$MJD_OBS,$filter,$obscode,$status); + if ($EXTVER_IS_1 == 1) { + # Specification of columns to write + $columns = [ + # matching rownum from detectability original request + { name => 'ROWNUM', type => 'V', writetype => TULONG }, + # any errors that occurred during processing + { name => 'ERROR_CODE', type => 'V', writetype => TULONG }, + # number of pixels used in hypothetical PSF for the query detection + { name => 'DETECT_N', type => 'V', writetype => TULONG }, + # detectibility, indicating the fraction of PSF pixels detetable by IPP + { name => 'DETECT_F', type => 'D', writetype => TDOUBLE }, + # flux of the target source + { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE }, + # error in the flux of the target source + { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE }, + ]; + + # Header translation table + $headers = { + 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, + comment => 'MOPS Query ID for this batch query' }, + 'FPA_ID' => { name => 'FPA_ID', type => TSTRING, + comment => 'original FPA_ID used at ingest' }, +# 'MJD-OBS' => { name => 'FPA_ID', type => TSTRING, +# comment => 'original FPA_ID used at ingest' }, +# 'FILTER' => { name => 'FPA_ID', type => TSTRING, +# comment => 'original FPA_ID used at ingest' }, +# 'OBSCODE' => { name => 'FPA_ID', type => TSTRING, +# comment => 'original FPA_ID used at ingest' }, + }; + } + else { + # Specification of columns to write + $columns = [ + # matching rownum from detectability original request + { name => 'ROWNUM', type => 'V', writetype => TULONG }, + # any errors that occurred during processing + { name => 'ERROR_CODE', type => 'V', writetype => TULONG }, + # number of pixels used in hypothetical PSF for the query detection + { name => 'DETECT_N', type => 'V', writetype => TULONG }, + # detectibility, indicating the fraction of PSF pixels detetable by IPP + { name => 'DETECT_F', type => 'D', writetype => TDOUBLE }, + # flux of the target source + { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE }, + # error in the flux of the target source + { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE }, + # The FPA That would be in the header if it were to be there. + { name => 'FPA_ID', type => '20A', writetype => TSTRING }, + ]; + + # Header translation table + $headers = { + 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, + comment => 'MOPS Query ID for this batch query' }, + }; + } # Parse the list of columns @@ -552,31 +634,36 @@ } - my $numRows = $#{ $rownum_ref } + 1; my $inHeader = { }; # Hack to force the data to match the detect_response_create formats + $inHeader->{QUERY_ID}->{value} = $query_id; - $inHeader->{FPA_ID}->{value} = $FPA_ID; - $inHeader->{MJD_OBS}->{value} = $MJD_OBS; - $inHeader->{FILTER}->{value} = $filter; - $inHeader->{OBSCODE}->{value} = $obscode; + if ($EXTVER_IS_1 == 1) { + my $fpa_id = (keys(%query))[0]; + $inHeader->{FPA_ID}->{value} = $fpa_id; + } # Fill the table columns with the data, making sure the flux is defined - for (my $i = 0; $i < $numRows; $i++) { - push @{$colData{'ROWNUM'}}, ${ $rownum_ref }[$i]; - push @{$colData{'ERROR_CODE'}}, ${ $out_err_ref }[$i]; - push @{$colData{'DETECT_N'}}, ${ $psphot_Npix_ref }[$i]; - push @{$colData{'DETECT_F'}}, ${ $psphot_Qfact_ref }[$i]; - push @{$colData{'TARGET_FLUX'}}, ${ $psphot_flux_ref }[$i]; - push @{$colData{'TARGET_FLUX_SIG'}}, ${ $psphot_error_ref }[$i]; - unless (defined(${ $colData{'TARGET_FLUX'}}[-1])) { - $colData{'TARGET_FLUX'}[-1] = 0.0; - } - unless (defined(${ $colData{'TARGET_FLUX_SIG'}}[-1])) { - $colData{'TARGET_FLUX_SIG'}[-1] = 0.0; - } - } - + foreach my $fpa_id (keys %query) { + $inHeader->{QUERY_ID}->{value} = $query{$fpa_id}{QUERY_ID}[0]; + if ($EXTVER_IS_1 == 1) { + my $fpa_id = (keys(%query))[0]; + $inHeader->{FPA_ID}->{value} = $fpa_id; + } + + push @{$colData{'ROWNUM'}}, @{ $query{$fpa_id}{ROWNUM} }; + push @{$colData{'ERROR_CODE'}}, @{ $query{$fpa_id}{PROC_ERROR} }; + push @{$colData{'DETECT_N'}}, @{ $query{$fpa_id}{NPIX} }; + push @{$colData{'DETECT_F'}}, @{ $query{$fpa_id}{QFACTOR} }; + push @{$colData{'TARGET_FLUX'}}, @{ $query{$fpa_id}{FLUX} }; + push @{$colData{'TARGET_FLUX_SIG'}}, @{ $query{$fpa_id}{FLUX_SIG} }; + if ($EXTVER_IS_1 != 1) { + push @{$colData{'FPA_ID'}}, (map {$fpa_id} @{ $query{$fpa_id}{ROWNUM} }); + } + } + my $numRows = 0; # Back to detect_response_create: + $status = 0; +# print "$output\n"; my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status ); check_fitsio( $status ); @@ -584,4 +671,5 @@ check_fitsio( $status ); my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE'; + $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, \@colNames, \@colTypes, undef, $EXTNAME, $status); check_fitsio( $status ); @@ -605,7 +693,9 @@ check_fitsio( $status ); } + for (my $i = 0; $i < scalar @colNames; $i++) { my $colName = $colNames[$i];# Column name my $writeType = $colWriteType[$i]; + my $numRows = scalar(@{$colData{$colName}}); unless(defined($writeType)) { print "write type undefined for $colName\n"; @@ -662,15 +752,9 @@ } -sub my_die_for_update { - my $state = shift; - my $stage = shift; - my $stage_id = shift; - my $component = shift; - my $need_magic = shift; - my $imagedb = shift; - my $exit_code = shift; - - print "$state $stage $stage_id $component $need_magic $imagedb\n"; - print STDERR "$state $stage $stage_id $component $need_magic $imagedb\n"; - exit($exit_code); -} +sub exit_with_failure { + my $status = shift; + my $message = shift; + carp("$message"); + exit($status); +} + Index: trunk/pstamp/scripts/dqueryparse.pl =================================================================== --- trunk/pstamp/scripts/dqueryparse.pl (revision 28776) +++ trunk/pstamp/scripts/dqueryparse.pl (revision 28777) @@ -102,5 +102,5 @@ if $extname ne "MOPS_DETECTABILITY_QUERY"; my_die("$req_file is version $extver expecting 1", $PS_EXIT_PROG_ERROR) - if $extver ne 1; + if ($extver ne 1) and ($extver ne 2); # Set up the workdir for this query. @@ -129,6 +129,9 @@ $fault = $error_code >> 8; if ($fault == $PSTAMP_NOT_AVAILABLE) { - $data_to_update = (split /\n/, (join "", @$stdout_buf))[-1]; + unless (-e "$outdir/update_request.dat") { + my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR); + } } + } @@ -158,27 +161,33 @@ # Failed to run correctly, which means that we need to queue a job and flag data for updating. # Get the dependency id for the data we're requesting be updated. - my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update); - - # Link this request to a job and link that job to any dependency - my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; - $command .= " -job_type detect_query -state run -fault 0"; - $command .= " -rownum 1"; - $command .= " -dep_id $dep_id" if $dep_id; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - if ($success) { - $job_id = join "", @$stdout_buf; - chomp $job_id; - if ($job_id && -e $response_file) { - # We shouldn't have a response file at this stage. - rename $response_file, "$outdir/response${job_id}.fits"; - } - $result = 0; - } else { - warn("Unable to perform $command error code: $error_code"); - $result = $error_code >> 8; - } + open(UPDATE_REQUEST,"$outdir/update_request.dat") || my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR); + while () { + my $data_to_update = $_; + chomp($data_to_update); + my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update); + + # Link this request to a job and link that job to any dependency + my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; + $command .= " -job_type detect_query -state run -fault 0"; + $command .= " -rownum 1"; + $command .= " -dep_id $dep_id" if $dep_id; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + if ($success) { + $job_id = join "", @$stdout_buf; + chomp $job_id; + if ($job_id && -e $response_file) { + # We shouldn't have a response file at this stage. + rename $response_file, "$outdir/response${job_id}.fits"; + } + $result = 0; + } else { + warn("Unable to perform $command error code: $error_code"); + $result = $error_code >> 8; + } + } + close(UPDATE_REQUEST); }