Index: /trunk/pstamp/scripts/detect_query_create =================================================================== --- /trunk/pstamp/scripts/detect_query_create (revision 28805) +++ /trunk/pstamp/scripts/detect_query_create (revision 28806) @@ -17,4 +17,5 @@ $query_id, $nostage, + $version, ); @@ -22,5 +23,6 @@ 'input|i=s' => \$input, 'output|o=s' => \$output, - 'query_id|q=s' => \$query_id, + 'query_id|q=s' => \$query_id, + 'version|v=s' => \$version, 'nostage' => \$nostage, ) or pod2usage( 2 ); @@ -30,5 +32,20 @@ -exitval => 3) unless defined $input and defined $output; -# The header kewords +# Read what we've been given +my $in; +if ($input eq '-') { + $in = \*STDIN; +} else { + open $in, "<$input" or die "cannot open $input for reading"; +} +my %colData; +my %headerData; +my $numRows = read_data_for_table($in,'\s+', \%colData, \%headerData); +if (!$numRows) { + print STDERR "no data in $input\n"; + exit 1; +} + +# The header keywords my $header = [ { name => 'QUERY_ID', @@ -62,16 +79,53 @@ value => undef }, - { name => 'STAGE', + { name => 'STAGE', writetype => TSTRING, comment => 'processing stage to examine', value => undef - } + } ]; -unless(defined($nostage)) { - push @{ $header }, { name => 'STAGE', - writetype => TSTRING, - comment => 'processing stage to examine', - value => undef - }; + +# Validate header. + +if (defined($query_id)) { + $headerData{QUERY_ID} = $query_id; +} +unless (exists($headerData{QUERY_ID})) { + die "No QUERY_ID specified for header."; +} + +unless (exists($headerData{EXTVER})) { + die "No EXTVER specified for header."; +} +if ($headerData{EXTVER} == 1) { + unless (exists($headerData{STAGE})) { + warn "No STAGE value specified in header. Assuming default value of 'diff'"; + $headerData{STAGE} = 'diff'; + } + foreach my $entry_ref (@{ $header }) { + my $name = $entry_ref->{name}; + unless (exists($headerData{$name})) { + die "Required header value $name not specified (try EXTVER=2?)."; + } + $entry_ref->{value} = $headerData{$name}; + } +} +elsif ($headerData{EXTVER} == 2) { + unless (exists($headerData{STAGE})) { + warn "No STAGE value specified in header. Assuming default value of 'diff'"; + $headerData{STAGE} = 'diff'; + } + my $tmp_header; + foreach my $entry_ref (@{ $header }) { + my $name = $entry_ref->{name}; + if (exists($headerData{$name})) { + $entry_ref->{value} = $headerData{$name}; + push @{ $tmp_header }, $entry_ref; + } + } + $header = $tmp_header; +} +else { + die "Unknown EXTVER = $headerData{EXTVER}."; } @@ -91,32 +145,51 @@ { name => 'MAG', type => 'D', writetype => TDOUBLE }, ]; - -my $in; -if ($input eq '-') { - $in = \*STDIN; -} else { - open $in, "<$input" or die "cannot open $input for reading"; -} - -my @colData; -foreach (@$columns) { - push @colData, []; -} - - -my $numRows = read_data_for_table($in,'\s+', \@colData, $header); -if (!$numRows) { - print STDERR "no data in $input\n"; - exit 1; -} - -# overwrite the QUERY_ID value from the input file with the command -# line argument - -if ($query_id) { - $header->[0]->{value} = $query_id; -} - -my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $columns, $header); +my $columns_v2 = [ + { name => 'FPA_ID', type => '20A', writetype => TSTRING }, + { name => 'MJD-OBS', type => 'D', writetype => TDOUBLE }, + { name => 'FILTER', type => '20A', writetype => TSTRING }, + { name => 'OBSCODE', type => '20A', writetype => TSTRING }, + { name => 'STAGE', type => '20A', writetype => TSTRING } + ]; +# Validate the data. +if ($headerData{EXTVER} == 1) { + foreach my $entry_ref (@{ $columns }) { + my $name = $entry_ref->{name}; + unless (exists($colData{$name})) { + die "Required data column $name not found (try EXTVER=2?)."; + } + } +} +if ($headerData{EXTVER} == 2) { + my $tmp_columns; + foreach my $entry_ref (@{ $columns }) { + my $name = $entry_ref->{name}; + if (exists($colData{$name})) { + push @{ $tmp_columns }, $entry_ref; + } + } + foreach my $entry_ref (@{ $columns_v2 }) { + my $name = $entry_ref->{name}; + if (exists($colData{$name})) { + push @{ $tmp_columns }, $entry_ref; + } + } + $columns = $tmp_columns; +} + +# Construct the array of arrays +my @colDataAoA = (); +foreach my $col_def (@{ $columns }) { + my $name = $col_def->{name}; + push @colDataAoA, $colData{$name}; +} + +# foreach (@$columns) { +# push @colData, []; +# } + + + +my $status = make_fits_table($output, EXTNAME, $numRows, \@colDataAoA, $columns, $header); exit $status; @@ -230,45 +303,49 @@ my $line_num = 0; - - # read data for header if any data is expected - if ($header) { - my $nhead = @$header; - while (my $line = <$in>) { - $line_num++; - chomp $line; - next if !$line; # skip blank lines - next if ($line =~ /^#/); # skip comment lines - my @vals = split /$sep/, $line; - my $nvals = @vals; - die "number of header columns in input $nvals does not equal expected number of header words $nhead" - if (@vals != @$header); - - for (my $i=0; $i < @$header; $i++) { - $header->[$i]->{value} = $vals[$i]; - } - - last; # only one header line - } - } - - my $row_num = 0; - my $ncols = @$colData; - while (my $line = <$in>) { - chomp $line; - $line_num++; - next if !$line; # skip blank lines - next if ($line =~ /^#/); # skip comment lines - - my @vals = split /$sep/, $line; - my $nvals = @vals; - die "number of columns $nvals in input does not equal expected number of header " - . " words $ncols on line $line_num" if ($nvals != $ncols); - - for (my $col = 0; $col < @$colData; $col++) { - $colData->[$col]->[$row_num] = $vals[$col]; - } - $row_num++; - } - + my @keywords = (); + my $row_num; + while (<$in>) { + chomp; + if ($line_num == 0) { + # Parse header information keywords + $_ =~ s/#//g; + @keywords = split /\s+/; + if ($keywords[0] eq '') { + shift(@keywords); + } + } + elsif ($line_num == 1) { + # Parse header information values + my @values = split /\s+/; + if ($#values != $#keywords) { + die "Number of header columns in input does not equal expected number of header words"; + } + for (my $i = 0; $i <= $#values; $i++) { + $header->{$keywords[$i]} = $values[$i]; + } + } + elsif ($line_num == 2) { + # Parse table information keywords, dumping old keywords + $_ =~ s/#//g; + @keywords = split /\s+/; + if ($keywords[0] eq '') { + shift(@keywords); + } + } + else { + # Parse table information values + unless ($_ =~ /^#/) { + my @values = split /\s+/; + if ($#values != $#keywords) { + die "Number of header columns in input does not equal expected number of header words"; + } + for (my $i = 0; $i <= $#values; $i++) { + push @{ $colData{$keywords[$i]} }, $values[$i]; + $row_num = $#{ $colData{$keywords[$i] } } + 1; + } + } + } + $line_num++; + } # we return the number of rows read return $row_num; Index: /trunk/pstamp/scripts/detect_query_read =================================================================== --- /trunk/pstamp/scripts/detect_query_read (revision 28805) +++ /trunk/pstamp/scripts/detect_query_read (revision 28806) @@ -20,4 +20,5 @@ use Math::Trig; use Data::Dumper; +use IPC::Cmd 0.36 qw( can_run run ); use constant EXTNAME => 'MOPS_DETECTABILITY_QUERY'; # Extension name for table @@ -31,9 +32,15 @@ $output, # Name of output table $save_temps, # Save temporary files? + $dbname, # needed to do camtool lookups. ); + +my $regtool = can_run('regtool') or (die "Can't find regtool"); +my $camtool = can_run('camtool') or (die "Can't find regtool"); + GetOptions( 'input|i=s' => \$input, 'output|o=s' => \$output, + 'dbname=s' => \$dbname, 'nolabel|l' => \$no_print_label, 'noheader|h' => \$no_print_header, @@ -65,60 +72,23 @@ # The keywords found in the header -# my $header = { -# 'EXTVER' => { name => 'EXTVER', -# writetype => TSTRING, -# comment => 'Extension version', -# value => undef -# }, -# 'EXTNAME' => { name => 'EXTNAME', -# writetype => TSTRING, -# comment => 'name of this binary table extension', -# value => undef -# }, -# 'QUERY_ID' => { name => 'QUERY_ID', -# writetype => TSTRING, -# comment => 'MOPS Query ID for this batch query', -# value => undef -# }, -# 'FPA_ID' => { name => 'FPA_ID', -# writetype => TSTRING, -# comment => 'orginal FPA_ID used at ingest', -# value => undef -# }, -# 'MJD_OBS' => { name => 'MJD-OBS', -# writetype => TDOUBLE, -# comment => 'starting time of the exposure, MJD', -# value => undef -# }, -# 'FILTER' => { name => 'FILTER', -# writetype => TSTRING, -# comment => 'effective filter use for the exposure', -# value => undef -# }, -# 'OBSCODE' => { name => 'OBSCODE', -# writetype => TSTRING, -# comment => 'site identifier (MPC observatory code)', -# value => undef -# }, -# 'STAGE' => { -# name => 'STAGE', -# writetype => TSTRING, -# comment => 'processing stage to examine', -# value => undef -# } -# }; my $parse_error = 0; -# Parse the header to determine what we expect to find. +# Parse the header to determine what we have to work with. +# Clean up empty space and remove unneeded single quotes. foreach my $header_key (keys %{ $inHeader }) { $inHeader->{$header_key} =~ s/\s+//g; $inHeader->{$header_key} =~ s/\'//g; } -my ($EXTVER,$headerEXTNAME,$QUERY_ID,$FPA_ID,$MJD_OBS,$FILTER,$OBSCODE,$STAGE) = - ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID},$inHeader->{FPA_ID}, - $inHeader->{'MJD-OBS'},$inHeader->{FILTER},$inHeader->{OBSCODE},$inHeader->{STAGE}); +my ($EXTVER,$headerEXTNAME,$QUERY_ID, + $FPA_ID,$MJD_OBS,$FILTER, + $OBSCODE,$STAGE) = + ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID}, + $inHeader->{FPA_ID},$inHeader->{'MJD-OBS'},$inHeader->{FILTER}, + $inHeader->{OBSCODE},$inHeader->{STAGE}); + unless(defined($EXTVER) && defined($headerEXTNAME) && (($EXTVER == 1)||($EXTVER == 2)) && - ($headerEXTNAME eq EXTNAME)) { + ($headerEXTNAME eq EXTNAME)&& + (defined($QUERY_ID))) { $parse_error = 1; } @@ -143,5 +113,5 @@ # apparent magnitude { name => 'MAG', type => 'D', writetype => TDOUBLE, version => 1 }, - # v2 query_id: MOPS query ID for this batch query + # v2 query_id: needs to be here. { name => 'QUERY_ID', type => '20A', writetype => TSTRING, version => 2, default => $QUERY_ID }, # v2 fpa_id: original FPA_ID used at ingest @@ -172,14 +142,109 @@ if ($col->{version} > $EXTVER) { - @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows); + @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1); next; } - $inFits->get_colnum(0, $col->{name}, $col_num, $status) and check_fitsio($status); - $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status); - $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status) - and check_fitsio($status); - $colData{$col->{name}} = $col_data; - if ($col->{name} eq 'MJD-OBS') { - print @{ $col_data } . "\n"; + else { + if (defined($col->{default})) { + @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1); + next; + } + } + + $inFits->get_colnum(0, $col->{name}, $col_num, $status); + if ($status == 0) { + $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status); + $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status) + and check_fitsio($status); + $colData{$col->{name}} = $col_data; + if ($col->{name} eq 'QUERY_ID') { + print @{ $col_data } . "\n"; + } + + } + elsif ($status == 219) { + @{ $colData{$col->{name}} } = map { "Not Set" } (0 .. $numRows - 1); + $status = 0; + } +} + +# Set things that aren't set. +for (my $i = 0; $i < $numRows; $i++) { +# Simple stuff first. + my %known_filters = (); + if ($colData{STAGE}[$i] eq "Not Set") { + $colData{STAGE}[$i] = 'diff'; + } + if ($colData{OBSCODE}[$i] eq "Not Set") { + $colData{OBSCODE}[$i] = 566; + } +# Define filter and MJD from FPA_ID + if (($colData{FILTER}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) { + if (exists($known_filters{$colData{FPA_ID}[$i]})) { + $colData{FILTER}[$i] = $known_filters{$colData{FPA_ID}[$i]}; + } + else { + my $cmd = "$regtool -processedexp -dbname $dbname -exp_name $colData{FPA_ID}[$i]"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => 0); + unless ($success) { + # This is a problem, because I'm not sure how we handle a failure to read something. + # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, + # we can't send that response back. + die("Unable to perform $cmd error code: $error_code"); + } + foreach my $entry (split /\n/, (join "", @$full_buf)) { + $entry =~ s/^\s+//; + my @line = split /\s+/, $entry; + if (scalar(@line) != 3) { + next; + } + my ($key,$type,$value) = @line; +# print "$entry => $key / $type / $value \n"; + if ($key =~ /filter/i) { + $value =~ s/.00000//; + $colData{FILTER}[$i] = $value; + $known_filters{$colData{FPA_ID}[$i]} = $value; + } + } + } + } + if (($colData{'MJD-OBS'}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) { + # HACK! + my $mjd = $colData{FPA_ID}[$i]; + $mjd =~ s/o(....)g.*/$1/; + $mjd += 50000; + $colData{'MJD-OBS'}[$i] = $mjd; + } + if (($colData{FPA_ID}[$i] eq "Not Set")&&(($colData{'FILTER'}[$i] ne "Not Set")&& + ($colData{'MJD-OBS'}[$i] ne "Not Set"))) { + my $dateobs_begin = mjd_to_dateobs($colData{'MJD-OBS'}[$i]); + my $dateobs_end = mjd_to_dateobs($colData{'MJD-OBS'}[$i] + 1); + my $ra = $colData{'RA1_DEG'}[$i]; + my $dec = $colData{'DEC1_DEG'}[$i]; + my $filter = $colData{'FILTER'}[$i] . ".00000"; + my $cmd = "$camtool -processedexp -dbname $dbname -filter $filter "; + $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end "; + $cmd .= " -ra $ra -decl $dec -limit 1"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => 0); + unless ($success) { + # This is a problem, because I'm not sure how we handle a failure to read something. + # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, + # we can't send that response back. + die("Unable to perform $cmd error code: $error_code"); + } + foreach my $entry (split /\n/, (join "", @$full_buf)) { + $entry =~ s/^\s+//; + my @line = split /\s+/, $entry; + if (scalar(@line) != 3) { + next; + } + my ($key,$type,$value) = @line; +# print "$entry => $key / $type / $value \n"; + if ($key =~ /exp_name/i) { + $colData{FPA_ID}[$i] = $value; + } + } } } @@ -189,4 +254,5 @@ foreach my $colName (@unique_fields) { my %counter = (); + my $i = 0; foreach my $row (@{ $colData{$colName} }) { $counter{$row} = 1; @@ -232,5 +298,5 @@ my $msg; # Message to output Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); - die "CFITSIO error: $msg\n"; + die "CFITSIO error: $status => $msg\n"; } } @@ -252,4 +318,15 @@ } +# Stolen from PStamp/Job.pm. Thanks, Bill. +sub mjd_to_dateobs { + my $mjd = shift; + + my $ticks = ($mjd - 40587.0) * 86400; + + my ($sec, $min, $hr, $day, $mon, $year) = gmtime($ticks); + + return sprintf "'%4d-%02d-%02dT%02d:%02d:%02dZ'", $year+1900, $mon+1, $day, $hr, $min, $sec; +} + Index: /trunk/pstamp/scripts/detectability_respond.pl =================================================================== --- /trunk/pstamp/scripts/detectability_respond.pl (revision 28805) +++ /trunk/pstamp/scripts/detectability_respond.pl (revision 28806) @@ -108,5 +108,5 @@ # Parse input request file using detect_query_read (as it's already written). # - my $dqr_command = "$detect_query_read --input $request_file"; + my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $dqr_command, verbose => $verbose);