Index: trunk/pstamp/scripts/detect_query_read
===================================================================
--- trunk/pstamp/scripts/detect_query_read	(revision 28777)
+++ trunk/pstamp/scripts/detect_query_read	(revision 28806)
@@ -20,4 +20,5 @@
 use Math::Trig;
 use Data::Dumper;
+use IPC::Cmd 0.36 qw( can_run run );
 
 use constant EXTNAME => 'MOPS_DETECTABILITY_QUERY'; # Extension name for table
@@ -31,9 +32,15 @@
      $output,			# Name of output table
      $save_temps,		# Save temporary files?
+     $dbname,                   # needed to do camtool lookups.
      );
+
+my $regtool = can_run('regtool') or (die "Can't find regtool");
+my $camtool = can_run('camtool') or (die "Can't find regtool");
+
 
 GetOptions(
 	   'input|i=s'    => \$input,
 	   'output|o=s'   => \$output,
+           'dbname=s'     => \$dbname,
            'nolabel|l'    => \$no_print_label,
            'noheader|h'   => \$no_print_header,
@@ -65,60 +72,23 @@
 
 # The keywords found in the header
-# my $header = {
-#         'EXTVER'   => { name => 'EXTVER',
-# 		       writetype => TSTRING,
-# 		       comment => 'Extension version',
-# 		       value => undef
-#                       },
-#         'EXTNAME'  => { name => 'EXTNAME',
-# 		       writetype => TSTRING,
-# 		       comment => 'name of this binary table extension',
-# 	  	       value => undef
-# 		      },
-#         'QUERY_ID' => { name => 'QUERY_ID', 
-#                         writetype => TSTRING, 
-#                         comment => 'MOPS Query ID for this batch query',
-#                         value => undef
-#                       },
-#         'FPA_ID'   => { name => 'FPA_ID', 
-#                         writetype => TSTRING, 
-#                         comment => 'orginal FPA_ID used at ingest',
-#                         value => undef
-#                       }, 
-#         'MJD_OBS'  => { name => 'MJD-OBS', 
-#                         writetype => TDOUBLE, 
-#                         comment => 'starting time of the exposure, MJD',
-#                         value => undef
-#                       },
-#         'FILTER'   => { name => 'FILTER', 
-#                         writetype => TSTRING, 
-#                         comment => 'effective filter use for the exposure',
-#                         value => undef
-#                       },
-#         'OBSCODE'  => { name => 'OBSCODE', 
-#                         writetype => TSTRING, 
-#                         comment => 'site identifier (MPC observatory code)',
-#                         value => undef
-#                       },
-#  	'STAGE'    => { 
-# 	                name => 'STAGE',
-# 			writetype => TSTRING,
-# 			comment => 'processing stage to examine',
-# 			value => undef
-# 	              }
-# };
 
 my $parse_error = 0;
-# Parse the header to determine what we expect to find.
+# Parse the header to determine what we have to work with.
+# Clean up empty space and remove unneeded single quotes.
 foreach my $header_key (keys %{ $inHeader }) {
     $inHeader->{$header_key} =~ s/\s+//g;
     $inHeader->{$header_key} =~ s/\'//g;
 }
-my ($EXTVER,$headerEXTNAME,$QUERY_ID,$FPA_ID,$MJD_OBS,$FILTER,$OBSCODE,$STAGE) = 
-    ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID},$inHeader->{FPA_ID},
-     $inHeader->{'MJD-OBS'},$inHeader->{FILTER},$inHeader->{OBSCODE},$inHeader->{STAGE});
+my ($EXTVER,$headerEXTNAME,$QUERY_ID,
+    $FPA_ID,$MJD_OBS,$FILTER,
+    $OBSCODE,$STAGE) = 
+    ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID},
+     $inHeader->{FPA_ID},$inHeader->{'MJD-OBS'},$inHeader->{FILTER},
+     $inHeader->{OBSCODE},$inHeader->{STAGE});
+
 unless(defined($EXTVER) && defined($headerEXTNAME) &&
        (($EXTVER == 1)||($EXTVER == 2)) && 
-       ($headerEXTNAME eq EXTNAME)) {
+       ($headerEXTNAME eq EXTNAME)&&
+       (defined($QUERY_ID))) {
     $parse_error = 1;
 }
@@ -143,5 +113,5 @@
         # apparent magnitude
         { name => 'MAG',      type => 'D',   writetype => TDOUBLE, version => 1 },
-    # v2 query_id: MOPS query ID for this batch query
+    # v2 query_id: needs to be here.
     { name => 'QUERY_ID', type => '20A', writetype => TSTRING, version => 2, default => $QUERY_ID },
     # v2 fpa_id: original FPA_ID used at ingest
@@ -172,14 +142,109 @@
 
     if ($col->{version} > $EXTVER) {
-	@{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows);
+	@{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1);
 	next;
     }
-    $inFits->get_colnum(0, $col->{name}, $col_num, $status) and check_fitsio($status);
-    $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status);
-    $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status)
-                                                                    and check_fitsio($status);
-    $colData{$col->{name}} = $col_data;
-    if ($col->{name} eq 'MJD-OBS') {
-	print @{ $col_data } . "\n";
+    else {
+	if (defined($col->{default})) {
+	    @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1);
+	    next;
+	}	 
+    }   
+
+    $inFits->get_colnum(0, $col->{name}, $col_num, $status);
+    if ($status == 0) {
+	$inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status);
+	$inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status)
+	    and check_fitsio($status);
+	$colData{$col->{name}} = $col_data;
+	if ($col->{name} eq 'QUERY_ID') {
+	    print @{ $col_data } . "\n";
+	}
+
+    }
+    elsif ($status == 219) {
+	@{ $colData{$col->{name}} } = map { "Not Set" } (0 .. $numRows - 1);
+	$status = 0;
+    }
+}
+
+# Set things that aren't set.
+for (my $i = 0; $i < $numRows; $i++) {
+# Simple stuff first.
+    my %known_filters = ();
+    if ($colData{STAGE}[$i] eq "Not Set") {
+	$colData{STAGE}[$i] = 'diff';
+    }
+    if ($colData{OBSCODE}[$i] eq "Not Set") {
+	$colData{OBSCODE}[$i] = 566;
+    }
+# Define filter and MJD from FPA_ID
+    if (($colData{FILTER}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) {
+	if (exists($known_filters{$colData{FPA_ID}[$i]})) {
+	    $colData{FILTER}[$i] = $known_filters{$colData{FPA_ID}[$i]};
+	}
+	else {
+	    my $cmd = "$regtool -processedexp -dbname $dbname -exp_name $colData{FPA_ID}[$i]";
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $cmd, verbose => 0);
+	    unless ($success) {
+		# This is a problem, because I'm not sure how we handle a failure to read something.
+		# We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 
+		# we can't send that response back.
+		die("Unable to perform $cmd error code: $error_code");
+	    }
+	    foreach my $entry (split /\n/, (join "", @$full_buf)) {
+		$entry =~ s/^\s+//;
+		my @line = split /\s+/, $entry;
+		if (scalar(@line) != 3) {
+		    next;
+		}
+		my ($key,$type,$value) = @line;
+#		print "$entry => $key / $type / $value \n";
+		if ($key =~ /filter/i) {
+		    $value =~ s/.00000//;
+		    $colData{FILTER}[$i] = $value;
+		    $known_filters{$colData{FPA_ID}[$i]} = $value;
+		}
+	    }
+	}
+    }
+    if (($colData{'MJD-OBS'}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) {
+	# HACK!
+	my $mjd = $colData{FPA_ID}[$i];
+	$mjd =~ s/o(....)g.*/$1/;
+	$mjd += 50000;
+	$colData{'MJD-OBS'}[$i] = $mjd;
+    }
+    if (($colData{FPA_ID}[$i] eq "Not Set")&&(($colData{'FILTER'}[$i] ne "Not Set")&&
+					      ($colData{'MJD-OBS'}[$i] ne "Not Set"))) {
+	my $dateobs_begin = mjd_to_dateobs($colData{'MJD-OBS'}[$i]);
+	my $dateobs_end = mjd_to_dateobs($colData{'MJD-OBS'}[$i] + 1);
+	my $ra = $colData{'RA1_DEG'}[$i];
+	my $dec = $colData{'DEC1_DEG'}[$i];
+	my $filter = $colData{'FILTER'}[$i] . ".00000";
+	my $cmd = "$camtool -processedexp -dbname $dbname -filter $filter ";
+	$cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
+	$cmd .= " -ra $ra -decl $dec -limit 1";
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $cmd, verbose => 0);
+	unless ($success) {
+	    # This is a problem, because I'm not sure how we handle a failure to read something.
+	    # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 
+	    # we can't send that response back.
+	    die("Unable to perform $cmd error code: $error_code");
+	}
+	foreach my $entry (split /\n/, (join "", @$full_buf)) {
+	    $entry =~ s/^\s+//;
+	    my @line = split /\s+/, $entry;
+	    if (scalar(@line) != 3) {
+		next;
+	    }
+	    my ($key,$type,$value) = @line;
+#		print "$entry => $key / $type / $value \n";
+	    if ($key =~ /exp_name/i) {
+		$colData{FPA_ID}[$i] = $value;
+	    }
+	}
     }
 }
@@ -189,4 +254,5 @@
 foreach my $colName (@unique_fields) {
     my %counter = ();
+    my $i = 0;
     foreach my $row (@{ $colData{$colName} }) {
 	$counter{$row} = 1;
@@ -232,5 +298,5 @@
 	my $msg;		# Message to output
 	Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
-	die "CFITSIO error: $msg\n";
+	die "CFITSIO error: $status => $msg\n";
     }
 }
@@ -252,4 +318,15 @@
 }
 
+# Stolen from PStamp/Job.pm.  Thanks, Bill.
+sub mjd_to_dateobs {
+    my $mjd = shift;
+
+    my $ticks = ($mjd - 40587.0) * 86400;
+
+    my ($sec, $min, $hr, $day, $mon, $year) = gmtime($ticks);
+
+    return sprintf "'%4d-%02d-%02dT%02d:%02d:%02dZ'", $year+1900, $mon+1, $day, $hr, $min, $sec;
+}
+