Changeset 27517 for branches/eam_branches/20100225/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

Timestamp:

Mar 29, 2010, 3:55:49 PM (16 years ago)

Author:

eugene

Message:

update merges from trunk

Location:

branches/eam_branches/20100225

Files:

• . (modified) (1 prop)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (22 diffs)

branches/eam_branches/20100225/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -16 +16 @@
                     locate_images
                     resolve_project
+                    getCamRunByCamID
                     );
 our %EXPORT_TAGS = (standard => [@EXPORT_OK]);
 
+
+use PS::IPP::PStamp::RequestFile qw( :standard );
 
 use IPC::Cmd 0.36 qw( can_run run );
@@ -24 +27 @@
 use PS::IPP::Config qw( :standard );
 use Carp;
+
+# caches of camProcessedExp objects. 
+# Key is $exp_id
+my %camRunByExpIDCache;
+# Key is $cam_id
+my %camRunByCamIDCache;
 
 ### my @images = locate_images($image_db, $req_type, $img_type, $id, $component,
@@ -36 +45 @@
     my $tess_id  = shift;
     my $component= shift;   # class_id or skycell_id
-    my $inverse  = shift;
+    my $option_mask  = shift;
     my $need_magic = shift;
     my $x        = shift;
@@ -43 +52 @@
     my $mjd_max  = shift;
     my $filter   = shift;
-    my $data_group    = shift;
+    my $data_group = shift;
     my $verbose  = shift;
 
-
     # we die in response to bad data in request files
+    # The wrapper script is responsible for updating the database
     die "Unknown req_type: $req_type" 
         if ($req_type ne "byid") and
@@ -55 +64 @@
            ($req_type ne "byskycell");
 
+
     my $dateobs_begin;
     my $dateobs_end;
@@ -65 +75 @@
     if (isnull($tess_id)) {
         $tess_id = undef;
+    }
+    if (isnull($data_group)) {
+        $data_group = undef;
+    }
+    if (isnull($filter)) {
+        $filter = undef;
+    }
+
+    if ($req_type eq "bycoord") {
+        my $results = lookup_bycoord($ipprc, $image_db, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
+        return $results;
     }
 
@@ -72 +93 @@
         # in one place
         $req_type = "bydiff";
-        my $results = lookup_diff($ipprc, $image_db, $id, $component, 1, $inverse, $img_type, $verbose);
+        my $results = lookup_diff($ipprc, $image_db, $id, $component, 1, $option_mask, $img_type, $verbose);
         return $results;
     }
 
     if ($req_type eq "bydiff") {
-        my $results = lookup_diff($ipprc, $image_db, $id, $component, 0, $inverse, $img_type, $verbose);
+        my $results = lookup_diff($ipprc, $image_db, $id, $component, 0, $option_mask, $img_type, $verbose);
         if (!$results) {
             return undef;
@@ -113 +134 @@
             return undef;
         }
-
-    } elsif ($req_type eq "bycoord") {
-
-        if ($img_type eq "stack") {
-            print STDERR "Error bycoord lookup not implemented for stage stack yet\n";
-            return undef;
-        }
     } elsif ($req_type eq "byskycell") {
         if (($img_type eq "raw") or ($img_type eq "chip")) {
@@ -131 +145 @@
     }
 
-    my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose);
+    my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic,
+        $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
 
     return $results;
@@ -146 +161 @@
     my $component= shift;
     my $need_magic = shift;
-    my $x        = shift;
-    my $y        = shift;
     my $dateobs_begin = shift;
     my $dateobs_end   = shift;
     my $filter = shift;
     my $data_group = shift;
+    my $option_mask = shift;
     my $verbose  = shift;
+
+    my $inverse = $option_mask & $PSTAMP_SELECT_INVERSE;
+    my $unconvolved = $option_mask & $PSTAMP_SELECT_UNCONV;
 
     my $missing_tools;
@@ -238 +255 @@
         $component_args = " -skycell_id $skycell_id" if $skycell_id;
 
-        $image_name  = "PPSTACK.OUTPUT";
-        $mask_name   = "PPSTACK.OUTPUT.MASK";
-        $weight_name = "PPSTACK.OUTPUT.VARIANCE";
+        if (!$unconvolved) {
+            $image_name  = "PPSTACK.OUTPUT";
+            $mask_name   = "PPSTACK.OUTPUT.MASK";
+            $weight_name = "PPSTACK.OUTPUT.VARIANCE";
+        } else {
+            $image_name  = "PPSTACK.UNCONV";
+            $mask_name   = "PPSTACK.UNCONV.MASK";
+            $weight_name = "PPSTACK.UNCONV.VARIANCE";
+        }
         # this is wrong but gets the right answer. Need to figure out how to find the
         # rule properly
@@ -267 +290 @@
         die "tess_id and component are required for byskycell" if !$tess_id or ! $skycell_id;
         $command .= " -tess_id $tess_id -skycell_id $skycell_id";
-    } elsif ($req_type eq "bycoord") {
-        $command .= " -radius 3.0 -ra $x -decl $y";
-        if ($img_type ne "raw") {
-            $command .= " -tess_id $tess_id" if $tess_id;
-            $command .= " -skycell_id $skycell_id" if $skycell_id;
-        }
     } else {
         die "Unknown req_type supplied: $req_type";
@@ -351 +368 @@
         } elsif ($img_type eq "diff") {
             $stage_id = $image->{diff_id};
+            if ($inverse && $image->{bothways}) {
+                $image_name  = "PPSUB.INVERSE";
+                $mask_name   = "PPSUB.INVERSE.MASK";
+                $weight_name = "PPSUB.INVERSE.VARIANCE";
+                $cmf_name    = "PPSUB.INVERSE.SOURCES";
+            }
         } elsif ($img_type eq "stack") {
             $stage_id = $image->{stack_id};
@@ -379 +402 @@
     my $skycell_id = shift;
     my $byid     = shift;
-    my $inverse  = shift;
+    my $option_mask  = shift;
     my $img_type = shift;
     my $verbose = shift;
+
+    my $inverse = $option_mask & $PSTAMP_SELECT_INVERSE;
 
     my $missing_tools;
@@ -436 +461 @@
         # so we interpret the requested image in that way
 
-        my $stack_id = $image->{stack2};
         # XXX difftool currently returns max long long for null
-        # this line is ready if we switch the code to return zero for null
-        if ($stack_id and ($stack_id != 9223372036854775807)) {
+        # these checks are ready if we switch the code to return zero for null
+        my $stack1 = $image->{stack1};
+        if (($stack1 == 0) or ($stack1 == 9223372036854775807)) {
+            $stack1 = undef
+        }
+        my $stack2 = $image->{stack2};
+        if (($stack2 == 0) or ($stack2 == 9223372036854775807)) {
+            $stack2 = undef
+        }
+        my $stack_id;
+        if ($stack1 and $stack2) {
+            # we have a stack - stack diff (well it might be stack - warp....)
+            # but at any rate we only handle image type diff and stack
+            if (($img_type ne "diff") and ($img_type ne "stack")) {
+                print STDERR "lookup_diff: cannot lookup IMG_TYPE $img_type bydiff from a stack stack diff run\n";
+                next;
+            }
+            # stack-stack diff
+            # XXX: define another flag for this
+            if (! $inverse) {
+                $stack_id = $stack1;
+            } else {
+                $stack_id = $stack2;
+            }
             $image->{stack_id} = $stack_id;
-        }
-
-        my ($warp_id, $exp_id, $exp_name, $chip_id, $cam_id);
-        if ($inverse and !$image->{bothways}) {
-            print STDERR "Inverse images requested for diffRun that is not bothways. Ignoring.\n";
-            $inverse = 0;
-        }
-        if ($inverse) {
-            $warp_id =  $image->{warp2};
-            $exp_id = $image->{exp_id_2};
-            $exp_name = $image->{exp_name_2};
-            $chip_id = $image->{chip_id_2};
-            $cam_id = $image->{cam_id_2};
         } else {
-            $warp_id =  $image->{warp1};
-            $exp_id = $image->{exp_id_1};
-            $exp_name = $image->{exp_name_1};
-            $chip_id = $image->{chip_id_1};
-            $cam_id = $image->{cam_id_1};
-        }
-        # XXX difftool currently returns max long long for null
-        # this line is ready if we switch the code to return zero for null
-        if ($warp_id and ($warp_id != 9223372036854775807)) {
-            $image->{warp_id} = $warp_id;
-            $image->{exp_id} = $exp_id;
-            $image->{exp_name} = $exp_name;
-            $image->{chip_id} = $chip_id;
-            $image->{cam_id} = $cam_id;
-        } else {
-            print STDERR "unexpected result warp_id not defined\n";
-            next;
-        }
+            my ($warp_id, $exp_id, $exp_name, $chip_id, $cam_id);
+            if ($inverse and !$image->{bothways}) {
+                print STDERR "Inverse images requested for diffRun that is not bothways. Ignoring inverse.\n";
+                $inverse = 0;
+            }
+            if ($inverse) {
+                $warp_id =  $image->{warp2};
+                $exp_id = $image->{exp_id_2};
+                $exp_name = $image->{exp_name_2};
+                $chip_id = $image->{chip_id_2};
+                $cam_id = $image->{cam_id_2};
+            } else {
+                $warp_id =  $image->{warp1};
+                $exp_id = $image->{exp_id_1};
+                $exp_name = $image->{exp_name_1};
+                $chip_id = $image->{chip_id_1};
+                $cam_id = $image->{cam_id_1};
+            }
+            # XXX difftool currently returns max long long for null
+            # this line is ready if we switch the code to return zero for null
+            if ($warp_id and ($warp_id != 9223372036854775807)) {
+                $image->{warp_id} = $warp_id;
+                $image->{exp_id} = $exp_id;
+                $image->{exp_name} = $exp_name;
+                $image->{chip_id} = $chip_id;
+                $image->{cam_id} = $cam_id;
+            }
+        }
+
+        # XXX: If difftool doesn't return a camera insert it here
+        if (!$image->{camera}) {
+            $image->{camera} = "GPC1";
+            # lie about the magicked status since stack stack diffs don't require destreaking
+            # we can remove this once diff_mode gets included
+            $image->{magicked} = 42;
+        }
+
         if ($img_type eq "diff") {
             # the $image is going to be returned directly in this case so we need to duplicate
@@ -482 +534 @@
                 $image->{mask}   = $ipprc->filename($filerule_base . ".MASK", $image->{path_base});
                 $image->{weight} = $ipprc->filename($filerule_base . ".VARIANCE", $image->{path_base});
+                $image->{cmf}    = $ipprc->filename($filerule_base . ".SOURCES", $image->{path_base});
+                $image->{psf}    = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $image->{path_base});
+                $image->{stage_id} = $image->{diff_id};
+                $image->{stage}    = "diff";
+                $image->{image_db} = $image_db;
+                $image->{component} = $image->{skycell_id};
             } else {
-                # XXX this will only happen if the minuend is not a warp
-                print STDERR "WARNING: cannot resolve camera so cannot get weight and mask images\n";
+                # XXX this will only happen if the minuend is not a warp. See hack above
+                print STDERR "WARNING: cannot resolve camera so cannot get resolve file rules\n";
                 next;
             }
@@ -494 +552 @@
 
 sub lookup_bycoord {
-    my $ipprc    = shift;
-    my $image_db = shift;
-    my $x        = shift;
-    my $y        = shift;
+    my $ipprc      = shift;
+    my $image_db   = shift;
+    my $img_type   = shift;
+    my $tess_id    = shift;
+    my $component  = shift;
+    my $need_magic = shift;
+    my $ra         = shift;
+    my $dec        = shift;
     my $dateobs_begin  = shift;
-    my $dateobs_end  = shift;
-    my $filter   = shift;
-    my $verbose  = shift;
+    my $dateobs_end    = shift;
+    my $filter     = shift;
+    my $data_group = shift;
+    my $option_mask = shift;
+    my $verbose    = shift;
 
     my $missing_tools;
     my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
+    my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+    my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
+    my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
     if ($missing_tools) {
         warn("Can't find required tools.");
@@ -510 +577 @@
     }
 
-    # XXX TODO:
-    # Full lookups by coordinate require looking at the DVO database. However, 
-    # dateobs and filter which is all that MOPS has asked for.
-
-    my $args;
-    if (!isnull($dateobs_begin)) {
-        $args .= " -dateobs_begin $dateobs_begin";
-    }
-    if (!isnull($dateobs_end)) {
-        $args .= " -dateobs_end $dateobs_end";
-    }
-    if (!isnull($filter)) {
-        $args .= " -filter $filter";
-    }
-
-    if (!$args) {
-        # avoid returning every exposure in the DB
-        print STDERR "no query arguments provided for bycoord\n";
+    my $results = ();
+    if (($img_type eq "raw") or ($img_type eq "chip")) {
+
+        my $runs = lookup_runs_by_camid_and_coords($ipprc, $image_db, $img_type,
+            $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose);
+
+        foreach my $run (@$runs) {
+            next if $component and ($run->{component} ne $component);
+            my $these_results = lookup($ipprc, $image_db, "byid", $img_type, $run->{id},
+                $tess_id, $run->{component}, $need_magic, 
+                $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
+
+            next if !$these_results;
+            push @$results, @$these_results;
+        }
+        
+    } else {
+        # this should be checked elsewhere
+        die "unexpected image type $img_type" if ($img_type ne "warp") 
+                                              and ($img_type ne "stack") and ($img_type ne "diff");
+
+        my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose);
+
+        foreach my $skycell (@$skycells) {
+            my $these_results = lookup($ipprc, $image_db, "byskycell", $img_type, undef,
+                $skycell->{tess_id}, $skycell->{component}, $need_magic, 
+                $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
+
+            next if !$these_results;
+            push @$results, @$these_results;
+        }
+    }
+
+    return $results;
+}
+
+# lookup_runs_by_camid_and_coords
+# given an ra, dec, and optionally other paramters, find camera runs for exposures
+# that are within some distance of the coordinates
+sub lookup_runs_by_camid_and_coords {
+    my $ipprc      = shift;
+    my $image_db   = shift;
+    my $img_type   = shift;
+    my $ra         = shift;
+    my $dec        = shift;
+    my $need_magic = shift;
+    my $dateobs_begin  = shift;
+    my $dateobs_end = shift;
+    my $filter     = shift;
+    my $data_group = shift;
+    my $verbose    = shift;
+
+    my $camruns;
+    my $missing_tools;
+    my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+    my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
+    if ($missing_tools) {
+        warn("Can't find required tools.");
+        exit ($PS_EXIT_CONFIG_ERROR);
+    }
+
+    {
+        my $command = "$camtool -dbname $image_db -processedexp";
+           $command .= " -ra $ra -decl $dec -radius 1.6";
+           $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
+           $command .= " -dateobs_end   $dateobs_end"   if $dateobs_end  ;
+           $command .= " -filter $filter" if $filter;
+           # NOTE: we are applying the data_group to the camera run.
+           # If we're looking for chip stage images there is no guarentee that 
+           # chipRun.data_group = camRun.data_group. In practice this is almost
+           # always the case. If this turns out to be a problem we can defer
+           # the data_group test to when we look up the chipProcessedImfiles
+           $command .= " -data_group $data_group" if $data_group;
+           $command .= " -destreaked" if $need_magic;
+        # run the tool and parse the output
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+        unless ($success) {
+            print STDERR @$stderr_buf;
+            return undef;
+        }
+        my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+        my $output = join "", @$stdout_buf;
+        if (!$output) {
+            print STDERR "no output returned from $command\n" if $verbose;
+            return undef;
+        }
+        $camruns = parse_md_fast($mdcParser, $output);
+    }
+    if (!$camruns) {
         return undef;
     }
-
-    # XXX TODO: This query doesn't work Something appears to be out of
-    # sync about the times I'm passing and what regtool and or the DB expect
-    # the query I used in the C version of locateimages used
-    # WHERE dateobs >= FROM_UNIXTIME(%ld) AND dateobs <= FROM_UNIXTIME(%ld + exp_time)
-
-    my $command = "$regtool -processedexp -dbname $image_db $args";
-
-    # run the tool and parse the output
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-    unless ($success) {
-        # not sure if we should die here
-        print STDERR @$stderr_buf;
-        return undef;
-    }
-    my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
-    my $output = join "", @$stdout_buf;
-    if (!$output) {
-        print STDERR "no output returned from $command\n" if $verbose;
-        return undef;
-    }
-    my $images = parse_md_fast($mdcParser, $output) or die ("Unable to parse metadata config doc");
-
-    return $images;
+    my $runs;
+    foreach my $camRun (@$camruns) {
+        my $cam_id = $camRun->{cam_id};
+
+        updateCamRunCache($camRun);
+
+        next if $camRun->{quality};
+        next if $camRun->{fault};
+        # XXX Use file rule
+        my $astrom = $camRun->{path_base} . ".smf";
+        my $astrom_resolved = $ipprc->file_resolve($astrom);
+        next if !$astrom_resolved;
+
+        my $command = "$dvoImagesAtCoords -astrom $astrom_resolved $ra $dec";
+        # run the tool and parse the output
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my $result_code = $error_code >> 8;
+            if ($result_code == $PSTAMP_NO_OVERLAP) {
+                print STDERR "no overlap for $astrom\n" if $verbose;
+                next;
+            }
+            # unexpected result code
+            die "unexpected result code: $result_code from $command\n";
+        }
+
+        my $output = join "", @$stdout_buf;
+        if (!$output) {
+            # this shouldn't happen when dvoImagesAtCoord exits with zero status
+            die "no output returned from $command\n";
+        }
+
+        # dvoImagesAtCoords should return a single line
+        # rownum ra dec class_id
+        my @lines = split "\n", $output;
+        my $n = scalar @lines;
+        if ($n != 1) {
+            print STDERR "unexpected number of lines returned by dvoImagesAtCoords: $n\n";
+
+            # XXX: There is a bug in dvo where each component is listed twice
+            # When that gets fixed remove the conditional and just die
+            die "unexpected number lines returned by dvoImagesAtCoords: $n"
+                if ($n != 2) or ($lines[0] ne $lines[1]);
+        }
+
+        my (undef, $ra_out, $dec_out, $class_id) = split " ", $lines[0];
+        if (!$class_id) {
+            die "unexpected output from dvoImagesAtCoords: $lines[0]";
+        }
+        # build the hash to return
+        my $run = {
+            exp_id    => $camRun->{exp_id},
+            exp_name  => $camRun->{exp_name},
+            chip_id   => $camRun->{chip_id},
+            cam_id    => $camRun->{cam_id},
+            astrom    => $astrom_resolved,
+            class_id  => $class_id,
+            component => $class_id
+        };
+        if ($img_type eq "chip") {
+            $run->{id} = $run->{chip_id};
+        } else {
+            $run->{id} = $run->{exp_id};
+        }
+        push @$runs, $run;
+    }
+
+    return $runs;
+}
+
+sub lookup_skycell_by_coords {
+    my $ipprc      = shift;
+    my $requested_tess_id    = shift;
+    my $requested_skycell  = shift;
+    my $ra         = shift;
+    my $dec        = shift;
+    my $verbose    = shift;
+
+    my $missing_tools;
+    my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
+    if ($missing_tools) {
+        warn("Can't find required tools.");
+        exit ($PS_EXIT_CONFIG_ERROR);
+    }
+
+    $requested_tess_id = "" if isnull($requested_tess_id);
+    $requested_skycell = "" if isnull($requested_skycell);
+
+    my @lines;
+    {
+        my $command = "$whichimage $ra $dec";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+        unless ($success) {
+            print STDERR @$stderr_buf;
+            return undef;
+        }
+
+        my $output = join "", @$stdout_buf;
+        if (!$output) {
+            print STDERR "no output returned from $command\n" if $verbose;
+            return undef;
+        }
+        @lines = split "\n", $output;
+        if (!scalar @lines) {
+            # can this happen? if $output is not null?
+            print STDERR "no output returned from $command\n" if $verbose;
+            return undef;
+        }
+    }
+    my $runs;
+    foreach my $line (@lines) {
+        my ($ra_out, $dec_out, $tess_id, $skycell_id) = split " ", $line;
+        die "unexpected output from whichimage" if !$tess_id or !$skycell_id;
+
+        if ($requested_tess_id) {
+            next if ($requested_tess_id ne $tess_id);
+        } else {
+            # skip these obsolete tesselations unless they were explicitly asked for
+            # should I do this?
+            next if $tess_id eq "FIXNS";
+            next if $tess_id eq "ALLSKY";
+        }
+        next if $requested_skycell and ($skycell_id ne $requested_skycell);
+
+        # build the hash to return
+        my $run = {
+            tess_id   => $tess_id,
+            component => $skycell_id,
+        };
+        push @$runs, $run;
+    }
+
+    return $runs;
 }
 
@@ -571 +820 @@
     my $verbose = shift;
 
+    my $mdcParser;
+
     my $exp_id = $image->{exp_id};
     if (($exp_id eq $last_exp_id) and $lastAstromFile) {
@@ -587 +838 @@
     }
 
-    my $command = "$camtool -dbname $image_db -processedexp -exp_id $exp_id";
-    # run the tool and parse the output
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-    unless ($success) {
-        print STDERR @$stderr_buf;
-        return undef;
-    }
-
-    my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
-    my $output = join "", @$stdout_buf;
-    if (!$output) {
-        print STDERR "no output returned from $command\n" if $verbose;
-        return undef;
-    }
-    my $camruns = parse_md_fast($mdcParser, $output);
-    if (!$camruns) {
-        return undef;
-    }
-
-    # If there are multiple cam runs for this exposure, take the last completed one
-    # on the assumption that it has the best astrometry.
-    my $camdata;
-    while ($camdata = pop @$camruns) {
-        last if (($camdata->{quality} eq 0) and ($camdata->{fault} eq 0));
-    }
-    if (!$camdata) {
-        # no cam runs for this exposure id therefore best astrometry is whatever is in the header
-        return undef;
-    }
-    my $camRoot = $camdata->{path_base};
+    my $camRun = getCamRunByExpID($exp_id);
+    if (!$camRun) {
+        my $command = "$camtool -dbname $image_db -processedexp -exp_id $exp_id";
+        # run the tool and parse the output
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+        unless ($success) {
+            print STDERR @$stderr_buf;
+            return undef;
+        }
+
+        $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+        my $output = join "", @$stdout_buf;
+        if (!$output) {
+            print STDERR "no output returned from $command\n" if $verbose;
+            return undef;
+        }
+        my $camruns = parse_md_fast($mdcParser, $output);
+        if (!$camruns) {
+            return undef;
+        }
+
+        # If there are multiple cam runs for this exposure, take the last completed one with good quality
+        # on the assumption that it has the best astrometry.
+        while ($camRun = pop @$camruns) {
+            last if (($camRun->{quality} eq 0) and ($camRun->{fault} eq 0));
+        }
+        # XXX: this looks like a bug at least if ASTROM.SOURCE eq PSASTRO.OUTPUT
+        if (!$camRun) {
+            # no cam runs for this exposure id therefore best astrometry is whatever is in the header
+            return undef;
+        }
+        updateCamRunCache($camRun);
+    }
+    my $camRoot = $camRun->{path_base};
     my $camera = $image->{camera};
     my $astromSource = $astromSources{$camera};
     if (!$astromSource) {
-       my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            die("Unable to perform ppConfigDump: $error_code");
-        }
-        my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-            die("Unable to parse metadata config doc");
-        $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
+        if ($camera eq "GPC1") {
+            # CHEATER !
+            $astromSource = "PSASTRO.OUTPUT";
+        } else {
+            my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                die("Unable to perform ppConfigDump: $error_code");
+            }
+            $mdcParser = PS::IPP::Metadata::Config->new if !$mdcParser; # Parser for metadata config files
+            my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+                die("Unable to parse metadata config doc");
+            $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
+        }
         $astromSources{$camera} = $astromSource;
     }
 
+    # XXX: Is this code correct if ASTROM.SOURCE ne "PSASTRO.OUTPUT"
     my $astromFile = $ipprc->filename($astromSource, $camRoot);
     if ($astromFile) {
@@ -686 +948 @@
     my $val = shift;
 
-    return (!defined($val) or (lc($val) eq "null"));
+    return (!defined($val) or ($val eq "") or (lc($val) eq "null"));
 }
 
@@ -746 +1008 @@
     return $last_project;
 }
+
+sub updateCamRunCache {
+    my $camRun = shift;
+    die "updateCamRunCache: camRun is nil" if !$camRun;
+
+    my $exp_id = $camRun->{exp_id};
+    die "updateCamRunCache: exp_id is nil" if !$exp_id;
+
+    my $cam_id = $camRun->{cam_id};
+    die "updateCamRunCache: cam_id is nil" if !$cam_id;
+
+    $camRunByCamIDCache{$cam_id} = $camRun;
+
+    # if this camRun is newer than previous update the cache.
+    # This assumes that the newest has the "best" astrometry
+    my $previous = $camRunByExpIDCache{$exp_id};
+    if ($previous) {
+        my $previous_cam_id = $previous->{cam_id};
+        return if $cam_id < $previous_cam_id;
+    }
+
+    $camRunByExpIDCache{$exp_id} = $camRun;
+}
+
+sub getCamRunByExpID {
+    my $exp_id = shift;
+    die "getCamRun: exp_id is nil" if !$exp_id;
+
+    return $camRunByExpIDCache{$exp_id};
+}
+sub getCamRunByCamID {
+    my $cam_id = shift;
+    die "getCamRun: cam_id is nil" if !$cam_id;
+
+    return $camRunByCamIDCache{$cam_id};
+}
 1;