Index: trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm
===================================================================
--- trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm	(revision 27589)
+++ trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm	(revision 27751)
@@ -27,4 +27,10 @@
 use PS::IPP::Config qw( :standard );
 use Carp;
+use DateTime;
+use File::Temp qw(tempfile);
+use File::Basename;
+use POSIX;
+
+my $save_temps = 0;
 
 # caches of camProcessedExp objects. 
@@ -34,10 +40,14 @@
 my %camRunByCamIDCache;
 
-### my @images = locate_images($image_db, $req_type, $img_type, $id, $component,
-###            $mjd_min, $mjd_max, $filter);
+# cache of last project looked up
+my $last_project = "";
+
+my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool);
+my ($pstamptool, $dvoImagesAtCoords, $whichimage, $ppConfigDump);
 
 sub locate_images {
     my $ipprc    = shift;   # required
-    my $image_db = shift;   # required
+    my $imagedb = shift;   # required
+    my $rowList  = shift;   # required
     my $req_type = shift;   # required
     my $img_type = shift;   # required
@@ -47,6 +57,4 @@
     my $option_mask  = shift;
     my $need_magic = shift;
-    my $x        = shift;
-    my $y        = shift;
     my $mjd_min  = shift;
     my $mjd_max  = shift;
@@ -56,6 +64,7 @@
 
     # we die in response to bad data in request files
-    # The wrapper script is responsible for updating the database
-    die "Unknown req_type: $req_type" 
+    # The caller is responsible for updating the database
+    # pstampparse.pl error checks now so this shouldn't happen
+    &my_die("Unknown req_type: $req_type", $PS_EXIT_PROG_ERROR) 
         if ($req_type ne "byid") and
            ($req_type ne "byexp") and 
@@ -84,5 +93,10 @@
 
     if ($req_type eq "bycoord") {
-        my $results = lookup_bycoord($ipprc, $image_db, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
+        my $num_rows = scalar @$rowList;
+        die "Unexpected number of rows found in rowList: $num_rows" if $num_rows != 1;
+        my $row = $rowList->[0];
+        my $x = $row->{CENTER_X};
+        my $y = $row->{CENTER_Y};
+        my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
         return $results;
     }
@@ -93,10 +107,12 @@
         # in one place
         $req_type = "bydiff";
-        my $results = lookup_diff($ipprc, $image_db, $id, $component, 1, $option_mask, $img_type, $verbose);
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $img_type, $verbose);
         return $results;
     }
 
     if ($req_type eq "bydiff") {
-        my $results = lookup_diff($ipprc, $image_db, $id, $component, 0, $option_mask, $img_type, $verbose);
+        # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or 
+        # the image from the diffRun
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $img_type, $verbose);
         if (!$results) {
             return undef;
@@ -130,5 +146,5 @@
             # fall though and lookup by stack_id
         } else {
-            # shouldn't I check this elsewhere?
+            # This is checked this elsewhere?
             print STDERR "Error: $img_type is an unknown image type\n";
             return undef;
@@ -145,16 +161,20 @@
     }
 
-    my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic,
-        $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
+    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component,
+        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
 
     return $results;
 }
+
+# The subroutine lookup handles byexp, byid, and byskycell lookups including lookups that are
+# triggered by a bydiff request
 
 sub lookup
 {
     my $ipprc    = shift;
-    my $image_db = shift;
+    my $rowList    = shift;
+    my $imagedb = shift;
     my $req_type = shift;
-    my $img_type = shift;
+    my $stage = shift;
     my $id       = shift;
     my $tess_id  = shift;
@@ -172,14 +192,4 @@
     my $use_imfile_id = ($req_type eq "byid") && ($option_mask & $PSTAMP_USE_IMFILE_ID);
 
-    my $missing_tools;
-    my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
-    my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
-    my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
-    my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
-    my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
     my $command;
     my $id_opt;     # option for the lookup
@@ -203,8 +213,12 @@
     my $magic_arg = $need_magic ? " -destreaked" : "";
 
+    # all rows have the same center type
+    my $skycenter = ! ($rowList->[0]->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+
     my $component_args;
-    if ($img_type eq "raw") {
+    my $choose_components = 0;
+    if ($stage eq "raw") {
         $class_id = $component;
-        $command = "$regtool -processedimfile -dbname $image_db";
+        $command = "$regtool -processedimfile -dbname $imagedb";
         # XXX: for now restrict lookups to type object 
         # are stamps of detrend exposures interesting?
@@ -214,9 +228,20 @@
         $want_astrom = 1;
         $set_class_id = 1;
-    } elsif ($img_type eq "chip") {
+    } elsif ($stage eq "chip") {
         $class_id = $component;
-        $command = "$chiptool -processedimfile -dbname $image_db";
-        $component_args = " -class_id $class_id" if $class_id;
+        # if the request is such that it will yield a single image per "run" or the
+        # center is specified in pixel coordinates 
+        # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then
+        # choose the chips containing the center pixel by calling selectComponets() below
+        if ($class_id or $use_imfile_id or !$skycenter) {
+            $command = "$chiptool -processedimfile -dbname $imagedb";
+            $component_args = " -class_id $class_id" if $class_id;
+        } else {
+            $command = "$chiptool -listrun -pstamp_order -dbname $imagedb";
+            $choose_components = 1;
+        }
         $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id";
+        # XXX: perhaps we should stop resolving images to this level here and do it at job run time.
+        # With the mdc file it has all of the information that it needs.
         $image_name   = "PPIMAGE.CHIP";
         $mask_name    = "PPIMAGE.CHIP.MASK";
@@ -228,8 +253,13 @@
         $want_astrom  = 1;
         $set_class_id = 1;
-    } elsif ($img_type eq "warp") {
+    } elsif ($stage eq "warp") {
         $skycell_id = $component;
-        $command = "$warptool -warped -dbname $image_db";
-        $component_args = " -skycell_id $skycell_id" if $skycell_id;
+        if ($skycell_id or $use_imfile_id or !$skycenter) {
+            $command = "$warptool -warped -dbname $imagedb";
+            $component_args = " -skycell_id $skycell_id" if $skycell_id;
+        } else {
+            $command = "$warptool -listrun -pstamp_order -dbname $imagedb";
+            $choose_components = 1;
+        }
         $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id";
         $image_name   = "PSWARP.OUTPUT";
@@ -239,8 +269,13 @@
         $psf_name     = "PSPHOT.PSF.SKY.SAVE";
         $base_name    = "path_base"; # name of the field for the warptool output
-    } elsif ($img_type eq "diff") {
+    } elsif ($stage eq "diff") {
         $skycell_id = $component;
-        $command = "$difftool -diffskyfile -dbname $image_db";
-        $component_args = " -skycell_id $skycell_id" if $skycell_id;
+        if ($skycell_id or $use_imfile_id or !$skycenter) {
+            $command = "$difftool -diffskyfile -dbname $imagedb";
+            $component_args = " -skycell_id $skycell_id" if $skycell_id;
+        } else {
+            $command = "$difftool -listrun -pstamp_order -dbname $imagedb";
+            $choose_components = 1;
+        }
         $id_opt = $use_imfile_id ? "-diff_skyfile_id" : "-diff_id";
         $image_name  = "PPSUB.OUTPUT";
@@ -250,7 +285,7 @@
         $psf_name     = "PSPHOT.PSF.SKY.SAVE";
         $base_name   = "path_base";
-    } elsif ($img_type eq "stack") {
+    } elsif ($stage eq "stack") {
         $skycell_id = $component;
-        $command = "$stacktool -sumskyfile -dbname $image_db";
+        $command = "$stacktool -sumskyfile -dbname $imagedb";
         $id_opt = "-stack_id";
         $component_args = " -skycell_id $skycell_id" if $skycell_id;
@@ -272,7 +307,8 @@
         $base_name   = "path_base";
     } else {
-        die "Unknown img_type supplied: $img_type";
-    }
-
+        die "Unknown IMG_TYPe supplied: $stage";
+    }
+
+    my $filter_runs = 0;
     if ($req_type eq "byid") {
         $command .= " $id_opt $id";
@@ -288,12 +324,15 @@
         # the reason as 'not destreaked'
         $magic_arg = "";
+        # remove duplicate runs for the same exposure.
+        $filter_runs = 1;
     } elsif ($req_type eq "byskycell") {
         die "tess_id and component are required for byskycell" if !$tess_id or ! $skycell_id;
         $command .= " -tess_id $tess_id -skycell_id $skycell_id";
     } else {
-        die "Unknown req_type supplied: $req_type";
-    }
-
-    if ($img_type ne "stack") {
+        # this should be caught by caller
+        &my_die("Unexpected req_type supplied: $req_type", $PS_EXIT_PROG_ERROR);
+    }
+
+    if ($stage ne "stack") {
         $command .= $magic_arg;
         $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
@@ -304,30 +343,32 @@
     $command .= " -data_group $data_group" if !isnull($data_group);
 
-    # run the tool and parse the output
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-    unless ($success) {
-        # not sure if we should die here
-        print STDERR @$stderr_buf;
+    my $images = runToolAndParse($command, $verbose);
+
+    if (!$images or scalar @$images == 0) {
         return undef;
     }
 
-    my $buf = join "", @$stdout_buf;
-    if (!$buf) {
-        return;
-    }
-
-    my $mdcParser = PS::IPP::Metadata::Config->new;
-    my $images = parse_md_fast($mdcParser, $buf)
-        or die ("Unable to parse metadata config doc");
-
+    if ($filter_runs) {
+        # The image selectors are such that multiple runs my have be returned for the same exposure.
+        # Return only the latest one.
+        $images = filterRuns($stage, $need_magic, $images, $verbose);
+    }
+    if ($choose_components) {
+        # the list of "images" is actually a list of "Runs"
+        # match the coords in the rows to the components in the runs
+        # returns an actual list of images
+        $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
+    } else {
+        # put each of the rows into all of the the images
+        setRowRefs($rowList, $images);
+    }
+
+    my $camera;
     my $output = [];
-
-    my $camera;
     foreach my $image (@$images) {
         my $base;
 
         next if $image->{fault};
-        next if ($img_type ne "raw") and $image->{quality};
+        next if ($stage ne "raw") and $image->{quality};
 
         if ($base_name) {
@@ -361,11 +402,11 @@
         }
         my $stage_id;
-        if ($img_type eq "raw") {
+        if ($stage eq "raw") {
             $stage_id = $image->{exp_id};
-        } elsif ($img_type eq "chip") {
+        } elsif ($stage eq "chip") {
             $stage_id = $image->{chip_id};
-        } elsif ($img_type eq "warp") {
+        } elsif ($stage eq "warp") {
             $stage_id = $image->{warp_id};
-        } elsif ($img_type eq "diff") {
+        } elsif ($stage eq "diff") {
             $stage_id = $image->{diff_id};
             if ($inverse && $image->{bothways}) {
@@ -375,10 +416,10 @@
                 $cmf_name    = "PPSUB.INVERSE.SOURCES";
             }
-        } elsif ($img_type eq "stack") {
+        } elsif ($stage eq "stack") {
             $stage_id = $image->{stack_id};
         }
         $out->{stage_id} = $stage_id;
-        $out->{stage}    = $img_type;
-        $out->{image_db} = $image_db;
+        $out->{stage}    = $stage;
+        $out->{imagedb} = $imagedb;
 
         if ($base) {
@@ -390,5 +431,7 @@
             $out->{backmdl}= $ipprc->filename($backmdl_name, $base, $class_id) if $backmdl_name;
         }
-        $out->{astrom} = find_astrometry($ipprc, $image_db, $image, $verbose) if $want_astrom;
+        if ($want_astrom and ! defined $out->{astrom}) {
+            $out->{astrom} = find_astrometry($ipprc, $imagedb, $image, $verbose);
+        }
 
         push @$output, $out;
@@ -397,7 +440,9 @@
     return $output;
 }
+
 sub lookup_diff {
     my $ipprc    = shift;
-    my $image_db = shift;
+    my $rowList  = shift;
+    my $imagedb = shift;
     my $id       = shift;
     my $skycell_id = shift;
@@ -409,12 +454,5 @@
     my $inverse = $option_mask & $PSTAMP_SELECT_INVERSE;
 
-    my $missing_tools;
-    my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
-
-    my $command = "$difftool -diffskyfile -dbname $image_db";
+    my $command = "$difftool -diffskyfile -dbname $imagedb";
     
     if ($byid) {
@@ -425,21 +463,5 @@
     $command .= " -skycell_id $skycell_id" if $skycell_id;
 
-    # run the tool and parse the output
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-    unless ($success) {
-        # not sure if we should die here
-        print STDERR @$stderr_buf;
-        return undef;
-    }
-
-    my $buf = join "", @$stdout_buf;
-    if (!$buf) {
-        return undef;
-    }
-
-    my $mdcParser = PS::IPP::Metadata::Config->new;
-    my $images = parse_md_fast($mdcParser, $buf)
-        or die ("Unable to parse metadata config doc");
+    my $images = runToolAndParse($command, $verbose);
 
     my $n = @$images;
@@ -527,4 +549,7 @@
 
         if ($img_type eq "diff") {
+            my @imageList = ($image);
+
+            setRowRefs($rowList, \@imageList);
             # the $image is going to be returned directly in this case so we need to duplicate
             # some of processing that lookup does for other img_types
@@ -539,5 +564,5 @@
                 $image->{stage_id} = $image->{diff_id};
                 $image->{stage}    = "diff";
-                $image->{image_db} = $image_db;
+                $image->{imagedb} = $imagedb;
                 $image->{component} = $image->{skycell_id};
             } else {
@@ -554,5 +579,6 @@
 sub lookup_bycoord {
     my $ipprc      = shift;
-    my $image_db   = shift;
+    my $rowList    = shift;
+    my $imagedb    = shift;
     my $img_type   = shift;
     my $tess_id    = shift;
@@ -568,23 +594,15 @@
     my $verbose    = shift;
 
-    my $missing_tools;
-    my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
-    my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
-    my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
-    my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
-
     my $results = ();
     if (($img_type eq "raw") or ($img_type eq "chip")) {
 
-        my $runs = lookup_runs_by_camid_and_coords($ipprc, $image_db, $img_type,
+        my $runs = lookup_runs_by_cam_id_and_coords($ipprc, $imagedb, $img_type,
             $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose);
 
+        # lookup is going to filter these we don't need to do it here
+#        $runs = filterRuns($img_type, $need_magic, $runs, $verbose);
         foreach my $run (@$runs) {
             next if $component and ($run->{component} ne $component);
-            my $these_results = lookup($ipprc, $image_db, "byid", $img_type, $run->{id},
+            my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $run->{id},
                 $tess_id, $run->{component}, $need_magic, 
                 $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
@@ -602,5 +620,5 @@
 
         foreach my $skycell (@$skycells) {
-            my $these_results = lookup($ipprc, $image_db, "byskycell", $img_type, undef,
+            my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef,
                 $skycell->{tess_id}, $skycell->{component}, $need_magic, 
                 $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
@@ -614,10 +632,10 @@
 }
 
-# lookup_runs_by_camid_and_coords
+# lookup_runs_by_cam_id_and_coords
 # given an ra, dec, and optionally other paramters, find camera runs for exposures
 # that are within some distance of the coordinates
-sub lookup_runs_by_camid_and_coords {
+sub lookup_runs_by_cam_id_and_coords {
     my $ipprc      = shift;
-    my $image_db   = shift;
+    my $imagedb   = shift;
     my $img_type   = shift;
     my $ra         = shift;
@@ -631,14 +649,7 @@
 
     my $camruns;
-    my $missing_tools;
-    my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
-    my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
 
     {
-        my $command = "$camtool -dbname $image_db -processedexp";
+        my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order";
            $command .= " -ra $ra -decl $dec -radius 1.6";
            $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
@@ -652,19 +663,7 @@
            $command .= " -data_group $data_group" if $data_group;
            $command .= " -destreaked" if $need_magic;
+
         # run the tool and parse the output
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-        unless ($success) {
-            print STDERR @$stderr_buf;
-            return undef;
-        }
-        my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
-        my $output = join "", @$stdout_buf;
-        if (!$output) {
-            print STDERR "no output returned from $command\n" if $verbose;
-            return undef;
-        }
-        $camruns = parse_md_fast($mdcParser, $output);
+        $camruns = runToolAndParse($command, $verbose);
     }
     if (!$camruns) {
@@ -672,11 +671,18 @@
     }
     my $runs;
+    my $last_exp_id = 0;
     foreach my $camRun (@$camruns) {
         my $cam_id = $camRun->{cam_id};
-
-        updateCamRunCache($camRun);
+        my $exp_id = $camRun->{exp_id};
+
+        next if $exp_id eq $last_exp_id;
 
         next if $camRun->{quality};
         next if $camRun->{fault};
+
+        updateCamRunCache($camRun);
+
+        $last_exp_id = $exp_id;
+
         # XXX Use file rule
         my $astrom = $camRun->{path_base} . ".smf";
@@ -684,12 +690,16 @@
         next if !$astrom_resolved;
 
+        my $start_dvo = DateTime->now->mjd;
         my $command = "$dvoImagesAtCoords -astrom $astrom_resolved $ra $dec";
         # run the tool and parse the output
+        my $dvo_verbose = 0;
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
+                    run(command => $command, verbose => $dvo_verbose);
+        my $dtime_dvo = (DateTime->now->mjd - $start_dvo)* 86400.;
+#        print "Time to run dvoImagesAtCoords: $dtime_dvo\n";
         unless ($success) {
             my $result_code = $error_code >> 8;
             if ($result_code == $PSTAMP_NO_OVERLAP) {
-                print STDERR "no overlap for $astrom\n" if $verbose;
+                print "no overlap for $astrom\n" if $verbose;
                 next;
             }
@@ -721,4 +731,7 @@
             die "unexpected output from dvoImagesAtCoords: $lines[0]";
         }
+
+        print "cam_id $cam_id exp_id $camRun->{exp_id} chip_id $camRun->{chip_id} $class_id\n";
+
         # build the hash to return
         my $run = {
@@ -732,7 +745,11 @@
         };
         if ($img_type eq "chip") {
-            $run->{id} = $run->{chip_id};
+            $run->{id} = $camRun->{chip_id};
+            $run->{state} = $camRun->{chip_state};
+            $run->{magicked} = $camRun->{chip_magicked};
         } else {
-            $run->{id} = $run->{exp_id};
+            $run->{id} = $camRun->{exp_id};
+            $run->{state} = 'full';
+            $run->{magicked} = $camRun->{raw_magicked};
         }
         push @$runs, $run;
@@ -749,11 +766,4 @@
     my $dec        = shift;
     my $verbose    = shift;
-
-    my $missing_tools;
-    my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
 
     $requested_tess_id = "" if isnull($requested_tess_id);
@@ -817,5 +827,5 @@
 sub find_astrometry {
     my $ipprc = shift;
-    my $image_db = shift;
+    my $imagedb = shift;
     my $image = shift;      # hashref to output of the lookup tool
     my $verbose = shift;
@@ -825,5 +835,5 @@
     my $exp_id = $image->{exp_id};
     if (($exp_id eq $last_exp_id) and $lastAstromFile) {
-        # running camtool 60 times is really expensive when the answer is the same
+        # running camtool 60 times is really expensive when the answer is the same every time
         return $lastAstromFile;
     }
@@ -831,15 +841,7 @@
     $lastAstromFile = undef;
 
-    my $missing_tools;
-    my $camtool = can_run("camtool") or (warn "Can't find camtool" and $missing_tools = 1);
-    my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
-
     my $camRun = getCamRunByExpID($exp_id);
     if (!$camRun) {
-        my $command = "$camtool -dbname $image_db -processedexp -exp_id $exp_id";
+        my $command = "$camtool -dbname $imagedb -processedexp -exp_id $exp_id -pstamp_order";
         # run the tool and parse the output
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -864,6 +866,6 @@
         # If there are multiple cam runs for this exposure, take the last completed one with good quality
         # on the assumption that it has the best astrometry.
-        while ($camRun = pop @$camruns) {
-            last if (($camRun->{quality} eq 0) and ($camRun->{fault} eq 0));
+        foreach $camRun (@$camruns) {
+            last if (($camRun->{state} eq 'full') and ($camRun->{quality} eq 0) and ($camRun->{fault} eq 0));
         }
         # XXX: this looks like a bug at least if ASTROM.SOURCE eq PSASTRO.OUTPUT
@@ -959,9 +961,10 @@
 # resolve_project()
 # get project specific information
-my $last_project = "";
 sub resolve_project {
     my $ipprc = shift;
     my $project_name = shift;
 
+    findTools();
+
     if (!$project_name) {
         carp ("project is not defined");
@@ -978,30 +981,13 @@
     my $verbose = 0;
 
-    my $missing_tools;
-    my $pstamptool = can_run("pstamptool") or (warn "Can't find pstamptool" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
-
     my $command = "$pstamptool -project -name $project_name";
     $command .= " -dbname $dbname" if defined $dbname;
     $command .= " -dbserver $dbserver" if defined $dbserver;
-    # run the tool and parse the output
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-    unless ($success) {
-        print STDERR @$stderr_buf;
+
+    my $proj_hash = runToolAndParse($command, $verbose);
+    if (!$proj_hash or scalar @$proj_hash == 0) {
+        print STDERR "Project $project_name not found\n";
         return undef;
     }
-
-    my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
-    my $output = join "", @$stdout_buf;
-    if (!$output) {
-        print STDERR "no output returned from $command\n" if $verbose;
-        return undef;
-    }
-    my $proj_hash = parse_md_fast($mdcParser, $output);
 
     $last_project = $proj_hash->[0];
@@ -1035,13 +1021,280 @@
 sub getCamRunByExpID {
     my $exp_id = shift;
-    die "getCamRun: exp_id is nil" if !$exp_id;
+    &my_die ("getCamRun: exp_id is nil", $PS_EXIT_PROG_ERROR) if !$exp_id;
 
     return $camRunByExpIDCache{$exp_id};
 }
+
 sub getCamRunByCamID {
     my $cam_id = shift;
-    die "getCamRun: cam_id is nil" if !$cam_id;
+    &my_die ("getCamRun: cam_id is nil", $PS_EXIT_PROG_ERROR) if !$cam_id;
 
     return $camRunByCamIDCache{$cam_id};
 }
+
+sub selectComponents {
+    my $ipprc = shift;
+    my $imagedb = shift;
+    my $req_type = shift;
+    my $stage  = shift;
+    my $rowList = shift;
+    my $runList = shift;
+    my $verbose = shift;
+    my $results = [];
+    
+    my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
+    my $npoints = 0;
+    foreach my $row (@$rowList) {
+        print $pointsList "$npoints $row->{CENTER_X} $row->{CENTER_Y}\n";
+        $npoints++;
+    }
+    close $pointsList;
+
+    # XXX: need to loop over these
+    # my $run = $runList->[0];
+    if (($req_type eq "byid") or ($req_type eq "byexp")) {
+        my ($last_tess_id, $tess_dir_abs, $astrom_file) = ("", "", "");
+        foreach my $run (@$runList) {
+            my $command = "$dvoImagesAtCoords -coords $pointsListName";
+            if (($stage eq "chip") or ($stage eq "raw")) {
+                # XXX: use file rule and handle cameras where the astrometry is solved at
+                # the chip stage.
+                $astrom_file = $run->{cam_path_base} . ".smf";
+                my $astrom_file_resolved = $ipprc->file_resolve($astrom_file);
+                $command .= " -astrom $astrom_file_resolved";
+            } else {
+                my $tess_id = $run->{tess_id};
+                if ($tess_id ne $last_tess_id) {
+                    $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id );
+                    $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs );
+                    $last_tess_id = $tess_id;
+                }
+                $command .= " -D CATDIR $tess_dir_abs"
+            }
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                 run(command => $command, verbose => $verbose);
+            unless ($success) {
+                # don't fail if the program exited normally and exit status was PSTAMP_NO_OVERLAP
+                #                     # That just means that the coordinate didn't match any image/skycell
+                if (!WIFEXITED($error_code) || (WEXITSTATUS($error_code) ne $PSTAMP_NO_OVERLAP)) {
+                    print STDERR @$stderr_buf;
+                    my $rc = WIFEXITED($error_code) ? WEXITSTATUS($error_code) : $PS_EXIT_SYS_ERROR;
+                    &my_die( "dvoImagesAtCoords failed: $rc", $rc);
+                }
+            }
+            my %components;
+            my @lines = split "\n", join "", @$stdout_buf;
+            foreach my $line (@lines) {
+                my ($ptnum, undef, undef, $component) = split " ", $line;
+                my $ref = $components{$component};
+                if (!$ref) {
+                    $ref = $components{$component} = [];
+                }
+                push @$ref, $ptnum;
+            }
+            if ($verbose) {
+                foreach my $c (keys %components) {
+                    my $ref = $components{$c};
+                    print "component $c contains: ";
+                    foreach my $i (@$ref) {
+                        print "$i ";
+                    }
+                    print "\n";
+                }
+            }
+            # now find the images for this run
+            foreach my $c (keys %components) {
+                my $command;
+                my $stage_id;
+                if ($stage eq 'raw') {
+                    $stage_id = $run->{exp_id};
+                    $command = "$regtool -processedimfile -exp_id $stage_id -class_id $c";
+                } elsif ($stage eq 'chip') {
+                    $stage_id = $run->{chip_id};
+                    $command = "$chiptool -processedimfile -chip_id $stage_id -class_id $c";
+                } elsif ($stage eq 'warp') {
+                    $stage_id = $run->{warp_id};
+                    $command = "$warptool -warped -warp_id $stage_id -skycell_id $c";
+                } elsif ($stage eq 'diff') {
+                    $stage_id = $run->{diff_id};
+                    $command = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $c";
+                }
+                $command .= " -dbname $imagedb";
+                my $images = runToolAndParse($command, $verbose);
+                if (!defined $images) {
+                    # XXX: need to get the set error_code to $PSTAMP_NO_OVERLAP
+                    print "No components containing coordinates found for ${stage}_id $stage_id\n";
+                    foreach my $row (@$rowList) {
+                        $row->{error_code} = $PSTAMP_NO_OVERLAP;
+                    }
+                    next;
+                }
+                if (scalar @$images != 1) {
+                    my $num_images = scalar @$images;
+                    &my_die ("unexpected number of images returned: $num_images\n", $PS_EXIT_PROG_ERROR);
+                }
+                my $image = $images->[0];
+                    
+                my $ref = $components{$c};
+                $image->{row_index} = $ref;
+                if (($stage eq "raw") or ($stage eq 'chip')) {
+                    $image->{astrom} = $astrom_file;
+                    $image->{cam_id} = $run->{cam_id};
+                    $image->{cam_path_base} = $run->{cam_path_base};
+                }
+                push @$results, $image;
+            }
+        }
+    } else {
+        &my_die ("sorry not done with $req_type\n", $PS_EXIT_PROG_ERROR);
+    }
+    return $results;
+}
+
+sub filterRuns {
+    my $stage      = shift;
+    my $need_magic = shift;
+    my $inputs     = shift;
+    my $verbose    = shift;
+
+    if ($inputs and (scalar @$inputs) == 1)  {
+        # one run nothing to do
+        return $inputs;
+    }
+
+    my $output = [];
+
+    return $output if (!$inputs or scalar @$inputs == 0);
+
+    my $id_name = $stage . "_id";
+
+    # input list is "order by exp_id, run_id DESC"   run_id is one of (chip_id, warp_id, diff_id)
+    print "Starting filterRuns\n";
+    my $last_exp_id = 0;
+    my $last_run_id = 0;
+    my $printed = 0;
+    foreach my $input (@$inputs) {
+        my $exp_id = $input->{exp_id};
+        my $run_id = $input->{$id_name};
+        my $magicked = $input->{magicked};  # this will be either stageRun.magicked or stage%file.magicked
+        my $state = $input->{state};
+
+        # can't process run in these states
+        if (($state eq 'new') or ($state eq 'purged') or ($state eq 'scrubbed')) {
+            print "skipping ${stage}Run $run_id for exp_id $exp_id in state $state\n";
+            next;
+        }
+
+        $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id));
+
+        # skip if we need magicked run and this one has never been magicked
+        if ($need_magic and !$magicked) {
+            print "skipping ${stage}Run $run_id for exp_id $exp_id not magicked\n" if !$printed;
+            next;
+        }
+
+        if (($exp_id == $last_exp_id) and ($run_id != $last_run_id)) {
+            print "Skipping duplicate ${stage}Run $run_id for $exp_id\n" if !$printed;
+            next;
+        }
+        print "Keeping ${stage}Run $run_id for $exp_id\n";
+        push @$output, $input;
+        $last_exp_id = $exp_id;
+        $last_run_id = $run_id;
+        $printed = 0;
+    }
+
+    my $num_runs = scalar @$output;
+    print "filterRuns returning $num_runs ${stage}Run\n";
+
+    return $output;
+}
+
+# run a command that produces metadata output and parse the results into an array of objects
+sub runToolAndParse {
+    my $command = shift;
+    my $verbose = shift;
+
+    my ($program) = split " ", $command;
+    $program = basename($program);
+
+    print "Running $command\n" if !$verbose;
+    my $start_tool = DateTime->now->mjd;
+    # run the command and parse the output
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+    unless ($success) {
+        # not sure if we should die here
+        print STDERR @$stderr_buf;
+        return undef;
+    }
+
+    my $now = DateTime->now->mjd;
+    my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.;
+    print "Time to run $program: $dtime_tool\n";
+
+    my $buf = join "", @$stdout_buf;
+    if (!$buf) {
+        return undef;
+    }
+
+    my $start_parse = DateTime->now->mjd;
+
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $results = parse_md_fast($mdcParser, $buf)
+        or die ("Unable to parse metadata config doc");
+
+    my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.;
+    print "Time to parse results from $program: $dtime_parse\n";
+
+    return $results;
+}
+
+sub findTools {
+    return if ($regtool);
+
+    my $missing_tools;
+    $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
+    $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
+    $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+    $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
+    $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+    $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
+    $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
+    $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" 
+                                                                and $missing_tools = 1);
+    $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
+    $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+    if ($missing_tools) {
+        warn("Can't find required tools.");
+        exit ($PS_EXIT_CONFIG_ERROR);
+    }
+}
+
+# add a row_index array to a set of images with entries for each of the rows in a list
+sub setRowRefs {
+    my $rowList = shift;
+    my $images  = shift;
+
+    my $row_index = [];
+    for (my $i = 0; $i < scalar @$rowList; $i++) {
+        push @$row_index, $i;
+    }
+    foreach my $image (@$images) {
+        $image->{row_index} = $row_index;
+    }
+}
+
+sub my_die
+{
+    my $msg = shift;
+    my $fault = shift;
+
+    carp $msg;
+
+    # we don't fault the request here pstamp_parser_run.pl handles that if necessary
+
+    return $fault;
+}
 1;