Changeset 27839 for branches/simmosaic_branches/pstamp/scripts/pstampparse.pl

Timestamp:

May 3, 2010, 8:45:22 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/simmosaic_branches

Files:

• . (modified) (1 prop)
• pstamp/scripts/pstampparse.pl (modified) (8 diffs)

branches/simmosaic_branches/pstamp/scripts/pstampparse.pl

@@ -13 +13 @@
 use PS::IPP::PStamp::RequestFile qw( :standard );
 use PS::IPP::PStamp::Job qw( :standard );
+use File::Temp qw(tempfile);
+use File::Basename qw(basename);
+use Carp;
+use POSIX;
+use DateTime;
 
 my $verbose;
@@ -22 +27 @@
 my $out_dir;
 my $product;
+my $label;
+my $save_temps;
+my $no_update;
 
 GetOptions(
@@ -28 +36 @@
     'out_dir=s' =>  \$out_dir,
     'product=s' =>  \$product,
+    'label=s'   =>  \$label,
     'mode=s'    =>  \$mode,
     'dbname=s'  =>  \$dbname,
     'dbserver=s'=>  \$dbserver,
     'verbose'   =>  \$verbose,
+    'save-temps'=>  \$save_temps,
+    'no-update' =>  \$no_update,
 );
 
-die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "list_job") or ($mode eq "queue_job");
+die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
 die "--file is required"     if !defined($request_file_name);
 
@@ -62 +73 @@
     exit ($PS_EXIT_CONFIG_ERROR);
 }
+
+# just deal with these arguments once and for all
+$pstamptool .= " -dbname $dbname" if $dbname;
+$pstamptool .= " -dbserver $dbserver" if $dbserver;
+
+# list_job is a deugging mode
+$no_update = 1 if $mode eq "list_job";
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -82 +100 @@
 
 # make sure the file contains what we are expecting
-
-die "$request_file_name is not a PS1_PS_REQEST" if $extname ne "PS1_PS_REQUEST";
-die "REQ_NAME not found in $request_file_name"  if (!$req_name);
-die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
-
-if ($req_id) {
+# This program shouldn't have been run if the request file is bogus.
+# No need to notify the client
+my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
+my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
+my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1");
+
+
+# check for duplicate request name
+my $duplicate_req_name = 0;
+if ($req_id and !$no_update) {
+    my $command = "$pstamptool -listreq  -name $req_name -not_req_id $req_id";
+    # no verbose so that error message about request not found doesn't appear in parse_error.txt
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 0);
+    my $exitStatus = $error_code >> 8;
+    if ($success) {
+        # -listreq succeeded duplicate request name
+        print STDERR "REQ_NAME $req_name has already been used\n";
+        insertFakeJobForRow(undef, 0, $PSTAMP_DUP_REQUEST);
+        $duplicate_req_name = 1;
+        my $datestr = strftime "%Y%m%d%H%M%S.$req_id", gmtime;
+        $req_name = "ERROR.$datestr";
+        #exit 0;
+    }
+}
+
+if ($req_id and !$no_update) {
+    # update the database with the request name. This will be used as the
+    # the output data store's product name
     my $command = "$pstamptool -updatereq -req_id $req_id  -name $req_name";
     $command .= " -outProduct $product";
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
     unless ($success) {
-        die "$command failed";
-    }
+        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+    }
+}
+if ($duplicate_req_name) {
+    exit 0;
 }
 
@@ -111 +153 @@
         print STDERR @$stderr_buf;
     }
-    my $table =  $mdcParser->parse(join "", @$stdout_buf) or
-        die("Unable to parse metdata config doc");
-
-    $rows = parse_md_list($table);
-}
-
+    if (@$stdout_buf) {
+        my $table =  $mdcParser->parse(join "", @$stdout_buf) or
+            my_die("Unable to parse metdata config doc", $PS_EXIT_UNKNOWN_ERROR);
+        $rows = parse_md_list($table);
+    }
+
+}
+
+#
+# Loop over rows in the request file collecting consecutive rows that have the "same images of interest"
+# in the sense that their selection parameters will yield the same "Runs".
+# Process the groups of rows together to reduce lookup time and to potentially make multiple
+# stamps from the same ppstamp process.
+#
+my @rowList;
 my $num_jobs = 0;
+my $imageList;
+my $stage;
+my $need_magic;
 foreach my $row (@$rows) {
+    # santiy check the paramaters
+    if (!checkRow($row)) {
+        # when it enconters an error checkRow adds a fake job with an appropriate error code to the database
+        $num_jobs++;
+        next;
+    }
+    # initialize counter for "job number"
+    $row->{job_num} = 0;
+    $row->{error_code} = 0;
+
+    if (scalar @rowList == 0) {
+        push @rowList, $row;
+        next;
+    }
+
+    my $firstRow = $rowList[0];
+    if (same_images_of_interest($firstRow, $row)) {
+        # add this row to the list and move on
+        push @rowList, $row;
+        next;
+    }
+
+    # the images of interest for this new row doesn't match the list. 
+    # process the list ...
+    $num_jobs += processRows(\@rowList);
+
+    # and reset the list to contain just the new row
+    @rowList = ($row);
+}
+
+# out of rows process the list
+if (scalar @rowList > 0) {
+    $num_jobs += processRows(\@rowList);
+}
+
+if (($mode eq "queue_job") and ($num_jobs eq 0)) {
+    print STDERR "no jobs created for $req_name\n" if $verbose;
+    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+}
+
+exit 0;
+
+sub checkRow {
+        
+    my $row = shift;
+
+    # If we encounter an error for a particular row add a job with the proper fault code.
+
     my $rownum   = $row->{ROWNUM};
+    if (!validID($rownum)) {
+        print STDERR "$rownum is not a valid ROWNUM\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
     my $job_type = $row->{JOB_TYPE};
+    if (($job_type ne "stamp") and ($job_type ne "get_image")) {
+        print STDERR "$job_type is not a valid JOB_TYPE\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+    
+    my $req_type = $row->{REQ_TYPE};
+    if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and
+        ($req_type ne "byskycell") and ($req_type ne "bydiff")) {
+        print STDERR "$req_type is not a valid REQ_TYPE\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+
+
+    my $component = $row->{COMPONENT};
+    if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
+        $row->{COMPONENT} = $component = "";
+    }
+    $row->{TESS_ID} = "" if !defined $row->{TESS_ID};
+
+    my $filter  = $row->{REQFILT};
+    if ($filter) {
+        if (length($filter) == 1) {
+            # allow single character filter cuts to work
+            $row->{REQFILT} .= '%';
+        }
+    }
+    my $mjd_min = $row->{MJD_MIN};
+    if (defined($mjd_min) and !validNumber($mjd_min)) {
+        print STDERR "$mjd_min is not a valid MJD_MIN\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+    my $mjd_max = $row->{MJD_MAX};
+    if (defined($mjd_max) and !validNumber($mjd_max)) {
+        print STDERR "$mjd_max is not a valid MJD_MAX\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+    my $data_group = $row->{DATA_GROUP};
+    if (!defined $data_group) {
+        # backwards compatability hook
+        $data_group = $row->{LABEL};
+        $data_group = "null" if !defined $data_group;
+        $row->{DATA_GROUP} = $data_group;
+    }
+        
+    # req_finish doesn't work if bit zero of option mask is not set;
+    $row->{OPTION_MASK} |= 1;
+
+    my $option_mask= $row->{OPTION_MASK};
+    my $inverse = ($option_mask & $PSTAMP_SELECT_INVERSE) ? 1 : 0;
+    $row->{inverse} = $inverse;
+    my $unconvolved = ($option_mask & $PSTAMP_SELECT_UNCONV) ? 1 : 0;
+    $row->{unconvolved} = $unconvolved;
+
+    my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+
+    if (!$skycenter and !$component) {
+        print STDERR "COMPONENT must be specified for pixel coordinate ROI center\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+
+    my $stage = $row->{IMG_TYPE};
+    if (!check_image_type($stage)) {
+        print STDERR "invalid IMG_TYPE for row $rownum\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+
+    if ((($job_type eq "stamp") or ($req_type eq "bycoord")) and ! validROI($row)) {
+        print STDERR "invalid ROI for row $rownum\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+
+    if (($req_type eq "byexp") and ($stage eq "stack")) {
+        print STDERR "byexp not implemented for stack stage. row: $rownum\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
+        return 0;
+    }
+
+    # $mode list_uri is a debugging mode (it may used by the http interface)
+    # if this happens just croak
+   # my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
+
+
+    if ($req_type eq "bycoord") {
+        if (!$skycenter) {
+            print STDERR "center must be specified in sky coordintes for bycoord" if $verbose;
+            insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+            return 0;
+        }
+    }
+
+    if (($req_type eq "byid") or ($req_type eq "bydiff")) {
+        if (!validID($row->{ID})) {
+            print STDERR "ID must be a positive integer for req_type $req_type\n" if $verbose;
+            insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+            return 0
+        }
+    }
+
+    return 1;
+}
+
+sub processRows {
+    my $rowList = shift;
+    my $num_jobs = 0;
+
+    # all rows in the list are compatible
+    my $row = $rowList->[0];
+
     my $project  = $row->{PROJECT};
-    my $req_type = $row->{REQ_TYPE};
-    my $img_type = $row->{IMG_TYPE};
-    my $id       = $row->{ID};
-    my $class_id = $row->{CLASS_ID};
-    my $filter   = $row->{REQFILT};
-    my $mjd_min = $row->{MJD_MIN};
-    my $mjd_max = $row->{MJD_MAX};
+
+    # note: resolve_project avoids running pstamptool every time by remembering the
+    # last project resolved
+    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
+    if (!$proj_hash) {
+        print STDERR "project $project not found\n"  if $verbose;
+        foreach $row (@$rowList) {
+            insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT);
+            $num_jobs++;
+        }
+        return $num_jobs;
+    }
+    my $req_type  = $row->{REQ_TYPE};
+    $stage        = $row->{IMG_TYPE};
+    my $id        = $row->{ID};
+    my $component = $row->{COMPONENT};
+    my $tess_id   = $row->{TESS_ID};
+
+    my $filter    = $row->{REQFILT};
+    my $mjd_min   = $row->{MJD_MIN};
+    my $mjd_max   = $row->{MJD_MAX};
+    my $data_group = $row->{DATA_GROUP};
+
+    my $rownum     = $row->{ROWNUM};
+    my $job_type   = $row->{JOB_TYPE};
+    my $option_mask= $row->{OPTION_MASK};
+    
+    my $image_db   = $proj_hash->{dbname};
+    my $camera     = $proj_hash->{camera};
+    $need_magic    = $proj_hash->{need_magic};
+
+    # Temporary hack so that MOPS can get at non-magicked data
+    my $allow_mops_unmagicked = 1;
+    if ($allow_mops_unmagicked) {
+        if ($product and (($product eq "mops-pstamp-results") or
+                          ($product eq "mops-pstamp-results2"))) {
+            $need_magic = 0;
+        }
+    }
+    
+    my $numRows = scalar @$rowList;
+
+#    $tess_id = "" if !defined $tess_id;
+#    $component = "" if !defined $component;
+
+    print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n";
+    
+    # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
+    # request specification. An array reference is returned.
+    my $start_locate = DateTime->now->mjd;
+
+    # XXX: perhaps we should get rid of most of this argument list.
+    # Now that we are passing down compatible rows all of the
+    # information required is contained there
+
+    $imageList = locate_images($ipprc, $image_db, \@rowList, $req_type, $stage, $id, $tess_id, $component,
+                $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
+
+    # XXX: why use mjd? It doesn't have great precision.
+    my $dtime_locate = (DateTime->now->mjd - $start_locate) * 86400.;
+    print "Time to locate_images for row $rownum $dtime_locate\n";
+
+    if (!$imageList or !@$imageList) {
+        print STDERR "no matching images found for row $rownum\n" if $verbose;
+        # note in this case queueJobs inserts the fake job for these rows
+    }
+    # handle this
+    $row->{need_magic} = $need_magic;
+
+    $num_jobs += queueJobs($mode, \@rowList, $imageList);
+
+    # if a row slipped through with no jobs add one
+    foreach my $row (@rowList) {
+        if ($row->{job_num} == 0) {
+            print "row $row->{ROWNUM} produced no jobs\n";
+            print STDERR "row $row->{ROWNUM} produced no jobs\n";
+            my $error_code = $row->{error_code};
+            $error_code =  $PSTAMP_NO_IMAGE_MATCH if !$error_code;
+            insertFakeJobForRow($row, ++$row->{job_num}, $error_code);
+        }
+    }
+
+    return $num_jobs;
+}
+
+sub queueJobForImage
+{
+    my $row = shift;
+    my $stage = shift;
+    my $image = shift;
+    my $need_magic = shift;
+    my $mode = shift;
+
+    my $rownum = $row->{ROWNUM};
+    my $option_mask = $row->{OPTION_MASK};
+    my $components = $row->{components};
+
+    my $roi_string;
+
+    # note values were checked by the function validROI()
     my $x = $row->{CENTER_X};
     my $y = $row->{CENTER_Y};
@@ -134 +449 @@
     my $h = $row->{HEIGHT};
     my $coord_mask = $row->{COORD_MASK};
-    my $option_mask= $row->{OPTION_MASK};
-
-    $class_id = "" if ($class_id eq "null" or $class_id eq "all");
-    
-    # XXX: TODO: sanity check all parameters
-
-    # XXX: TODO: We shouldn't just die in this loop.
-    # If we encounter an error for a particular row
-    # add a job with the proper fault code. If there is a db or config error we should probably just
-    # trash the request.
-    die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri");
-
-    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
-    die "project $project not found\n" unless $proj_hash;
-
-    my $image_db = $proj_hash->{dbname};
-    my $camera = $proj_hash->{camera};
-    my $need_magic = $proj_hash->{need_magic};
-
-    my $roi_string;
-    # XXX we're depending on other code to insure valid values for roi components
-    # this is checked in and ppstamp but I should check here so that we don't get that far,
-    # but not today
-    if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
-        if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
-            $roi_string = "-pixcenter $x $y";
-        } else {
-            $roi_string = "-skycenter $x $y";
-        }
-        if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
-            $roi_string .= " -pixrange $w $h";
-        } else {
-            $roi_string .= " -arcrange $w $h";
-        }
-    }
-
-    die "region of interest is required to make postage stamps"
-        if (($job_type eq "stamp") && !defined($roi_string));
-
-    # find the uris for this request
-
-    if ($req_type eq "bycoord") {
-        die "region of interest is required for request type bycoord" if !defined($roi_string) ;
-        die "center must be specified in sky coordintes for bycoord" 
-            if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS);
-    }
-
-    # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this request specification
-    my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $class_id,
-            $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
-
-    if (!$images) {
-        print STDERR "no matching images found for row $rownum\n";
-        next;
-    }
+    if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
+        $roi_string = "-pixcenter $x $y";
+    } else {
+        $roi_string = "-skycenter $x $y";
+    }
+    if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
+        $roi_string .= " -pixrange $w $h";
+    } else {
+        $roi_string .= " -arcrange $w $h";
+    }
+
+    my $component = $image->{component};
+
+    my $job_num = ++($row->{job_num});
+
+    my $imagefile = $image->{image};
+    if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
+        # XXX: should we add a faulted job so the client can know what happened if no images come back?
+        # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback
+        # to users, but speeds up processing significantly
+        print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
+
+        # for now assume yes.
+
+        insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
+        return 1;
+    } elsif ($stage eq "stack") {
+        # unconvolved stack images weren't available prior to some point in time.
+        # XXX: handle this more correctly by examining the stack run's config dump file.
+        # It looks like # the feature was turned on sometime around November 11, 2009. stackRun 30067 is the lowest
+        # one that I found with an unconvolved image.
+        my $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES = 30067;
+        if ($row->{unconvolved} and ($row->{PROJECT} eq 'gpc1') and 
+            ($image->{stack_id} < $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES)) {
+            print STDERR "Unconvolved stack image is not available for stackRun.stack_id: $image->{stack_id}\n";
+            insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_AVAILABLE);
+            return 1;
+        }
+    }
+    my $exp_id = $image->{exp_id};
+            
+    my $args = $roi_string ? $roi_string : "";
+    if ($stage eq "raw" or $stage eq "chip") {
+        $args .= " -class_id $component" if $component;
+    }
+
+    # add astrometry file for raw and chip images if one is available
+    if (($stage eq "chip") || ($stage eq "raw")) {
+        $args .= " -astrom $image->{astrom}" if $image->{astrom};
+    }
+
+    $image->{job_args} = $args;
+
+    # XXX: we can get rid of the following everything that we need is
+    # in the params file
+
+    $args .= " -file $imagefile";
+
+    if (($option_mask & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
+        $args .= " -mask $image->{mask}";
+    }
+    if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
+        $args .= " -variance $image->{weight}";
+    }
+
+    my $base = basename($image->{image});
+    if (! $base =~ /.fits$/ ) {
+        my_die("unexpected image file name found $image->{image}", $PS_EXIT_PROG_ERROR);
+    }
+    $base =~ s/.fits$//;
+            
+    my $output_base = "$out_dir/${rownum}_${job_num}_${base}";
+    my $argslist = "${output_base}.args";
+
+    # copy the argument list to a file
+    open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR);
+    print ARGSLIST "$args\n";
+    close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR);
+
+    write_params($output_base, $image);
+
+    my $newState = "run";
+    my $fault = 0;
+    my $dep_id;
+
+    # XXX: this code is repeated in queueGetImageJobs we should encapsulate it in a subroutine and share it
+    if ($stage ne 'raw') {
+        # updates for stack stage not supported yet
+        my $allow_wait_for_update = ($stage ne 'stack');
+        my $run_state = $image->{state};
+        my $data_state = $image->{data_state};
+        $data_state = $run_state if $stage eq 'stack';
+        if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or
+            ($run_state eq 'drop') or 
+            ($run_state eq 'error_cleaned') or ($data_state eq 'error_cleaned') or
+            ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) {
+            # image is gone and it's not coming back
+            $newState = 'stop';
+            $fault = $PSTAMP_GONE;
+        } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) {
+            if ($stage eq 'chip') {
+                my $burntool_state = $image->{burntool_state};
+                if ($burntool_state and (abs($burntool_state) < 14)) {
+                    $newState = 'stop';
+                    $fault = $PSTAMP_NOT_AVAILABLE;
+                }
+            } 
+            if (!$allow_wait_for_update) {
+                print STDERR "wait for update not supported for stage $stage yet\n";
+                $newState = 'stop';
+                $fault = $PSTAMP_NOT_AVAILABLE;
+            }
+            if (!$fault) {
+                # wait for update unless the customer asks us not to
+                if (($option_mask & $PSTAMP_NO_WAIT_FOR_UPDATE)) {
+                    $newState = 'stop';
+                    $fault = $PSTAMP_NOT_AVAILABLE;
+                } elsif (!$image->{magicked}) {
+                    $newState = 'stop';
+                    $fault = $PSTAMP_NOT_DESTREAKED;
+                } else {
+                    # cause the image to be re-made
+                    # set up to queue an update run
+                    queue_update_run(\$newState, \$fault, \$dep_id, $image->{imagedb}, 
+                        $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic);
+                }
+            }
+        }
+    }
+
+    my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
+                    . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask";
+    $command .= " -fault $fault" if $fault;
+    $command .= " -exp_id $exp_id" if $exp_id;
+    $command .= " -dep_id $dep_id" if $dep_id;
+
+    if (!$no_update) {
+        # mode eq "queue_job"
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            print STDERR @$stderr_buf;
+            # XXX TODO: now what? Should we mark the error state for the request?
+            # should we keep going for other uris? If so how do we report that some
+            # of the work that the request wanted isn't going to get done
+            my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print "skipping command: $command\n";
+    }
+
+    return 1;
+}
+
+# queue jobs for a collection of request specifications that have the same Images of Interest
+sub queueJobs
+{
+    my $mode = shift;
+    my $rowList = shift;
+    my $imageList = shift;
+
+    my $firstRow = $rowList[0];
+    my $stage    = $firstRow->{IMG_TYPE};
+    my $job_type = $firstRow->{JOB_TYPE};
+    my $need_magic = $firstRow->{need_magic};
+
+    my $num_jobs = 0;
+
     if ($mode eq "list_uri") {
-        foreach my $image (@$images) {
+        foreach my $image (@$imageList) {
             print "$image->{image}\n";
         }
     } elsif ($job_type eq "get_image") {
-        # XXX TODO: Get rid of this block and use the same code as the stamp jobs
-        # XXX This doesn't work to get the mask and weight images
-        my $listfile = "$out_dir/filelist";
-
-
-        # map our img_type to the Data Store file types.
-        my %filelist_img_types = ( "raw" => "chip", 
-                                   "chip" => "chipproc",
-                                   "warp" => "warp", 
-                                   "stack"=>"stack",
-                                   "diff" => "diff");
-
-        my $filelist_img_type = $filelist_img_types{$img_type};
-
-        open LISTFILE, ">$listfile"
-            or die "failed to open file list: $listfile while parsing get_image request $req_id";
-
-        foreach my $image (@$images) {
-            my $class_id = $image->{class_id} ? $image->{class_id} : "";
-            print LISTFILE "$image->{image}|$filelist_img_type|$class_id|\n";
-        }
-        close LISTFILE;
+        my $n = scalar @$rowList;
+
+        my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1;
+
+        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+
+    } else {
+        if (!$imageList or (scalar @$imageList eq 0)) {
+            # We didn't find any images for this set of rows. Insert a fake job to carry
+            # the status back to the requestor.
+            foreach my $row (@$rowList) {
+                my $error_code = $row->{error_code};
+                $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code;
+                insertFakeJobForRow($row, ++$row->{job_num}, $error_code);
+                $num_jobs++;
+            }
+            return $num_jobs;
+        }
+
+        foreach my $image (@$imageList) {
+            # get the array of row indices that touch this image
+            my $row_index = $image->{row_index};
+            if (!$row_index or scalar @$row_index == 0) {
+                # XXX should this happen? Why did something get returned.
+                print "image ${stage}_id: $image->{stage_id} component: $image->{component} matched no rows\n";
+                next;
+            }
+            # XXX: TODO: eventually we may change ppstamp to be able to make multiple stamps per invocation
+
+            foreach my $i (@$row_index) {
+                my $row = $rowList->[$i];
+
+                $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $mode);
+            }
+        }
+    }
+
+    return $num_jobs;
+}
+
+#        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+sub queueGetImageJobs
+{
+    my $row = shift;
+    my $imageList = shift;
+    my $stage = shift;
+    my $need_magic = shift;
+    my $mode = shift;
+
+    my $num_jobs = 0;
+    my $rownum = $row->{ROWNUM};
+    my $option_mask = $row->{OPTION_MASK};
+
+    # For dist_bundle we need
+    #  --camera from $image
+    #  --stage 
+    #  --stage_id from $image
+    #  --component from $image
+    #  --path_base 
+    #  --outdir global to this script
+
+    # loop over images
+    foreach my $image (@$imageList) {
+        my $stage_id = $image->{stage_id};
+        my $component = $image->{component};
+
+        # skip faulted components for now. Should we even be here?
+        if ($image->{fault} > 0) {
+            printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose;
+            next;
+        }
+
+        my $job_num = ++($row->{job_num});
+
+        my $imagefile = $image->{image};
+        if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
+            # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed
+            # in locate_images because it's much more efficient to do the test in the database.
+            # For these two modes we fall through to here in order to give feedback to the requestor as
+            # to why the request failed to queue jobs.
+            print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
+            insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
+            $num_jobs++;
+
+            next;
+        }
+        my $exp_id = $image->{exp_id};
+            
+        my $output_base = "$out_dir/${rownum}_${job_num}";
+
+        write_params($output_base, $image);
+
+        my $newState = "run";
+        my $fault = 0;
+        my $dep_id;
+
+        if ($stage ne 'raw') {
+            my $run_state = $image->{state};
+            my $data_state = $image->{data_state};
+            $data_state = $run_state if $stage eq "stack";
+            if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or
+                ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) {
+                # image is gone and it's not coming back
+                $newState = 'stop';
+                $fault = $PSTAMP_GONE;
+            } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) {
+                # wait for update unless the customer asks us to not to
+                if ($option_mask & $PSTAMP_NO_WAIT_FOR_UPDATE) {
+                    $newState = 'stop';
+                    $fault = $PSTAMP_NOT_AVAILABLE;
+                } else {
+                    # cause the image to be re-made
+                    # set up to queue an update run
+                    queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, 
+                        $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic);
+                }
+            }
+        }
+
         $num_jobs++;
-        my $command = "$pstamptool -addjob -req_id $req_id -job_type get_image"
-                    . " -uri $listfile -outputBase $out_dir -rownum $rownum";
-        $command .= " -dbname $dbname" if $dbname;
-        $command .= " -dbserver $dbserver" if $dbserver;
-        if ($mode eq "list_job") {
-            print "$command\n";
-        } else {
+        my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
+                        . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask";
+        $command .= " -fault $fault" if $fault;
+        $command .= " -exp_id $exp_id" if $exp_id;
+        $command .= " -dep_id $dep_id" if $dep_id;
+
+        if (!$no_update) {
+            # mode eq "queue_job"
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                 run(command => $command, verbose => $verbose);
@@ -229 +755 @@
                 print STDERR @$stderr_buf;
                 # XXX TODO: now what? Should we mark the error state for the request?
-                die "failed to queue job for request $req_id: rownum: $rownum";
+                # should we keep going for other uris? If so how do we report that some
+                # of the work that the request wanted isn't going to get done
+                my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
             }
+        } else {
+            print "skipping command: $command\n";
+        }
+    }
+    if ( $num_jobs == 0 ) {
+        print STDERR "no jobs for row $rownum\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_NO_JOBS_QUEUED);
+        $num_jobs = 1;
+    }
+    return $num_jobs;
+}
+sub insertFakeJobForRow
+{
+    my $row = shift;
+    my $job_num = shift;
+    my $fault = shift;
+
+    my ($job_type, $rownum);
+    if ($row) {
+        $job_type = $row->{JOB_TYPE};
+        $rownum = $row->{ROWNUM};
+        $rownum = 0 if !defined $rownum;
+        if ($job_type) {
+            if (($job_type ne "stamp") and ($job_type ne "get_image")) {
+                print STDERR "invalid job type: $job_type found in row $rownum\n";
+                $job_type = "none";
+            }
+        } else {
+            print STDERR "undefined job type found in row $rownum\n";
+            $job_type = "none";
         }
     } else {
-        # sequence number for the the job for a request spec. Not to be confused with job_id
-        my $job_num = 0;
-
-        # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images
-        # use -list instead of -file
-        foreach my $image (@$images) {
-            my $uri = $image->{image};
-            if (($img_type ne "stack") and ($need_magic and !$image->{magicked})) {
-                print STDERR "skippping non-magicked image $uri\n" if $verbose;
-                next;
-            }
-            my $exp_id = $image->{exp_id};
-            
-            my $args = $roi_string ? $roi_string : "";
-            $args .= " -class_id $class_id" if $class_id;
-
-            $job_num++;
-
-            my $output_base = "$out_dir/${rownum}_${job_num}";
-
-            # add astrometry file for raw and chip images if one is available
-            if (($img_type eq "chip") || ($img_type eq "raw")) {
-                $args .= " -astrom $image->{astrom}" if $image->{astrom};
-            }
-
-            if (($option_mask & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
-                $args .= " -mask $image->{mask}";
-            }
-            if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
-                $args .= " -weight $image->{weight}";
-            }
-
-            # XXX: TODO: here is where we need to check whether or not the source inputs still exist
-            # and if not, queue an update job and set the job state appropriately.
-
-            my $newState = "run";
-
-            $num_jobs++;
-            my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type"
-                . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum"
-                . " -state $newState";
-            $command .= " -exp_id $exp_id" if $exp_id;
-            $command .= " -dbname $dbname" if $dbname;
-            $command .= " -dbserver $dbserver" if $dbserver;
-
-            if ($mode eq "list_job") { 
-                # this is a debugging mode, just print the pstamptool that would have run
-                # this is sort of like the mode -noupdate that some other tools support
-                print "$command\n";
-            } else {
-                # mode eq "queue_job"
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    print STDERR @$stderr_buf;
-                    # XXX TODO: now what? Should we mark the error state for the request?
-                    # should we keep going for other uris? If so how do we report that some
-                    # of the work that the request wanted isn't going to get done
-                    die "failed to queue job for request $req_id";
-                }
-            }
-        }
-    }
-}
-
-if ($mode eq "queue_jobs") {
-    if ($num_jobs) {
-        exit 0;
+        $job_type = "none";
+        $rownum = 0;
+    }
+
+    my $command = "$pstamptool -addjob  -req_id $req_id -job_type $job_type"
+                        . " -rownum $rownum -state stop -fault $fault";
+
+    if (!$no_update) {
+        # mode eq "queue_job"
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            print STDERR @$stderr_buf;
+            # XXX TODO: now what? Should we mark the error state for the request?
+            # should we keep going for other uris? If so how do we report that some
+            # of the work that the request wanted isn't going to get done
+            my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
+        }
     } else {
-        # no jobs queued return error to get the request stopped
-        # TODO: need to define meaningful error codes
-        exit $PS_EXIT_UNKNOWN_ERROR;
-    }
-} else {
-    exit 0;
-}
+        print "skipping command: $command\n";
+    }
+}
+
+sub same_images_of_interest {
+    my $r1 = shift;
+    my $r2 = shift;
+
+    return 0 if (($r1->{REQ_TYPE} eq "bycoord")   or ($r2->{REQ_TYPE} eq "bycoord"));
+    return 0 if (($r1->{JOB_TYPE} eq "get_image") or ($r2->{JOB_TYPE} eq "get_image"));
+    return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE});
+    return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE});
+    return 0 if ($r1->{ID}       ne $r2->{ID});
+    return 0 if ($r1->{TESS_ID}  ne $r2->{TESS_ID});
+    return 0 if ($r1->{COMPONENT}  ne $r2->{COMPONENT});
+    return 0 if ($r1->{REQFILT}  ne $r2->{REQFILT});
+    return 0 if ($r1->{DATA_GROUP}    ne $r2->{DATA_GROUP});
+    return 0 if ($r1->{MJD_MIN}  ne $r2->{MJD_MAX});
+    return 0 if ($r1->{MJD_MAX}  ne $r2->{MJD_MAX});
+    return 0 if ($r1->{OPTION_MASK}  ne $r2->{OPTION_MASK});
+    return 0 if ($r1->{PROJECT}  ne $r2->{PROJECT});
+    return 0 if ($r1->{inverse}  ne $r2->{inverse});
+    return 0 if ($r1->{unconvolved}  ne $r2->{unconvolved});
+    # don't combine requests in pixel coordinates
+    return 0 if (($r1->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS) || ($r2->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS));
+
+    return 1;
+}
+
+sub validNumber
+{
+    my $val = shift;
+
+    return 0 if !defined $val;
+
+    return ($val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/);
+}
+sub validID
+{
+    my $val = shift;
+
+    return 0 if !$val;
+
+    return  ! ($val =~ /\D/);
+}
+
+sub validROI
+{
+    my $row = shift;
+    return 0 if !validNumber($row->{CENTER_X});
+    return 0 if !validNumber($row->{CENTER_Y});
+    return 0 if !validNumber($row->{WIDTH});
+    return 0 if !validNumber($row->{HEIGHT});
+
+    return 1;
+}
+
+sub findRow
+{
+    my $rownum = shift;
+    my $rowList = shift;
+
+    foreach my $row (@$rowList) {
+        return $row if $row->{ROWNUM} eq $rownum;
+    }
+    return undef;
+}
+
+sub queue_update_run 
+{
+    my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
+
+    if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) {
+        my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR);
+    }
+
+    my $dep_id;
+    my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component";
+    $command .= " -need_magic" if $need_magic;
+
+    # compute rlabel for the run.
+    # XXX: This bit of policy shouldn't be buried so deeply in the code
+    # For now use one that implies 'postage stamp server' 'update' 'request_label"
+    my $rlabel = "ps_ud_" . $label if $label;
+    $command .= " -rlabel $rlabel" if $rlabel;
+
+    if (!$no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+        my $output = join "", @$stdout_buf;
+        chomp $output;
+        $dep_id = $output;
+        #
+        # XXX: need to fault the request or something
+        my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id;
+    } else {
+        print STDERR "skipping $command\n";
+        $dep_id = 42;
+    }
+    
+    $$r_dep_id = $dep_id;
+    $$r_fault = 0;
+    $$r_jobState = 'run';
+}
+
+sub write_params {
+    my $output_base = shift;
+    my $image = shift;
+
+    # write the contents of this "image" as a metadata config doc
+    # Simply treat all values as strings. This is ok since we are only going to
+    # read it from another perl script
+    my $mdc_file = "${output_base}.mdc";
+    open P, ">$mdc_file" or my_die("failed to open $mdc_file", $PS_EXIT_UNKNOWN_ERROR);
+
+    print P "params METADATA\n";
+
+    foreach my $key (keys %$image) {
+        my $value = $image->{$key};
+        if (defined $value) {
+            printf P "  %-20s STR     %s\n", $key, $value;
+        } else {
+            printf P "  %-20s STR     NULL\n", $key;
+        }
+    }
+
+    print P "END\n";
+    close P or my_die("failed to close $mdc_file", $PS_EXIT_UNKNOWN_ERROR);
+}
+
+sub check_image_type
+{
+    my $img_type = shift;
+    if (!$img_type) {
+            print STDERR "NULL IMG_TYPE supplied\n";
+            return 0;
+    }
+    if (($img_type eq "raw") or ($img_type eq "chip") or ($img_type eq "warp") or
+        ($img_type eq "stack") or ($img_type eq "diff")) {
+        return 1;
+    } else {
+        print STDERR "$img_type is not a valid IMG_TYPE\n";
+        return 0;
+    }
+}
+
+sub my_die
+{
+    my $msg = shift;
+    my $fault = shift;
+
+    carp $msg;
+
+    # we don't fault the request here pstamp_parser_run.pl handles that if necessary
+
+    exit $fault;
+}