Changeset 25406 for branches/eam_branches/20090715/pstamp/scripts/pstampparse.pl

Timestamp:

Sep 15, 2009, 4:02:42 PM (17 years ago)

Author:

eugene

Message:

updates from trunk

File:

• branches/eam_branches/20090715/pstamp/scripts/pstampparse.pl (modified) (24 diffs)

branches/eam_branches/20090715/pstamp/scripts/pstampparse.pl

@@ -14 +14 @@
 use PS::IPP::PStamp::Job qw( :standard );
 use File::Temp qw(tempfile);
+use File::Basename qw(basename);
+use Carp;
 
 my $verbose;
@@ -91 +93 @@
 # make sure the file contains what we are expecting
 
-die "$request_file_name is not a PS1_PS_REQEST" if $extname ne "PS1_PS_REQUEST";
-die "REQ_NAME not found in $request_file_name"  if (!$req_name);
-die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
-
+# we shouldn't get here if this is the case so just die. No need to notify the client
+my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
+my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
+my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1");
+
+# check for duplicate request name
 if ($req_id and !$no_update) {
+    my $command = "$pstamptool -listreq  -name $req_name -not_req_id $req_id";
+    $command .= " -dbname $dbname" if $dbname;
+    $command .= " -dbserver $dbserver" if $dbserver;
+    # no verbose so that error message about request not found doesn't appear in parse_error.txt
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 0);
+    my $exitStatus = $error_code >> 8;
+    if ($success) {
+        # -listreq succeeded duplicate request name
+        insertFakeJobForRow(undef, 0, $PSTAMP_DUP_REQUEST);
+        exit 0;
+    }
+}
+
+if ($req_id and !$no_update) {
+    # update the database with the request name. This will be used as the
+    # the output data store's product name
     my $command = "$pstamptool -updatereq -req_id $req_id  -name $req_name";
     $command .= " -outProduct $product";
@@ -103 +124 @@
         run(command => $command, verbose => $verbose);
     unless ($success) {
-        die "$command failed";
+        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
     }
 }
@@ -120 +141 @@
     }
     my $table =  $mdcParser->parse(join "", @$stdout_buf) or
-        die("Unable to parse metdata config doc");
+        my_die("Unable to parse metdata config doc", $PS_EXIT_UNKNOWN_ERROR);
 
     $rows = parse_md_list($table);
@@ -143 +164 @@
     # parameters that select the images of interest
     my $project  = $row->{PROJECT};
+
+    # note: resolve_project avoids running pstamptool every time by remembering the
+    # last project resolved
+    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
+    if (!$proj_hash) {
+        print STDERR "project $project not found\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT);
+        $num_jobs++;
+        next;
+    }
     my $req_type = $row->{REQ_TYPE};
     $stage = $row->{IMG_TYPE};
@@ -153 +184 @@
 
     my $option_mask= $row->{OPTION_MASK};
-
-    die "valid region of interest is required to make postage stamps"
-        if (($job_type eq "stamp") and ! validROI($row));
+    my $inverse = ($option_mask & $PSTAMP_SELECT_INVERSE) ? 1 : 0;
+    $row->{inverse} = $inverse;
 
     my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
-    $component = "" if (defined($component) and ($component eq "null" or $component eq "all"));
+
+    my $search_component = (!defined($component) or ($component eq "null")) ? "" : $component;
     
-    die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri");
-
-
-    # note: resolve_project avoids running pstamptool every time by remembering the
-    # last project resolved
-    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
-    die "project $project not found\n" unless $proj_hash;
+    if (!$skycenter and !$search_component) {
+        print STDERR "COMPONENT must be specified for pixel coordinate ROI\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        $num_jobs++;
+        next;
+    }
+
+    $search_component = "" if $search_component eq "all";
+
+    if (($job_type eq "stamp") and ! validROI($row)) {
+        print STDERR "invalid ROI for row $rownum\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        $num_jobs++;
+        next;
+    }
+
+    
+    # $mode list_uri is a debugging mode (it may used by the http interface)
+    # if this happens just croak
+    my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
+
 
     my $image_db   = $proj_hash->{dbname};
@@ -172 +217 @@
     $need_magic = $proj_hash->{need_magic};
 
+    # Temporary hack so that MOPS can get at non-magicked data
+    if ($product eq "mops-pstamp-results") {
+        $need_magic = 0;
+    }
+
     # collect rows with the same images of interest in a list so that they
-    # can be processed optimially
+    # can be looked up together
     if (@rowList) {
         my $firstRow = $rowList[0];
-        # XXX: the collecting might work with !$skycenter but I need to think about it
-        if ($skycenter and same_images_of_interest($row, $firstRow)) {
+        # note order of these parameters matters
+        if (same_images_of_interest($firstRow, $row)) {
+
+            # add this row to the list and move on
             push @rowList, $row;
-            # On to the next row
+
             next;
+
         } else {
-            # queue the jobs for this set of rows
+            # this row has different selectors
+            # queue the jobs for the ones we've collected
             $num_jobs += queueJobs($mode, \@rowList, $imageList);
             @rowList = ();
@@ -188 +242 @@
     }
 
+    # look up images for the current row
     if ($req_type eq "bycoord") {
-        die "center must be specified in sky coordintes for bycoord" if ( ! $skycenter);
-        die "lookup bycoord is not yet implemented";
+        if (!$skycenter) {
+            print STDERR "center must be specified in sky coordintes for bycoord" if $verbose;
+            insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+            $num_jobs++;
+            next;
+        }
+        print STDERR "lookup bycoord is not yet implemented" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
+        $num_jobs++;
+        next;
     } else {
-
-        # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this
-        # request specification. An array reference is returned
+        # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
+        # request specification. An array reference is returned.
         my ($x, $y);
-        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $component, $skycenter,
-                $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
-    }
-
-    if (!$imageList or !@$imageList) {
-        print STDERR "no matching images found for row $rownum\n";
-        next;
-    }
-    $row->{need_magic} = $need_magic;
-    push @rowList, $row;
+
+        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $search_component,
+                $inverse, $skycenter, $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
+
+        if (!$imageList or !@$imageList) {
+            print STDERR "no matching images found for row $rownum\n" if $verbose;
+            # note in this case queueJobs inserts the fake job for these rows
+        }
+        $row->{need_magic} = $need_magic;
+        push @rowList, $row;
+    }
 }
 
@@ -212 +275 @@
 }
 
-if ($mode eq "queue_jobs") {
-    if ($num_jobs) {
-        exit 0;
-    } else {
-        # no jobs queued return error to get the request stopped
-        # TODO: need to define meaningful error codes
-        exit $PS_EXIT_UNKNOWN_ERROR;
-    }
-} else {
-    exit 0;
-}
+if (($mode eq "queue_jobs") and ($num_jobs eq 0)) {
+    print STDERR "no jobs created for $req_name\n" if $verbose;
+    insertFakeJobForRow(undef, 0, $PSTAMP_UNKNOWN_ERROR);
+}
+
+# PAU
+
+exit 0;
 
 
@@ -232 +292 @@
     my $have_skycells = shift;
     my $need_magic = shift;
+
+    my $num_jobs = 0;
+    my $rownum = $row->{ROWNUM};
+    my $components = $row->{components};
+
+    my $roi_string;
+
+    # note values were insured to be numbers in validROI()
+    my $x = $row->{CENTER_X};
+    my $y = $row->{CENTER_Y};
+    my $w = $row->{WIDTH};
+    my $h = $row->{HEIGHT};
+    my $coord_mask = $row->{COORD_MASK};
+    if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
+        if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
+            $roi_string = "-pixcenter $x $y";
+        } else {
+            $roi_string = "-skycenter $x $y";
+        }
+        if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
+            $roi_string .= " -pixrange $w $h";
+        } else {
+            $roi_string .= " -arcrange $w $h";
+        }
+    }
 
     # loop over images
@@ -244 +329 @@
 
         # skip this component if it is not in the list for this row
-        next if !$row->{components}->{$component};
-
-        my $rownum = $row->{ROWNUM};
-
-        my $roi_string;
-
-        # note values were insured to be numbers in validROI()
-        my $x = $row->{CENTER_X};
-        my $y = $row->{CENTER_Y};
-        my $w = $row->{WIDTH};
-        my $h = $row->{HEIGHT};
-        my $coord_mask = $row->{COORD_MASK};
-        if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
-            if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
-                $roi_string = "-pixcenter $x $y";
-            } else {
-                $roi_string = "-skycenter $x $y";
-            }
-            if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
-                $roi_string .= " -pixrange $w $h";
-            } else {
-                $roi_string .= " -arcrange $w $h";
-            }
-        }
+        next if ! $components->{$component};
+
+        $job_num++;
 
         my $imagefile = $image->{image};
         if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
-            # XXX: should we add a faulted job so the client can know what happened?
-            print STDERR "skippping non-magicked image $imagefile\n" if $verbose;
+            # XXX: should we add a faulted job so the client can know what happened if no images come back?
+            print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
+
+            # for now assume yes.
+
+            insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
+            $num_jobs++;
+
             next;
         }
@@ -282 +352 @@
         }
 
-        $job_num++;
-
         # add astrometry file for raw and chip images if one is available
         if (($stage eq "chip") || ($stage eq "raw")) {
@@ -298 +366 @@
         }
 
-        my $output_base = "$out_dir/${rownum}_${job_num}";
+        my $base = basename($image->{path_base});
+        # XXX: TODO use filerule for this. I don't have a camera defined here
+        if (($stage eq 'chip') and ($image->{camera} eq 'GPC1')) {
+            $base = "${base}.${component}";
+        }
+
+        my $output_base = "$out_dir/${rownum}_${job_num}_${base}";
         my $argslist = "${output_base}.args";
 
         # copy the argument list to a file
-        open ARGSLIST, ">$argslist" or die "failed to open $argslist";
+        open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR);
         print ARGSLIST "$args\n";
-        close ARGSLIST or die "failed to close $argslist";
-
-        # XXX: TODO: here is where we need to check whether or not the source inputs still exist
-        # and if not, queue an update job and set the job state appropriately.
+        close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR);
 
         my $newState = "run";
+        my $fault = 0;
+
+        if (($stage ne 'stack') and ($stage ne 'raw')) {
+            if (($image->{state} eq 'goto_purged') or ($image->{data_state} eq 'purged')) {
+                $newState = 'stop';
+                $fault = $PSTAMP_GONE;
+            } elsif (($image->{data_state} ne 'full') or ($image->{state} ne 'full' )) {
+                # XXX here is where we need to queue an update job
+                # for now just say that the image is not available
+                $newState = 'stop';
+                $fault = $PSTAMP_NOT_AVAILABLE;
+            }
+        }
 
         $num_jobs++;
         my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
                         . " -outputBase $output_base -rownum $rownum -state $newState";
+        $command .= " -fault $fault" if $fault;
         $command .= " -exp_id $exp_id" if $exp_id;
         $command .= " -dbname $dbname" if $dbname;
@@ -331 +416 @@
                 # should we keep going for other uris? If so how do we report that some
                 # of the work that the request wanted isn't going to get done
-                die "failed to queue job for request $req_id";
+                my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
             }
         } else {
@@ -337 +422 @@
         }
     }
+    if ( $num_jobs == 0 ) {
+        print STDERR "no jobs for row $rownum\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_NO_OVERLAP);
+        $num_jobs = 1;
+    }
     return $num_jobs;
 }
 
+# queue jobs for a collection of request specifications that have the same Images of Interest
 sub queueJobs
 {
@@ -358 +449 @@
         }
     } elsif ($job_type eq "get_image") {
-        die "get_image jobs not implemented yet";
+        print STDERR "get_image jobs not implemented yet" if $verbose;
+        foreach my $row (@$rowList) {
+            insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
+            $num_jobs++;
+        }
     } else {
-        my $thisRun;
-
-        my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
-        foreach my $row (@$rowList) {
-            print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n";
-            $row->{components} = {};
-        }
-        close $pointsList;
+        if (!$imageList or (scalar @$imageList eq 0)) {
+            # we didn't find any images for this set of rows. Insert a fake job to carry
+            # the status back to the requestor
+            foreach my $row (@$rowList) {
+                insertFakeJobForRow($row, 1, $PSTAMP_NO_IMAGE_MATCH);
+                $num_jobs++;
+            }
+            return $num_jobs;
+        }
 
         my $have_skycells;
@@ -376 +472 @@
         }
         
+        my $thisRun;
+
+        my $npoints = 0;
+        my ($pointsList, $pointsListName);
+        if (scalar @$imageList > 1) {
+            ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
+            foreach my $row (@$rowList) {
+                $row->{components} = {};
+                if ($row->{skycenter}) {
+                    print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n";
+                    $npoints++;
+                } else {
+                    # this row's center is in pixel coordinates add all images to the component list for this row
+                    foreach my $i (@$imageList) {
+                        my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id};
+                        my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR)
+                                if !$component;
+                        $row->{components}->{$component} = 1;
+                    }
+                }
+            }
+            close $pointsList;
+        } else {
+            # only one image. Avoid the expense of dvoImagesAtCoords. 
+            # queue the job and let ppstamp figure out if the center is valid for the image
+            # the resulting result code will be the same.
+            foreach my $row (@$rowList) {
+                $row->{components} = {};
+                my $i = $imageList->[0];
+                my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id};
+                my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR) if !$component;
+                $row->{components}->{$component} = 1;
+            }
+        }
+
         my $tess_dir_abs;
         my $last_tess_id = "";
         while ($thisRun = getOneRun($stage, $imageList)) {
-            {
+            if ($npoints) {
+                my_die( "pointsListName is not defined", $PS_EXIT_PROG_ERROR) if !$pointsListName;
+                # we collected a set of sky coordinates above.
+                # filter the images so that only those that contain the centers of the ROIs are processed
                 my $command = "$dvoImagesAtCoords $pointsListName";
                 if ($have_skycells) {
@@ -391 +525 @@
                 } else {
                     my $astrom = $thisRun->[0]->{astrom};
-                    die "no astrometry file found" if !$astrom;
+                    my_die( "no astrometry file found", $PS_EXIT_UNKNOWN_ERROR) if !$astrom;
                     my $astrom_resolved = $ipprc->file_resolve($astrom);
                     $command .= " -astrom $astrom_resolved";
@@ -400 +534 @@
                     print STDERR @$stderr_buf;
                     my $rc = $error_code >> 8;
-                    die "dvoImagesAtCoords failed: $rc";
+                    my_die( "dvoImagesAtCoords failed: $rc", $PS_EXIT_UNKNOWN_ERROR);
                 }
                 # now we have a list of row numbers and components
@@ -410 +544 @@
                     my ($rownum, undef, undef, $component) = split " ", $line;
 
-                    # I guess since we need this function we should be useing a hash for rowList 
+                    # I guess since we need this function we should be using a hash for rowList 
                     my $row = findRow($rownum, $rowList);
                     $row->{components}->{$component} = 1;
@@ -422 +556 @@
     }
     return $num_jobs;
+}
+sub insertFakeJobForRow
+{
+    my $row = shift;
+    my $job_num = shift;
+    my $fault = shift;
+
+    my ($job_type, $rownum);
+    if ($row) {
+        $job_type = $row->{JOB_TYPE};
+        $rownum = $row->{ROWNUM};
+    } else {
+        $job_type = "none";
+        $rownum = 0;
+    }
+
+    my $command = "$pstamptool -addjob  -req_id $req_id -job_type $job_type"
+                        . " -rownum $rownum -state stop -fault $fault";
+    $command .= " -dbname $dbname" if $dbname;
+    $command .= " -dbserver $dbserver" if $dbserver;
+
+    if ($mode eq "list_job") { 
+        # this is a debugging mode, just print the pstamptool that would have run
+        # this is sort of like the mode -noupdate that some other tools support
+        print "$command\n";
+    } elsif (!$no_update) {
+        # mode eq "queue_job"
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            print STDERR @$stderr_buf;
+            # XXX TODO: now what? Should we mark the error state for the request?
+            # should we keep going for other uris? If so how do we report that some
+            # of the work that the request wanted isn't going to get done
+            my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print "skipping command: $command\n";
+    }
 }
 
@@ -440 +613 @@
         return $image->{diff_id};
     } else {
-        die "unenexpected stage: $stage found";
+        my_die("unenexpected stage: $stage found", $PS_EXIT_PROG_ERROR);
     }
 }
@@ -476 +649 @@
     return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE});
     return 0 if ($r1->{ID} ne $r2->{ID});
+    return 0 if ($r1->{inverse} ne $r2->{inverse});
 
     if (defined($r1->{COMPONENT})) {
         return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT});
     } elsif (defined($r2->{COMPONENT})) {
+        # if first row has no component all of the images will be retrieved, so
+        # the fact that this row has a component is ok. Fall through to return 1
+        # XXX Nope this doesn't work. It is consistent with the other logic in some way
+        # that i haven't fully thought through
         return 0;
     }
@@ -514 +692 @@
     return undef;
 }
+
+sub my_die
+{
+    my $msg = shift;
+    my $fault = shift;
+
+    carp $msg;
+
+    # we don't fault the request here pstamp_parser_run.pl handles that if necessary
+
+    return $fault;
+}