Changeset 26236 for branches/eam_branches/20091113/pstamp/scripts/pstampparse.pl

Timestamp:

Nov 20, 2009, 4:13:39 PM (17 years ago)

Author:

eugene

Message:

updates from head

File:

• branches/eam_branches/20091113/pstamp/scripts/pstampparse.pl (modified) (9 diffs)

branches/eam_branches/20091113/pstamp/scripts/pstampparse.pl

@@ -98 +98 @@
 
 # make sure the file contains what we are expecting
-
-# we shouldn't get here if this is the case so just die. No need to notify the client
+# This program shouldn't have been run if the request file is bogus.
+# No need to notify the client
 my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
 my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
@@ -165 +165 @@
     # XXX: TODO: sanity check all parameters
 
-    # XXX: TODO: We shouldn't really just die in this loop.
-    # If we encounter an error for a particular row
-    # add a job with the proper fault code. If there is a db or config error we should probably just
-    # trash the request.
+    # If we encounter an error for a particular row add a job with the proper fault code.
+    # If we encounter an error in this loop we shouldn't really just die.
+    # We only do that now in the case of I/O or DB errors or the like.
 
     my $rownum   = $row->{ROWNUM};
@@ -236 +235 @@
     }
 
-    # collect rows with the same images of interest in a list so that they
-    # can be looked up together
+    # For "stamp" and "list_uri" jobs collect rows with the same images of interest in a list so that they
+    # can be looked up together.
     if (@rowList) {
         my $firstRow = $rowList[0];
-        # note order of these parameters matters
-        if (same_images_of_interest($firstRow, $row)) {
+        if (($firstRow->{JOB_TYPE} ne "get_image") and same_images_of_interest($firstRow, $row)) {
 
             # add this row to the list and move on
@@ -305 +303 @@
     my $have_skycells = shift;
     my $need_magic = shift;
+    my $mode = shift;
 
     my $num_jobs = 0;
@@ -395 +394 @@
         print ARGSLIST "$args\n";
         close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR);
+
+        write_params($output_base, $image);
 
         my $newState = "run";
@@ -476 +477 @@
         }
     } elsif ($job_type eq "get_image") {
-        print STDERR "get_image jobs not implemented yet" if $verbose;
-        foreach my $row (@$rowList) {
-            insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
-            $num_jobs++;
-        }
+        my $n = scalar @$rowList;
+
+        my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1;
+
+        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+
     } else {
         if (!$imageList or (scalar @$imageList eq 0)) {
@@ -582 +584 @@
             
             foreach my $row (@$rowList) {
-                $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic);
+                $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic, $mode);
             }
         }
+    }
+    return $num_jobs;
+}
+
+#        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+sub queueGetImageJobs
+{
+    my $row = shift;
+    my $imageList = shift;
+    my $stage = shift;
+    my $need_magic = shift;
+    my $mode = shift;
+
+    my $num_jobs = 0;
+    my $rownum = $row->{ROWNUM};
+    my $option_mask = $row->{OPTION_MASK};
+
+    # For dist_bundle we need
+    #  --camera from $image
+    #  --stage 
+    #  --stage_id from $image
+    #  --component from $image
+    #  --path_base 
+    #  --outdir global to this script
+
+    # loop over images
+    my $job_num = 0;
+    foreach my $image (@$imageList) {
+        my $stage_id = $image->{stage_id};
+        my $component = $image->{component};
+
+        # skip faulted components for now. Should we even be here?
+        if ($image->{fault} > 0) {
+            printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose;
+            next;
+        }
+
+        $job_num++;
+
+        my $imagefile = $image->{image};
+        if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
+            # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed
+            # in locate_images because it's much more efficient to do the test in the database.
+            # For these two modes we fall through to here in order to give feedback to the requestor as
+            # to why the request failed to queue jobs.
+            print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
+            insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
+            $num_jobs++;
+
+            next;
+        }
+        my $exp_id = $image->{exp_id};
+            
+        my $output_base = "$out_dir/${rownum}_${job_num}";
+
+        write_params($output_base, $image);
+
+        my $newState = "run";
+        my $fault = 0;
+        my $dep_id;
+
+        if ($stage ne 'raw') {
+            my $run_state = $image->{state};
+            my $data_state = $image->{data_state};
+            $data_state = $run_state if $stage eq "stack";
+            if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or
+                ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) {
+                # image is gone and it's not coming back
+                $newState = 'stop';
+                $fault = $PSTAMP_GONE;
+            } elsif (($data_state ne 'full') or ($run_state ne 'full' )) {
+                # don't wait for update unless the caller asks us to
+                if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) {
+                    $newState = 'stop';
+                    $fault = $PSTAMP_NOT_AVAILABLE;
+                } else {
+                    # cause the image to be re-made
+                    # set up to queue an update run
+                    queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, 
+                        $run_state, $stage, $image->{stage_id}, $need_magic, $image->{label});
+                }
+            }
+        }
+
+        $num_jobs++;
+        my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
+                        . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask";
+        $command .= " -fault $fault" if $fault;
+        $command .= " -exp_id $exp_id" if $exp_id;
+        $command .= " -dep_id $dep_id" if $dep_id;
+
+        if ($mode eq "list_job") { 
+            # this is a debugging mode, just print the pstamptool that would have run
+            # this is sort of like the mode -noupdate that some other tools support
+            print "$command\n";
+        } elsif (!$no_update) {
+            # mode eq "queue_job"
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                print STDERR @$stderr_buf;
+                # XXX TODO: now what? Should we mark the error state for the request?
+                # should we keep going for other uris? If so how do we report that some
+                # of the work that the request wanted isn't going to get done
+                my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping command: $command\n";
+        }
+    }
+    if ( $num_jobs == 0 ) {
+        print STDERR "no jobs for row $rownum\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_NO_JOBS_QUEUED);
+        $num_jobs = 1;
     }
     return $num_jobs;
@@ -677 +793 @@
     return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE});
     return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE});
-    return 0 if ($r1->{ID} ne $r2->{ID});
-    return 0 if ($r1->{TESS_ID} ne $r2->{TESS_ID});
-    return 0 if ($r1->{REQFILT} ne $r2->{REQFILT});
-    return 0 if ($r1->{LABEL} ne $r2->{LABEL});
-    return 0 if ($r1->{MJD_MIN} ne $r2->{MJD_MAX});
-    return 0 if ($r1->{MJD_MAX} ne $r2->{MJD_MAX});
-    return 0 if ($r1->{inverse} ne $r2->{inverse});
+    return 0 if ($r1->{ID}       ne $r2->{ID});
+    return 0 if ($r1->{TESS_ID}  ne $r2->{TESS_ID});
+    return 0 if ($r1->{REQFILT}  ne $r2->{REQFILT});
+    return 0 if ($r1->{LABEL}    ne $r2->{LABEL});
+    return 0 if ($r1->{MJD_MIN}  ne $r2->{MJD_MAX});
+    return 0 if ($r1->{MJD_MAX}  ne $r2->{MJD_MAX});
+    return 0 if ($r1->{inverse}  ne $r2->{inverse});
 
     if (defined($r1->{COMPONENT})) {
@@ -759 +875 @@
 }
 
+sub write_params {
+    my $output_base = shift;
+    my $image = shift;
+
+    # write the contents of this "image" as a metadata config doc
+    # Simply treat all values as strings. This is ok since we are only going to
+    # read it from another perl script
+    my $mdc_file = "${output_base}.mdc";
+    open P, ">$mdc_file" or my_die("failed to open $mdc_file", $PS_EXIT_UNKNOWN_ERROR);
+
+    print P "params METADATA\n";
+
+    foreach my $key (keys %$image) {
+        my $value = $image->{$key};
+        if (defined $value) {
+            printf P "  %-20s STR     %s\n", $key, $value;
+        } else {
+            printf P "  %-20s STR     NULL\n", $key;
+        }
+    }
+
+    print P "END\n";
+    close P or my_die("failed to close $mdc_file", $PS_EXIT_UNKNOWN_ERROR);
+}
+
 sub my_die
 {