Changeset 27839 for branches/simmosaic_branches/pstamp/scripts/pstamp_finish.pl

Timestamp:

May 3, 2010, 8:45:22 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/simmosaic_branches

Files:

• . (modified) (1 prop)
• pstamp/scripts/pstamp_finish.pl (modified) (22 diffs)

branches/simmosaic_branches/pstamp/scripts/pstamp_finish.pl

@@ -8 +8 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
+use Carp;
 
 use Time::Local;
@@ -47 +48 @@
 
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
-
 if ($redirect_output) {
     my $logDest = "$out_dir/psfinish.$req_id.log";
@@ -63 +63 @@
 my $pstamp_results = can_run('pstamp_results_file.pl') 
                             or (warn "Can't find pstamp_results_file.pl" and $missing_tools = 1);
+my $pstampdump = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -68 +69 @@
 }
 
+if ($product eq "NULL") {
+    stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
+}
 
 my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
@@ -78 +82 @@
     my $fileset = $req_name;
 
-
-    # Here we invoke the assumption that the output for the request is placed in the
-    # fileset directory directly
-#    my $out_dir = "$outputDataStoreRoot/$product/$fileset";
-
-    # now we are assuming that the output directory is the dirname of the request file
-    # XXX: put this in the database
-
     print STDERR "product: $product  REQ_NAME: $req_name $out_dir\n" if $verbose;
 
@@ -94 +90 @@
         if (!mkdir $out_dir) {
             print STDERR "cannot create output directory $out_dir";
-            stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR);
+            stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
         }
 
@@ -102 +98 @@
         # request
         print STDERR "output directory $out_dir exists but is not a directory";
-        stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR);
+        stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
     }
 
     if (! -e $req_file ) {
         print STDERR "request file $req_file is missing\n";
-        stop_request($req_id, $PS_EXIT_CONFIG_ERROR);
+        stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR);
     }
 
@@ -117 +113 @@
         # point 
         print STDERR "failed to read request_file $req_file" ;
-        stop_request($req_id, $PS_EXIT_CONFIG_ERROR);
+        stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR);
     }
 
@@ -128 +124 @@
     my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps);
     print $rlf "results.fits|||table|\n";
+    print $rlf "results.mdc|||text|\n";
 
     my $err_file = "parse_error.txt";
@@ -154 +151 @@
                 print STDERR "Request $req_id produced no jobs.\n"
             }
-            # No Jobs. 
-            # XXXX Ouch. We need results for each rownum (each request specification) in the request file 
-            # including those that produced no jobs.
-            # for now add an entry for rownum 1 and a phony error code.
-            # we've included parse_error.txt to the fileset if it exists
-            #
-            my $rownum = 0;
-            my $fault = 42; # get a real error code
-            print $tdf "$rownum|$fault|0|0|";
-            print $tdf "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|\n";
+            # This should not happen. A fake job should be queued
+            stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
         } else {
             my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
@@ -173 +162 @@
     }
 
+    my $exp_info;
+    my $last_exp_id = -1;
     foreach my $job (@jobs) {
         my $job_id = $job->{job_id};
@@ -180 +171 @@
         my $exp_id = $job->{exp_id};
 
+
+        if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) {
+            # this request had a duplicate request name. We can't put the results
+            # on the data store since the product name is already used
+            # maybe we could be more clever about this and choose a fileset name
+            stop_request_and_exit($req_id, $fault);
+        }
         my ($row, $req_info, $project) = get_request_info($rows, $rownum);
-        my $proj_hash = resolve_project($ipprc, $project, $dbname);
+
+        my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
         my $image_db = $proj_hash->{dbname};
-
-        # get the metadata for the exposure (if any i.e. stack) 
-        # returns an appropriate string if !$exp_id
-        my $exp_info = get_exposure_info($image_db, $exp_id);
-
-        if (($job_type eq "stamp") || ($job_type eq "get_image")) {
+        if (!$image_db) {
+            carp("failed to find imagedb for project: $project");
+            if (!$fault) {
+                stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR);
+            }
+        }
+
+        my $job_params = get_job_parameters($job);
+        my $stage = "";
+        if ($job_params) {
+            $stage = $job_params->{stage};
+        }
+
+        if ($stage ne 'stack') {
+            # get the metadata for the exposure (if any i.e. stack) 
+            # returns an appropriate string if !$exp_id
+            $exp_info = get_exposure_info($image_db, $exp_id);
+            $last_exp_id = $exp_id;
+        } else {
+            my $filter = $job_params->{filter};
+            $filter = "0" if !$filter;
+            $exp_info = "0|0|0|$filter|0|0";
+            $last_exp_id = -1;
+        }
+
+        if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) {
             my $jreglist = "$out_dir/reglist$job_id";
             if (open JRL, "<$jreglist") {;
@@ -203 +222 @@
 
                     # ra_deg and dec_deg are the coordinates of center of the stamp
-                    # XXX: parse the stamp header to find it
-                    print $tdf "0.0|0.0|";
+                    # first assume that the image is compressed and check the first extension.
+                    # If not found check the PHU. If that doesn't work just set them to zero.
+                    # XXX do this more cleanly
+                    my (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
+                    if (!defined $ra_deg) {
+                        (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields RA_DEG DEC_DEG`;
+                    }
+                    $ra_deg = 0.0 if (!$ra_deg);
+                    $dec_deg = 0.0 if (!$dec_deg);
+                    print $tdf "$ra_deg|$dec_deg|";
 
                     print $tdf "$exp_info|";
@@ -217 +244 @@
                 print $tdf "$exp_info|";
                 print $tdf "$req_info|";
+                print $tdf "\n";
             }
         } else {
@@ -236 +264 @@
             print STDERR "Unable to perform $command error code: $error_code\n";
             $request_fault = $error_code >> 8;
+        } else {
+            # dump a textual representation
+            my $command = "$pstampdump $out_dir/results.fits > $out_dir/results.mdc";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                print STDERR "Unable to perform $command error code: $error_code\n";
+                $request_fault = $error_code >> 8;
+            }
         }
     }
@@ -241 +278 @@
         # register the fileset
         my $command = "$dsreg --list $reglist_name --add $fileset --product $product --type PSRESULTS";
-        $command .= " --link --datapath $out_dir";
+        $command .= " --link --datapath $out_dir --ps0 $req_id";
         $command .= " --dbname $dbname" if $dbname;
 
@@ -267 +304 @@
 }
 
-sub stop_request {
+sub stop_request_and_exit {
     my $req_id = shift;
     my $fault  = shift;
@@ -287 +324 @@
     my $rownum = shift;
 
+    if ($rownum eq 0) {
+        my $dummy_rowinfo = "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|";
+        return (undef, $dummy_rowinfo, "none");
+    }
     my $row = $rows->{$rownum};
+
+
+    # these may be set to null during processing
+    my $component = $row->{COMPONENT};
+    $component = "null" if !$component;
+    my $tess_id = $row->{TESS_ID};
+    $tess_id = "null" if !$tess_id;
+    my $comment = $row->{COMMENT};
+    $comment = "null" if !$comment;
+    my $data_group = $row->{DATA_GROUP};
+    if (!defined $data_group) {
+        # XXX: backwards compatibility hook. Remove "soon".
+        $data_group = $row->{LABEL};
+    }
+    $data_group = "null" if !$data_group;
 
     # This is ugly, error prone and hard to change.
     # Create a results file module and provide a list of the names (we have the data in the columns)
     my $rowinfo = "$row->{PROJECT}|$row->{JOB_TYPE}|$row->{REQ_TYPE}|$row->{IMG_TYPE}|";
-    $rowinfo   .= "$row->{ID}|$row->{CLASS_ID}|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|";
+    $rowinfo   .= "$row->{ID}|$tess_id|$component|$data_group|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|";
     $rowinfo   .= "$row->{REQFILT}|$row->{COORD_MASK}|$row->{CENTER_X}|$row->{CENTER_Y}|";
-    $rowinfo   .= "$row->{WIDTH}|$row->{HEIGHT}";
+    $rowinfo   .= "$row->{WIDTH}|$row->{HEIGHT}|";
+    $rowinfo   .= $comment;
 
     return ($row, $rowinfo, $row->{PROJECT});
+}
+
+sub get_job_parameters {
+    my $job = shift;
+    if (!$job->{outputBase}) {
+        print "get_job_parameters: $job->{job_id} has no outputBase\n";
+        return undef;
+    }
+    my $params_file = $job->{outputBase} . '.mdc';
+    if (! -e $params_file ) {
+        print "get_job_parameters: $job->{job_id} has no parameters file\n";
+        return undef;
+    }
+    open IN, "<$params_file" or die "unable to open $params_file";
+    my $data = join "", (<IN>);
+    close IN;
+    if (! $data ) {
+        print "get_job_parameters: parameters file is empty\n";
+        return undef;
+    }
+    my $metadata = $mdcParser->parse($data) or die("Unable to parse metdata config doc");
+
+    my $results = parse_md_list($metadata);
+    if (scalar @$results != 1) {
+        print STDERR "get_job_params: failed to parse_md_list\n";
+        return undef;
+    }
+    return $results->[0];
 }
 
@@ -303 +388 @@
     my $exp_id = shift;
 
-    if (!$exp_id) {
+    if (!$exp_id or !$image_db) {
         # no exposure id just return zeros
         # XXX: we could put a value in for filter, but we don't have a good place to find it
@@ -347 +432 @@
     my $ra_deg   = $exp->{ra} * RADIANS_TO_DEGREES;
     my $decl_deg = $exp->{decl} * RADIANS_TO_DEGREES;
-    my $mjd_obs = dateobs_to_mjd($exp->{dateobs}, $exp->{exp_time});
+    my $mjd_obs = dateobs_to_mjd($exp->{dateobs});
 
     my $info = "$mjd_obs|$ra_deg|$decl_deg|$exp->{filter}|$exp->{exp_time}|$exp->{exp_name}";
@@ -356 +441 @@
 sub dateobs_to_mjd {
     my $dateobs = shift;
-    my $exp_time = shift;
 
     # dateobs is in format: 1970-01-01T00:00:00
@@ -366 +450 @@
     my $ticks = timegm($sec, $min, $hr, $day, $mon-1, $year-1900);
 
-    $ticks += $exp_time / 2.0;
-
     return 40587.0 + ($ticks/86400.);
 }