Changeset 27840 for branches/simtest_nebulous_branches/pstamp/scripts/pstamp_job_run.pl

Timestamp:

May 3, 2010, 8:50:52 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/simtest_nebulous_branches

Files:

• . (modified) (1 prop)
• pstamp/scripts/pstamp_job_run.pl (modified) (10 diffs)

branches/simtest_nebulous_branches/pstamp/scripts/pstamp_job_run.pl

@@ -11 +11 @@
 use Sys::Hostname;
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Carp;
 use File::Basename;
+use File::Copy;
 use Digest::MD5::File qw( file_md5_hex );
 use PS::IPP::PStamp::RequestFile qw( :standard );
 use IPC::Cmd 0.36 qw( can_run run );
+use POSIX;
 
 use PS::IPP::Metadata::Config;
 #use PS::IPP::Metadata::Stats;
-#use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Metadata::List qw( parse_md_list );
 
 use PS::IPP::Config qw( :standard );
 
-my ($job_id, $redirect_output, $outputBase, $rownum, $jobType); 
+my ($job_id, $redirect_output, $outputBase, $rownum, $jobType, $options); 
 my ($verbose, $dbname, $dbserver, $no_update);
 
@@ -30 +33 @@
     'rownum=s'          =>  \$rownum,
     'output_base=s'     =>  \$outputBase,
+    'options=s'         =>  \$options,
     'redirect-output'   =>  \$redirect_output,
     'dbname=s'          =>  \$dbname,
@@ -49 +53 @@
 my_die("rownum is required", $job_id, $PS_EXIT_PROG_ERROR) if !$rownum;
 my_die("output_base is required", $job_id, $PS_EXIT_PROG_ERROR) if !$outputBase;
+
+$options = 1 if !$options;
 
 
@@ -92 +98 @@
         run(command => $command, verbose => $verbose);
 
-    if ($success) {
+    my $exitStatus;
+    if (WIFEXITED($error_code)) {
+        $exitStatus = WEXITSTATUS($error_code);
+    } else {
+        print STDERR "ppstamp failed error_code: $error_code\n";
+        $exitStatus = $PS_EXIT_SYS_ERROR;
+    }
+
+    if ($exitStatus == 0) {
         my $dir = dirname($outputBase);
 
         my $reglist = "$dir/reglist$job_id";
 
-        open F, ">$reglist" or my_die( "can't open $reglist for output", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+        my $F;
+        open $F, ">$reglist" or my_die( "can't open $reglist for output", $job_id, $PS_EXIT_UNKNOWN_ERROR);
 
         # Figure out what output images were produced
@@ -106 +121 @@
                            $PSTAMP_SELECT_WEIGHT => "wt.fits");
 
-        # we always create a stamp of the image
-        my $output_mask = $PSTAMP_SELECT_IMAGE;
-
-        # we search the argString for -mask and -variance. 
-        # searching the arg string allows us to avoid adding a column in pstampJob
-        $output_mask   |= $PSTAMP_SELECT_MASK   if ($argString =~ /-mask/);
-        $output_mask   |= $PSTAMP_SELECT_WEIGHT if ($argString =~ /-variance/);
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_WEIGHT);
 
         foreach my $key (keys (%extensions)) {
@@ -127 +136 @@
 
             # XXX is pstamp always the right file type, if not where do we get the right one?
-            print F file_registration_line($filename, $path, "pstamp") . "\n";
-        }
-
-        close F;
+            print $F file_registration_line($filename, $path, "pstamp") . "\n";
+        }
+
+        get_other_outputs($F, $outputBase, $options);
+
+        close $F;
         $jobStatus = $PS_EXIT_SUCCESS;
-    } else {
-        $jobStatus = $error_code >> 8;
-        my_die( "ppstamp failed with error code: $jobStatus", $job_id, $jobStatus);
+    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP) {
+        $jobStatus = $PSTAMP_NO_OVERLAP;
+    } else {
+        my_die( "ppstamp failed with error code: $exitStatus", $job_id, $exitStatus);
     }
 } elsif ($jobType eq "get_image") {
-    my_die( "get_image jobs not working right now", $job_id, $PS_EXIT_CONFIG_ERROR);
 
     my $uri = "";
-    my $command = "$pstamp_get_image_job --job_id $job_id --uri $uri --out_dir $outputBase --rownum $rownum";
+    my $command = "$pstamp_get_image_job --job_id $job_id --output_base $outputBase --rownum $rownum";
     $command .= " --dbname $dbname" if $dbname;
     $command .= " --dbserver $dbserver" if $dbserver;
+    $command .= " --verbose" if $verbose;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -161 +173 @@
 {
     my $command = "$pstamptool -updatejob -job_id $job_id -state stop"; 
+    $command .= " -fault $jobStatus" if $jobStatus;
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -194 +207 @@
     }
 }
+        
+sub get_other_outputs {
+    my $f = shift;
+    my $output_base = shift;
+    my $options = shift;
+
+    if ($options & ( $PSTAMP_SELECT_CMF | $PSTAMP_SELECT_PSF | $PSTAMP_SELECT_BACKMDL)) {
+        my $comp = read_params_file($output_base);
+
+        my $stage = $comp->{stage};
+
+        # raw files don't have any other data products
+        return 1 if $stage eq "raw";
+
+        # add the other data products if they are selected and exist.
+        # silently skip them if they don't exist. Perhaps this should be
+        # detected in pstampparse so that the user can be notified with 
+        # a message in parse_error.txt ("warp do not have a background model")
+        my $cmf_file = $comp->{cmf} if ($options & $PSTAMP_SELECT_CMF);
+        my $psf_file = $comp->{psf} if ($options & $PSTAMP_SELECT_PSF);
+        my $backmdl_file = $comp->{backmdl} if ($options & $PSTAMP_SELECT_BACKMDL);
+        my $pattern_file = $comp->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
+
+        my $outdir = dirname($output_base);
+        my $basename = basename($output_base);
+        my ($rownum, $jobnum, $therest) = split /_/, $basename;
+        &my_die("failed to split basename: $basename", $job_id, $PS_EXIT_CONFIG_ERROR) 
+            if (!$therest or !$rownum or !$jobnum);
+
+        my $prefix = "${rownum}_${jobnum}_";
+
+        if ($cmf_file) {
+            print "cmf file is $cmf_file\n";
+            copy_and_register_file($f, $cmf_file, $outdir, $prefix);
+        }
+        if ($psf_file) {
+            print "psf_file is $psf_file\n";
+            copy_and_register_file($f, $psf_file, $outdir, $prefix);
+        }
+        if ($backmdl_file) {
+            print "backmdl_file is $backmdl_file\n";;
+            copy_and_register_file($f, $backmdl_file, $outdir, $prefix);
+        }
+       if (0) {
+        # don't enable this yet
+        if ($pattern_file) {
+            print "pattern_file is $pattern_file\n";;
+            copy_and_register_file($f, $pattern_file, $outdir, $prefix);
+        }
+       }
+    }
+}
+
+sub read_params_file {
+    my $output_base = shift;
+
+    my $params_file = $output_base . ".mdc";
+    open (IN, "<$params_file") 
+        or my_die("failed to open params file: $params_file", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $data = $mdcParser->parse(join "", (<IN>))
+        or my_die("failed to parse params file: $params_file", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $components = parse_md_list($data);
+
+    my $n = scalar @$components;
+    if ($n != 1) {
+        my_die("params file $params_file contains unexpected number of components: $n",
+                $job_id, $PS_EXIT_PROG_ERROR);
+    }
+    return $components->[0];
+}
+
+# copy_and_register_file ($f, $src, $destdir, $prefix);
+sub copy_and_register_file {
+    my $F = shift;
+    my $src = shift;
+    my $destdir = shift;
+    my $prefix = shift;
+
+    my $fn = $prefix . basename($src);
+    my $dst = "$destdir/$fn";
+
+    my $resolved = $ipprc->file_resolve($src);
+
+    my_die("failed to resolve $src", $job_id, $PS_EXIT_UNKNOWN_ERROR) if !$resolved;
+
+    if (!-e $resolved) {
+        print STDERR "$src does not exist, skipping\n";
+        return;
+    }
+
+
+    copy($resolved, $dst) or my_die("failed to copy $resolved to $dst", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    print $F file_registration_line($fn, $dst, "fits") . "\n";
+}
 
 sub my_die
@@ -201 +311 @@
     my $exit_code = shift;      # Exit code to add
 
-    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+    $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code;
 
     carp($msg);
@@ -208 +318 @@
         $command .= " -job_id $job_id";
         $command .= " -fault $exit_code";
+        # XXX: fix pstamptool to not require -state when -fault with nonzero value is provided
+        $command .= " -state run";
         $command .= " -dbname $dbname" if defined $dbname;
         $command .= " -dbserver $dbserver" if defined $dbserver;