Changeset 27839 for branches/simmosaic_branches/pstamp/scripts/pstamp_job_run.pl

Timestamp:

May 3, 2010, 8:45:22 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/simmosaic_branches

Files:

• . (modified) (1 prop)
• pstamp/scripts/pstamp_job_run.pl (modified) (5 diffs)

branches/simmosaic_branches/pstamp/scripts/pstamp_job_run.pl

@@ -11 +11 @@
 use Sys::Hostname;
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Carp;
 use File::Basename;
+use File::Copy;
 use Digest::MD5::File qw( file_md5_hex );
 use PS::IPP::PStamp::RequestFile qw( :standard );
-
-my $verbose;
-my $dbname;
-my $dbserver;
-my $job_id;
-my $redirect_output;
-my $output_base;
-
-my $host = hostname();
-
+use IPC::Cmd 0.36 qw( can_run run );
+use POSIX;
+
+use PS::IPP::Metadata::Config;
+#use PS::IPP::Metadata::Stats;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config qw( :standard );
+
+my ($job_id, $redirect_output, $outputBase, $rownum, $jobType, $options); 
+my ($verbose, $dbname, $dbserver, $no_update);
 
 GetOptions(
     'job_id=s'          =>  \$job_id,
-    'output_base=s'     =>  \$output_base,
-    'redirect-output'   => \$redirect_output,
+    'job_type=s'        =>  \$jobType,
+    'rownum=s'          =>  \$rownum,
+    'output_base=s'     =>  \$outputBase,
+    'options=s'         =>  \$options,
+    'redirect-output'   =>  \$redirect_output,
     'dbname=s'          =>  \$dbname,
     'dbserver=s'        =>  \$dbserver,
     'verbose'           =>  \$verbose,
+    'no-update'         =>  \$no_update,
 );
 
 
+my $host = hostname();
 if ($verbose) {
     print "\n\n";
@@ -41 +49 @@
 
 die "job_id is required" if !$job_id;
-die "output_base is required" if !$output_base;
-
-use IPC::Cmd 0.36 qw( can_run run );
-
-use PS::IPP::Metadata::Config;
-use PS::IPP::Metadata::Stats;
-use PS::IPP::Metadata::List qw( parse_md_list );
-
-use PS::IPP::Config qw($PS_EXIT_SUCCESS
-                       $PS_EXIT_UNKNOWN_ERROR
-                       $PS_EXIT_SYS_ERROR
-                       $PS_EXIT_CONFIG_ERROR
-                       $PS_EXIT_PROG_ERROR
-                       $PS_EXIT_DATA_ERROR
-                       $PS_EXIT_TIMEOUT_ERROR
-                       metadataLookupStr
-                       metadataLookupBool
-                       caturi
-                       );
+
+my_die( "job_type is required", $job_id, $PS_EXIT_PROG_ERROR) if !$jobType;
+my_die("rownum is required", $job_id, $PS_EXIT_PROG_ERROR) if !$rownum;
+my_die("output_base is required", $job_id, $PS_EXIT_PROG_ERROR) if !$outputBase;
+
+$options = 1 if !$options;
+
+
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
-
 if ($redirect_output) {
-    my $logDest = "$output_base.log";
-    $ipprc->redirect_output($logDest);
-}
+    my $logDest = "$outputBase.log";
+    $ipprc->redirect_output($logDest)
+        or my_die ("unable to redirect output to $logDest", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+
 
 if (!$dbserver) {
@@ -78 +76 @@
 
 if ($missing_tools) {
-    warn("Can't find required tools.");
-    exit ($PS_EXIT_CONFIG_ERROR);
+    my_die("Can't find required tools", $job_id, $PS_EXIT_CONFIG_ERROR);
 }
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my $psjob;
-#Look up the uri for the given job
-{
-    my $command = "$pstamptool -pendingjob -job_id $job_id";
+my $jobStatus;
+if ($jobType eq "stamp") {
+    my $argslist = "$outputBase.args";
+    open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+    my $argString = <ARGSLIST>;
+    close ARGSLIST;
+    chomp $argString;
+
+    # XXX: should we do any other sanity checking?
+    my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+
+    my $command = "$ppstamp $outputBase $argString";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-    unless ($success) {
-        die("Unable to perform pstamptool -pendingjob: $error_code");
-    }
-
-    if (@$stdout_buf == 0) {
-        print STDERR "pending pstamp job id $job_id not found\n";
-        exit 0;
-    }
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        die("Unable to parse metdata config doc");
-
-    my $jobs = parse_md_list($metadata);
-    my $num = @$jobs;
-    die "unexpected number of jobs $num found for job: $job_id" if $num > 1;
-
-    $psjob = $jobs->[0];
-}
-
-if (!$psjob) {
-    print STDERR "postage stamp job $job_id not found\n";
-    exit 1;
-}
-
-my $rownum = $psjob->{rownum};
-my $uri = $psjob->{uri};
-my $outputBase = $psjob->{outputBase};
-my $argString = $psjob->{args};
-my $jobType = $psjob->{jobType};
-
-my $jobStatus;
-if ($jobType eq "stamp") {
-    my $command = "$ppstamp -file $uri $outputBase $argString";
+
+    my $exitStatus;
+    if (WIFEXITED($error_code)) {
+        $exitStatus = WEXITSTATUS($error_code);
+    } else {
+        print STDERR "ppstamp failed error_code: $error_code\n";
+        $exitStatus = $PS_EXIT_SYS_ERROR;
+    }
+
+    if ($exitStatus == 0) {
+        my $dir = dirname($outputBase);
+
+        my $reglist = "$dir/reglist$job_id";
+
+        my $F;
+        open $F, ">$reglist" or my_die( "can't open $reglist for output", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+        # Figure out what output images were produced
+
+        # Note: we are assuming the contents of the PSTAMP filerules here.
+        my %extensions = ( $PSTAMP_SELECT_IMAGE  => "fits", 
+                           $PSTAMP_SELECT_MASK   => "mk.fits",
+                           $PSTAMP_SELECT_WEIGHT => "wt.fits");
+
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_WEIGHT);
+
+        foreach my $key (keys (%extensions)) {
+            my $do_this_one = $key & $output_mask;
+
+            next if (! $do_this_one);
+
+            my $extension = $extensions{$key};
+
+            my $basename = basename($outputBase);
+
+            my $filename = "${basename}.${extension}";
+            my $path  = "${outputBase}.${extension}";
+
+            # XXX is pstamp always the right file type, if not where do we get the right one?
+            print $F file_registration_line($filename, $path, "pstamp") . "\n";
+        }
+
+        get_other_outputs($F, $outputBase, $options);
+
+        close $F;
+        $jobStatus = $PS_EXIT_SUCCESS;
+    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP) {
+        $jobStatus = $PSTAMP_NO_OVERLAP;
+    } else {
+        my_die( "ppstamp failed with error code: $exitStatus", $job_id, $exitStatus);
+    }
+} elsif ($jobType eq "get_image") {
+
+    my $uri = "";
+    my $command = "$pstamp_get_image_job --job_id $job_id --output_base $outputBase --rownum $rownum";
+    $command .= " --dbname $dbname" if $dbname;
+    $command .= " --dbserver $dbserver" if $dbserver;
+    $command .= " --verbose" if $verbose;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+
+    if ($success) {
+        $jobStatus = $PS_EXIT_SUCCESS;
+    } else {
+        $jobStatus = $error_code >> 8;
+        my_die( "pstamp_get_image_job failed with error code: $jobStatus", $job_id, $jobStatus);
+    }
+} elsif ($jobType eq "detect_query") {
+    my_die("detect_query jobs not supported yet", $job_id,$PS_EXIT_CONFIG_ERROR);
+} else {
+    my_die("unknown jobType $jobType found", $job_id, $PS_EXIT_PROG_ERROR);
+}
+
+# mark the job stopped in the database
+{
+    my $command = "$pstamptool -updatejob -job_id $job_id -state stop"; 
+    $command .= " -fault $jobStatus" if $jobStatus;
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
-    if ($success) {
-        # XXX shouldn't need to do this, review schema`
-        my $dir = dirname($outputBase);
-
-        my $reglist = "$dir/reglist$job_id";
-
-        open F, ">$reglist" or die "can't open $reglist for output";
-
-        # figure out what output images we should expect
-
-        # Note: we are assuming the contents of the filerules here.
-        my @extensions = ( "fits", "mk.fits", "wt.fits");
-
-        # we always require an image
-        my $output_mask = $PSTAMP_SELECT_IMAGE;
-
-        # we search the argString for -mask and -weight. 
-        # searching the arg string allows us to avoid adding a column in pstampJob
-        # XXX yeah but does this make us susceptible to user input fouling us up?
-        # For example, what if somebody sets the output base to file-mask-weight
-        # TODO: clean this up
-        $output_mask |= $PSTAMP_SELECT_MASK   if ($argString =~ /-mask/);
-        $output_mask |= $PSTAMP_SELECT_WEIGHT if ($argString =~ /-weight/);
-
-        # XXX we're getting a bit intimate with the bit field definitions here.
-        # do better use a hash
-
-        my $m = 1;
-        foreach my $extension (@extensions) {
-            my $do_this_one = $m & $output_mask;
-            $m = $m << 1;
-            next if (! $do_this_one);
-
-            my $basename = basename($outputBase);
-
-            my $filename = "${basename}.${extension}";
-            my $path  = "${outputBase}.${extension}";
-
-            # XXX is pstamp always the right file type, if not where do we get the right one?
-            print F file_registration_line($filename, $path, "pstamp") . "\n";
-        }
-
-        close F;
-        $jobStatus = $PS_EXIT_SUCCESS;
-    } else {
-        $jobStatus = $error_code >> 8;
-        print STDERR "ppstamp failed with error code: $jobStatus\n";
-    }
-} elsif ($jobType eq "get_image") {
-    my $command = "$pstamp_get_image_job --job_id $job_id --uri $uri --out_dir $outputBase --rownum $rownum";
-    $command .= " --dbname $dbname" if $dbname;
-    $command .= " --dbserver $dbserver" if $dbserver;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
-    if ($success) {
-        $jobStatus = $PS_EXIT_SUCCESS;
-    } else {
-        $jobStatus = $error_code >> 8;
-        print STDERR "ppstamp failed with error code: $jobStatus\n";
-    }
-} elsif ($jobType eq "detect_query") {
-    die("multiple detect_query jobs not supported yet");
-} else {
-    die("unknown jobType $jobType found");
-}
-
-# stop the job and set the result value
-{
-    my $command = "$pstamptool -updatejob -job_id $job_id -state stop -fault $jobStatus";
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        die("Unable to perform $command: $error_code");
-    }
-}
-
-exit $jobStatus;
+    if (!$no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            die("Unable to perform $command: $error_code");
+        }
+    } else {
+        print STDERR "skipping command: $command\n"
+    }
+}
+
+exit 0;
 
 # create a string to be passed as input to dsreg when registering this file in a fileset
@@ -219 +198 @@
     if (-e $path) {
         my @finfo = stat($path);
-        die "failed to stat $path" unless (@finfo);    # XXX clean up
+        my_die("failed to stat $path", $job_id, $PS_EXIT_UNKNOWN_ERROR) unless (@finfo); 
         my $bytes = $finfo[7];
         my $md5sum = file_md5_hex($path);
@@ -225 +204 @@
         return "$filename|$bytes|$md5sum|$filetype|";
     } else {
-        die "$filename not found at $path";
-    }
-}
+        my_die("$filename not found at $path", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+    }
+}
+        
+sub get_other_outputs {
+    my $f = shift;
+    my $output_base = shift;
+    my $options = shift;
+
+    if ($options & ( $PSTAMP_SELECT_CMF | $PSTAMP_SELECT_PSF | $PSTAMP_SELECT_BACKMDL)) {
+        my $comp = read_params_file($output_base);
+
+        my $stage = $comp->{stage};
+
+        # raw files don't have any other data products
+        return 1 if $stage eq "raw";
+
+        # add the other data products if they are selected and exist.
+        # silently skip them if they don't exist. Perhaps this should be
+        # detected in pstampparse so that the user can be notified with 
+        # a message in parse_error.txt ("warp do not have a background model")
+        my $cmf_file = $comp->{cmf} if ($options & $PSTAMP_SELECT_CMF);
+        my $psf_file = $comp->{psf} if ($options & $PSTAMP_SELECT_PSF);
+        my $backmdl_file = $comp->{backmdl} if ($options & $PSTAMP_SELECT_BACKMDL);
+        my $pattern_file = $comp->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
+
+        my $outdir = dirname($output_base);
+        my $basename = basename($output_base);
+        my ($rownum, $jobnum, $therest) = split /_/, $basename;
+        &my_die("failed to split basename: $basename", $job_id, $PS_EXIT_CONFIG_ERROR) 
+            if (!$therest or !$rownum or !$jobnum);
+
+        my $prefix = "${rownum}_${jobnum}_";
+
+        if ($cmf_file) {
+            print "cmf file is $cmf_file\n";
+            copy_and_register_file($f, $cmf_file, $outdir, $prefix);
+        }
+        if ($psf_file) {
+            print "psf_file is $psf_file\n";
+            copy_and_register_file($f, $psf_file, $outdir, $prefix);
+        }
+        if ($backmdl_file) {
+            print "backmdl_file is $backmdl_file\n";;
+            copy_and_register_file($f, $backmdl_file, $outdir, $prefix);
+        }
+       if (0) {
+        # don't enable this yet
+        if ($pattern_file) {
+            print "pattern_file is $pattern_file\n";;
+            copy_and_register_file($f, $pattern_file, $outdir, $prefix);
+        }
+       }
+    }
+}
+
+sub read_params_file {
+    my $output_base = shift;
+
+    my $params_file = $output_base . ".mdc";
+    open (IN, "<$params_file") 
+        or my_die("failed to open params file: $params_file", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $data = $mdcParser->parse(join "", (<IN>))
+        or my_die("failed to parse params file: $params_file", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $components = parse_md_list($data);
+
+    my $n = scalar @$components;
+    if ($n != 1) {
+        my_die("params file $params_file contains unexpected number of components: $n",
+                $job_id, $PS_EXIT_PROG_ERROR);
+    }
+    return $components->[0];
+}
+
+# copy_and_register_file ($f, $src, $destdir, $prefix);
+sub copy_and_register_file {
+    my $F = shift;
+    my $src = shift;
+    my $destdir = shift;
+    my $prefix = shift;
+
+    my $fn = $prefix . basename($src);
+    my $dst = "$destdir/$fn";
+
+    my $resolved = $ipprc->file_resolve($src);
+
+    my_die("failed to resolve $src", $job_id, $PS_EXIT_UNKNOWN_ERROR) if !$resolved;
+
+    if (!-e $resolved) {
+        print STDERR "$src does not exist, skipping\n";
+        return;
+    }
+
+
+    copy($resolved, $dst) or my_die("failed to copy $resolved to $dst", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    print $F file_registration_line($fn, $dst, "fits") . "\n";
+}
+
+sub my_die
+{
+    my $msg = shift;            # Warning message on die
+    my $job_id = shift;         # job identifier
+    my $exit_code = shift;      # Exit code to add
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code;
+
+    carp($msg);
+    if (defined $job_id and not $no_update) {
+        my $command = "$pstamptool -updatejob";
+        $command .= " -job_id $job_id";
+        $command .= " -fault $exit_code";
+        # XXX: fix pstamptool to not require -state when -fault with nonzero value is provided
+        $command .= " -state run";
+        $command .= " -dbname $dbname" if defined $dbname;
+        $command .= " -dbserver $dbserver" if defined $dbserver;
+        system($command);
+    }
+    exit $exit_code;
+}
+