Changeset 26151 for trunk/pstamp

Timestamp:

Nov 15, 2009, 1:53:33 PM (17 years ago)

Author:

bills

Message:

implementation for "get_image" jobs

Location:

trunk/pstamp/scripts

Files:

• pstamp_get_image_job.pl (modified) (3 diffs)
• pstamp_job_run.pl (modified) (1 diff)
• pstamp_parser_run.pl (modified) (2 diffs)
• pstampparse.pl (modified) (7 diffs)

trunk/pstamp/scripts/pstamp_get_image_job.pl

@@ -16 +16 @@
 use File::Basename;
 use Digest::MD5::File qw( file_md5_hex );
+use IPC::Cmd 0.36 qw( can_run run );
 
-use PS::IPP::Config qw($PS_EXIT_SUCCESS
-                       $PS_EXIT_UNKNOWN_ERROR
-                       $PS_EXIT_SYS_ERROR
-                       $PS_EXIT_CONFIG_ERROR
-                       $PS_EXIT_PROG_ERROR
-                       $PS_EXIT_DATA_ERROR
-                       $PS_EXIT_TIMEOUT_ERROR
-                       metadataLookupStr
-                       metadataLookupBool
-                       caturi
-                       );
+
+use PS::IPP::Config qw( :standard );
+
 my $product;
 my $fileset;
 
-my $uri;
-my $out_dir;
+my $output_base;
 
-my $verbose = 1;
+my $verbose;
 my $ipprc;
 my $dbname;
+my $dbserver;
 my $job_id;
 my $rownum;
@@ -47 +40 @@
         'job_id=s'        =>      \$job_id,
         'rownum=s'        =>      \$rownum,
-        'uri=s'           =>      \$uri,
-        'out_dir=s'       =>      \$out_dir,
+        'output_base=s'   =>      \$output_base,
         'dbname=s'        =>      \$dbname,
+        'dbserver=s'      =>      \$dbserver,
+        'verbose'         =>      \$verbose,
 ) or pod2usage(2);
 
@@ -55 +49 @@
 $err .= "--job_id is required\n" if (!$job_id);
 $err .= "--rownum is required\n" if (!$rownum);
-$err .= "--uri is required to specify the file list\n" if (!$uri);
-$err .= "--out_dir is required to specify the output fileset\n" if (!$out_dir);
+$err .= "--output_base is required to specify the output fileset\n" if (!$output_base);
 
 die $err if $err;
 
-# collapse any multiple slashes in output_dir
-$out_dir = File::Spec->canonpath($out_dir);
+my $params_file = $output_base . ".params";
+if (! open(INPUT, "<$params_file") ) {
+    die("failed to open params file: $params_file");
+}
 
-my @path = split(/\//, $out_dir);
+my $params = join "", (<INPUT>);
 
-my $nelem = @path;
-if ($nelem < 2) {
-    die("$out_dir is not a valid output fileset directory");
+print STDERR "\nparams is $params\n";
+
+my ($stage, $stage_id, $component, $path_base, $camera) = split " ", $params;
+if (!$camera or !$path_base or !$component or !$stage_id or !$stage) {
+   die "failed to parse params: $params";
 }
-$fileset = $path[$nelem-1];
-$product = $path[$nelem-2];
 
-$_ = $out_dir;
-my ($ds_dir) = m%(.*)/$product/$fileset%;
+print STDERR "\nCAMERA is $camera\n";
 
-if ($verbose) {
-    print STDERR "DS root:  $ds_dir\n";
-    print STDERR "Product:  $product\n";
-    print STDERR "Fileset:  $fileset\n";
-    print STDERR "Filelist: $uri\n";
+my $out_dir = dirname($output_base);
+my $prefix = basename($output_base);
+my $results_file = $output_base . ".bundle_results";
+
+# Look for programs we need
+my $missing_tools;
+my $dist_bundle   = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
 }
 
 
-if (!stat("$ds_dir/index.txt")) {
-    $err .= "Data Store not found at '$ds_dir'.\n"
+{
+    my $command = "$dist_bundle --camera $camera --stage $stage --stage_id $stage_id --component $component";
+    $command .= " --path_base $path_base --outdir $out_dir --results_file $results_file";
+    $command .= " --prefix $prefix";
+    # ignore magic for now
+    $command .= " --no_magic";
+
+    $command .= " --dbname $dbname" if $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        die("Unable to perform $command: $error_code");
+    }
 }
 
-show_usage("Invalid product '$product'.")
-    unless defined stat("$ds_dir/$product/index.txt");
-
-if (! open(INPUT, "<$uri") ) {
-    die("failed to open file list: $uri");
+if (! open(RESULTS, "<$results_file")) {
+    die("failed to open bundle results file: $results_file");
 }
 
-my $reglist = "$out_dir/reglist";
-if (! open(REGLIST, ">$reglist$job_id") ) {
+my $file_name;
+my $bytes;
+my $md5sum;
+foreach my $line (<RESULTS>) {
+    chomp $line;
+    next if !$line;
+    next if $line =~ /bundleResults/;
+    last if $line =~ /END/;
+
+    my ($tag, $type, $val) = split " ", $line;
+    if ($tag eq "name") {
+        $file_name = $val;
+    } elsif ($tag eq "bytes") {
+        $bytes = $val;
+    } elsif ($tag eq "md5sum") {
+        $md5sum = $val;
+    } else {
+        die("unexpected tag: $tag  found in results file: $results_file");
+    }
+}
+
+my $reglist = "$out_dir/reglist$job_id";
+if (! open(REGLIST, ">$reglist") ) {
     die("failed to open registration list: $reglist");
 }
 
-my @files;
-my $num = 0;
-while (<INPUT>) {
-    chomp;
-    my @fields = split /\|/;
-    my $pathname = shift @fields;
-    my $filetype = shift @fields;
-    my $class_id = shift @fields;
-
-    ($pathname , my $filename) = resolvepath($pathname);
-
-    if ($verbose) {
-        print STDERR "$pathname @fields\n";
-    }
-
-    $num++;
-    $filename = "${rownum}_${num}_$filename";
-    my $dest_path = "$out_dir/$filename";
-    if (! copy $pathname, "$dest_path" ) {
-        die("failed trying to copy $pathname to $out_dir");
-    }
-
-    # XXX is pstamp always the right file type, if not where can we get the right one?
-    print REGLIST file_registration_line($filename, $dest_path, $filetype, $class_id);
-
-    foreach my $f (@fields) {
-        print REGLIST "$f|";
-    }
-    print REGLIST "\n";
-}
+print REGLIST "$file_name|$bytes|$md5sum|tgz|\n";
 
 close(REGLIST);
 
 exit 0;
-
-sub resolvepath {
-    my $pathname = $_[0];
-
-    # use the basename of the unresolved file as the name of the file
-    # in order to avoid nebulous name mangled paths in the datastore
-    my $file = basename($pathname);
-
-    my $slash = index($pathname, "/");
-    if ($slash != 0) {
-        if (!$ipprc) {
-            $ipprc = PS::IPP::Config->new();
-        }
-        $pathname = $ipprc->file_resolve($pathname);
-    }
-
-
-    return ($pathname, $file);
-}
-
-# XXX move this to a module so the code can be shared 
-sub file_registration_line {
-    my $filename = shift;
-    my $path     = shift;
-    my $filetype = shift;
-    my $chipname = shift;
-    $chipname = $chipname ? "$chipname|" : "";
-
-    if (-e $path) {
-        my @finfo = stat($path);
-        die "failed to stat $path" unless (@finfo);    # XXX clean up
-        my $bytes = $finfo[7];
-        my $md5sum = file_md5_hex($path);
-
-        return "$filename|$bytes|$md5sum|$filetype|$chipname";
-    } else {
-        die "$filename not found at $path";
-    }
-}

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -148 +148 @@
     }
 } elsif ($jobType eq "get_image") {
-    my_die( "get_image jobs not working right now", $job_id, $PS_EXIT_PROG_ERROR);
 
     my $uri = "";
-    my $command = "$pstamp_get_image_job --job_id $job_id --uri $uri --out_dir $outputBase --rownum $rownum";
+    my $command = "$pstamp_get_image_job --job_id $job_id --output_base $outputBase --rownum $rownum";
     $command .= " --dbname $dbname" if $dbname;
     $command .= " --dbserver $dbserver" if $dbserver;

trunk/pstamp/scripts/pstamp_parser_run.pl

@@ -13 +13 @@
 use Getopt::Long qw( GetOptions );
 use File::Basename qw( basename dirname);
+use File::Copy;
 use POSIX qw( strftime );
 use Carp;
@@ -119 +120 @@
         unlink $new_uri or my_die("failed to unlink $new_uri", $req_id, $PS_EXIT_UNKNOWN_ERROR);
     }
-    if (! symlink $uri, $new_uri) {
-        my_die ("failed to link request file $uri to workdir $workdir", $req_id, $PS_EXIT_UNKNOWN_ERROR);
+    if (! copy $uri, $new_uri) {
+        my_die ("failed to copy request file $uri to workdir $workdir", $req_id, $PS_EXIT_UNKNOWN_ERROR);
     }
 }

trunk/pstamp/scripts/pstampparse.pl

@@ -98 +98 @@
 
 # make sure the file contains what we are expecting
-
-# we shouldn't get here if this is the case so just die. No need to notify the client
+# This program shouldn't have been run if the request file is bogus.
+# No need to notify the client
 my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
 my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
@@ -165 +165 @@
     # XXX: TODO: sanity check all parameters
 
-    # XXX: TODO: We shouldn't really just die in this loop.
-    # If we encounter an error for a particular row
-    # add a job with the proper fault code. If there is a db or config error we should probably just
-    # trash the request.
+    # If we encounter an error for a particular row add a job with the proper fault code.
+    # If we encounter an error in this loop we shouldn't really just die.
+    # We only do that now in the case of I/O or DB errors or the like.
 
     my $rownum   = $row->{ROWNUM};
@@ -236 +235 @@
     }
 
-    # collect rows with the same images of interest in a list so that they
-    # can be looked up together
+    # For "stamp" and "list_uri" jobs collect rows with the same images of interest in a list so that they
+    # can be looked up together.
     if (@rowList) {
         my $firstRow = $rowList[0];
-        # note order of these parameters matters
-        if (same_images_of_interest($firstRow, $row)) {
+        if (($firstRow->{JOB_TYPE} ne "get_image") and same_images_of_interest($firstRow, $row)) {
 
             # add this row to the list and move on
@@ -305 +303 @@
     my $have_skycells = shift;
     my $need_magic = shift;
+    my $mode = shift;
 
     my $num_jobs = 0;
@@ -476 +475 @@
         }
     } elsif ($job_type eq "get_image") {
-        print STDERR "get_image jobs not implemented yet" if $verbose;
-        foreach my $row (@$rowList) {
-            insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
-            $num_jobs++;
-        }
+        my $n = scalar @$rowList;
+
+        my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1;
+
+        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+
     } else {
         if (!$imageList or (scalar @$imageList eq 0)) {
@@ -582 +582 @@
             
             foreach my $row (@$rowList) {
-                $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic);
+                $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic, $mode);
             }
         }
+    }
+    return $num_jobs;
+}
+
+#        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+sub queueGetImageJobs
+{
+    my $row = shift;
+    my $imageList = shift;
+    my $stage = shift;
+    my $need_magic = shift;
+    my $mode = shift;
+
+    my $num_jobs = 0;
+    my $rownum = $row->{ROWNUM};
+    my $option_mask = $row->{OPTION_MASK};
+
+    # For dist_bundle we need
+    #  --camera from $image
+    #  --stage 
+    #  --stage_id from $image
+    #  --component from $image
+    #  --path_base 
+    #  --outdir global to this script
+
+    # loop over images
+    my $job_num = 0;
+    foreach my $image (@$imageList) {
+        my $stage_id = $image->{stage_id};
+        my $component = $image->{component};
+
+        # skip faulted components for now. Should we even be here?
+        if ($image->{fault} > 0) {
+            printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose;
+            next;
+        }
+
+        $job_num++;
+
+        my $imagefile = $image->{image};
+        if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
+            # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed
+            # in locate_images because it's much more efficient to do the test in the database.
+            # For these two modes we fall through to here in order to give feedback to the requestor as
+            # to why the request failed to queue jobs.
+            print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
+            insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
+            $num_jobs++;
+
+            next;
+        }
+        my $exp_id = $image->{exp_id};
+            
+        my $output_base = "$out_dir/${rownum}_${job_num}_";
+        my $params = "${output_base}.params";
+
+        # copy the argument list to a file
+        open PARAMS, ">$params" or my_die("failed to open $params", $PS_EXIT_UNKNOWN_ERROR);
+        print PARAMS "$stage $image->{stage_id} $image->{component} $image->{path_base} $image->{camera}\n";
+        close PARAMS or my_die("failed to close $params", $PS_EXIT_UNKNOWN_ERROR);
+
+        my $newState = "run";
+        my $fault = 0;
+        my $dep_id;
+
+        if ($stage ne 'raw') {
+            my $run_state = $image->{state};
+            my $data_state = $image->{data_state};
+            $data_state = $run_state if $stage eq "stack";
+            if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or
+                ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) {
+                # image is gone and it's not coming back
+                $newState = 'stop';
+                $fault = $PSTAMP_GONE;
+            } elsif (($data_state ne 'full') or ($run_state ne 'full' )) {
+                # don't wait for update unless the caller asks us to
+                if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) {
+                    $newState = 'stop';
+                    $fault = $PSTAMP_NOT_AVAILABLE;
+                } else {
+                    # cause the image to be re-made
+                    # set up to queue an update run
+                    queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, 
+                        $run_state, $stage, $image->{stage_id}, $need_magic, $image->{label});
+                }
+            }
+        }
+
+        $num_jobs++;
+        my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
+                        . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask";
+        $command .= " -fault $fault" if $fault;
+        $command .= " -exp_id $exp_id" if $exp_id;
+        $command .= " -dep_id $dep_id" if $dep_id;
+
+        if ($mode eq "list_job") { 
+            # this is a debugging mode, just print the pstamptool that would have run
+            # this is sort of like the mode -noupdate that some other tools support
+            print "$command\n";
+        } elsif (!$no_update) {
+            # mode eq "queue_job"
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                print STDERR @$stderr_buf;
+                # XXX TODO: now what? Should we mark the error state for the request?
+                # should we keep going for other uris? If so how do we report that some
+                # of the work that the request wanted isn't going to get done
+                my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping command: $command\n";
+        }
+    }
+    if ( $num_jobs == 0 ) {
+        print STDERR "no jobs for row $rownum\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_NO_JOBS_QUEUED);
+        $num_jobs = 1;
     }
     return $num_jobs;
@@ -677 +795 @@
     return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE});
     return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE});
-    return 0 if ($r1->{ID} ne $r2->{ID});
-    return 0 if ($r1->{TESS_ID} ne $r2->{TESS_ID});
-    return 0 if ($r1->{REQFILT} ne $r2->{REQFILT});
-    return 0 if ($r1->{LABEL} ne $r2->{LABEL});
-    return 0 if ($r1->{MJD_MIN} ne $r2->{MJD_MAX});
-    return 0 if ($r1->{MJD_MAX} ne $r2->{MJD_MAX});
-    return 0 if ($r1->{inverse} ne $r2->{inverse});
+    return 0 if ($r1->{ID}       ne $r2->{ID});
+    return 0 if ($r1->{TESS_ID}  ne $r2->{TESS_ID});
+    return 0 if ($r1->{REQFILT}  ne $r2->{REQFILT});
+    return 0 if ($r1->{LABEL}    ne $r2->{LABEL});
+    return 0 if ($r1->{MJD_MIN}  ne $r2->{MJD_MAX});
+    return 0 if ($r1->{MJD_MAX}  ne $r2->{MJD_MAX});
+    return 0 if ($r1->{inverse}  ne $r2->{inverse});
 
     if (defined($r1->{COMPONENT})) {