Changeset 32852 for branches/eam_branches/ipp-20111122/pstamp/scripts

Timestamp:

Dec 2, 2011, 11:45:13 AM (15 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20111122/pstamp/scripts

Files:

• pstamp_checkdependent.pl (modified) (1 diff)
• pstamp_cleanup.pl (modified) (2 diffs)
• pstamp_get_image_job.pl (modified) (7 diffs)
• pstamp_job_run.pl (modified) (2 diffs)
• pstamp_parser_run.pl (modified) (1 diff)

branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_checkdependent.pl

@@ -255 +255 @@
             # caller will fault the jobs
             return $error_code;
-        } elsif ($chip->{dsRun_state} eq 'failed_revert') {
+        } elsif ($chip->{dsRun_state} =~ /failed_revert/) {
             # XXX: revert failures are rarely fixed. give up but say it's just not available not GONE 
             print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update\n";

branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_cleanup.pl

@@ -23 +23 @@
 
 use PS::IPP::Config qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 
 my $req_id;
@@ -95 +96 @@
 }
 
+{
+    my $command = "$pstamptool -listfile -req_id $req_id";
+    $command   .= " -dbname $dbname" if $dbname;
+    $command   .= " -dbserver $dbserver" if $dbserver;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        die("Unable to perform $command error code: $error_code");
+    }
+    my $output = join "", @$stdout_buf;
+    if ($output) {
+        my $files = parse_md_fast($mdcParser, $output);
+        foreach my $file (@$files) {
+            $ipprc->file_delete($file->{path}); 
+        }
+    }
+}
+
 # now go find the workdir for this request 
 # XXX: we finally *have* to store this in the database

branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_get_image_job.pl

@@ -25 +25 @@
 
 my $output_base;
+my $bundleroot;
 
 my $verbose;
-my $ipprc;
+my $imagedbname;
 my $dbname;
 my $dbserver;
@@ -41 +42 @@
         'rownum=s'        =>      \$rownum,
         'output_base=s'   =>      \$output_base,
+        'bundleroot=s'    =>      \$bundleroot,
+        'imagedbname=s'   =>      \$imagedbname,
         'dbname=s'        =>      \$dbname,
         'dbserver=s'      =>      \$dbserver,
@@ -52 +55 @@
 
 my_die( $err, $PS_EXIT_PROG_ERROR) if $err;
+
+my $ipprc = PS::IPP::Config->new();
 
 my $params_file = $output_base . ".mdc";
@@ -91 +96 @@
 my $missing_tools;
 my $dist_bundle   = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1);
+my $pstamptool   = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -96 +102 @@
 }
 
+$pstamptool .= " -dbname $dbname" if $dbname;
+$pstamptool .= " -dbserver $dbserver" if $dbserver;
+
 my $outdir = dirname($output_base);
 my $basename = basename($path_base);
-my $outroot = $output_base ."_" . $basename;
 my $results_file = $output_base . ".bundle_results";
+my $outroot;
+if ($bundleroot) {
+    my (undef, undef, undef, $mday, $month, $year) = gmtime(time());
+    $month += 1;
+    $year += 1900;
+
+    # This will generate an outroot like:
+    # neb://any/pstamp/bundles/2011/11/22/14026916_1_1_o5739g0358o.353654.wrp.215092.skycell.2083.025
+
+    $outroot = $bundleroot . sprintf("/%4d/%02d/%02d/${job_id}_", $year, $month, $mday) . basename($output_base) . "_". $basename;
+} else {
+    $outroot = $output_base ."_" . $basename;
+}
 
 {
@@ -105 +126 @@
     $command .= " --results_file $results_file";
     $command .= " --component $component --path_base $path_base --outroot $outroot";
-#    XXX: we need to do some work if we want to support muggle bundles
-#    $command .= " --no_magic if $no_magic";
+    #    XXX: we need to do some work if we want to support muggle bundles
+    #    $command .= " --no_magic if $no_magic";
     $command .= " --magicked" if $magicked;
-    $command .= " --dbname $dbname" if $dbname;
+    $command .= " --dbname $imagedbname" if $imagedbname;
     $command .= " --verbose" if $verbose;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -147 +168 @@
 }
 
+if ($bundleroot) {
+    {
+        # delete any existing pstampFile in case this job has faulted and
+        # been reverted
+        my $command = "$pstamptool -deletefile -job_id $job_id";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("Unable to perform $command: $error_code", $error_code >> 8);
+        }
+    }
+    my $linkname = "$outdir/$file_name";
+    if (-l $linkname or -e $linkname) {
+        unlink $linkname or my_die("Failed to unlink existing symlink: $linkname", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    my $bundle_name = dirname($outroot) . "/$file_name";
+    my $resolved = $ipprc->file_resolve($bundle_name);
+    if (!$resolved) {
+        my_die("failed to resolve $bundle_name", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    symlink $resolved, $linkname or my_die("failed to create symlink to $resolved in $outdir", 
+        $PS_EXIT_UNKNOWN_ERROR);
+
+    my $command = "$pstamptool -addfile -job_id $job_id -path $bundle_name";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        my_die("Unable to perform $command: $error_code", $error_code >> 8);
+    }
+}
+
+
 print REGLIST "$file_name|$bytes|$md5sum|tgz|\n";
 

branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_job_run.pl

@@ -88 +88 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
+my $params = read_params_file($outputBase);
+
 my $jobStatus;
 if ($jobType eq "stamp") {
-    my $params = read_params_file($outputBase);
 
     my $argString;
@@ -272 +273 @@
 } elsif ($jobType eq "get_image") {
 
-    my $uri = "";
+    my $pstamp_bundle_root = metadataLookupStr($ipprc->{_siteConfig}, "PSTAMP_BUNDLE_ROOT");
+    my $imagedb = $params->{imagedb};
+
     my $command = "$pstamp_get_image_job --job_id $job_id --output_base $outputBase --rownum $rownum";
+    $command .= " --bundleroot $pstamp_bundle_root" if $pstamp_bundle_root;
+    $command .= " --imagedb $imagedb" if $imagedb;
     $command .= " --dbname $dbname" if $dbname;
     $command .= " --dbserver $dbserver" if $dbserver;

branches/eam_branches/ipp-20111122/pstamp/scripts/pstamp_parser_run.pl

@@ -89 +89 @@
     # outdir is where all of the files generated for this request are placed
     # NOTE: this location needs to be kept in sync with the web interface ( request.php )
-    my $datestr = strftime "%Y%m%d", gmtime;
+    my $datestr = strftime "%Y/%m/%d", gmtime;
     my $datedir = "$pstamp_workdir/$datestr";
     if (! -e $datedir ) {
-        mkdir $datedir or my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
-            $PS_EXIT_CONFIG_ERROR);
+        my $rc = system "mkdir -p $datedir";
+        if ($rc) {
+            my $status = $rc >> 8;
+            $status = $PS_EXIT_CONFIG_ERROR if !$status;
+            my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
+                $status);
+        }
     }