Changeset 33098 for branches/meh_branches/ppsub_test/pstamp/scripts

Timestamp:

Jan 11, 2012, 11:19:21 PM (15 years ago)

Author:

mhuber

Message:

merged latest r33093 trunk changes to meh_branches/ppsub_test

Location:

branches/meh_branches/ppsub_test

Files:

• . (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (10 diffs)
• pstamp/scripts/pstamp_cleanup.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_get_image_job.pl (modified) (7 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (5 diffs)
• pstamp/scripts/pstamp_parser_run.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_server_status (modified) (1 diff)
• pstamp/scripts/pstampparse.pl (modified) (5 diffs)

branches/meh_branches/ppsub_test/pstamp/scripts/pstamp_checkdependent.pl

@@ -117 +117 @@
     }
 }
+
+# magic is no longer rquired
+$need_magic = 0;
+$magic_ok =  1;
 my $status = 0;
-if ((($it->{state} eq 'full') or ($it->{state} eq 'update')) and ($it->{data_state} eq 'full')
-        and (!$need_magic or $magic_ok or $it->{magicked} > 0)) {
+if ((($it->{state} eq 'full') or ($it->{state} eq 'update')) and ($it->{data_state} eq 'full')) {
 
     # This Dependency is satisfied. All done. Release the pstampJobs
@@ -197 +200 @@
     }
 
+if (0) {
     if (!$job_fault and ($stage eq 'chip')) {
         # what about "error_cleaned" ?
@@ -210 +214 @@
         }
     }
+}
 
     if ($job_fault >= $PSTAMP_FIRST_ERROR_CODE) {
@@ -240 +245 @@
         my $state = $chip->{state};
         my $data_state = $chip->{data_state};
+        my $fault = $chip->{fault};
         if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
-            ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) {
-
-            print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state cannot update\n";
+            ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop') or
+            ($fault eq $PSTAMP_GONE)) {
+
+            print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state fault: $fault cannot update\n";
             my $error_code;
             if (($state eq 'error_cleaned') or ($data_state ='error_cleaned')) {
@@ -253 +260 @@
             # caller will fault the jobs
             return $error_code;
-        } elsif ($chip->{dsRun_state} eq 'failed_revert') {
-            # XXX: revert failures are rarely fixed. give up but say it's just not available not GONE 
-            print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update\n";
-            return $PSTAMP_NOT_AVAILABLE;
         } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
 
@@ -282 +285 @@
             } elsif ($fault_count > $max_fault_count) {
                 print "$stage $stage_id has faulted $fault_count times. Giving up\n";
-                $fault = $PSTAMP_GONE;
-                faultComponent('chip', $chip->{chip_id}, $chip->{class_id}, $PSTAMP_GONE);
-                return $PSTAMP_GONE;
+                # XXX: stop faulting chips for now
+#                $fault = $PSTAMP_GONE;
+#                faultComponent('chip', $chip->{chip_id}, $chip->{class_id}, $PSTAMP_GONE);
+                $fault = $PSTAMP_NOT_AVAILABLE;
+                return $fault;
             }
             # fault the dependent
             my_die("chip $chip->{chip_id} $chip->{class_id} faulted: $chip->{fault}", $chip->{fault});
-        } elsif ($chip->{dsFile_fault} eq $PSTAMP_GONE) {
-            print STDERR "magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} is GONE";
-            return $PSTAMP_GONE;
-        } elsif ($chip->{dsFile_fault} and ($chip->{dsFile_data_state} eq 'update')) {
-            # fault the dependent
-            my_die("magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} faulted: $chip->{dsFile_fault}", $chip->{dsFile_fault});
-        }
-    }
-
-    my $status = 0;
-    if (!$queued_update) {
-        $status = check_states_magicDSRun('chip', $chip_id, \@chips, $rlabel, $need_magic, $raw_all_magicked, $magic_ds_id, $dsRun_state);
-    }
-
-    return $status;
+        }
+    }
+
+    return 0;
 }
 
@@ -321 +315 @@
     # The update system currently requires that data that has been magicked be destreaked
     # at chip stage so it needs magic even if the customer doesn't.
+if (0) {
     if ($metadata->{magicked} != 0) {
         $need_magic = 1;
     }
-
+}
     if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
          ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) {
@@ -369 +364 @@
         # if chip has been magicked before require it to be magicked again
         # because the warp pending query requires it.
+if (0) {
         if ($chip->{magicked} < 0) {
             print "Input has been destreaked so we must destreak before warping\n";
             $need_magic = 1 
         }
+}
 
         if ($need_magic and ($chip->{magicked} eq 0)) {
@@ -421 +418 @@
     my $metadata = shift;
     my $rlabel = shift;     # if defined a new label for the chipRun
-    my $need_magic = shift; 
+    my $need_magic = 0; 
 
     my $diff_id   = $metadata->{diff_id};
@@ -427 +424 @@
     my $skycell = $metadata;
     my $skycell_id = $skycell->{skycell_id};
-
-    # The update system currently requires that data that has been magicked be destreaked
-    # at chip stage so it needs magic even if the customer doesn't.
-    if ($metadata->{magicked} != 0) {
-        $need_magic = 1;
-    }
 
     if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {

branches/meh_branches/ppsub_test/pstamp/scripts/pstamp_cleanup.pl

@@ -23 +23 @@
 
 use PS::IPP::Config qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 
 my $req_id;
@@ -95 +96 @@
 }
 
+{
+    my $command = "$pstamptool -listfile -req_id $req_id";
+    $command   .= " -dbname $dbname" if $dbname;
+    $command   .= " -dbserver $dbserver" if $dbserver;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        die("Unable to perform $command error code: $error_code");
+    }
+    my $output = join "", @$stdout_buf;
+    if ($output) {
+        my $files = parse_md_fast($mdcParser, $output);
+        foreach my $file (@$files) {
+            $ipprc->file_delete($file->{path}); 
+        }
+    }
+}
+
 # now go find the workdir for this request 
 # XXX: we finally *have* to store this in the database

branches/meh_branches/ppsub_test/pstamp/scripts/pstamp_get_image_job.pl

@@ -25 +25 @@
 
 my $output_base;
+my $bundleroot;
 
 my $verbose;
-my $ipprc;
+my $imagedbname;
 my $dbname;
 my $dbserver;
@@ -41 +42 @@
         'rownum=s'        =>      \$rownum,
         'output_base=s'   =>      \$output_base,
+        'bundleroot=s'    =>      \$bundleroot,
+        'imagedbname=s'   =>      \$imagedbname,
         'dbname=s'        =>      \$dbname,
         'dbserver=s'      =>      \$dbserver,
@@ -52 +55 @@
 
 my_die( $err, $PS_EXIT_PROG_ERROR) if $err;
+
+my $ipprc = PS::IPP::Config->new();
 
 my $params_file = $output_base . ".mdc";
@@ -91 +96 @@
 my $missing_tools;
 my $dist_bundle   = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1);
+my $pstamptool   = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -96 +102 @@
 }
 
+$pstamptool .= " -dbname $dbname" if $dbname;
+$pstamptool .= " -dbserver $dbserver" if $dbserver;
+
 my $outdir = dirname($output_base);
 my $basename = basename($path_base);
-my $outroot = $output_base ."_" . $basename;
 my $results_file = $output_base . ".bundle_results";
+my $outroot;
+if ($bundleroot) {
+    my (undef, undef, undef, $mday, $month, $year) = gmtime(time());
+    $month += 1;
+    $year += 1900;
+
+    # This will generate an outroot like:
+    # neb://any/pstamp/bundles/2011/11/22/14026916_1_1_o5739g0358o.353654.wrp.215092.skycell.2083.025
+
+    $outroot = $bundleroot . sprintf("/%4d/%02d/%02d/${job_id}_", $year, $month, $mday) . basename($output_base) . "_". $basename;
+} else {
+    $outroot = $output_base ."_" . $basename;
+}
 
 {
@@ -105 +126 @@
     $command .= " --results_file $results_file";
     $command .= " --component $component --path_base $path_base --outroot $outroot";
-#    XXX: we need to do some work if we want to support muggle bundles
-#    $command .= " --no_magic if $no_magic";
-    $command .= " --magicked" if $magicked;
-    $command .= " --dbname $dbname" if $dbname;
+    # DING DONG ....
+    $command .= " --no_magic";
+    $command .= " --dbname $imagedbname" if $imagedbname;
     $command .= " --verbose" if $verbose;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -147 +167 @@
 }
 
+if ($bundleroot) {
+    {
+        # delete any existing pstampFile in case this job has faulted and
+        # been reverted
+        my $command = "$pstamptool -deletefile -job_id $job_id";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("Unable to perform $command: $error_code", $error_code >> 8);
+        }
+    }
+    my $linkname = "$outdir/$file_name";
+    if (-l $linkname or -e $linkname) {
+        unlink $linkname or my_die("Failed to unlink existing symlink: $linkname", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    my $bundle_name = dirname($outroot) . "/$file_name";
+    my $resolved = $ipprc->file_resolve($bundle_name);
+    if (!$resolved) {
+        my_die("failed to resolve $bundle_name", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    symlink $resolved, $linkname or my_die("failed to create symlink to $resolved in $outdir", 
+        $PS_EXIT_UNKNOWN_ERROR);
+
+    my $command = "$pstamptool -addfile -job_id $job_id -path $bundle_name";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        my_die("Unable to perform $command: $error_code", $error_code >> 8);
+    }
+}
+
+
 print REGLIST "$file_name|$bytes|$md5sum|tgz|\n";
 

branches/meh_branches/ppsub_test/pstamp/scripts/pstamp_job_run.pl

@@ -59 +59 @@
 $options = 1 if !$options;
 
+# We don't need to muggle anymore. Ignore those options
+$options &=  ~($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED);
 
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
@@ -88 +90 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
+my $params = read_params_file($outputBase);
+
 my $jobStatus;
 if ($jobType eq "stamp") {
-    my $params = read_params_file($outputBase);
 
     my $argString;
@@ -107 +110 @@
         $nan_masked = 0;
         $muggle = 1;
+    }
+
+    if ($stage eq "raw") {
+        $options &= ~($PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE); 
     }
 
@@ -214 +221 @@
     }
 
+    # MAGIC IS DEAD
+    $nan_masked = 0;
+
     my $command = "$ppstamp $outputBase $argString $fileArgs";
     $command .= " -dbname $dbname" if $dbname;
@@ -272 +282 @@
 } elsif ($jobType eq "get_image") {
 
-    my $uri = "";
+    my $pstamp_bundle_root = metadataLookupStr($ipprc->{_siteConfig}, "PSTAMP_BUNDLE_ROOT");
+    my $imagedb = $params->{imagedb};
+
     my $command = "$pstamp_get_image_job --job_id $job_id --output_base $outputBase --rownum $rownum";
+    $command .= " --bundleroot $pstamp_bundle_root" if $pstamp_bundle_root;
+    $command .= " --imagedb $imagedb" if $imagedb;
     $command .= " --dbname $dbname" if $dbname;
     $command .= " --dbserver $dbserver" if $dbserver;

branches/meh_branches/ppsub_test/pstamp/scripts/pstamp_parser_run.pl

@@ -55 +55 @@
 }
 
+$need_magic = 0;
+
 my $missing_tools;
 
@@ -89 +91 @@
     # outdir is where all of the files generated for this request are placed
     # NOTE: this location needs to be kept in sync with the web interface ( request.php )
-    my $datestr = strftime "%Y%m%d", gmtime;
+    my $datestr = strftime "%Y/%m/%d", gmtime;
     my $datedir = "$pstamp_workdir/$datestr";
     if (! -e $datedir ) {
-        mkdir $datedir or my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
-            $PS_EXIT_CONFIG_ERROR);
+        my $rc = system "mkdir -p $datedir";
+        if ($rc) {
+            my $status = $rc >> 8;
+            $status = $PS_EXIT_CONFIG_ERROR if !$status;
+            my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
+                $status);
+        }
     }
 

branches/meh_branches/ppsub_test/pstamp/scripts/pstamp_server_status

@@ -71 +71 @@
         $error_code = (($error_code >> 8) or 1);
         if ($error_code == 12) {
-            #print "Postage Stamp Server is not running\n";
-            print "Postage Stamp Server will be down for maintenance it will back shortly.\n";
+            print "Postage Stamp Server is not running\n";
+            #print "Postage Stamp Server will be down for maintenance it will back shortly.\n";
             exit 0;
         } else {

branches/meh_branches/ppsub_test/pstamp/scripts/pstampparse.pl

@@ -148 +148 @@
     } elsif ($lcname =~ /qub/) {
         $label = "QUB";
+        $label_changed = 1;
+    } elsif ($lcname =~ /pypstamp/) {
+        $label = "PY";
         $label_changed = 1;
     }
@@ -251 +254 @@
     # If we encounter an error for a particular row add a job with the proper fault code.
 
+    my $stage = $row->{IMG_TYPE};
+
     my $rownum   = $row->{ROWNUM};
     if (!validID($rownum)) {
@@ -272 +277 @@
     }
     if ($job_type eq 'get_image') {
-        unless ($req_type eq 'byid' or $req_type eq 'byexp') {
-            print STDERR "REQ_TYPE must be 'byid' or 'byexp' for JOB_TYPE 'get_image'\n";
+        unless ($req_type eq 'byid' or $req_type eq 'byexp' or ($req_type eq 'byskycell' and $stage eq 'stack')) {
+            print STDERR "REQ_TYPE must be 'byid' or 'byexp' or byskcyell for stacks for JOB_TYPE 'get_image'\n";
             insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
             return 0;
@@ -334 +339 @@
     }
 
-    my $stage = $row->{IMG_TYPE};
     if (!check_image_type($stage)) {
         print STDERR "invalid IMG_TYPE for row $rownum\n";
@@ -460 +464 @@
     my $numRows = scalar @$rowList;
 
+    if (($req_type eq 'byskycell') or ($req_type eq 'bycoord')) {
+        # avoid error from print below if $id isn't needed
+        $id = "" if !$id;
+    }
     print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n";