Changeset 30812 for branches/eam_branches/ipp-20110213/pstamp

Timestamp:

Mar 6, 2011, 2:34:55 PM (15 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/eam_branches/ipp-20110213/pstamp/scripts

Files:

• pstamp_checkdependent.pl (modified) (6 diffs)
• pstampparse.pl (modified) (5 diffs)
• pstampstopfaulted (modified) (1 diff)

branches/eam_branches/ipp-20110213/pstamp/scripts/pstamp_checkdependent.pl

@@ -27 +27 @@
 my $IPP_DIFF_MODE_STACK_STACK = 4;
 
-my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count);
+my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count, $logfile);
 my ($dbname, $ps_dbserver, $verbose, $save_temps, $no_update);
 
@@ -40 +40 @@
     'fault_count=i' =>  \$fault_count,
     'max_fault_count=i' =>  \$max_fault_count,
+    'logfile=s'     =>  \$logfile,
     'dbname=s'      =>  \$dbname,       # postage stamp server dbname
     'dbserver=s'    =>  \$ps_dbserver,  # postage stamp server dbserver
@@ -68 +69 @@
 my $ipprc = PS::IPP::Config->new();
 
+if ($logfile) {
+   $ipprc->redirect_output($logfile);
+}
+
 if (!$ps_dbserver) {
     $ps_dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
@@ -314 +319 @@
     my $data_state = $metadata->{data_state};
 
+    # The update system currently requires that data that has been magicked be destreaked
+    # at chip stage so it needs magic even if the customer doesn't.
+    if ($metadata->{magicked} != 0) {
+        $need_magic = 1;
+    }
+
     if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
          ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) {
@@ -350 +361 @@
     foreach my $chip (@$data) {
         $chip_id = $chip->{chip_id};
-        if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
+
+        # if chip has been magicked before require it to be magicked again
+        # because the warp pending query requires it.
+        if ($chip->{magicked} < 0) {
+            print "Input has been destreaked so we must destreak before warping\n";
+            $need_magic = 1 
+        }
+
+        if ($need_magic and ($chip->{magicked} eq 0)) {
+            my_die("Client requires magic, but chip never magicked. How did this dependent get queued?", $PS_EXIT_PROG_ERROR);
+        }
+
+        if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} < 0))) {
             $chips_ready = 0;
             $chip->{fault} = $chip->{chip_fault};
@@ -399 +422 @@
     my $skycell = $metadata;
     my $skycell_id = $skycell->{skycell_id};
+
+    # The update system currently requires that data that has been magicked be destreaked
+    # at chip stage so it needs magic even if the customer doesn't.
+    if ($metadata->{magicked} != 0) {
+        $need_magic = 1;
+    }
 
     if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {

branches/eam_branches/ipp-20110213/pstamp/scripts/pstampparse.pl

@@ -31 +31 @@
 my $no_update;
 my $dest_requires_magic;
+
+# set this to true to disable update processing
+my $no_updates_allowed = 0;
 
 GetOptions(
@@ -424 +427 @@
                 # and this user's data store destination is allowed uncensored stamps, so accept the request
                 $need_magic = 0;
+
+                # Since user can get unmagicked data "by coordinate" requests can go back in time
+                # to dredge unusable data from the "dark days"...
+                if ($req_type eq 'bycoord' and $mjd_min eq 0) {
+                    # ... so unless the user sets mjd_min clamp it to 2009-04-01
+                    # XXX: This value should live in the pstampProject table not be hardcoded here
+                    $mjd_min = 54922;
+                }
             } else {
                 print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
@@ -465 +476 @@
     print "Time to locate_images for row $rownum $dtime_locate\n";
 
-    # handle this XXX: what did I mean by this comment
+    # handle this 
+    # XXX: what did I mean by that comment?
     $row->{need_magic} = $need_magic;
 
     $num_jobs += queueJobs($mode, \@rowList, $imageList);
 
-    # if a row slipped through with no jobs add one
+    # if a row slipped through with no jobs add a faulted one
     foreach my $row (@rowList) {
         if ($row->{job_num} == 0) {
@@ -495 +507 @@
     my $option_mask = $row->{OPTION_MASK};
     my $components = $row->{components};
+
+    $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_updates_allowed;
 
     my $roi_string;
@@ -668 +682 @@
     my $rownum = $row->{ROWNUM};
     my $option_mask = $row->{OPTION_MASK};
+
+    $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_updates_allowed;
 
     # For dist_bundle we need

branches/eam_branches/ipp-20110213/pstamp/scripts/pstampstopfaulted

@@ +1 @@
+#!/bin/bash
+#
+# script executed by the task pstamp.stopfaulted to stop proceessing of jobs
+# and dependents that have fault_count >= some maximum value. That value is expected
+# to be passed in as an argument otherwise the default in pstamptool is used
+# All arguments are passed to pstamptool
 
-# script executed by the task pstamp.stopfaulted to stop proceessing of jobs
-# and dependents that have faulted 5 or more times
-# all arguments to this script are passed to pstamptool
-
-# 25 is PSTAMP_NOT_AVAILABLE
+# note: 25 is PSTAMP_NOT_AVAILABLE
 fault_code=25
-fault_count=3
 
 date
 echo stopping faulted dependent jobs
-pstamptool -stopdependentjob -set_fault $fault_code -fault_count $fault_count $*
+cmd="pstamptool -stopdependentjob -set_fault $fault_code $*"
+echo $cmd
+$cmd
 
 echo stopping faulted jobs
-pstamptool -updatejob -state run -set_state stop -set_fault $fault_code -fault_count $fault_count $*
+cmd="pstamptool -updatejob -state run -set_state stop -set_fault $fault_code $*"
+echo $cmd
+$cmd