Changeset 37403 for branches/eam_branches/ps2-tc3-20130727/ippTasks

Timestamp:

Sep 19, 2014, 4:05:27 PM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ps2-tc3-20130727

Files:

• . (modified) (1 prop)
• ippTasks/Makefile.am (modified) (1 diff)
• ippTasks/diff.pro (modified) (2 diffs)
• ippTasks/ipphosts.mhpcc.config (modified) (2 diffs)
• ippTasks/pantasks.pro (modified) (3 diffs)
• ippTasks/pstamp.pro (modified) (4 diffs)
• ippTasks/remote.pro (copied) (copied from trunk/ippTasks/remote.pro )
• ippTasks/stack.pro (modified) (1 diff)
• ippTasks/summit.copy.pro (modified) (1 diff)
• ippTasks/survey.pro (modified) (9 diffs)
• ippTasks/warp.pro (modified) (2 diffs)

branches/eam_branches/ps2-tc3-20130727/ippTasks/Makefile.am

@@ -49 +49 @@
         vp.pro \
         bg.regeneration.pro \
+        remote.pro \
         fullforce.pro
 

branches/eam_branches/ps2-tc3-20130727/ippTasks/diff.pro

@@ -61 +61 @@
   end
   task diff.revert
+    active false
+  end
+end
+
+macro diff.summary.on
+  task diff.summary.load
+    active true
+  end
+  task diff.summary.run
+    active true
+  end
+end
+
+macro diff.summary.off
+  task diff.summary.load
+    active false
+  end
+  task diff.summary.run
     active false
   end
@@ -125 +143 @@
     add_poll_args run
     add_poll_labels run
+    # increase the poll limit for warp over the default to
+    # help it keep up with chip processing
+    # NOTE : it is not a problem for difftool to have multiple 
+    # -limit entries: the last one is used
+    $run = $run -limit {$POLL_LIMIT * 2}
     command $run
   end

branches/eam_branches/ps2-tc3-20130727/ippTasks/ipphosts.mhpcc.config

@@ -18 +18 @@
 ipphosts METADATA
   camera STR skycell
-  count S32  30
+  count S32  60
   sky00 STR  stsci00.0
   sky01 STR  stsci00.1
@@ -49 +49 @@
   sky28 STR  stsci09.1
   sky29 STR  stsci09.2
+  sky30 STR  stsci10.0
+  sky31 STR  stsci10.1
+  sky32 STR  stsci10.2
+  sky33 STR  stsci11.0
+  sky34 STR  stsci11.1
+  sky35 STR  stsci11.2
+  sky36 STR  stsci12.0
+  sky37 STR  stsci12.1
+  sky38 STR  stsci12.2
+  sky39 STR  stsci13.0
+  sky40 STR  stsci13.1
+  sky41 STR  stsci13.2
+  sky42 STR  stsci14.0
+  sky43 STR  stsci14.1
+  sky44 STR  stsci14.2
+  sky45 STR  stsci15.0
+  sky46 STR  stsci15.1
+  sky47 STR  stsci15.2
+  sky48 STR  stsci16.0
+  sky49 STR  stsci16.1
+  sky50 STR  stsci16.2
+  sky51 STR  stsci17.0
+  sky52 STR  stsci17.1
+  sky53 STR  stsci17.2
+  sky54 STR  stsci18.0
+  sky55 STR  stsci18.1
+  sky56 STR  stsci18.2
+  sky57 STR  stsci19.0
+  sky58 STR  stsci19.1
+  sky59 STR  stsci19.2
 END
 

branches/eam_branches/ps2-tc3-20130727/ippTasks/pantasks.pro

@@ -645 +645 @@
     host anyhost
   else
+    # echo host $host
     host $host
   end
@@ -655 +656 @@
   end
 
-  local skycellID varname count host skyname skyhash
+  local skycellID varname count host skyname skyhash fullhost
   $skycellID = $1
 
@@ -676 +677 @@
 
   # missing ipphosts table results in host = NULL
-  book getword ipphosts skycell $skyname -var host
+  book getword ipphosts skycell $skyname -var fullhost
+  list word -splitbychar . $fullhost
+  $host = $word:0
 
   if ("$host" == "NULL")
     host anyhost
   else
+    # echo host $host
     host $host
   end

branches/eam_branches/ps2-tc3-20130727/ippTasks/pstamp.pro

@@ -175 +175 @@
 macro pstamp.find.off
     task pstamp.request.find
+        active false
+    end
+end
+macro pstamp.parse.on
+    task pstamp.request.load
+        active true
+    end
+    task pstamp.request.run
+        active true
+    end
+end
+macro pstamp.parse.off
+    task pstamp.request.load
+        active false
+    end
+    task pstamp.request.run
+        active false
+    end
+end
+macro pstamp.job.on
+    task pstamp.job.load
+        active true
+    end
+    task pstamp.job.run
+        active true
+    end
+end
+macro pstamp.job.off
+    task pstamp.job.load
+        active false
+    end
+    task pstamp.job.run
+        active false
+    end
+end
+macro pstamp.finish.on
+    task pstamp.finish.load
+        active true
+    end
+    task pstamp.finish.run
+        active true
+    end
+end
+macro pstamp.finish.off
+    task pstamp.finish.load
+        active false
+    end
+    task pstamp.finish.run
         active false
     end
@@ -469 +517 @@
         add_poll_args run
         add_poll_labels run
+        # limit query for finished requests to small number
+        # because we only run 4 at a time. Using the default queue
+        # depth fouls up the priority orer
+        # XXX: use a varaible
+        $run = $run -limit 8
         command $run
     end
@@ -499 +552 @@
     periods     -timeout 2400
     host        anyhost
+    npending    4
 
     task.exec
@@ -951 +1005 @@
     # overloading nfs
 
-    npending    5
+    npending    2
 
     task.exec

branches/eam_branches/ps2-tc3-20130727/ippTasks/stack.pro

@@ -72 +72 @@
     active false
   end
+  task stack.summary.load
+    active false
+  end
+  task stack.summary.run
+    active false
+  end
+end
+
+macro stack.summary.on
+  task stack.summary.load
+    active true
+  end
+  task stack.summary.run
+    active true
+  end
+end
+
+macro stack.summary.off
   task stack.summary.load
     active false

branches/eam_branches/ps2-tc3-20130727/ippTasks/summit.copy.pro

@@ -567 +567 @@
         command pztool -clearcommonfaults -dbname $DB:$pztoolClearFault_DB
 
+        # loop over the multiple databases quickly, then pause for 5 min
         $pztoolClearFault_DB ++
-        if ($pztoolClearFault_DB >= $DB:n) set pztoolClearFault_DB = 0
+        if ($pztoolClearFault_DB >= $DB:n) 
+          set pztoolClearFault_DB = 0
+          periods -exec 300
+        else
+          periods -exec 30
+        end
       end
-      periods -exec 1800
     end
 

branches/eam_branches/ps2-tc3-20130727/ippTasks/survey.pro

@@ -668 +668 @@
     end
     
-#    echo $run
     command $run
   end
@@ -698 +697 @@
   periods      -exec $SURVEY_EXEC
   periods      -timeout $SURVEY_TIMEOUT
+  ## time in UT -- want to start ~0900 HST and run until next night starts. 24h boundary so two trange needed?
+  trange        19:00:00 23:59:00
+  trange        00:00:00 05:00:00
   npending     1
 
@@ -752 +754 @@
     end
     
-#    echo $run
     command $run
   end
@@ -838 +839 @@
     end
     
-    # echo $run
     command $run
   end
@@ -1437 +1437 @@
   host local
  
-  periods      -poll $SURVEY_POLL
-  periods      -exec $SURVEY_EXEC
+  # we are going to use a long timeout (900 sec) between passes of the list
+  # and a short timeout (10 sec) while we are doing the full list
+  periods      -exec 10
+  periods      -poll 2
   periods      -timeout $SURVEY_TIMEOUT
   npending     1
@@ -1449 +1451 @@
     book npages SURVEY_PUBLISH -var N
     if ($N == 0)
-    #  echo "No labels for processing"
+      # echo "No labels for processing"
       break
     endif
+
+    # short timeout while we have something to do
+    periods -exec 10
 
     # survey.publish allows multiple entries per label.
@@ -1465 +1470 @@
       book getpage SURVEY_PUBLISH 0 -var tag -key STATE NEW
 
-      # Select different database
-      $SURVEY_PUBLISH ++
-      if ($SURVEY_PUBLISH >= $DB:n) set SURVEY_PUBLISH = 0
+      # Select different database, set a long timeout
+      $SURVEY_PUBLISH_DB ++
+      if ($SURVEY_PUBLISH_DB >= $DB:n) set SURVEY_PUBLISH_DB = 0
+      periods -exec 900
+      date -var mytime
+      echo "done with publish list @ $mytime"
     end
 
@@ -1484 +1492 @@
       option DEFAULT
     else
-      $run = $run -dbname $DB:$SURVEY_PUBLISH
-      option $DB:$SURVEY_PUBLISH
+      $run = $run -dbname $DB:$SURVEY_PUBLISH_DB
+      option $DB:$SURVEY_PUBLISH_DB
     end
     
     # echo $run
-
     command $run
     
@@ -1859 +1866 @@
     end
     
-     echo $run
     command $run
   end

branches/eam_branches/ps2-tc3-20130727/ippTasks/warp.pro

@@ -99 +99 @@
   end
   task warp.revert.overlap
+    active false
+  end
+end
+
+macro warp.summary.on
+  task warp.summary.load
+    active true
+  end
+  task warp.summary.run
+    active true
+  end
+end
+
+macro warp.summary.off
+  task warp.summary.load
+    active false
+  end
+  task warp.summary.run
     active false
   end
@@ -261 +279 @@
     add_poll_args run
     add_poll_labels run
+    # increase the poll limit for warp over the default to
+    # help it keep up with chip processing
+    # NOTE : it is not a problem for warptool to have multiple 
+    # -limit entries: the last one is used
+    $run = $run -limit {$POLL_LIMIT * 2}
     command $run
   end