Changeset 28794 for branches/eam_branches/ipp-20100621/ippTasks

Timestamp:

Jul 30, 2010, 9:31:50 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/eam_branches/ipp-20100621/ippTasks

Files:

• . (modified) (1 prop)
• Makefile.am (modified) (1 diff)
• background.pro (copied) (copied from trunk/ippTasks/background.pro )
• burntool.pro (copied) (copied from trunk/ippTasks/burntool.pro )
• detrend.norm.pro (modified) (2 diffs)
• detrend.process.pro (modified) (2 diffs)
• detrend.resid.pro (modified) (2 diffs)
• detrend.stack.pro (modified) (2 diffs)
• dist.pro (modified) (3 diffs)
• dqstats.pro (modified) (3 diffs)
• ipphosts.mhpcc.config (modified) (1 diff)
• nightly_stacks.pro (modified) (2 diffs)
• pantasks.pro (modified) (4 diffs)
• pstamp.pro (modified) (1 diff)
• rcserver.pro (modified) (3 diffs)
• simtest.pro (modified) (1 diff)
• stack.pro (modified) (2 diffs)

branches/eam_branches/ipp-20100621/ippTasks/Makefile.am

@@ -34 +34 @@
         minidvodb.pro \
         nightly_stacks.pro \
+        burntool.pro \
         lossy_compress.pro \
+        background.pro \
         diffphot.pro
 

branches/eam_branches/ipp-20100621/ippTasks/detrend.norm.pro

@@ -64 +64 @@
 end
 
+macro detnorm.revert.off
+  task detrend.norm.revert
+    active false
+  end
+  task detrend.normexp.revert
+    active false
+  end
+  task detrend.normstat.revert
+    active false
+  end
+end
+
+macro detnorm.revert.on
+  task detrend.norm.revert
+    active true
+  end
+  task detrend.normexp.revert
+    active true
+  end
+  task detrend.normstat.revert
+    active true
+  end
+end
+
+
 # these variables will cycle through the known database names
 $detPendingNormStatImfile_DB = 0
 $detPendingNormImfile_DB = 0
 $detPendingNormExp_DB = 0
+$detPendingNormStatImfile_DB_revert = 0
+$detPendingNormImfile_DB_revert = 0
+$detPendingNormExp_DB_revert = 0
 
 # select images ready for copy 
@@ -440 +468 @@
   end
 end
+
+task detrend.norm.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+
+    $run = dettool -revertnormalizedimfile -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingNormImfile_DB_revert
+      $run = $run -dbname $DB:$detPendingNormImfile_DB_revert
+      $detPendingNormImfile_DB_revert ++
+      if ($detPendingNormImfile_DB_revert >= $DB:n) set detPendingNormImfile_DB_revert = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task detrend.normexp.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+
+    $run = dettool -revertnormalizedexp -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingNormExp_DB_revert
+      $run = $run -dbname $DB:$detPendingNormExp_DB_revert
+      $detPendingNormExp_DB_revert ++
+      if ($detPendingNormExp_DB_revert >= $DB:n) set detPendingNormExp_DB_revert = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task detrend.normstat.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+  
+    $run = dettool -revertnormalizedstat -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingNormStatImfile_DB_revert
+      $run = $run -dbname $DB:$detPendingNormStatImfile_DB_revert
+      $detPendingNormStatImfile_DB_revert ++
+      if ($detPendingNormStatImfile_DB_revert >= $DB:n) set detPendingNormStatImfile_DB_revert = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end

branches/eam_branches/ipp-20100621/ippTasks/detrend.process.pro

@@ -51 +51 @@
 end
 
+macro detproc.revert.off
+  task detrend.process.revert
+    active false
+  end
+  task detrend.processexp.revert
+    active false
+  end
+end
+
+macro detproc.revert.on
+  task detrend.process.revert
+    active true
+  end
+  task detrend.processexp.revert
+    active true
+  end
+end
 
 # these variables will cycle through the known database names
 $detPendingProcessedImfile_DB = 0
 $detPendingProcessedExp_DB = 0
+$detPendingProcessedImfile_revert_DB = 0
+$detPendingProcessedExp_revert_DB = 0
 
 # select images ready for copy 
@@ -313 +332 @@
   end
 end
+
+task detrend.process.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+ 
+    $run = dettool -revertprocessedimfile -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingProcessedImfile_revert_DB
+      $run = $run -dbname $DB:$detPendingProcessedImfile_revert_DB
+      $detPendingProcessedImfile_revert_DB ++
+      if ($detPendingProcessedImfile_revert_DB >= $DB:n) set detPendingProcessedImfile_revert_DB = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task detrend.processexp.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+  
+    $run = dettool -revertprocessedexp -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingProcessedExp_revert_DB
+      $run = $run -dbname $DB:$detPendingProcessedExp_revert_DB
+      $detPendingProcessedExp_revert_DB ++
+      if ($detPendingProcessedExp_revert_DB >= $DB:n) set detPendingProcessedExp_revert_DB = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end

branches/eam_branches/ipp-20100621/ippTasks/detrend.resid.pro

@@ -49 +49 @@
 end
 
+macro detresid.revert.on
+  task detrend.resid.revert
+    active true
+  end
+  task detrend.residexp.revert
+    active true
+  end
+end
+
+macro detresid.revert.off
+  task detrend.resid.revert
+    active false
+  end
+  task detrend.residexp.revert
+    active false
+  end
+end
+
 # these variables will cycle through the known database names
 $detPendingResidImfile_DB = 0
 $detPendingResidExp_DB = 0
+$detPendingResidImfile_revert_DB = 0
+$detPendingResidExp_revert_DB = 0
 
 # select images ready for copy 
@@ -316 +336 @@
   end
 end
+
+task detrend.resid.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+
+    $run = dettool -revertresidimfile -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingResidImfile_revert_DB
+      $run = $run -dbname $DB:$detPendingResidImfile_revert_DB
+      $detPendingResidImfile_revert_DB ++
+      if ($detPendingResidImfile_revert_DB >= $DB:n) set detPendingResidImfile_revert_DB = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task detrend.residexp.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+  
+
+    $run = dettool -revertresidexp -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingResidExp_revert_DB
+      $run = $run -dbname $DB:$detPendingResidExp_revert_DB
+      $detPendingResidExp_revert_DB ++
+      if ($detPendingResidExp_revert_DB >= $DB:n) set detPendingResidExp_revert_DB = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end

branches/eam_branches/ipp-20100621/ippTasks/detrend.stack.pro

@@ -35 +35 @@
 end
 
+macro detstack.revert.off
+  task detrend.stack.revert
+    active false
+  end
+end
+
+macro detstack.revert.on
+  task detrend.stack.revert
+    active true
+  end
+end
+
 # this variable will cycle through the known database names
 $detPendingStackedImfile_DB = 0
+$detPendingStackedImfile_revert_DB = 0
 
 # select images ready for detrend_stack.pl
@@ -165 +178 @@
   end
 end
+
+
+task detrend.stack.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+  
+    $run = dettool -revertstacked -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingStackedImfile_revert_DB
+      $run = $run -dbname $DB:$detPendingStackedImfile_revert_DB
+      $detPendingStackedImfile_revert_DB ++
+      if ($detPendingStackedImfile_revert_DB >= $DB:n) set detPendingStackedImfile_revert_DB = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end

branches/eam_branches/ipp-20100621/ippTasks/dist.pro

@@ -28 +28 @@
 list DIST_STAGE -add "stack"
 list DIST_STAGE -add "SSdiff"
+list DIST_STAGE -add "chip_bg"
+list DIST_STAGE -add "warp_bg"
 
 $currentStage = 0
@@ -145 +147 @@
         # to compute an index into the host table
         strlen $stage_id length
-        $start = $length - 2
-        substr $stage_id $start 2 index
+        if ($length >= 2) 
+            $start = $length - 2
+            substr $stage_id $start 2 index
+        else
+            # stage_id < 10 caused a very annoying couple of hours for bills to debug
+            substr $stage_id 0 1 index
+        end
         $component_id = $index % $count
         book getword ipphosts distribution $component_id -var myhost
@@ -409 +416 @@
 
     # using $DIST_ID as the "component" works fine here since we only look
-    # at the last two digits
-    set.dist.workdir.by.component $STAGE_ID "exposure" $OUTDIR_TEMPLATE OUTDIR 
+    # at the last two digits. But make sure that there 2 digits
+    $fake_component = $STAGE_ID + 10
+    set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR 
     if ("$OUTDIR" == "NULL")
         echo ERROR failed to set workdir for $DIST_ID

branches/eam_branches/ipp-20100621/ippTasks/dqstats.pro

@@ -37 +37 @@
   task dqstats.revert
     active false
+  end
+end
+
+macro dqstats.revert.on
+  task dqstats.revert
+    active true
+  end
+end
+
+macro dqstats.revert.off
+  task dqstats.revert
+    active true
   end
 end
@@ -109 +121 @@
 
   ## we want only a single outstanding dqstats job.  
-  host         local
   npending     1
 
@@ -187 +198 @@
   periods      -timeout 120.0
   npending     1
+  active       false
 
   stdout NULL

branches/eam_branches/ipp-20100621/ippTasks/ipphosts.mhpcc.config

@@ -253 +253 @@
 END
 
-# this list is no longer used
-# XXX : delete if this does not cause trouble
-## ipphosts METADATA
-##   camera STR distribution
-##   count  S32  26
-##   0      STR ipp021
-##   1      STR ipp023
-##   2      STR ipp024
-##   3      STR ipp026
-##   4      STR ipp028
-##   5      STR ipp029
-##   6      STR ipp030
-##   7      STR ipp031
-##   8      STR ipp032
-##   9      STR ipp033
-##  10      STR ipp034
-##  11      STR ipp035
-##  12      STR ipp036
-##  13      STR ipp038
-##  14      STR ipp039
-##  15      STR ipp040
-##  16      STR ipp041
-##  17      STR ipp043
-##  18      STR ipp044
-##  19      STR ipp045
-##  20      STR ipp046
-##  21      STR ipp047
-##  22      STR ipp048
-##  23      STR ipp050
-##  24      STR ipp051
-##  25      STR ipp052
-## END
+ipphosts METADATA
+  camera STR distribution
+  count  S32  26
+  0      STR ipp021
+  1      STR ipp023
+  2      STR ipp024
+  3      STR ipp026
+  4      STR ipp028
+  5      STR ipp029
+  6      STR ipp030
+  7      STR ipp031
+  8      STR ipp032
+  9      STR ipp033
+ 10      STR ipp034
+ 11      STR ipp035
+ 12      STR ipp036
+ 13      STR ipp038
+ 14      STR ipp039
+ 15      STR ipp040
+ 16      STR ipp041
+ 17      STR ipp043
+ 18      STR ipp044
+ 19      STR ipp045
+ 20      STR ipp046
+ 21      STR ipp047
+ 22      STR ipp048
+ 23      STR ipp050
+ 24      STR ipp051
+ 25      STR ipp052
+END
 
 ipphosts METADATA

branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro

@@ -16 +16 @@
     ns.initday.off
     ns.detrends.off
+    ns.dqstats.off
     ns.registration.off
     ns.burntool.off
@@ -23 +24 @@
 
 macro ns.on
-    ns.initday.on
-    ns.detrends.off
-    ns.registration.on
-    ns.burntool.on
-    ns.chips.on
-    ns.stacks.on
+    nightly.stacks.on
 end
 
 macro ns.off
-    ns.initday.off
-    ns.detrends.off
-    ns.registration.off
-    ns.burntool.off
-    ns.chips.off
-    ns.stacks.off
+    nightly.stacks.off
 end
 

branches/eam_branches/ipp-20100621/ippTasks/pantasks.pro

@@ -242 +242 @@
 end
 
+macro detrend.revert.off
+  detproc.revert.off
+  detstack.revert.off
+  detnorm.revert.off
+  detresid.revert.off
+end
+
+macro detrend.revert.on
+  detproc.revert.on
+  detstack.revert.on
+  detnorm.revert.on
+  detresid.revert.on
+end
+
 macro detrend.reset
   detproc.reset
@@ -251 +265 @@
 
 macro all.on
-  register.on
   detrend.on
   flatcorr.on
@@ -264 +277 @@
 
 macro all.off
-  register.off
   detrend.off
   flatcorr.off
@@ -721 +733 @@
 end
 
+macro tasks.revert.off
+    chip.revert.off
+    camera.revert.off
+    fake.revert.off
+    warp.revert.off
+    diff.revert.off
+    stack.revert.off
+end
+macro tasks.revert.on
+    chip.revert.on
+    camera.revert.on
+    fake.revert.on
+    warp.revert.on
+    diff.revert.on
+    stack.revert.on
+end
+

branches/eam_branches/ipp-20100621/ippTasks/pstamp.pro

@@ -188 +188 @@
         add_poll_args run
         add_poll_labels run
+        # limit number of requests in the queue to avoid blocking everything
+        # when jobs take an infinite amount of time to parse
+        $run = $run -limit 10
         command $run
     end

branches/eam_branches/ipp-20100621/ippTasks/rcserver.pro

@@ -12 +12 @@
 ### Database lists
 $rcPendingFS_DB = 0
+$rcRevertFS_DB = 0
 
 ### Check status of tasks
@@ -38 +39 @@
   task rcserver.makefileset.run
     active false
+  end
+  task rcserver.revert
+    active false
+  end
+end
+
+macro rcserver.revert.on
+  task rcserver.revert
+    active true
   end
 end
@@ -156 +166 @@
   end
 end
+
+
+task rcserver.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGSUBDIR/revert.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = disttool -revertfileset
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$rcRevertFS_DB
+      $run = $run -dbname $DB:$rcRevertFS_DB
+      $rcRevertFS_DB ++
+      if ($rcRevertFS_DB >= $DB:n) set rcRevertFS_DB = 0
+    end
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end

branches/eam_branches/ipp-20100621/ippTasks/simtest.pro

@@ -159 +159 @@
   $SIMTEST_AUTO = simtest.flatcorr.auto
 end
-
-macro tasks.revert.off
-    chip.revert.off
-    camera.revert.off
-    fake.revert.off
-    warp.revert.off
-    diff.revert.off
-    stack.revert.off
-end
-macro tasks.revert.on
-    chip.revert.on
-    camera.revert.on
-    fake.revert.on
-    warp.revert.on
-    diff.revert.on
-    stack.revert.on
-end

branches/eam_branches/ipp-20100621/ippTasks/stack.pro

@@ -126 +126 @@
       # save the DB name for the exit tasks
       option $DB:$stackSkycell_DB
-      $run = $run -dbname $DB:$stackSkycell_DB -limit 40
+      $run = $run -dbname $DB:$stackSkycell_DB
       $stackSkycell_DB ++
       if ($stackSkycell_DB >= $DB:n) set stackSkycell_DB = 0
@@ -132 +132 @@
     add_poll_args run
     add_poll_labels run
+    # reduce the limit (the last one takes precedence)
+    $run = $run -limit 40
     command $run
   end