Changeset 29515 for branches/eam_branches/ipp-20100823/ippTasks/destreak.pro

Timestamp:

Oct 21, 2010, 2:45:13 PM (16 years ago)

Author:

eugene

Message:

merge changes from trunk

File:

• branches/eam_branches/ipp-20100823/ippTasks/destreak.pro (modified) (11 diffs)

branches/eam_branches/ipp-20100823/ippTasks/destreak.pro

@@ -10 +10 @@
 book init magicToDS
 book init magicDSToRevert
-book init magicDSToCleanup
 
 ### indexes into Database lists
@@ -30 +29 @@
 $magicDSRevertStage = 0
 
+macro destreak.show.stages
+    echo $DS_STAGE:n stages enabled for destreak processing
+    for i 0 $DS_STAGE:n
+        echo $i $DS_STAGE:$i
+    end
+    echo current magicDSStage = $magicDSStage
+    echo current magicDSRevertStage = $magicDSRevertStage
+end
+
 ### Check status of tasks
 macro destreak.status
-    # since echo and listbook output go to different files these do not come in order
-    echo magicToDS
     book listbook magicToDS
-    echo magicDSToRevert
     book listbook magicDSToRevert
-    echo magicDSToCleanup
-    book listbook magicDSToCleanup
 end
 
@@ -45 +48 @@
     book init magicToDS
     book init magicDSToRevert
-    book init magicDSToCleanup
 end
 
@@ -73 +75 @@
 end
 
-macro destreak.cleanup.on
-    task destreak.cleanup.load
-        active true
-    end
-    task destreak.cleanup.run
-        active true
-    end
-end
-
 ### Turn tasks off
 macro destreak.off
@@ -106 +99 @@
     end
 end
-macro destreak.cleanup.off
-    task destreak.cleanup.load
-        active false
-    end
-    task destreak.cleanup.run
-        active false
-    end
-end
-
 
 task           destreak.load
@@ -203 +187 @@
     book getword magicToDS $pageName camera -var CAMERA
     book getword magicToDS $pageName streaks_uri -var STREAKS
+    book getword magicToDS $pageName streaks_path_base -var STREAKS_PATH_BASE
     book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS
+    book getword magicToDS $pageName inv_streaks_path_base -var INV_STREAKS_PATH_BASE
     book getword magicToDS $pageName stage -var STAGE
     book getword magicToDS $pageName stage_id -var STAGE_ID
@@ -229 +215 @@
     # TODO: do not add recoveryroot or replace if they are null or zero
 
-    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED
+    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED
 
     add_standard_args run
@@ -274 +260 @@
 
   periods      -poll $LOADPOLL
-  #periods      -exec $LOADEXEC
   periods      -exec 30
-  periods      -timeout 60
+  periods      -timeout 300
   npending     1
 
@@ -293 +278 @@
     add_poll_args run
     add_poll_labels run
+    # since this command can be expensive, reduce the limit
+    $run = $run -limit 24
     command $run
   end
@@ -317 +304 @@
 task           destreak.revert.load
   host         local
-
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 20
+  periods      -poll 60.0
+  periods      -exec 120.
+  periods      -timeout 120
   npending     1
   active       false
@@ -518 +504 @@
 end
 
-task           destreak.cleanup.load
-  host         local
-
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 20
-  npending     1
-  active       false
-
-  stdout NULL
-  stderr $LOGSUBDIR/destreak.cleanup.load.log
-
-  task.exec
-    $run = magicdstool -tocleanup
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-
-      # save the DB name for the exit tasks
-      option $DB:$magicDSToCleanup_DB
-      $run = $run -dbname $DB:$magicDSToCleanup_DB
-
-      # only bump database number after we have gone through all of the stages
-      $magicDSToCleanup_DB ++
-      if ($magicDSToCleanup_DB >= $DB:n) set magicDSToCleanup_DB = 0
-    end
-    add_poll_args run
-    add_poll_labels run
-    command $run
-  end
-
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout magicDSToCleanup -key magic_ds_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook magicDSToCleanup
-    end
-
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup magicDSToCleanup
-  end
-
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-
-  task.exit    crash
-    showcommand crash
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-
-task           destreak.cleanup.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       false
-
-  task.exec
-    stdout $LOGSUBDIR/destreak.cleanup.run.log
-    stderr $LOGSUBDIR/destreak.cleanup.run.log
-
-    # if we are unable to run the 'exec', use a long retry time
-    periods -exec $RUNEXEC
-
-    book npages magicDSToCleanup -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    
-    # look for new images (pantaskState == INIT)
-    book getpage magicDSToCleanup 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-
-    book setword magicDSToCleanup $pageName pantaskState RUN
-    book getword magicDSToCleanup $pageName magic_ds_id -var MAGIC_DS_ID
-    book getword magicDSToCleanup $pageName camera -var CAMERA
-    book getword magicDSToCleanup $pageName stage -var STAGE
-    book getword magicDSToCleanup $pageName outroot -var OUTROOT
-    book getword magicDSToCleanup $pageName dbname -var DBNAME
-
-    sprintf logfile "%s/mds.%s.cleanup.log" $OUTROOT $MAGIC_DS_ID
-
-    host anyhost
-
-    $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --stage $STAGE --camera $CAMERA --logfile $logfile
-
-    add_standard_args run
-
-    # save the pageName for future reference below
-    options $pageName
-
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    # if we are ready to run, drop the retry timeout low so we fill up the queue
-    periods -exec 0.05
-    command $run
-  end
-
-  # default exit status
-  task.exit    0
-    process_exit magicDSToCleanup $options:0 $JOB_STATUS
-   end
-
-  # locked list
-  task.exit    default
-    showcommand failure
-    process_exit magicDSToCleanup $options:0 $JOB_STATUS
-  end
-
-  task.exit    crash
-    showcommand crash
-    book setword magicDSToCleanup $options:0 pantaskState CRASH
-  end
-
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword magicDSToCleanup $options:0 pantaskState TIMEOUT
-  end
-end
-