Index: /branches/eam_branches/20090715/ippTasks/Makefile.am
===================================================================
--- /branches/eam_branches/20090715/ippTasks/Makefile.am	(revision 25401)
+++ /branches/eam_branches/20090715/ippTasks/Makefile.am	(revision 25402)
@@ -13,4 +13,5 @@
 	chip.pro \
 	camera.pro \
+	addstar.pro \
 	fake.pro \
 	warp.pro \
@@ -25,5 +26,6 @@
 	pstamp.pro \
 	receive.pro \
-	publish.pro
+	publish.pro \
+	science.cleanup.pro
 
 other_files = \
Index: /branches/eam_branches/20090715/ippTasks/addstar.pro
===================================================================
--- /branches/eam_branches/20090715/ippTasks/addstar.pro	(revision 25402)
+++ /branches/eam_branches/20090715/ippTasks/addstar.pro	(revision 25402)
@@ -0,0 +1,227 @@
+## addstar.pro : globals and support macros : -*- sh -*-
+## this file contains the tasks for running the addstar analysis stage
+## these tasks use the book addPendingExp
+
+# test for required global variables
+check.globals
+
+book init addPendingExp
+
+macro addstar.status
+  book listbook addPendingExp
+end
+
+macro addstar.reset
+  book init addPendingExp
+end
+
+macro addstar.on
+  task addstar.exp.load
+    active true
+  end
+  task addstar.exp.run
+    active true
+  end
+end
+
+macro addstar.off
+  task addstar.exp.load
+    active false
+  end
+  task addstar.exp.run
+    active false
+  end
+end
+
+# this variable will cycle through the known database names
+$addstar_DB = 0
+$addstar_revert_DB = 0
+
+# select images ready for addstar analysis
+# new entries are added to addPendingExp
+# skip already-present entries
+task	       addstar.exp.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/addstar.exp.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = addtool -pendingexp
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$addstar_DB
+      $run = $run -dbname $DB:$addstar_DB
+      $addstar_DB ++
+      if ($addstar_DB >= $DB:n) set addstar_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook addPendingExp
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup addPendingExp
+  end
+
+  # default exit status
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# run the addstar script on pending exposures
+task	       addstar.exp.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 10
+
+  task.exec
+    book npages addPendingExp -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images in addPendingExp (pantaskState == INIT)
+    book getpage addPendingExp 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword addPendingExp $pageName pantaskState RUN
+    book getword addPendingExp $pageName camera -var CAMERA
+    book getword addPendingExp $pageName exp_tag -var EXP_TAG
+    book getword addPendingExp $pageName add_id -var ADD_ID
+    book getword addPendingExp $pageName cam_id -var CAM_ID
+    book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE
+    book getword addPendingExp $pageName dvodb  -var DVODB
+    book getword addPendingExp $pageName dbname -var DBNAME
+    book getword addPendingExp $pageName reduction -var REDUCTION
+    book getword addPendingExp $pageName state -var RUN_STATE
+
+    # specify choice of remote host based on camera and chip (class_id)
+    set.host.for.camera $CAMERA FPA
+
+    # set the WORKDIR variable
+    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
+
+    # notes on how this works:
+    # -- raw workdir examples:
+    # file://data/@HOST@.0/gpc1/20080130
+    # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?)
+    # -- out workdir examples:
+    # file://data/ipp004.0/gpc1/20080130
+    # neb:///ipp004-vol0/gpc1/20080130
+
+    ## generate outroot specific to this exposure (& chip)
+    sprintf outroot "%s/%s/%s.add.%s" $WORKDIR $EXP_TAG $EXP_TAG $ADD_ID
+    sprintf camroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID
+
+    stdout $LOGDIR/addstar.exp.log
+    stderr $LOGDIR/addstar.exp.log
+
+    $run = addstar_run.pl --exp_tag $EXP_TAG --add_id $ADD_ID --camera $CAMERA --outroot $outroot --camroot $camroot --redirect-output --run-state $RUN_STATE
+    if ("$REDUCTION" != "NULL")
+      $run = $run --reduction $REDUCTION
+    end
+    if ("$DVODB" != "NULL")
+      $run = $run --dvodb $DVODB
+    end
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # success
+  task.exit default
+    process_exit addPendingExp $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword addPendingExp $options:0 pantaskState CRASH
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+    book setword addPendingExp $options:0 pantaskState TIMEOUT
+  end
+end
+
+task addstar.revert
+  host         local
+
+  periods      -poll 5.0
+  periods      -exec 60.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = addtool -revertprocessedexp
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$addstar_revert_DB
+      $run = $run -dbname $DB:$addstar_revert_DB
+      $addstar_revert_DB ++
+      if ($addstar_revert_DB >= $DB:n) set addstar_revert_DB = 0
+    end
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
Index: /branches/eam_branches/20090715/ippTasks/detrend.cleanup.pro
===================================================================
--- /branches/eam_branches/20090715/ippTasks/detrend.cleanup.pro	(revision 25401)
+++ /branches/eam_branches/20090715/ippTasks/detrend.cleanup.pro	(revision 25402)
@@ -234,5 +234,5 @@
     book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID 
     book getword detCleanupProcessedImfile $pageName camera   -var CAMERA
-    book getword detCleanupProcessedImfile $pageName state    -var CLEANUP_MODE
+    book getword detCleanupProcessedImfile $pageName data_state    -var CLEANUP_MODE
     book getword detCleanupProcessedImfile $pageName dbname   -var DBNAME
 
@@ -244,5 +244,5 @@
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE
     add_standard_args run
 
@@ -352,5 +352,5 @@
     book getword detCleanupProcessedExp $pageName exp_id   -var EXP_ID   
     book getword detCleanupProcessedExp $pageName camera -var CAMERA
-    book getword detCleanupProcessedExp $pageName state -var CLEANUP_MODE
+    book getword detCleanupProcessedExp $pageName data_state -var CLEANUP_MODE
     book getword detCleanupProcessedExp $pageName dbname -var DBNAME
 
@@ -362,5 +362,5 @@
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --exp_id $EXP_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE
     add_standard_args run
 
@@ -470,5 +470,5 @@
     book getword detCleanupStackedImfile $pageName class_id -var CLASS_ID 
     book getword detCleanupStackedImfile $pageName camera -var CAMERA
-    book getword detCleanupStackedImfile $pageName state -var CLEANUP_MODE
+    book getword detCleanupStackedImfile $pageName data_state -var CLEANUP_MODE
     book getword detCleanupStackedImfile $pageName dbname -var DBNAME
 
@@ -480,5 +480,5 @@
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE
     add_standard_args run
 
@@ -588,5 +588,5 @@
     book getword detCleanupNormStatImfile $pageName iteration -var ITERATION
     book getword detCleanupNormStatImfile $pageName camera -var CAMERA
-    book getword detCleanupNormStatImfile $pageName state -var CLEANUP_MODE
+    book getword detCleanupNormStatImfile $pageName data_state -var CLEANUP_MODE
     book getword detCleanupNormStatImfile $pageName dbname -var DBNAME
 
@@ -598,5 +598,5 @@
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE
     add_standard_args run
 
@@ -704,5 +704,5 @@
     book getword detCleanupNormImfile $pageName class_id -var CLASS_ID 
     book getword detCleanupNormImfile $pageName camera -var CAMERA
-    book getword detCleanupNormImfile $pageName state -var CLEANUP_MODE
+    book getword detCleanupNormImfile $pageName data_state -var CLEANUP_MODE
     book getword detCleanupNormImfile $pageName dbname -var DBNAME
 
@@ -714,5 +714,5 @@
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE
     add_standard_args run
 
@@ -819,5 +819,5 @@
     book getword detCleanupNormExp $pageName iteration -var ITERATION
     book getword detCleanupNormExp $pageName camera -var CAMERA
-    book getword detCleanupNormExp $pageName state -var CLEANUP_MODE
+    book getword detCleanupNormExp $pageName data_state -var CLEANUP_MODE
     book getword detCleanupNormExp $pageName dbname -var DBNAME
 
@@ -829,5 +829,5 @@
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE
     add_standard_args run
 
@@ -937,5 +937,5 @@
     book getword detCleanupResidImfile $pageName iteration -var ITERATION     
     book getword detCleanupResidImfile $pageName camera -var CAMERA
-    book getword detCleanupResidImfile $pageName state -var CLEANUP_MODE
+    book getword detCleanupResidImfile $pageName data_state -var CLEANUP_MODE
     book getword detCleanupResidImfile $pageName dbname -var DBNAME
 
@@ -947,5 +947,5 @@
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID  --camera $CAMERA --mode $CLEANUP_MODE
     add_standard_args run
 
@@ -1056,5 +1056,5 @@
     book getword detCleanupResidExp $pageName iteration -var ITERATION
     book getword detCleanupResidExp $pageName camera 	-var CAMERA
-    book getword detCleanupResidExp $pageName state  	-var CLEANUP_MODE
+    book getword detCleanupResidExp $pageName data_state  	-var CLEANUP_MODE
     book getword detCleanupResidExp $pageName dbname 	-var DBNAME
 
@@ -1066,5 +1066,5 @@
 
     # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE
     add_standard_args run
 
Index: /branches/eam_branches/20090715/ippTasks/dist.pro
===================================================================
--- /branches/eam_branches/20090715/ippTasks/dist.pro	(revision 25401)
+++ /branches/eam_branches/20090715/ippTasks/dist.pro	(revision 25402)
@@ -4,5 +4,5 @@
 check.globals
 
-#$LOGSUBDIR = $LOGDIR/dist
+$LOGSUBDIR = $LOGDIR/dist
 mkdir $LOGSUBDIR
 
@@ -122,4 +122,5 @@
 
   task.exec
+    if ($NETWORK == 0) break
     book npages distToProcess -var N
     if ($N == 0) 
@@ -128,6 +129,4 @@
     end
     periods -exec 0.05
-
-    if ($NETWORK == 0) break
     
     # look for new components to process (pantaskState == INIT)
Index: /branches/eam_branches/20090715/ippTasks/ipphosts.mhpcc.config
===================================================================
--- /branches/eam_branches/20090715/ippTasks/ipphosts.mhpcc.config	(revision 25401)
+++ /branches/eam_branches/20090715/ippTasks/ipphosts.mhpcc.config	(revision 25402)
@@ -114,6 +114,6 @@
   XY73  STR  ipp015
   XY74  STR  ipp015
-  XY75  STR  ipp053
-  XY76  STR  ipp053
+  XY75  STR  ipp025
+  XY76  STR  ipp025
 END
 
@@ -193,5 +193,5 @@
   ota73  STR  ipp015
   ota74  STR  ipp015
-  ota75  STR  ipp053
-  ota76  STR  ipp053
+  ota75  STR  ipp025
+  ota76  STR  ipp025
 END
Index: /branches/eam_branches/20090715/ippTasks/magic.pro
===================================================================
--- /branches/eam_branches/20090715/ippTasks/magic.pro	(revision 25401)
+++ /branches/eam_branches/20090715/ippTasks/magic.pro	(revision 25402)
@@ -104,4 +104,29 @@
     end
 end
+macro magic.ds.off
+    task magic.destreak.load
+        active false
+    end
+    task magic.destreak.run
+        active false
+    end
+end
+macro magic.ds.on
+    task magic.destreak.load
+        active true
+    end
+    task magic.destreak.run
+        active true
+    end
+end
+
+macro magic.ds.status
+    echo magicToDS
+    book listbook magicToDS
+    echo ""
+    echo magicDSToRevert
+    book listbook magicDSToRevert
+end
+
 
 
Index: /branches/eam_branches/20090715/ippTasks/pantasks.pro
===================================================================
--- /branches/eam_branches/20090715/ippTasks/pantasks.pro	(revision 25401)
+++ /branches/eam_branches/20090715/ippTasks/pantasks.pro	(revision 25402)
@@ -213,4 +213,5 @@
   chip.on
   camera.on
+  addstar.on
   fake.on
   warp.on
@@ -225,4 +226,5 @@
   chip.off
   camera.off
+  addstar.off
   fake.off
   warp.off
@@ -242,4 +244,5 @@
   module chip.pro
   module camera.pro
+  module addstar.pro
   module fake.pro
   module warp.pro
Index: /branches/eam_branches/20090715/ippTasks/pstamp.pro
===================================================================
--- /branches/eam_branches/20090715/ippTasks/pstamp.pro	(revision 25401)
+++ /branches/eam_branches/20090715/ippTasks/pstamp.pro	(revision 25402)
@@ -10,9 +10,10 @@
 
 # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config
-# warning: no quotes around the two words. it cause it to get passed to pstamptool as one word
-# tricky to debug problem ensues
+# warning: no quotes around the two words. That causes the variable to get passed to pstamptool as one word
+# and a tricky to debug problem ensues
 # example:
-# $PSDBSERVER = -dbserver hostname
-if ($?PSDBSERVER == 0)
+# $PS_DBSERVER = -dbserver hostname
+
+if ($?PS_DBSERVER == 0)
     $PS_DBSERVER = ""
 end
@@ -134,4 +135,5 @@
         end
         add_poll_args run
+        # add_poll_labels run
         command $run
     end
@@ -178,4 +180,6 @@
         book getword pstampRequest $pageName ds_outProduct -var PRODUCT
 
+        host anyhost
+
         $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --redirect-output
 
@@ -219,13 +223,16 @@
 
     task.exec
+        $run = pstamptool  -completedreq
         if ($DB:n == 0)
             option DEFAULT
-            command pstamptool  -completedreq
         else 
             option $DB:$pstampFin_DB
-            command pstamptool  -completedreq -dbname $DB:$pstampFin_DB $PS_DBSERVER
+            $run = $run -dbname $DB:$pstampFin_DB $PS_DBSERVER
             $pstampFin_DB ++
             if ($pstampFin_DB >= $DB:n) set pstampFin_DB = 0
         end
+        add_poll_args run
+        # add_poll_labels run
+        command $run
     end
 
@@ -313,13 +320,16 @@
 
     task.exec
+        $run = pstamptool -pendingjob
         if ($DB:n == 0)
             option DEFAULT
-            command pstamptool -pendingjob -limit 10
         else
             option $DB:$pstampJob_DB
-            command pstamptool -pendingjob -limit 10 -dbname $DB:$pstampJob_DB $PS_DBSERVER
+            $run = $run -dbname $DB:$pstampJob_DB $PS_DBSERVER
             $pstampJob_DB ++
             if ($pstampJob_DB >= $DB:n) set pstampJob_DB = 0
         end
+        add_poll_args run
+        # add_poll_labels run 
+        command $run
     end
 
@@ -356,5 +366,9 @@
     task.exec
         book npages pstampJob -var N
-        if ($N == 0) break
+        if ($N == 0) 
+            periods -exec $RUNEXEC
+            break
+        end
+        periods -exec 0.05
         
         book getpage pstampJob 0 -var pageName -key pantaskState INIT
@@ -374,4 +388,5 @@
         end
 
+        host anyhost
 
         $run = pstamp_job_run.pl --job_id $JOB_ID --job_type $JOB_TYPE --rownum $ROWNUM --output_base $OUTPUT_BASE --redirect-output 
Index: /branches/eam_branches/20090715/ippTasks/publish.pro
===================================================================
--- /branches/eam_branches/20090715/ippTasks/publish.pro	(revision 25401)
+++ /branches/eam_branches/20090715/ippTasks/publish.pro	(revision 25402)
@@ -161,5 +161,4 @@
     book getword publishRun $pageName stage -var STAGE
     book getword publishRun $pageName stage_id -var STAGE_ID
-    book getword publishRun $pageName exp_id -var EXP_ID
     book getword publishRun $pageName dbname -var DBNAME
 
@@ -170,5 +169,5 @@
     strsub $WORKDIR_TEMPLATE @HOST@ $default_host -var WORKDIR
 
-    $run = publish_file.pl --pub_id $PUB_ID --exp_id $EXP_ID --camera $CAMERA --workdir $WORKDIR --product $PRODUCT --stage $STAGE --stage_id $STAGE_ID --redirect-output
+    $run = publish_file.pl --pub_id $PUB_ID --camera $CAMERA --workdir $WORKDIR --product $PRODUCT --stage $STAGE --stage_id $STAGE_ID --redirect-output
     add_standard_args run
 
Index: /branches/eam_branches/20090715/ippTasks/site.mhpcc.pro
===================================================================
--- /branches/eam_branches/20090715/ippTasks/site.mhpcc.pro	(revision 25401)
+++ /branches/eam_branches/20090715/ippTasks/site.mhpcc.pro	(revision 25402)
@@ -14,4 +14,5 @@
   controller host add ipp023
   controller host add ipp024
+  controller host add ipp025
   controller host add ipp026
   controller host add ipp028
@@ -39,5 +40,5 @@
   controller host add ipp051
   controller host add ipp052
-  controller host add ipp053
+#  controller host add ipp053
 end
 
Index: /branches/eam_branches/20090715/ippTasks/summit.copy.pro
===================================================================
--- /branches/eam_branches/20090715/ippTasks/summit.copy.pro	(revision 25401)
+++ /branches/eam_branches/20090715/ippTasks/summit.copy.pro	(revision 25402)
@@ -380,6 +380,10 @@
     periods      -poll     0.05
     periods      -timeout  650
-    # trage       16:00 23:59
-    # trage       00:00 04:00
+    trange        -reset
+    # only active in the night (18:00 to 06:00 HST, times are UT):
+    # trange      04:00 16:00
+    # only active in the day (06:00 to 18:00 HST, times are UT):
+    # trange        16:00 23:59
+    # trange        00:00 04:00
 
     task.exec
@@ -449,4 +453,5 @@
         end
         if (("$MD5SUM" != "NULL") && ("$MD5SUM" != "0") && (not($COMPRESS)))
+# && (($YEAR > 2008) || (("$YEAR" = "2007") && ($MONTH > 8))))
             $run = $run --md5 $MD5SUM
         end