Index: trunk/ippTasks/Makefile.am =================================================================== --- trunk/ippTasks/Makefile.am (revision 23479) +++ trunk/ippTasks/Makefile.am (revision 23480) @@ -1,4 +1,3 @@ task_files = \ - detrend.mkruns.pro \ detrend.norm.pro \ detrend.process.pro \ @@ -7,4 +6,5 @@ detrend.stack.pro \ detrend.correct.pro \ + detrend.cleanup.pro \ flatcorr.pro \ automate.pro \ @@ -36,4 +36,6 @@ simtest.flatcorr.config \ simtest.flatcorr.auto \ + simtest.ctemask.config \ + simtest.ctemask.auto \ simtest.stack.config \ simtest.stack.auto Index: trunk/ippTasks/detrend.cleanup.pro =================================================================== --- trunk/ippTasks/detrend.cleanup.pro (revision 23480) +++ trunk/ippTasks/detrend.cleanup.pro (revision 23480) @@ -0,0 +1,1098 @@ +## detrend.process.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingProcessedImfile and detPendingProcessedExp + +# test for required global variables +check.globals + +book init detCleanupProcessedImfile +book init detCleanupProcessedExp +book init detCleanupStackedImfile +book init detCleanupNormStatImfile +book init detCleanupNormImfile +book init detCleanupNormExp +book init detCleanupResidImfile +book init detCleanupResidExp + +macro detclean.reset + book init detCleanupProcessedImfile + book init detCleanupProcessedExp + book init detCleanupStackedImfile + + book init detCleanupNormStatImfile + book init detCleanupNormImfile + book init detCleanupNormExp + + book init detCleanupResidImfile + book init detCleanupResidExp +end + +macro detclean.status + echo detCleanupProcessedImfile + book listbook detCleanupProcessedImfile + echo detCleanupProcessedExp + book listbook detCleanupProcessedExp + echo detCleanupStackedImfile + book listbook detCleanupStackedImfile + + book listbook detCleanupNormStatImfile + book listbook detCleanupNormImfile + book listbook detCleanupNormExp + book listbook detCleanupResidImfile + book listbook detCleanupResidExp +end + +macro detclean.on + task detrend.cleanup.process.load + active true + end + task detrend.cleanup.process.run + active true + end + task detrend.cleanup.processexp.load + active true + end + task detrend.cleanup.processexp.run + active true + end + task detrend.cleanup.stack.load + active true + end + task detrend.cleanup.stack.run + active true + end + + task detrend.cleanup.norm.load + active true + end + task detrend.cleanup.norm.run + active true + end + task detrend.cleanup.normexp.load + active true + end + task detrend.cleanup.normexp.run + active true + end + task detrend.cleanup.normstat.load + active true + end + task detrend.cleanup.normstat.run + active true + end + + task detrend.cleanup.resid.load + active true + end + task detrend.cleanup.resid.run + active true + end + task detrend.cleanup.residexp.load + active true + end + task detrend.cleanup.residexp.run + active true + end +end + +macro detclean.off + task detrend.cleanup.process.load + active false + end + task detrend.cleanup.process.run + active false + end + task detrend.cleanup.processexp.load + active false + end + task detrend.cleanup.processexp.run + active false + end + task detrend.cleanup.stack.load + active false + end + task detrend.cleanup.stack.run + active false + end + task detrend.cleanup.norm.load + active false + end + task detrend.cleanup.norm.run + active false + end + task detrend.cleanup.normexp.load + active false + end + task detrend.cleanup.normexp.run + active false + end + task detrend.cleanup.normstat.load + active false + end + task detrend.cleanup.normstat.run + active false + end + task detrend.cleanup.resid.load + active false + end + task detrend.cleanup.resid.run + active false + end + task detrend.cleanup.residexp.load + active false + end + task detrend.cleanup.residexp.run + active false + end +end + + +# these variables will cycle through the known database names +$detCleanupProcessedImfile_DB = 0 +$detCleanupProcessedExp_DB = 0 +$detCleanupStackedImfile_DB = 0 +$detCleanupNormStatImfile_DB = 0 +$detCleanupNormImfile_DB = 0 +$detCleanupNormExp_DB = 0 +$detCleanupResidImfile_DB = 0 +$detCleanupResidExp_DB = 0 + +######## cleanup process imfile ######## +task detrend.cleanup.process.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.process.imfile.log + + task.exec + $run = dettool -pendingcleanup_processedimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupProcessedImfile_DB + $run = $run -dbname $DB:$detCleanupProcessedImfile_DB + $detCleanupProcessedImfile_DB ++ + if ($detCleanupProcessedImfile_DB >= $DB:n) set detCleanupProcessedImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupProcessedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupProcessedImfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.process.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupProcessedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupProcessedImfile (pantaskState == INIT) + book getpage detCleanupProcessedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupProcessedImfile $pageName pantaskState RUN + book getword detCleanupProcessedImfile $pageName det_id -var DET_ID + book getword detCleanupProcessedImfile $pageName exp_id -var EXP_ID + book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID + book getword detCleanupProcessedImfile $pageName camera -var CAMERA + book getword detCleanupProcessedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.process.imfile.log + stderr $LOGDIR/detrend.cleanup.process.imfile.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupProcessedImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detCleanupProcessedImfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupProcessedImfile $options:0 pantaskState TIMEOUT + end +end + + +######## cleanup process exp ######## +task detrend.cleanup.processexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.process.exp.log + + task.exec + $run = dettool -pendingcleanup_processedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupProcessedExp_DB + $run = $run -dbname $DB:$detCleanupProcessedExp_DB + $detCleanupProcessedExp_DB ++ + if ($detCleanupProcessedExp_DB >= $DB:n) set detCleanupProcessedExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupProcessedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupProcessedExp + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.processexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupProcessedExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupProcessedExp (pantaskState == INIT) + book getpage detCleanupProcessedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupProcessedExp $pageName pantaskState RUN + book getword detCleanupProcessedExp $pageName det_id -var DET_ID + book getword detCleanupProcessedExp $pageName exp_id -var EXP_ID + book getword detCleanupProcessedExp $pageName camera -var CAMERA + book getword detCleanupProcessedExp $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.process.exp.log + stderr $LOGDIR/detrend.cleanup.process.exp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --exp_id $EXP_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupProcessedExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detCleanupProcessedExp $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupProcessedExp $options:0 pantaskState TIMEOUT + end +end + +########## cleanup stack ########### +task detrend.cleanup.stack.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.stack.log + + task.exec + $run = dettool -pendingcleanup_stacked + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupStackedImfile_DB + $run = $run -dbname $DB:$detCleanupStackedImfile_DB + $detCleanupStackedImfile_DB ++ + if ($detCleanupStackedImfile_DB >= $DB:n) set detCleanupStackedImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupStackedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupStackedImfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.stack.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupStackedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupStackedImfile (pantaskState == INIT) + book getpage detCleanupStackedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupStackedImfile $pageName pantaskState RUN + book getword detCleanupStackedImfile $pageName det_id -var DET_ID + book getword detCleanupStackedImfile $pageName iteration -var ITERATION + book getword detCleanupStackedImfile $pageName class_id -var CLASS_ID + book getword detCleanupStackedImfile $pageName camera -var CAMERA + book getword detCleanupStackedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupStackedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.stack.log + stderr $LOGDIR/detrend.cleanup.stack.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupStackedImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detCleanupStackedImfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupStackedImfile $options:0 pantaskState TIMEOUT + end +end + + +########## cleanup normstat ########### +task detrend.cleanup.normstat.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.normstat.log + + task.exec + $run = dettool -pendingcleanup_normalizedstat + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormStatImfile_DB + $run = $run -dbname $DB:$detCleanupNormStatImfile_DB + $detCleanupNormStatImfile_DB ++ + if ($detCleanupNormStatImfile_DB >= $DB:n) set detCleanupNormStatImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormStatImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormStatImfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.normstat.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormStatImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormStatImfile (pantaskState == INIT) + book getpage detCleanupNormStatImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormStatImfile $pageName pantaskState RUN + book getword detCleanupNormStatImfile $pageName det_id -var DET_ID + book getword detCleanupNormStatImfile $pageName iteration -var ITERATION + book getword detCleanupNormStatImfile $pageName camera -var CAMERA + book getword detCleanupNormStatImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormStatImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.normstat.log + stderr $LOGDIR/detrend.cleanup.normstat.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormStatImfile $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword detCleanupNormStatImfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormStatImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup norm (normalized.imfile) ########### +task detrend.cleanup.norm.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.norm.log + + task.exec + $run = dettool -pendingcleanup_normalizedimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormImfile_DB + $run = $run -dbname $DB:$detCleanupNormImfile_DB + $detCleanupNormImfile_DB ++ + if ($detCleanupNormImfile_DB >= $DB:n) set detCleanupNormImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormImfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.norm.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormImfile (pantaskState == INIT) + book getpage detCleanupNormImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormImfile $pageName pantaskState RUN + book getword detCleanupNormImfile $pageName det_id -var DET_ID + book getword detCleanupNormImfile $pageName iteration -var ITERATION + book getword detCleanupNormImfile $pageName class_id -var CLASS_ID + book getword detCleanupNormImfile $pageName camera -var CAMERA + book getword detCleanupNormImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.norm.log + stderr $LOGDIR/detrend.cleanup.norm.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormImfile $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword detCleanupNormImfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup normexp ########### +task detrend.cleanup.normexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.normexp.cleanup.log + + task.exec + $run = dettool -pendingcleanup_normalizedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormExp_DB + $run = $run -dbname $DB:$detCleanupNormExp_DB + $detCleanupNormExp_DB ++ + if ($detCleanupNormExp_DB >= $DB:n) set detCleanupNormExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormExp + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.normexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormExp (pantaskState == INIT) + book getpage detCleanupNormExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormExp $pageName pantaskState RUN + book getword detCleanupNormExp $pageName det_id -var DET_ID + book getword detCleanupNormExp $pageName iteration -var ITERATION + book getword detCleanupNormExp $pageName camera -var CAMERA + book getword detCleanupNormExp $pageName state -var CLEANUP_MODE + book getword detCleanupNormExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.normexp.log + stderr $LOGDIR/detrend.cleanup.normexp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormExp $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword detCleanupNormExp $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormExp $options:0 pantaskState TIMEOUT + end +end + + +######## cleanup resid imfile ######## +task detrend.cleanup.resid.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.resid.imfile.log + + task.exec + $run = dettool -pendingcleanup_residimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupResidImfile_DB + $run = $run -dbname $DB:$detCleanupResidImfile_DB + $detCleanupResidImfile_DB ++ + if ($detCleanupResidImfile_DB >= $DB:n) set detCleanupResidImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupResidImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupResidImfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.resid.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupResidImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupResidImfile (pantaskState == INIT) + book getpage detCleanupResidImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupResidImfile $pageName pantaskState RUN + book getword detCleanupResidImfile $pageName det_id -var DET_ID + book getword detCleanupResidImfile $pageName exp_id -var EXP_ID + book getword detCleanupResidImfile $pageName class_id -var CLASS_ID + book getword detCleanupResidImfile $pageName iteration -var ITERATION + book getword detCleanupResidImfile $pageName camera -var CAMERA + book getword detCleanupResidImfile $pageName state -var CLEANUP_MODE + book getword detCleanupResidImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.resid.imfile.log + stderr $LOGDIR/detrend.cleanup.resid.imfile.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupResidImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detCleanupResidImfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupResidImfile $options:0 pantaskState TIMEOUT + end +end + + +######## cleanup resid exp ######## +task detrend.cleanup.residexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.resid.exp.log + + task.exec + $run = dettool -pendingcleanup_residexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupResidExp_DB + $run = $run -dbname $DB:$detCleanupResidExp_DB + $detCleanupResidExp_DB ++ + if ($detCleanupResidExp_DB >= $DB:n) set detCleanupResidExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupResidExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupResidExp + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.residexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupResidExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupResidExp (pantaskState == INIT) + book getpage detCleanupResidExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupResidExp $pageName pantaskState RUN + book getword detCleanupResidExp $pageName det_id -var DET_ID + book getword detCleanupResidExp $pageName exp_id -var EXP_ID + book getword detCleanupResidExp $pageName iteration -var ITERATION + book getword detCleanupResidExp $pageName camera -var CAMERA + book getword detCleanupResidExp $pageName state -var CLEANUP_MODE + book getword detCleanupResidExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.resid.exp.log + stderr $LOGDIR/detrend.cleanup.resid.exp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupResidExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detCleanupResidExp $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupResidExp $options:0 pantaskState TIMEOUT + end +end Index: trunk/ippTasks/detrend.norm.pro =================================================================== --- trunk/ippTasks/detrend.norm.pro (revision 23479) +++ trunk/ippTasks/detrend.norm.pro (revision 23480) @@ -9,8 +9,4 @@ book init detPendingNormImfile book init detPendingNormExp - -book init detCleanupNormStatImfile -book init detCleanupNormImfile -book init detCleanupNormExp macro detnorm.reset @@ -18,8 +14,4 @@ book init detPendingNormImfile book init detPendingNormExp - - book init detCleanupNormStatImfile - book init detCleanupNormImfile - book init detCleanupNormExp end @@ -28,8 +20,4 @@ book listbook detPendingNormImfile book listbook detPendingNormExp - - book listbook detCleanupNormStatImfile - book listbook detCleanupNormImfile - book listbook detCleanupNormExp end @@ -53,23 +41,4 @@ active true end - - task detrend.cleanup.norm.load - active true - end - task detrend.cleanup.norm.run - active true - end - task detrend.cleanup.normexp.load - active true - end - task detrend.cleanup.normexp.run - active true - end - task detrend.cleanup.normstat.load - active true - end - task detrend.cleanup.normstat.run - active true - end end @@ -91,22 +60,4 @@ end task detrend.normstat.run - active false - end - task detrend.cleanup.norm.load - active false - end - task detrend.cleanup.norm.run - active false - end - task detrend.cleanup.normexp.load - active false - end - task detrend.cleanup.normexp.run - active false - end - task detrend.cleanup.normstat.load - active false - end - task detrend.cleanup.normstat.run active false end @@ -117,8 +68,4 @@ $detPendingNormImfile_DB = 0 $detPendingNormExp_DB = 0 - -$detCleanupNormStatImfile_DB = 0 -$detCleanupNormImfile_DB = 0 -$detCleanupNormExp_DB = 0 # select images ready for copy @@ -493,349 +440,2 @@ end end - -########## cleanup normstat ########### -task detrend.cleanup.normstat.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - active true - - stdout NULL - stderr $LOGDIR/detrend.cleanup.normstat.log - - task.exec - $run = dettool -pendingcleanup_normalizedstat - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$detCleanupNormStatImfile_DB - $run = $run -dbname $DB:$detCleanupNormStatImfile_DB - $detCleanupNormStatImfile_DB ++ - if ($detCleanupNormStatImfile_DB >= $DB:n) set detCleanupNormStatImfile_DB = 0 - end - add_poll_args run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout detCleanupNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook detCleanupNormStatImfile - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupNormStatImfile - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# run the ipp_cleanup.pl script on pending images -task detrend.cleanup.normstat.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active true - - task.exec - book npages detCleanupNormStatImfile -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images in detCleanupNormStatImfile (pantaskState == INIT) - book getpage detCleanupNormStatImfile 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword detCleanupNormStatImfile $pageName pantaskState RUN - book getword detCleanupNormStatImfile $pageName det_id -var DET_ID - book getword detCleanupNormStatImfile $pageName iteration -var ITERATION - book getword detCleanupNormStatImfile $pageName camera -var CAMERA - book getword detCleanupNormStatImfile $pageName state -var CLEANUP_MODE - book getword detCleanupNormStatImfile $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) - set.host.for.camera $CAMERA FPA - - stdout $LOGDIR/detrend.cleanup.normstat.log - stderr $LOGDIR/detrend.cleanup.normstat.log - - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit default - process_exit detCleanupNormStatImfile $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword detCleanupNormStatImfile $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword detCleanupNormStatImfile $options:0 pantaskState TIMEOUT - end -end - -########## cleanup norm (normalized.imfile) ########### -task detrend.cleanup.norm.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - active true - - stdout NULL - stderr $LOGDIR/detrend.cleanup.norm.log - - task.exec - $run = dettool -pendingcleanup_normalizedimfile - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$detCleanupNormImfile_DB - $run = $run -dbname $DB:$detCleanupNormImfile_DB - $detCleanupNormImfile_DB ++ - if ($detCleanupNormImfile_DB >= $DB:n) set detCleanupNormImfile_DB = 0 - end - add_poll_args run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout detCleanupNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook detCleanupNormImfile - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupNormImfile - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# run the ipp_cleanup.pl script on pending images -task detrend.cleanup.norm.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active true - - task.exec - book npages detCleanupNormImfile -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images in detCleanupNormImfile (pantaskState == INIT) - book getpage detCleanupNormImfile 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword detCleanupNormImfile $pageName pantaskState RUN - book getword detCleanupNormImfile $pageName det_id -var DET_ID - book getword detCleanupNormImfile $pageName iteration -var ITERATION - book getword detCleanupNormImfile $pageName class_id -var CLASS_ID - book getword detCleanupNormImfile $pageName camera -var CAMERA - book getword detCleanupNormImfile $pageName state -var CLEANUP_MODE - book getword detCleanupNormImfile $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) - set.host.for.camera $CAMERA FPA - - stdout $LOGDIR/detrend.cleanup.norm.log - stderr $LOGDIR/detrend.cleanup.norm.log - - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit default - process_exit detCleanupNormImfile $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword detCleanupNormImfile $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword detCleanupNormImfile $options:0 pantaskState TIMEOUT - end -end - -########## cleanup normexp ########### -task detrend.cleanup.normexp.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - active true - - stdout NULL - stderr $LOGDIR/detrend.normexp.cleanup.log - - task.exec - $run = dettool -pendingcleanup_normalizedexp - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$detCleanupNormExp_DB - $run = $run -dbname $DB:$detCleanupNormExp_DB - $detCleanupNormExp_DB ++ - if ($detCleanupNormExp_DB >= $DB:n) set detCleanupNormExp_DB = 0 - end - add_poll_args run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout detCleanupNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook detCleanupNormExp - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupNormExp - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# run the ipp_cleanup.pl script on pending images -task detrend.cleanup.normexp.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active true - - task.exec - book npages detCleanupNormExp -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images in detCleanupNormExp (pantaskState == INIT) - book getpage detCleanupNormExp 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword detCleanupNormExp $pageName pantaskState RUN - book getword detCleanupNormExp $pageName det_id -var DET_ID - book getword detCleanupNormExp $pageName iteration -var ITERATION - book getword detCleanupNormExp $pageName camera -var CAMERA - book getword detCleanupNormExp $pageName state -var CLEANUP_MODE - book getword detCleanupNormExp $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) - set.host.for.camera $CAMERA FPA - - stdout $LOGDIR/detrend.cleanup.normexp.log - stderr $LOGDIR/detrend.cleanup.normexp.log - - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit default - process_exit detCleanupNormExp $options:0 $JOB_STATUS - end - - task.exit crash - showcommand crash - book setword detCleanupNormExp $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword detCleanupNormExp $options:0 pantaskState TIMEOUT - end -end - Index: trunk/ippTasks/detrend.process.pro =================================================================== --- trunk/ippTasks/detrend.process.pro (revision 23479) +++ trunk/ippTasks/detrend.process.pro (revision 23480) @@ -8,12 +8,8 @@ book init detPendingProcessedImfile book init detPendingProcessedExp -book init detCleanupProcessedImfile -book init detCleanupProcessedExp macro detproc.reset book init detPendingProcessedImfile book init detPendingProcessedExp - book init detCleanupProcessedImfile - book init detCleanupProcessedExp end @@ -23,8 +19,4 @@ echo detPendingProcessedExp book listbook detPendingProcessedExp - echo detCleanupProcessedImfile - book listbook detCleanupProcessedImfile - echo detCleanupProcessedExp - book listbook detCleanupProcessedExp end @@ -42,16 +34,4 @@ active true end - task detrend.cleanup.process.load - active true - end - task detrend.cleanup.process.run - active true - end - task detrend.cleanup.processexp.load - active true - end - task detrend.cleanup.processexp.run - active true - end end @@ -69,16 +49,4 @@ active false end - task detrend.cleanup.process.load - active false - end - task detrend.cleanup.process.run - active false - end - task detrend.cleanup.processexp.load - active false - end - task detrend.cleanup.processexp.run - active false - end end @@ -87,6 +55,4 @@ $detPendingProcessedImfile_DB = 0 $detPendingProcessedExp_DB = 0 -$detCleanupProcessedImfile_DB = 0 -$detCleanupProcessedExp_DB = 0 # select images ready for copy @@ -347,238 +313,2 @@ end end - -######## cleanup process imfile ######## -task detrend.cleanup.process.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - active true - - stdout NULL - stderr $LOGDIR/detrend.cleanup.process.imfile.log - - task.exec - $run = dettool -pendingcleanup_processedimfile - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$detCleanupProcessedImfile_DB - $run = $run -dbname $DB:$detCleanupProcessedImfile_DB - $detCleanupProcessedImfile_DB ++ - if ($detCleanupProcessedImfile_DB >= $DB:n) set detCleanupProcessedImfile_DB = 0 - end - add_poll_args run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout detCleanupProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook detCleanupProcessedImfile - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupProcessedImfile - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# run the ipp_cleanup.pl script on pending images -task detrend.cleanup.process.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active true - - task.exec - book npages detCleanupProcessedImfile -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images in detCleanupProcessedImfile (pantaskState == INIT) - book getpage detCleanupProcessedImfile 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword detCleanupProcessedImfile $pageName pantaskState RUN - book getword detCleanupProcessedImfile $pageName det_id -var DET_ID - book getword detCleanupProcessedImfile $pageName exp_id -var EXP_ID - book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID - book getword detCleanupProcessedImfile $pageName camera -var CAMERA - book getword detCleanupProcessedImfile $pageName state -var CLEANUP_MODE - book getword detCleanupProcessedImfile $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) - set.host.for.camera $CAMERA FPA - - stdout $LOGDIR/detrend.cleanup.process.imfile.log - stderr $LOGDIR/detrend.cleanup.process.imfile.log - - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit default - process_exit detCleanupProcessedImfile $options:0 $JOB_STATUS - end - - # locked list - task.exit crash - showcommand crash - echo "hostname: $JOB_HOSTNAME" - book setword detCleanupProcessedImfile $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword detCleanupProcessedImfile $options:0 pantaskState TIMEOUT - end -end - - -######## cleanup process exp ######## -task detrend.cleanup.processexp.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - active true - - stdout NULL - stderr $LOGDIR/detrend.cleanup.process.exp.log - - task.exec - $run = dettool -pendingcleanup_processedexp - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$detCleanupProcessedExp_DB - $run = $run -dbname $DB:$detCleanupProcessedExp_DB - $detCleanupProcessedExp_DB ++ - if ($detCleanupProcessedExp_DB >= $DB:n) set detCleanupProcessedExp_DB = 0 - end - add_poll_args run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout detCleanupProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook detCleanupProcessedExp - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupProcessedExp - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# run the ipp_cleanup.pl script on pending images -task detrend.cleanup.processexp.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active true - - task.exec - book npages detCleanupProcessedExp -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images in detCleanupProcessedExp (pantaskState == INIT) - book getpage detCleanupProcessedExp 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword detCleanupProcessedExp $pageName pantaskState RUN - book getword detCleanupProcessedExp $pageName det_id -var DET_ID - book getword detCleanupProcessedExp $pageName exp_id -var EXP_ID - book getword detCleanupProcessedExp $pageName camera -var CAMERA - book getword detCleanupProcessedExp $pageName state -var CLEANUP_MODE - book getword detCleanupProcessedExp $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) - set.host.for.camera $CAMERA FPA - - stdout $LOGDIR/detrend.cleanup.process.exp.log - stderr $LOGDIR/detrend.cleanup.process.exp.log - - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --exp_id $EXP_ID --camera $CAMERA --mode $CLEANUP_MODE - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit default - process_exit detCleanupProcessedExp $options:0 $JOB_STATUS - end - - # locked list - task.exit crash - showcommand crash - echo "hostname: $JOB_HOSTNAME" - book setword detCleanupProcessedExp $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword detCleanupProcessedExp $options:0 pantaskState TIMEOUT - end -end Index: trunk/ippTasks/detrend.resid.pro =================================================================== --- trunk/ippTasks/detrend.resid.pro (revision 23479) +++ trunk/ippTasks/detrend.resid.pro (revision 23480) @@ -8,12 +8,8 @@ book init detPendingResidImfile book init detPendingResidExp -book init detCleanupResidImfile -book init detCleanupResidExp macro detresid.reset book init detPendingResidImfile book init detPendingResidExp - book init detCleanupResidImfile - book init detCleanupResidExp end @@ -21,6 +17,4 @@ book listbook detPendingResidImfile book listbook detPendingResidExp - book listbook detCleanupResidImfile - book listbook detCleanupResidExp end @@ -38,16 +32,4 @@ active true end - task detrend.cleanup.resid.load - active true - end - task detrend.cleanup.resid.run - active true - end - task detrend.cleanup.residexp.load - active true - end - task detrend.cleanup.residexp.run - active true - end end @@ -65,16 +47,4 @@ active false end - task detrend.cleanup.resid.load - active false - end - task detrend.cleanup.resid.run - active false - end - task detrend.cleanup.residexp.load - active false - end - task detrend.cleanup.residexp.run - active false - end end @@ -82,6 +52,4 @@ $detPendingResidImfile_DB = 0 $detPendingResidExp_DB = 0 -$detCleanupResidImfile_DB = 0 -$detCleanupResidExp_DB = 0 # select images ready for copy @@ -348,240 +316,2 @@ end end - -######## cleanup resid imfile ######## -task detrend.cleanup.resid.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - active true - - stdout NULL - stderr $LOGDIR/detrend.cleanup.resid.imfile.log - - task.exec - $run = dettool -pendingcleanup_residimfile - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$detCleanupResidImfile_DB - $run = $run -dbname $DB:$detCleanupResidImfile_DB - $detCleanupResidImfile_DB ++ - if ($detCleanupResidImfile_DB >= $DB:n) set detCleanupResidImfile_DB = 0 - end - add_poll_args run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout detCleanupResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook detCleanupResidImfile - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupResidImfile - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# run the ipp_cleanup.pl script on pending images -task detrend.cleanup.resid.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active true - - task.exec - book npages detCleanupResidImfile -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images in detCleanupResidImfile (pantaskState == INIT) - book getpage detCleanupResidImfile 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword detCleanupResidImfile $pageName pantaskState RUN - book getword detCleanupResidImfile $pageName det_id -var DET_ID - book getword detCleanupResidImfile $pageName exp_id -var EXP_ID - book getword detCleanupResidImfile $pageName class_id -var CLASS_ID - book getword detCleanupResidImfile $pageName iteration -var ITERATION - book getword detCleanupResidImfile $pageName camera -var CAMERA - book getword detCleanupResidImfile $pageName state -var CLEANUP_MODE - book getword detCleanupResidImfile $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) - set.host.for.camera $CAMERA FPA - - stdout $LOGDIR/detrend.cleanup.resid.imfile.log - stderr $LOGDIR/detrend.cleanup.resid.imfile.log - - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit default - process_exit detCleanupResidImfile $options:0 $JOB_STATUS - end - - # locked list - task.exit crash - showcommand crash - echo "hostname: $JOB_HOSTNAME" - book setword detCleanupResidImfile $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword detCleanupResidImfile $options:0 pantaskState TIMEOUT - end -end - - -######## cleanup resid exp ######## -task detrend.cleanup.residexp.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - active true - - stdout NULL - stderr $LOGDIR/detrend.cleanup.resid.exp.log - - task.exec - $run = dettool -pendingcleanup_residexp - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$detCleanupResidExp_DB - $run = $run -dbname $DB:$detCleanupResidExp_DB - $detCleanupResidExp_DB ++ - if ($detCleanupResidExp_DB >= $DB:n) set detCleanupResidExp_DB = 0 - end - add_poll_args run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout detCleanupResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook detCleanupResidExp - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupResidExp - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# run the ipp_cleanup.pl script on pending images -task detrend.cleanup.residexp.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active true - - task.exec - book npages detCleanupResidExp -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images in detCleanupResidExp (pantaskState == INIT) - book getpage detCleanupResidExp 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword detCleanupResidExp $pageName pantaskState RUN - book getword detCleanupResidExp $pageName det_id -var DET_ID - book getword detCleanupResidExp $pageName exp_id -var EXP_ID - book getword detCleanupResidExp $pageName iteration -var ITERATION - book getword detCleanupResidExp $pageName camera -var CAMERA - book getword detCleanupResidExp $pageName state -var CLEANUP_MODE - book getword detCleanupResidExp $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) - set.host.for.camera $CAMERA FPA - - stdout $LOGDIR/detrend.cleanup.resid.exp.log - stderr $LOGDIR/detrend.cleanup.resid.exp.log - - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit default - process_exit detCleanupResidExp $options:0 $JOB_STATUS - end - - # locked list - task.exit crash - showcommand crash - echo "hostname: $JOB_HOSTNAME" - book setword detCleanupResidExp $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword detCleanupResidExp $options:0 pantaskState TIMEOUT - end -end Index: trunk/ippTasks/detrend.stack.pro =================================================================== --- trunk/ippTasks/detrend.stack.pro (revision 23479) +++ trunk/ippTasks/detrend.stack.pro (revision 23480) @@ -7,9 +7,7 @@ book init detPendingStackedImfile -book init detCleanupStackedImfile macro detstack.reset book init detPendingStackedImfile - book init detCleanupStackedImfile end @@ -17,6 +15,4 @@ echo detPendingStackedImfile book listbook detPendingStackedImfile - echo detCleanupStackedImfile - book listbook detCleanupStackedImfile end @@ -26,10 +22,4 @@ end task detrend.stack.run - active true - end - task detrend.cleanup.stack.load - active true - end - task detrend.cleanup.stack.run active true end @@ -43,15 +33,8 @@ active false end - task detrend.cleanup.stack.load - active false - end - task detrend.cleanup.stack.run - active false - end end # this variable will cycle through the known database names $detPendingStackedImfile_DB = 0 -$detCleanupStackedImfile_DB = 0 # select images ready for detrend_stack.pl @@ -182,121 +165,2 @@ end end - -########## cleanup stack ########### -task detrend.cleanup.stack.load - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - active true - - stdout NULL - stderr $LOGDIR/detrend.cleanup.stack.log - - task.exec - $run = dettool -pendingcleanup_stacked - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$detCleanupStackedImfile_DB - $run = $run -dbname $DB:$detCleanupStackedImfile_DB - $detCleanupStackedImfile_DB ++ - if ($detCleanupStackedImfile_DB >= $DB:n) set detCleanupStackedImfile_DB = 0 - end - add_poll_args run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout detCleanupStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook detCleanupStackedImfile - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup detCleanupStackedImfile - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# run the ipp_cleanup.pl script on pending images -task detrend.cleanup.stack.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC - periods -timeout 60 - active true - - task.exec - book npages detCleanupStackedImfile -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new images in detCleanupStackedImfile (pantaskState == INIT) - book getpage detCleanupStackedImfile 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword detCleanupStackedImfile $pageName pantaskState RUN - book getword detCleanupStackedImfile $pageName det_id -var DET_ID - book getword detCleanupStackedImfile $pageName iteration -var ITERATION - book getword detCleanupStackedImfile $pageName class_id -var CLASS_ID - book getword detCleanupStackedImfile $pageName camera -var CAMERA - book getword detCleanupStackedImfile $pageName state -var CLEANUP_MODE - book getword detCleanupStackedImfile $pageName dbname -var DBNAME - - # specify choice of local or remote host based on camera and chip (class_id) - set.host.for.camera $CAMERA FPA - - stdout $LOGDIR/detrend.cleanup.stack.log - stderr $LOGDIR/detrend.cleanup.stack.log - - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE - add_standard_args run - - # save the pageName for future reference below - options $pageName - - # create the command line - if ($VERBOSE > 1) - echo command $run - end - command $run - end - - # default exit status - task.exit default - process_exit detCleanupStackedImfile $options:0 $JOB_STATUS - end - - # locked list - task.exit crash - showcommand crash - echo "hostname: $JOB_HOSTNAME" - book setword detCleanupStackedImfile $options:0 pantaskState CRASH - end - - # operation timed out? - task.exit timeout - showcommand timeout - book setword detCleanupStackedImfile $options:0 pantaskState TIMEOUT - end -end -