Index: trunk/ippTasks/detrend.norm.pro =================================================================== --- trunk/ippTasks/detrend.norm.pro (revision 18989) +++ trunk/ippTasks/detrend.norm.pro (revision 19089) @@ -9,4 +9,8 @@ book init detPendingNormImfile book init detPendingNormExp + +book init detCleanupNormStatImfile +book init detCleanupNormImfile +book init detCleanupNormExp macro detnorm.reset @@ -14,4 +18,8 @@ book init detPendingNormImfile book init detPendingNormExp + + book init detCleanupNormStatImfile + book init detCleanupNormImfile + book init detCleanupNormExp end @@ -20,4 +28,8 @@ book listbook detPendingNormImfile book listbook detPendingNormExp + + book listbook detCleanupNormStatImfile + book listbook detCleanupNormImfile + book listbook detCleanupNormExp end @@ -41,4 +53,23 @@ active true end + + task detrend.cleanup.norm.load + active true + end + task detrend.cleanup.norm.run + active true + end + task detrend.cleanup.normexp.load + active true + end + task detrend.cleanup.normexp.run + active true + end + task detrend.cleanup.normstat.load + active true + end + task detrend.cleanup.normstat.run + active true + end end @@ -60,4 +91,22 @@ end task detrend.normstat.run + active false + end + task detrend.cleanup.norm.load + active false + end + task detrend.cleanup.norm.run + active false + end + task detrend.cleanup.normexp.load + active false + end + task detrend.cleanup.normexp.run + active false + end + task detrend.cleanup.normstat.load + active false + end + task detrend.cleanup.normstat.run active false end @@ -68,4 +117,8 @@ $detPendingNormImfile_DB = 0 $detPendingNormExp_DB = 0 + +$detCleanupNormStatImfile_DB = 0 +$detCleanupNormImfile_DB = 0 +$detCleanupNormExp_DB = 0 # select images ready for copy @@ -81,5 +134,5 @@ stdout NULL - stderr $LOGDIR/detrend.normstat.load.log + stderr $LOGDIR/detrend.normstat.log task.exec @@ -191,5 +244,5 @@ stdout NULL - stderr $LOGDIR/detrend.norm.load.log + stderr $LOGDIR/detrend.norm.log task.exec @@ -264,6 +317,6 @@ sprintf outroot "%s/%s.%s.%s/%s.%s.norm.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION - stdout $LOGDIR/detrend.norm.run.log - stderr $LOGDIR/detrend.norm.run.log + stdout $LOGDIR/detrend.norm.log + stderr $LOGDIR/detrend.norm.log $run = detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE --outroot $outroot --redirect-output --verbose @@ -304,5 +357,5 @@ stdout NULL - stderr $LOGDIR/detrend.normexp.load.log + stderr $LOGDIR/detrend.normexp.log task.exec @@ -374,6 +427,6 @@ sprintf outroot "%s/%s.%s.%s/%s.%s.normexp.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION - stdout $LOGDIR/detrend.norm.run.log - stderr $LOGDIR/detrend.norm.run.log + stdout $LOGDIR/detrend.normexp.log + stderr $LOGDIR/detrend.normexp.log $run = detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE --outroot $outroot @@ -401,2 +454,316 @@ end end + +########## cleanup normstat ########### +task detrend.cleanup.normstat.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.normstat.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_normalizedstat -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormStatImfile_DB + command dettool -pendingcleanup_normalizedstat -limit 20 -dbname $DB:$detCleanupNormStatImfile_DB + $detCleanupNormStatImfile_DB ++ + if ($detCleanupNormStatImfile_DB >= $DB:n) set detCleanupNormStatImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormStatImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormStatImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.normstat.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormStatImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormStatImfile (pantaskState == INIT) + book getpage detCleanupNormStatImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormStatImfile $pageName pantaskState RUN + book getword detCleanupNormStatImfile $pageName det_id -var DET_ID + book getword detCleanupNormStatImfile $pageName iteration -var ITERATION + book getword detCleanupNormStatImfile $pageName camera -var CAMERA + book getword detCleanupNormStatImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormStatImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.normstat.log + stderr $LOGDIR/detrend.cleanup.normstat.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormStatImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormStatImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup norm (normalized.imfile) ########### +task detrend.cleanup.norm.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.norm.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_normalizedimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormImfile_DB + command dettool -pendingcleanup_normalizedimfile -limit 20 -dbname $DB:$detCleanupNormImfile_DB + $detCleanupNormImfile_DB ++ + if ($detCleanupNormImfile_DB >= $DB:n) set detCleanupNormImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.norm.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormImfile (pantaskState == INIT) + book getpage detCleanupNormImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormImfile $pageName pantaskState RUN + book getword detCleanupNormImfile $pageName det_id -var DET_ID + book getword detCleanupNormImfile $pageName iteration -var ITERATION + book getword detCleanupNormImfile $pageName class_id -var CLASS_ID + book getword detCleanupNormImfile $pageName camera -var CAMERA + book getword detCleanupNormImfile $pageName state -var CLEANUP_MODE + book getword detCleanupNormImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.norm.log + stderr $LOGDIR/detrend.cleanup.norm.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup normexp ########### +task detrend.cleanup.normexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.normexp.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_normalizedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormExp_DB + command dettool -pendingcleanup_normalizedexp -limit 20 -dbname $DB:$detCleanupNormExp_DB + $detCleanupNormExp_DB ++ + if ($detCleanupNormExp_DB >= $DB:n) set detCleanupNormExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.normexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupNormExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormExp (pantaskState == INIT) + book getpage detCleanupNormExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormExp $pageName pantaskState RUN + book getword detCleanupNormExp $pageName det_id -var DET_ID + book getword detCleanupNormExp $pageName iteration -var ITERATION + book getword detCleanupNormExp $pageName camera -var CAMERA + book getword detCleanupNormExp $pageName state -var CLEANUP_MODE + book getword detCleanupNormExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.normexp.log + stderr $LOGDIR/detrend.cleanup.normexp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormExp $options:0 pantaskState TIMEOUT + end +end + Index: trunk/ippTasks/detrend.resid.pro =================================================================== --- trunk/ippTasks/detrend.resid.pro (revision 18989) +++ trunk/ippTasks/detrend.resid.pro (revision 19089) @@ -8,8 +8,12 @@ book init detPendingResidImfile book init detPendingResidExp +book init detCleanupResidImfile +book init detCleanupResidExp macro detresid.reset book init detPendingResidImfile book init detPendingResidExp + book init detCleanupResidImfile + book init detCleanupResidExp end @@ -17,4 +21,6 @@ book listbook detPendingResidImfile book listbook detPendingResidExp + book listbook detCleanupResidImfile + book listbook detCleanupResidExp end @@ -32,4 +38,16 @@ active true end + task detrend.cleanup.resid.load + active true + end + task detrend.cleanup.resid.run + active true + end + task detrend.cleanup.residexp.load + active true + end + task detrend.cleanup.residexp.run + active true + end end @@ -45,4 +63,16 @@ end task detrend.residexp.run + active false + end + task detrend.cleanup.resid.load + active false + end + task detrend.cleanup.resid.run + active false + end + task detrend.cleanup.residexp.load + active false + end + task detrend.cleanup.residexp.run active false end @@ -52,4 +82,6 @@ $detPendingResidImfile_DB = 0 $detPendingResidExp_DB = 0 +$detCleanupResidImfile_DB = 0 +$detCleanupResidExp_DB = 0 # select images ready for copy @@ -65,5 +97,5 @@ stdout NULL - stderr $LOGDIR/detresid.load.log + stderr $LOGDIR/detrend.resid.imfile.log task.exec @@ -142,6 +174,6 @@ sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION - stdout $LOGDIR/detresid.run.log - stderr $LOGDIR/detresid.run.log + stdout $LOGDIR/detrend.resid.imfile.log + stderr $LOGDIR/detrend.resid.imfile.log $run = detrend_resid_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE --outroot $outroot --redirect-output --verbose @@ -186,5 +218,5 @@ stdout NULL - stderr $LOGDIR/detresid.exp.load.log + stderr $LOGDIR/detrend.resid.exp.log task.exec @@ -260,6 +292,6 @@ sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION - stdout $LOGDIR/detresid.exp.run.log - stderr $LOGDIR/detresid.exp.run.log + stdout $LOGDIR/detrend.resid.exp.log + stderr $LOGDIR/detrend.resid.exp.log $run = detrend_resid_exp.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose @@ -288,2 +320,214 @@ end end + +######## cleanup resid imfile ######## +task detrend.cleanup.resid.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.resid.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_residimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupResidImfile_DB + command dettool -pendingcleanup_residimfile -limit 20 -dbname $DB:$detCleanupResidImfile_DB + $detCleanupResidImfile_DB ++ + if ($detCleanupResidImfile_DB >= $DB:n) set detCleanupResidImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupResidImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupResidImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.resid.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupResidImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupResidImfile (pantaskState == INIT) + book getpage detCleanupResidImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupResidImfile $pageName pantaskState RUN + book getword detCleanupResidImfile $pageName det_id -var DET_ID + book getword detCleanupResidImfile $pageName exp_id -var EXP_ID + book getword detCleanupResidImfile $pageName class_id -var CLASS_ID + book getword detCleanupResidImfile $pageName iteration -var ITERATION + book getword detCleanupResidImfile $pageName camera -var CAMERA + book getword detCleanupResidImfile $pageName state -var CLEANUP_MODE + book getword detCleanupResidImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.resid.imfile.log + stderr $LOGDIR/detrend.cleanup.resid.imfile.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupResidImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupResidImfile $options:0 pantaskState TIMEOUT + end +end + + +######## cleanup resid exp ######## +task detrend.cleanup.residexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.resid.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_residexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupResidExp_DB + command dettool -pendingcleanup_residexp -limit 20 -dbname $DB:$detCleanupResidExp_DB + $detCleanupResidExp_DB ++ + if ($detCleanupResidExp_DB >= $DB:n) set detCleanupResidExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupResidExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupResidExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.residexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupResidExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupResidExp (pantaskState == INIT) + book getpage detCleanupResidExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupResidExp $pageName pantaskState RUN + book getword detCleanupResidExp $pageName det_id -var DET_ID + book getword detCleanupResidExp $pageName exp_id -var EXP_ID + book getword detCleanupResidExp $pageName iteration -var ITERATION + book getword detCleanupResidExp $pageName camera -var CAMERA + book getword detCleanupResidExp $pageName state -var CLEANUP_MODE + book getword detCleanupResidExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.resid.exp.log + stderr $LOGDIR/detrend.cleanup.resid.exp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupResidExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupResidExp $options:0 pantaskState TIMEOUT + end +end Index: trunk/ippTasks/detrend.stack.pro =================================================================== --- trunk/ippTasks/detrend.stack.pro (revision 18989) +++ trunk/ippTasks/detrend.stack.pro (revision 19089) @@ -7,7 +7,9 @@ book init detPendingStackedImfile +book init detCleanupStackedImfile macro detstack.reset book init detPendingStackedImfile + book init detCleanupStackedImfile end @@ -15,4 +17,6 @@ echo detPendingStackedImfile book listbook detPendingStackedImfile + echo detCleanupStackedImfile + book listbook detCleanupStackedImfile end @@ -24,4 +28,10 @@ active true end + task detrend.cleanup.stack.load + active true + end + task detrend.cleanup.stack.run + active true + end end @@ -33,8 +43,15 @@ active false end + task detrend.cleanup.stack.load + active false + end + task detrend.cleanup.stack.run + active false + end end # this variable will cycle through the known database names $detPendingStackedImfile_DB = 0 +$detCleanupStackedImfile_DB = 0 # select images ready for detrend_stack.pl @@ -50,5 +67,5 @@ stdout NULL - stderr $LOGDIR/detrend.stack.load.log + stderr $LOGDIR/detrend.stack.log task.exec @@ -122,8 +139,8 @@ sprintf outroot "%s/%s.%s.%s/%s.%s.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION - stdout $LOGDIR/detrend.stack.run.log - stderr $LOGDIR/detrend.stack.run.log - - $run = detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose + stdout $LOGDIR/detrend.stack.log + stderr $LOGDIR/detrend.stack.log + + $run = detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION @@ -152,2 +169,108 @@ end end + +########## cleanup stack ########### +task detrend.cleanup.stack.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.stack.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_stacked -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupStackedImfile_DB + command dettool -pendingcleanup_stacked -limit 20 -dbname $DB:$detCleanupStackedImfile_DB + $detCleanupStackedImfile_DB ++ + if ($detCleanupStackedImfile_DB >= $DB:n) set detCleanupStackedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupStackedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupStackedImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.stack.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupStackedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupStackedImfile (pantaskState == INIT) + book getpage detCleanupStackedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupStackedImfile $pageName pantaskState RUN + book getword detCleanupStackedImfile $pageName det_id -var DET_ID + book getword detCleanupStackedImfile $pageName iteration -var ITERATION + book getword detCleanupStackedImfile $pageName class_id -var CLASS_ID + book getword detCleanupStackedImfile $pageName camera -var CAMERA + book getword detCleanupStackedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupStackedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.stack.log + stderr $LOGDIR/detrend.cleanup.stack.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupStackedImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupStackedImfile $options:0 pantaskState TIMEOUT + end +end + Index: trunk/ippTasks/diff.pro =================================================================== --- trunk/ippTasks/diff.pro (revision 18989) +++ trunk/ippTasks/diff.pro (revision 19089) @@ -8,16 +8,16 @@ check.globals -$LOGSUBDIR = $LOGDIR/diff -exec mkdir -p $LOGSUBDIR - ### Initialise the books containing the tasks to do book init diffSkyfile +book init diffCleanup ### Database lists $diffSkycell_DB = 0 +$diffCleanup_DB = 0 ### Check status of diffing tasks macro diff.status book listbook diffSkyfile + book listbook diffCleanup end @@ -25,4 +25,5 @@ macro diff.reset book init diffSkyfile + book init diffCleanup end @@ -35,4 +36,10 @@ active true end + task diff.cleanup.load + active true + end + task diff.cleanup.run + active true + end end @@ -43,4 +50,10 @@ end task diff.skycell.run + active false + end + task diff.cleanup.load + active false + end + task diff.cleanup.run active false end @@ -59,5 +72,5 @@ stdout NULL - stderr $LOGSUBDIR/diff.skycell.log + stderr $LOGDIR/diff.skycell.log task.exec @@ -122,4 +135,5 @@ ### XXX Instead of setting these by camera, need to set by skycell_id + ### XXX change these to use the --redirect scheme # set.host.for.camera $CAMERA $DIFF_ID # set.workdir.by.camera $CAMERA $DIFF_ID $WORKDIR_TEMPLATE $default_host WORKDIR @@ -167,2 +181,106 @@ end +# select images ready for diff analysis +# new entries are added to diffPendingImfile +# skip already-present entries +task diff.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/diff.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command difftool -pendingcleanuprun -limit 20 + else + # save the DB name for the exit tasks + option $DB:$diffCleanup_DB + command difftool -pendingcleanuprun -limit 20 -dbname $DB:$diffCleanup_DB + $diffCleanup_DB ++ + if ($diffCleanup_DB >= $DB:n) set diffCleanup_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout diffCleanup -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task diff.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages diffCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in diffCleanup (pantaskState == INIT) + book getpage diffCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffCleanup $pageName pantaskState RUN + book getword diffCleanup $pageName camera -var CAMERA + book getword diffCleanup $pageName state -var CLEANUP_MODE + book getword diffCleanup $pageName diff_id -var DIFF_ID + book getword diffCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and diff (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/diff.cleanup.log + stderr $LOGDIR/diff.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit diffCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffCleanup $options:0 pantaskState TIMEOUT + end +end Index: trunk/ippTasks/stack.pro =================================================================== --- trunk/ippTasks/stack.pro (revision 18989) +++ trunk/ippTasks/stack.pro (revision 19089) @@ -8,16 +8,16 @@ check.globals -$LOGSUBDIR = $LOGDIR/stack -exec mkdir -p $LOGSUBDIR - ### Initialise the books containing the tasks to do book init stackSumSkyfile +book init stackCleanup ### Database lists $stackSkycell_DB = 0 +$stackCleanup_DB = 0 ### Check status of stacking tasks macro stack.status book listbook stackSumSkyfile + book listbook stackCleanup end @@ -25,4 +25,5 @@ macro stack.reset book init stackSumSkyfile + book init stackCleanup end @@ -35,4 +36,10 @@ active true end + task stack.cleanup.load + active true + end + task stack.cleanup.run + active true + end end @@ -43,4 +50,10 @@ end task stack.skycell.run + active false + end + task stack.cleanup.load + active false + end + task stack.cleanup.run active false end @@ -59,5 +72,5 @@ stdout NULL - stderr $LOGSUBDIR/stack.skycell.log + stderr $LOGDIR/stack.skycell.log task.exec @@ -168,2 +181,107 @@ end + +# select images ready for stack analysis +# new entries are added to stackCleanup +# skip already-present entries +task stack.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/stack.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command stacktool -pendingcleanuprun -limit 20 + else + # save the DB name for the exit tasks + option $DB:$stackCleanup_DB + command stacktool -pendingcleanuprun -limit 20 -dbname $DB:$stackCleanup_DB + $stackCleanup_DB ++ + if ($stackCleanup_DB >= $DB:n) set stackCleanup_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackCleanup -key stack_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task stack.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages stackCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in stackCleanup (pantaskState == INIT) + book getpage stackCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackCleanup $pageName pantaskState RUN + book getword stackCleanup $pageName camera -var CAMERA + book getword stackCleanup $pageName state -var CLEANUP_MODE + book getword stackCleanup $pageName stack_id -var STACK_ID + book getword stackCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and stack (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/stack.cleanup.log + stderr $LOGDIR/stack.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit stackCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackCleanup $options:0 pantaskState TIMEOUT + end +end