Index: trunk/ippTasks/camera.pro =================================================================== --- trunk/ippTasks/camera.pro (revision 19089) +++ trunk/ippTasks/camera.pro (revision 19090) @@ -7,11 +7,14 @@ book init camPendingExp +book init camPendingCleanup macro camera.status book listbook camPendingExp + book listbook camPendingCleanup end macro camera.reset book init camPendingExp + book init camPendingCleanup end @@ -23,4 +26,10 @@ active true end + task camera.cleanup.load + active false + end + task camera.cleanup.run + active false + end end @@ -30,4 +39,10 @@ end task camera.exp.run + active false + end + task camera.cleanup.load + active false + end + task camera.cleanup.run active false end @@ -164,3 +179,109 @@ end -## XXX add a global path to output files +# this variable will cycle through the known database names +$camera_cleanup_DB = 0 + +# select images ready for cam analysis +# new entries are added to camPendingImfile +# skip already-present entries +task camera.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/camera.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command camtool -pendingcleanuprun -limit 20 + else + # save the DB name for the exit tasks + option $DB:$camera_cleanup_DB + command camtool -pendingcleanuprun -limit 20 -dbname $DB:$camera_cleanup_DB + $camera_cleanup_DB ++ + if ($camera_cleanup_DB >= $DB:n) set camera_cleanup_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout camPendingCleanup -key cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook camPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup camPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task camera.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages camPendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in camPendingCleanup (pantaskState == INIT) + book getpage camPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword camPendingCleanup $pageName pantaskState RUN + book getword camPendingCleanup $pageName camera -var CAMERA + book getword camPendingCleanup $pageName state -var CLEANUP_MODE + book getword camPendingCleanup $pageName cam_id -var CAM_ID + book getword camPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and cam (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/camera.cleanup.log + stderr $LOGDIR/camera.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit camPendingCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword camPendingCleanup $options:0 pantaskState TIMEOUT + end +end Index: trunk/ippTasks/detrend.process.pro =================================================================== --- trunk/ippTasks/detrend.process.pro (revision 19089) +++ trunk/ippTasks/detrend.process.pro (revision 19090) @@ -8,8 +8,12 @@ book init detPendingProcessedImfile book init detPendingProcessedExp +book init detCleanupProcessedImfile +book init detCleanupProcessedExp macro detproc.reset book init detPendingProcessedImfile book init detPendingProcessedExp + book init detCleanupProcessedImfile + book init detCleanupProcessedExp end @@ -19,4 +23,8 @@ echo detPendingProcessedExp book listbook detPendingProcessedExp + echo detCleanupProcessedImfile + book listbook detCleanupProcessedImfile + echo detCleanupProcessedExp + book listbook detCleanupProcessedExp end @@ -34,4 +42,16 @@ active true end + task detrend.cleanup.process.load + active true + end + task detrend.cleanup.process.run + active true + end + task detrend.cleanup.processexp.load + active true + end + task detrend.cleanup.processexp.run + active true + end end @@ -47,4 +67,16 @@ end task detrend.processexp.run + active false + end + task detrend.cleanup.process.load + active false + end + task detrend.cleanup.process.run + active false + end + task detrend.cleanup.processexp.load + active false + end + task detrend.cleanup.processexp.run active false end @@ -55,4 +87,6 @@ $detPendingProcessedImfile_DB = 0 $detPendingProcessedExp_DB = 0 +$detCleanupProcessedImfile_DB = 0 +$detCleanupProcessedExp_DB = 0 # select images ready for copy @@ -68,5 +102,5 @@ stdout NULL - stderr $LOGDIR/detproc.imfile.load.log + stderr $LOGDIR/detrend.process.imfile.log task.exec @@ -142,6 +176,6 @@ sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID - stdout $LOGDIR/detproc.imfile.run.log - stderr $LOGDIR/detproc.imfile.run.log + stdout $LOGDIR/detrend.process.imfile.log + stderr $LOGDIR/detrend.process.imfile.log $run = detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --det_type $DET_TYPE --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --camera $CAMERA --outroot $outroot --redirect-output @@ -185,5 +219,5 @@ stdout NULL - stderr $LOGDIR/detproc.exp.load.log + stderr $LOGDIR/detrend.process.exp.log task.exec @@ -257,6 +291,6 @@ sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID - stdout $LOGDIR/detproc.exp.run.log - stderr $LOGDIR/detproc.exp.run.log + stdout $LOGDIR/detrend.process.exp.log + stderr $LOGDIR/detrend.process.exp.log $run = detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose @@ -287,2 +321,212 @@ end end + +######## cleanup process imfile ######## +task detrend.cleanup.process.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.process.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_processedimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupProcessedImfile_DB + command dettool -pendingcleanup_processedimfile -limit 20 -dbname $DB:$detCleanupProcessedImfile_DB + $detCleanupProcessedImfile_DB ++ + if ($detCleanupProcessedImfile_DB >= $DB:n) set detCleanupProcessedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupProcessedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupProcessedImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.process.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupProcessedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupProcessedImfile (pantaskState == INIT) + book getpage detCleanupProcessedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupProcessedImfile $pageName pantaskState RUN + book getword detCleanupProcessedImfile $pageName det_id -var DET_ID + book getword detCleanupProcessedImfile $pageName exp_id -var EXP_ID + book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID + book getword detCleanupProcessedImfile $pageName camera -var CAMERA + book getword detCleanupProcessedImfile $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.process.imfile.log + stderr $LOGDIR/detrend.cleanup.process.imfile.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupProcessedImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupProcessedImfile $options:0 pantaskState TIMEOUT + end +end + + +######## cleanup process exp ######## +task detrend.cleanup.processexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.process.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -pendingcleanup_processedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detCleanupProcessedExp_DB + command dettool -pendingcleanup_processedexp -limit 20 -dbname $DB:$detCleanupProcessedExp_DB + $detCleanupProcessedExp_DB ++ + if ($detCleanupProcessedExp_DB >= $DB:n) set detCleanupProcessedExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupProcessedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupProcessedExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.processexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupProcessedExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupProcessedExp (pantaskState == INIT) + book getpage detCleanupProcessedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupProcessedExp $pageName pantaskState RUN + book getword detCleanupProcessedExp $pageName det_id -var DET_ID + book getword detCleanupProcessedExp $pageName exp_id -var EXP_ID + book getword detCleanupProcessedExp $pageName camera -var CAMERA + book getword detCleanupProcessedExp $pageName state -var CLEANUP_MODE + book getword detCleanupProcessedExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.process.exp.log + stderr $LOGDIR/detrend.cleanup.process.exp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --exp_id $EXP_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupProcessedExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupProcessedExp $options:0 pantaskState TIMEOUT + end +end Index: trunk/ippTasks/fake.pro =================================================================== --- trunk/ippTasks/fake.pro (revision 19089) +++ trunk/ippTasks/fake.pro (revision 19090) @@ -7,11 +7,14 @@ book init fakePendingImfile +book init fakePendingCleanup macro fake.status book listbook fakePendingImfile + book listbook fakePendingCleanup end macro fake.reset book init fakePendingImfile + book init fakePendingCleanup end @@ -23,4 +26,10 @@ active true end + task fake.cleanup.load + active false + end + task fake.cleanup.run + active false + end end @@ -30,4 +39,10 @@ end task fake.imfile.run + active false + end + task fake.cleanup.load + active false + end + task fake.cleanup.run active false end @@ -154,3 +169,109 @@ end - +# this variable will cycle through the known database names +$fake_cleanup_DB = 0 + +# select images ready for fake analysis +# new entries are added to fakePendingImfile +# skip already-present entries +task fake.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/fake.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command faketool -pendingcleanuprun -limit 20 + else + # save the DB name for the exit tasks + option $DB:$fake_cleanup_DB + command faketool -pendingcleanuprun -limit 20 -dbname $DB:$fake_cleanup_DB + $fake_cleanup_DB ++ + if ($fake_cleanup_DB >= $DB:n) set fake_cleanup_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fakePendingCleanup -key fake_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fakePendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fakePendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task fake.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages fakePendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in fakePendingCleanup (pantaskState == INIT) + book getpage fakePendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fakePendingCleanup $pageName pantaskState RUN + book getword fakePendingCleanup $pageName camera -var CAMERA + book getword fakePendingCleanup $pageName state -var CLEANUP_MODE + book getword fakePendingCleanup $pageName fake_id -var FAKE_ID + book getword fakePendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and fake (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/fake.cleanup.log + stderr $LOGDIR/fake.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage fake --stage_id $FAKE_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit fakePendingCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fakePendingCleanup $options:0 pantaskState TIMEOUT + end +end Index: trunk/ippTasks/warp.pro =================================================================== --- trunk/ippTasks/warp.pro (revision 19089) +++ trunk/ippTasks/warp.pro (revision 19090) @@ -12,8 +12,10 @@ book init warpInputExp book init warpPendingSkyCell +book init warpPendingCleanup ### Database lists $warpExp_DB = 0 $warpSkycell_DB = 0 +$warpCleanup_DB = 0 ### Check status of warping tasks @@ -21,4 +23,5 @@ book listbook warpInputExp book listbook warpPendingSkyCell + book listbook warpPendingCleanup end @@ -27,4 +30,5 @@ book init warpInputExp book init warpPendingSkyCell + book init warpPendingCleanup end @@ -43,4 +47,10 @@ active true end + task warp.cleanup.load + active true + end + task warp.cleanup.run + active true + end end @@ -57,4 +67,10 @@ end task warp.skycell.run + active false + end + task warp.cleanup.load + active false + end + task warp.cleanup.run active false end @@ -283,2 +299,106 @@ end +# select images ready for warp analysis +# new entries are added to warpPendingImfile +# skip already-present entries +task warp.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/warp.cleanup.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command warptool -pendingcleanuprun -limit 20 + else + # save the DB name for the exit tasks + option $DB:$warpCleanup_DB + command warptool -pendingcleanuprun -limit 20 -dbname $DB:$warpCleanup_DB + $warpCleanup_DB ++ + if ($warpCleanup_DB >= $DB:n) set warpCleanup_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpPendingCleanup -key warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task warp.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages warpPendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in warpPendingCleanup (pantaskState == INIT) + book getpage warpPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingCleanup $pageName pantaskState RUN + book getword warpPendingCleanup $pageName camera -var CAMERA + book getword warpPendingCleanup $pageName state -var CLEANUP_MODE + book getword warpPendingCleanup $pageName warp_id -var WARP_ID + book getword warpPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and warp (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/warp.cleanup.log + stderr $LOGDIR/warp.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit warpPendingCleanup $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingCleanup $options:0 pantaskState TIMEOUT + end +end