Index: /branches/ccl_branches/ipponly-20191108/ippTasks/Makefile.am =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/Makefile.am (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/Makefile.am (revision 41106) @@ -0,0 +1,82 @@ +task_files = \ + detrend.norm.pro \ + detrend.process.pro \ + detrend.reject.pro \ + detrend.resid.pro \ + detrend.stack.pro \ + detrend.correct.pro \ + detrend.cleanup.pro \ + flatcorr.pro \ + automate.pro \ + pantasks.pro \ + register.pro \ + chip.pro \ + camera.pro \ + addstar.pro \ + fake.pro \ + warp.pro \ + magic.pro \ + magic.cleanup.pro \ + destreak.pro \ + destreak.cleanup.pro \ + diff.pro \ + skycal.pro \ + stack.pro \ + staticsky.pro \ + summit.copy.pro \ + survey.pro \ + replicate.pro \ + rawcheck.pro \ + diskbalance.pro \ + dist.pro \ + dist.cleanup.pro \ + rcserver.pro \ + pstamp.pro \ + receive.pro \ + publish.pro \ + dqstats.pro \ + science.cleanup.pro \ + minidvodb.pro \ + minidvodbcopy.pro \ + mergedvodb.pro \ + nightly_science.pro \ + burntool.pro \ + lossy_compress.pro \ + background.pro \ + diffphot.pro \ + lap.pro \ + lapgroup.pro \ + vp.pro \ + bg.regeneration.pro \ + remote.pro \ + fullforce.pro + +other_files = \ + ipphosts.manoa.config \ + ipphosts.mhpcc.config \ + site.manoa.pro \ + site.mhpcc.pro \ + surveys.mhpcc.config \ + simtest.pro \ + simtest.basic.config \ + simtest.basic.auto \ + simtest.nebulous.basic.auto \ + simtest.detverify.config \ + simtest.detverify.auto \ + simtest.flatcorr.config \ + simtest.flatcorr.auto \ + simtest.ctemask.config \ + simtest.ctemask.auto \ + simtest.stack.config \ + simtest.stack.auto + +pantasksdir = $(datadir)/pantasks +ipptasksdir = $(pantasksdir)/modules +ipptasks_DATA = $(task_files) $(other_files) + +install-data-hook: + chmod 0755 $(panstasksdir) $(ipptasksdir) + +EXTRA_DIST = $(task_files) $(other_files) + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/ccl_branches/ipponly-20191108/ippTasks/addstar.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/addstar.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/addstar.pro (revision 41106) @@ -0,0 +1,1029 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + +if ($?ADDSTAGES:n == 0) set ADDSTAGES:n = 0 +if ($?MULTIADDSTAGES:n == 0) set MULTIADDSTAGES:n = 0 + +book init addPendingExp +book init addPendingMultiExp + +macro addstar.status + book listbook addPendingExp +end + +macro addstar.multi.status + book listbook addPendingMultiExp +end + +macro addstar.reset + book init addPendingExp +end + +macro addstar.multi.reset + book init addPendingMultiExp +end + +macro addstar.on + task addstar.exp.load + active true + end + task addstar.exp.run + active true + end +end +macro addstar.multi.on + task addstar.multiexp.load + active true + end + task addstar.multiexp.run + active true + end +end + +macro addstar.off + task addstar.exp.load + active false + end + task addstar.exp.run + active false + end +end +macro addstar.multi.off + task addstar.multiexp.load + active false + end + task addstar.multiexp.run + active false + end +end + +macro addstar.revert.off + task addstar.revert.cam + active false + end + task addstar.revert.stack + active false + end + task addstar.revert.staticsky + active false + end + task addstar.revert.skycal + active false + end + task addstar.revert.diff + active false + end + task addstar.revert.fullforce + active false + end + task addstar.revert.fullforce_summary + active false + end +end + +macro addstar.revert.on + task addstar.revert.cam + active true + end + task addstar.revert.stack + active true + end + task addstar.revert.staticsky + active true + end + task addstar.revert.skycal + active true + end + task addstar.revert.diff + active true + end + task addstar.revert.fullforce + active true + end + task addstar.revert.fullforce_summary + active true + end +end + + +#addstar stages + +macro add.addstages + if ($0 != 2) + echo "USAGE: add.addstages (addstages)" + break + end + if ($?ADDSTAGES:n == 0) + list ADDSTAGES -add $1 + return + end + + local found + $found = 0 + for i 0 $ADDSTAGES:n + if ($ADDSTAGES:$i == $1) + $found = 1 + echo "$ADDSTAGES:$i set" + last + end + end + + if ($found == 0) + list ADDSTAGES -add $1 + end +end + + + +macro del.addstages + if ($0 != 2) + echo "USAGE: del.addstages (addstages)" + break + end + if ($?ADDSTAGES:n == 0) + return + end + + list ADDSTAGES -del $1 +end + +macro show.addstages + if ($0 != 1) + echo "USAGE: show.addstages" + break + end + if ($?ADDSTAGES:n == 0) + echo "no addstar stages defined" + end + if ($ADDSTAGES:n == 0) + echo "no addstar stages defined" + end + + local i + for i 0 $ADDSTAGES:n + echo $ADDSTAGES:$i + end +end + +macro add.multiaddstages + if ($0 != 2) + echo "USAGE: add.multiaddstages (multiaddstages)" + break + end + if ($?MULTIADDSTAGES:n == 0) + list MULTIADDSTAGES -add $1 + return + end + + local found + $found = 0 + for i 0 $MULTIADDSTAGES:n + if ($MULTIADDSTAGES:$i == $1) + $found = 1 + echo "$MULTIADDSTAGES:$i set" + last + end + end + + if ($found == 0) + list MULTIADDSTAGES -add $1 + end +end + + + +macro del.multiaddstages + if ($0 != 2) + echo "USAGE: del.multiaddstages (multiaddstages)" + break + end + if ($?MULTIADDSTAGES:n == 0) + return + end + + list MULTIADDSTAGES -del $1 +end + +macro show.multiaddstages + if ($0 != 1) + echo "USAGE: show.multiaddstages" + break + end + if ($?MULTIADDSTAGES:n == 0) + echo "no addstar stages defined" + end + if ($MULTIADDSTAGES:n == 0) + echo "no addstar stages defined" + end + + local i + for i 0 $MULTIADDSTAGES:n + echo $MULTIADDSTAGES:$i + end +end + +macro stuff + echo $LOADEXEC_ADD + echo $LOADPOLL +end + +# this variable will cycle through the known database names +$addstar_DB = 0 +$addstar_stages_DB = 0 + +$addstar_multi_DB = 0 +$addstar_multi_stages_DB = 0 + +# this may not work for more databases (addstar) will do that later) +$addstar_revert_DB_C = 0 +$addstar_revert_DB_S = 0 +$addstar_revert_DB_SS = 0 +$addstar_revert_DB_SSM = 0 +$addstar_revert_DB_SC = 0 +$addstar_revert_DB_FF = 0 +$addstar_revert_DB_FFS = 0 +$addstar_revert_DB_DF = 0 + +#$LOADEXEC_ADD = 300 +$LOADEXEC_ADD = 10 + +if ($?addstar_multiadd_limit == 0) set addstar_multiadd_limit = 0 + +macro set.multiadd.limit + if ($0 != 2) + echo "USAGE: set.multiadd.limit (Nentry)" + break + end + + $addstar_multiadd_limit = $1 +end + +macro get.multiadd.limit + echo "addstar multi limit : $addstar_multiadd_limit" +end + +# select images ready for addstar analysis +# new entries are added to addPendingExp +# skip already-present entries +task addstar.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC_ADD + periods -timeout 300 + npending 1 + + stdout NULL + stderr $LOGDIR/addstar.exp.log + + task.exec + #if no stages defined we can't run + if ($ADDSTAGES:n == 0) echo "what" + # if ($ADDSTAGES:n == 0) break + $run = addtool -pendingexp + + #option $ADDSTAGES:$addstar_stages_DB + $run = $run -stage $ADDSTAGES:$addstar_stages_DB + option $DB:$addstar_DB + $run = $run -dbname $DB:$addstar_DB + $addstar_stages_DB ++ + if ($addstar_stages_DB >= $ADDSTAGES:n) + set addstar_stages_DB = 0 + $addstar_DB ++ + if ($addstar_DB >= $DB:n) set addstar_DB = 0 + #go to next stage. if run out of stages, go to 0 stage and + #cycle through dbname, if run out of dbnames, go to 0 dbname + end + add_poll_args run + add_poll_labels run + echo $run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook addPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup addPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +# select images ready for addstar analysis +# new entries are added to addPendingExp +# skip already-present entries +task addstar.multiexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC_ADD + periods -timeout 300 + npending 1 + + stdout NULL + stderr $LOGDIR/addstar.multiexp.log + + task.exec + #if no stages defined we can't run + if ($MULTIADDSTAGES:n == 0) echo "no stages for addstar.multi, use add.addstagesmulti" + # if ($MULTIADDSTAGES:n == 0) break + $run = addtool -pendingexp + + #option $ADDSTAGES:$addstar_stages_DB + $run = $run -stage $MULTIADDSTAGES:$addstar_multi_stages_DB -multiadd + option $DB:$addstar_multi_DB + $run = $run -dbname $DB:$addstar_multi_DB + $addstar_multi_stages_DB ++ + if ($addstar_multi_stages_DB >= $MULTIADDSTAGES:n) + set addstar_multi_stages_DB = 0 + $addstar_multi_DB ++ + if ($addstar_multi_DB >= $DB:n) set addstar_multi_DB = 0 + #go to next stage. if run out of stages, go to 0 stage and + #cycle through dbname, if run out of dbnames, go to 0 dbname + end + add_poll_args run + add_poll_labels run + echo $run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout addPendingMultiExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook addPendingMultiExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup addPendingMultiExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the addstar script on pending exposures +task addstar.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 1200 + + ## we want only a single outstanding addstar job. + host local + npending 1 + + task.exec + book npages addPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in addPendingExp (pantaskState == INIT) + book getpage addPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword addPendingExp $pageName pantaskState RUN + book getword addPendingExp $pageName camera -var CAMERA + book getword addPendingExp $pageName exp_tag -var EXP_TAG + book getword addPendingExp $pageName add_id -var ADD_ID + book getword addPendingExp $pageName stage_extra1 -var STAGE_EXTRA1 + book getword addPendingExp $pageName stageroot -var STAGEROOT + book getword addPendingExp $pageName stage -var STAGE + book getword addPendingExp $pageName stage_id -var STAGE_ID + book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword addPendingExp $pageName reduction -var REDUCTION + book getword addPendingExp $pageName dvodb -var DVODB + book getword addPendingExp $pageName minidvodb -var MINIDVODB + book getword addPendingExp $pageName minidvodb_name -var MINIDVODB_NAME + book getword addPendingExp $pageName minidvodb_group -var MINIDVODB_GROUP + book getword addPendingExp $pageName image_only -var IMAGE_ONLY + book getword addPendingExp $pageName dbname -var DBNAME + + + # specify choice of remote host based on camera and chip (class_id) + # set.host.for.camera $CAMERA FPA + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + + if ("$STAGE" == "cam") + sprintf outroot "%s/%s/%s.add.%s" $WORKDIR $EXP_TAG $EXP_TAG $ADD_ID + end + if ("$STAGE" == "staticsky") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "stack") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "skycal") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "diff") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "fullforce") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "fullforce_summary") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + + + + stdout $LOGDIR/addstar.exp.log + stderr $LOGDIR/addstar.exp.log + + $run = addstar_run.pl --add_id $ADD_ID --camera $CAMERA --dvodb $DVODB --stage $STAGE --stageroot $STAGEROOT --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$STAGE" == "staticsky") + $run = $run --stage_extra1 $STAGE_EXTRA1 --stage_id $STAGE_ID + end + if ("$STAGE" == "cam") + $run = $run --stage_id $STAGE_ID + end + if ("$STAGE" == "skycal") + $run = $run --stage_id $STAGE_ID + end + if ("$STAGE" == "diff") + #$run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1 + $run = $run --use_diff_inv --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1 + end + if ("$STAGE" == "fullforce") + $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1 + end + if ("$STAGE" == "fullforce_summary") + # This shouldn't need a stage_extra1, as there are no subcomponents to the ff summary. + $run = $run --stage_id $STAGE_ID + end + + + + if ("$IMAGE_ONLY" == "T") + $run = $run --image-only + end + if ("$MINIDVODB" == "T") + $run = $run --minidvodb + $run = $run --minidvodb_group $MINIDVODB_GROUP + if (("$MINIDVODB_NAME" != "NULL") && ("$MINIDVODB_NAME" != "(null)")) + $run = $run --minidvodb_name $MINIDVODB_NAME + #have addstar_run.pl grab the 'active' name if it is NULL + end + end + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit addPendingExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword addPendingExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword addPendingExp $options:0 pantaskState TIMEOUT + end +end + + + +# run the addstar script on pending exposures +task addstar.multiexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 3600 + # addstar MD fields with 250 exp takes about 2400 sec to load + + ## we want only a single outstanding addstar job. + host local + npending 1 + + task.exec + book npages addPendingMultiExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in addPendingExp (pantaskState == INIT) + book getpage addPendingMultiExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword addPendingMultiExp $pageName pantaskState RUN + book getword addPendingMultiExp $pageName camera -var CAMERA + book getword addPendingMultiExp $pageName exp_tag -var EXP_TAG + book getword addPendingMultiExp $pageName add_id -var ADD_ID + book getword addPendingMultiExp $pageName stage_extra1 -var STAGE_EXTRA1 + book getword addPendingMultiExp $pageName stageroot -var STAGEROOT + book getword addPendingMultiExp $pageName stage -var STAGE + book getword addPendingMultiExp $pageName stage_id -var STAGE_ID + book getword addPendingMultiExp $pageName workdir -var WORKDIR_TEMPLATE + book getword addPendingMultiExp $pageName reduction -var REDUCTION + book getword addPendingMultiExp $pageName dvodb -var DVODB + book getword addPendingMultiExp $pageName minidvodb -var MINIDVODB + book getword addPendingMultiExp $pageName minidvodb_name -var MINIDVODB_NAME + book getword addPendingMultiExp $pageName minidvodb_group -var MINIDVODB_GROUP + book getword addPendingMultiExp $pageName image_only -var IMAGE_ONLY + book getword addPendingMultiExp $pageName dbname -var DBNAME + book getword addPendingMultiExp $pageName label -var LABEL + + # specify choice of remote host based on camera and chip (class_id) + # set.host.for.camera $CAMERA FPA + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + + if ("$STAGE" == "cam") + sprintf outroot "%s/%s/%s.add.%s" $WORKDIR $EXP_TAG $EXP_TAG $ADD_ID + end + if ("$STAGE" == "staticsky") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "stack") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "skycal") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "diff") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "fullforce") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "fullforce_summary") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + + + + stdout $LOGDIR/addstar.multiexp.log + stderr $LOGDIR/addstar.multiexp.log + + $run = addstar_multi_run.pl --stage_id $STAGE_ID --label $LABEL --camera $CAMERA --dvodb $DVODB --stage $STAGE --stageroot $STAGEROOT --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$STAGE" == "staticsky") + $run = $run --stage_extra1 $STAGE_EXTRA1 --stage_id $STAGE_ID + end + if ("$STAGE" == "cam") + $run = $run --stage_id $STAGE_ID + end + if ("$STAGE" == "skycal") + $run = $run --stage_id $STAGE_ID + end + #if ("$STAGE" == "diff") + # $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1 + # end + #if ("$STAGE" == "fullforce") + # $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1 + # end + + if ($addstar_multiadd_limit > 0) + $run = $run --limit $addstar_multiadd_limit + end + + if ("$IMAGE_ONLY" == "T") + $run = $run --image-only + end + + if ("$MINIDVODB" == "T") + $run = $run --minidvodb + $run = $run --minidvodb_group $MINIDVODB_GROUP + if (("$MINIDVODB_NAME" != "NULL") && ("$MINIDVODB_NAME" != "(null)")) + $run = $run --minidvodb_name $MINIDVODB_NAME + #have addstar_run.pl grab the 'active' name if it is NULL + end + end + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit addPendingMultiExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword addPendingMultiExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword addPendingMultiExp $options:0 pantaskState TIMEOUT + end +end + +task addstar.revert.cam + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage cam + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_C + $run = $run -dbname $DB:$addstar_revert_DB_C + $addstar_revert_DB_C ++ + if ($addstar_revert_DB_C >= $DB:n) set addstar_revert_DB_C = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end +task addstar.revert.stack + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage stack + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_S + $run = $run -dbname $DB:$addstar_revert_DB_S + $addstar_revert_DB_S ++ + if ($addstar_revert_DB_S >= $DB:n) set addstar_revert_DB_S = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task addstar.revert.staticsky + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage staticsky + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_SSM + $run = $run -dbname $DB:$addstar_revert_DB_SSM + $addstar_revert_DB_SSM ++ + if ($addstar_revert_DB_SSM >= $DB:n) set addstar_revert_DB_SSM = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task addstar.revert.skycal + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage skycal + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_SC + $run = $run -dbname $DB:$addstar_revert_DB_SC + $addstar_revert_DB_SC ++ + if ($addstar_revert_DB_SC >= $DB:n) set addstar_revert_DB_SC = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +task addstar.revert.diff + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage diff + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_DF + $run = $run -dbname $DB:$addstar_revert_DB_DF + $addstar_revert_DB_DF ++ + if ($addstar_revert_DB_DF >= $DB:n) set addstar_revert_DB_DF = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +task addstar.revert.fullforce + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage fullforce + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_FF + $run = $run -dbname $DB:$addstar_revert_DB_FF + $addstar_revert_DB_FF ++ + if ($addstar_revert_DB_FF >= $DB:n) set addstar_revert_DB_FF = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task addstar.revert.fullforce_summary + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage fullforce_summary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_FFS + $run = $run -dbname $DB:$addstar_revert_DB_FFS + $addstar_revert_DB_FFS ++ + if ($addstar_revert_DB_FFS >= $DB:n) set addstar_revert_DB_FFS = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/autogen.sh =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/autogen.sh (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/autogen.sh (revision 41106) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ippTasks +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +#($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $LIBTOOLIZE installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" +# DIE=1 +#} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +#($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $AUTOHEADER installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" +# DIE=1 +#} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +#$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +#$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/ccl_branches/ipponly-20191108/ippTasks/automate.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/automate.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/automate.pro (revision 41106) @@ -0,0 +1,476 @@ +## automate.pro : autorun the detrend analysis : -*- sh -*- + +# example automation stage +# automate METADATA +# name STR DARK +# check STR "detselect -search -inst SIMTEST -det_type BIAS -dbname eamtest" +# ncheck S32 1 +# launch STR "dettool -definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK -dbname eamtest" +# block STR "dettool -runs -active -det_type DARK -dbname eamtest" +# END + +# an entry in the 'automate' book goes through the following states: +# INIT (book loaded from input file) +# CHECK (stage prereqs being checked) +# READY (prereqs satisfield, ready to be launched) +# LAUNCH (action being performed) +# DONE (action finished) + +macro automate.reset + ## probably should not always init + book init automate +end + +macro automate.load + if ($0 != 4) + echo "USAGE: automate.load (filename) (camera) (dbname)" + break + end + queueload tmp -x "cat $MODULES:0/$1" + + pwd -var cwd + + ## interpolate standard values + queuesubstr tmp @CAMERA@ $2 + queuesubstr tmp @DBNAME@ $3 + queuesubstr tmp @CWD@ $cwd + + $username = `whoami` + queuesubstr tmp @USER@ $username + + ipptool2book tmp automate -key name -uniq -setword pantaskState INIT.BLOCK + queuedelete tmp + + ## Pull out the ones that are to be run regularly + local npages + book npages automate -var npages + for i 0 $npages + book getpage automate $i -var pageName + if ("$pageName" != "NULL") + book getword automate $pageName regular -var regularCommand + if ("$regularCommand" != "NULL") + book setword automate $pageName pantaskState INIT.REGULAR + end + end + end + +end + +macro automate.on + task automate.block + active true + end + task automate.check + active true + end + task automate.launch + active true + end + task automate.regular + active true + end +end + +macro automate.off + task automate.block + active false + end + task automate.check + active false + end + task automate.launch + active false + end + task automate.regular + active false + end +end + +$automate_Nblock = -1 +task automate.block + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.BLOCK + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.BLOCK state" + end + break + end + + # cycle over the number of INIT.BLOCK pages: no point to loop over the others + $automate_Nblock ++ + if ($automate_Nblock >= $Npage) set automate_Nblock = 0 + + # search the automate book for an entry which is unstarted (state INIT.BLOCK) + book getpage automate $automate_Nblock -var pageName -key pantaskState INIT.BLOCK + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Nblock not in INIT.BLOCK state" + end + break + end + + book getword automate $pageName block -var blockCommand + if (("$blockCommand" == "NULL") || ("$blockCommand" == "NONE")) + # if there is no block needed, we can immediate progress to the next stage (INIT.CHECK) + book setword automate $pageName pantaskState INIT.CHECK + if ($VERBOSE >= 2) + echo "$pageName is ready : INIT.CHECK" + end + break + end + + book setword automate $pageName pantaskState RUN.BLOCK + + if ($VERBOSE >= 2) + echo "starting automate block for $pageName" + echo "command $blockCommand" + end + + options $pageName + command $blockCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + # convert 'stdout' to book format + # XXX to use other tests, we'll need to modify this + ipptool2book stdout tmpBlock + if ($VERBOSE > 2) + book listbook tmpBlock + end + + # if the block test returns any valid pages (valid results), the block is set, don't move to check + book npages tmpBlock -var Npage + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "$pageName is not blocked, ready for CHECK" + end + book setword automate $pageName pantaskState INIT.CHECK + else + if ($VERBOSE >= 2) + echo "$pageName is blocked, not ready for CHECK" + end + book setword automate $pageName pantaskState INIT.BLOCK + end + + # drop the detExt book after we've grabbed the state + book delete tmpBlock + end + + # all failures here (what state?) + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +$automate_Ncheck = -1 +task automate.check + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.CHECK + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.CHECK state" + end + break + end + + # cycle over the number of INIT.CHECK pages: no point to loop over the others + $automate_Ncheck ++ + if ($automate_Ncheck >= $Npage) set automate_Ncheck = 0 + + # search the automate book for an entry which is unstarted (state INIT.CHECK) + book getpage automate $automate_Ncheck -var pageName -key pantaskState INIT.CHECK + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Ncheck not in INIT.CHECK state" + end + break + end + + book getword automate $pageName check -var checkCommand + if (("$checkCommand" == "NULL") || ("$checkCommand" == "NONE")) + # if there is no check needed, we can immediate progress to the next stage (INIT.LAUNCH) + book setword automate $pageName pantaskState INIT.LAUNCH + if ($VERBOSE >= 2) + echo "$pageName is ready : INIT.LAUNCH" + end + break + end + + book setword automate $pageName pantaskState RUN.CHECK + + if ($VERBOSE >= 2) + echo "starting automate check for $pageName" + echo "command $checkCommand" + end + + options $pageName + command $checkCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + # convert 'stdout' to book format + # XXX to use other tests, we'll need to modify this + ipptool2book stdout tmpCheck + if ($VERBOSE > 2) + book listbook tmpCheck + end + + book getword automate $pageName ncheck -var Ncheck + if ("$Ncheck" == "NULL") + $Ncheck = 1 + end + book npages tmpCheck -var Npage + if ($Npage < $Ncheck) + if ($VERBOSE >= 2) + echo "$pageName not ready for LAUNCH" + end + book setword automate $pageName pantaskState INIT.CHECK + else + if ($VERBOSE >= 2) + echo "$pageName is ready for LAUNCH" + end + book setword automate $pageName pantaskState INIT.LAUNCH + + if ($Ncheck == 1) + # XXX this is a somewhat hackish way to carry information from the 'check' to the 'launch' + book getword tmpCheck page.000 det_id -var DET_ID + book getword tmpCheck page.000 iteration -var ITERATION + book setword automate $pageName det_id $DET_ID + book setword automate $pageName iteration $ITERATION + end + end + + # drop the detExt book after we've grabbed the state + book delete tmpCheck + end + + # all failures here (what state?) + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +$automate_Nlaunch = 0 +task automate.launch + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.LAUNCH + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.LAUNCH state" + end + break + end + + # cycle over the number of INIT.LAUNCH pages: no point to loop over the others + $automate_Nlaunch ++ + if ($automate_Nlaunch >= $Npage) set automate_Nlaunch = 0 + + # search the automate book for an entry which is unstarted (state INIT.LAUNCH) + book getpage automate $automate_Nlaunch -var pageName -key pantaskState INIT.LAUNCH + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Nlaunch not in INIT.LAUNCH state" + end + break + end + + book getword automate $pageName launch -var launchCommand + if (("$launchCommand" == "NULL") || ("$launchCommand" == "NONE")) + # if there is no launch needed, we can immediate progress to the next stage (INIT.LAUNCH) + book setword automate $pageName pantaskState INIT.LAUNCH + if ($VERBOSE >= 2) + echo "$pageName is ready : INIT.LAUNCH" + end + break + end + + # modify the launch command to replace certain elements from the page + book getword automate $pageName det_id -var DET_ID + if ("$DET_ID" != "NULL") + echo '$launchCommand' + strsub "$launchCommand" @det_id@ $DET_ID -var launchCommand + end + + book getword automate $pageName iteration -var ITERATION + if ("$ITERATION" != "NULL") + echo '$launchCommand' + strsub "$launchCommand" @iteration@ $ITERATION -var launchCommand + end + + echo '$launchCommand' + + book setword automate $pageName pantaskState RUN.LAUNCH + + if ($VERBOSE >= 2) + echo "starting automate launch for $pageName" + echo "command $launchCommand" + end + + options $pageName + command $launchCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + book setword automate $pageName pantaskState DONE.LAUNCH + end + + # all failures here (what state?) + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +$automate_Nregular = 0 +task automate.regular + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + task.exec + local Npage pageName + + # how many pages are waiting to be started? + book npages automate -var Npage -key pantaskState INIT.REGULAR + if ($Npage == 0) + if ($VERBOSE >= 2) + echo "no entries in INIT.REGULAR state" + end + break + end + + # cycle over the number of INIT.REGULAR pages: no point to loop over the others + $automate_Nregular ++ + if ($automate_Nregular >= $Npage) set automate_Nregular = 0 + + # search the automate book for an entry which is unstarted (state INIT.REGULAR) + book getpage automate $automate_Nregular -var pageName -key pantaskState INIT.REGULAR + if ("$pageName" == "NULL") + if ($VERBOSE >= 2) + echo "entry $automate_Nregular not in INIT.REGULAR state" + end + break + end + + book getword automate $pageName regular -var regularCommand + if (("$regularCommand" == "NULL") || ("$regularCommand" == "NONE")) + if ($VERBOSE >= 2) + echo "Warning: $pageName has no regular command" + end + break + end + + book setword automate $pageName pantaskState RUN.REGULAR + + if ($VERBOSE >= 2) + echo "starting automate regular for $pageName" + echo "command $regularCommand" + end + + options $pageName + command $regularCommand + end + + task.exit $EXIT_SUCCESS + local pageName Npage + + $pageName = $options:0 + + book setword automate $pageName pantaskState INIT.REGULAR + end + + # all failures here (what state?) + task.exit default + book setword automate $pageName pantaskState FAIL.REGULAR + showcommand failure + end + + task.exit crash + book setword automate $pageName pantaskState INIT.REGULAR + showcommand crash + end + + # operation times out? + task.exit timeout + book setword automate $pageName pantaskState INIT.REGULAR + showcommand timeout + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/background.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/background.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/background.pro (revision 41106) @@ -0,0 +1,535 @@ +## background.pro : globals and support macros : -*- sh -*- + +check.globals + +book init chipBackgroundRuns +book init warpBackgroundRuns + +macro bg.status + book listbook chipBackgroundRuns + book listbook warpBackgroundRuns +end + +macro bg.reset + book init chipBackgroundRuns + book init warpBackgroundRuns +end + +macro bg.on + task bg.chip.load + active true + end + task bg.chip.run + active true + end + task bg.chip.advance + active true + end + task bg.chip.revert + active true + end + task bg.warp.load + active true + end + task bg.warp.run + active true + end + task bg.warp.advance + active true + end + task bg.warp.revert + active true + end +end + +macro bg.off + task bg.chip.load + active false + end + task bg.chip.run + active false + end + task bg.chip.advance + active false + end + task bg.chip.revert + active false + end + task bg.warp.load + active false + end + task bg.warp.run + active false + end + task bg.warp.advance + active false + end + task bg.warp.revert + active false + end +end + +macro bg.revert.on + task bg.chip.revert + active true + end + task bg.warp.revert + active true + end +end + +macro bg.revert.off + task bg.chip.revert + active false + end + task bg.warp.revert + active false + end +end + + +# These variables cycle through the known database names +$bg_chip_load_DB = 0 +$bg_chip_advance_DB = 0 +$bg_chip_revert_DB = 0 +$bg_warp_load_DB = 0 +$bg_warp_advance_DB = 0 +$bg_warp_revert_DB = 0 + +task bg.chip.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.chip.load.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -tochip + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_chip_load_DB + $run = $run -dbname $DB:$bg_chip_load_DB + $bg_chip_load_DB ++ + if ($bg_chip_load_DB >= $DB:n) set bg_chip_load_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipBackgroundRuns -key chip_bg_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipBackgroundRuns + end + process_cleanup chipBackgroundRuns + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.chip.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + periods -exec $RUNEXEC + + book npages chipBackgroundRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + book getpage chipBackgroundRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipBackgroundRuns $pageName pantaskState RUN + book getword chipBackgroundRuns $pageName camera -var CAMERA + book getword chipBackgroundRuns $pageName chip_bg_id -var CHIP_BG_ID + book getword chipBackgroundRuns $pageName class_id -var CLASS_ID + book getword chipBackgroundRuns $pageName exp_tag -var EXP_TAG + book getword chipBackgroundRuns $pageName workdir -var WORKDIR_TEMPLATE + book getword chipBackgroundRuns $pageName dbname -var DBNAME + book getword chipBackgroundRuns $pageName reduction -var REDUCTION + + set.host.for.camera $CAMERA $CLASS_ID + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf outroot "%s/%s/%s.bgc.%s" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_BG_ID + stdout $LOGDIR/bg.chip.run.log + stderr $LOGDIR/bg.chip.run.log + + $run = background_chip.pl --threads @MAX_THREADS@ --chip_bg_id $CHIP_BG_ID --class_id $CLASS_ID --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit chipBackgroundRuns $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + exec bgtool -addchip -dbname $DBNAME -chip_bg_id $CHIP_BG_ID -class_id $CLASS_ID -fault $EXIT_CRASH_ERR + process_exit chipBackgroundRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipBackgroundRuns $options:0 pantaskState TIMEOUT + end +end + +# advance exposures for which all imfiles have completed processing +# sets the exposure state to full and queues warp processing if requested +task bg.chip.advance + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.chip.advance.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -advancechip + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_chip_advance_DB + $run = $run -dbname $DB:$bg_chip_advance_DB + $bg_chip_advance_DB ++ + if ($bg_chip_advance_DB >= $DB:n) set bg_chip_advance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.chip.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.chip.revert.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -revertchip + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_chip_revert_DB + $run = $run -dbname $DB:$bg_chip_revert_DB + $bg_chip_revert_DB ++ + if ($bg_chip_revert_DB >= $DB:n) set bg_chip_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.warp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.warp.load.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -towarp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_warp_load_DB + $run = $run -dbname $DB:$bg_warp_load_DB + $bg_warp_load_DB ++ + if ($bg_warp_load_DB >= $DB:n) set bg_warp_load_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpBackgroundRuns -key warp_bg_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpBackgroundRuns + end + process_cleanup warpBackgroundRuns + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.warp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + periods -exec $RUNEXEC + + book npages warpBackgroundRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + book getpage warpBackgroundRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpBackgroundRuns $pageName pantaskState RUN + book getword warpBackgroundRuns $pageName camera -var CAMERA + book getword warpBackgroundRuns $pageName warp_bg_id -var WARP_BG_ID + book getword warpBackgroundRuns $pageName skycell_id -var SKYCELL_ID + book getword warpBackgroundRuns $pageName tess_id -var TESS_DIR + book getword warpBackgroundRuns $pageName exp_tag -var EXP_TAG + book getword warpBackgroundRuns $pageName workdir -var WORKDIR_TEMPLATE + book getword warpBackgroundRuns $pageName dbname -var DBNAME + book getword warpBackgroundRuns $pageName reduction -var REDUCTION + + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf outroot "%s/%s/%s.bgw.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_BG_ID $SKYCELL_ID + stdout $LOGDIR/bg.warp.run.log + stderr $LOGDIR/bg.warp.run.log + + $run = background_warp.pl --threads @MAX_THREADS@ --warp_bg_id $WARP_BG_ID --skycell_id $SKYCELL_ID --tess_dir $TESS_DIR --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit warpBackgroundRuns $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + exec bgtool -addwarp -dbname $DBNAME -warp_bg_id $WARP_BG_ID -skycell_id $SKYCELL_ID -fault $EXIT_CRASH_ERR + process_exit warpBackgroundRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpBackgroundRuns $options:0 pantaskState TIMEOUT + end +end + +# advance exposures for which all imfiles have completed processing +# sets the exposure state to full and queues warp processing if requested +task bg.warp.advance + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.warp.advance.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -advancewarp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_warp_advance_DB + $run = $run -dbname $DB:$bg_warp_advance_DB + $bg_warp_advance_DB ++ + if ($bg_warp_advance_DB >= $DB:n) set bg_warp_advance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.warp.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.warp.revert.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -revertwarp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_warp_revert_DB + $run = $run -dbname $DB:$bg_warp_revert_DB + $bg_warp_revert_DB ++ + if ($bg_warp_revert_DB >= $DB:n) set bg_warp_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/bg.regeneration.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/bg.regeneration.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/bg.regeneration.pro (revision 41106) @@ -0,0 +1,306 @@ +## bg.regeneration.pro : tasks for regenerating the background model for warps and stacks : -*- sh -*- + +check.globals + +if ($?POLL_LIMIT_BGREG == 0) set POLL_LIMIT_BGREG = 40 + +macro set.bgreg.poll + if ($0 != 2) + echo "USAGE:set.bgreg.poll (value)" + break; + end + + $POLL_LIMIT_BGREG = $1 +end +macro get.bgreg.poll + echo $POLL_LIMIT_BGREG +end + +## Initialize the books for the tasks to do +book init bgRegWarp +book init bgRegStack + +## Database lists +$bgRegWarp_DB = 0 +$bgRegStack_DB = 0 + +macro bgreg.warp.status + book listbook bgRegWarp +end +macro bgreg.stack.status + book listbook bgRegStack +end + +## Reset tasks +macro bgreg.reset + book init bgRegWarp + book init bgRegStack +end + +## Turn tasks on +macro bgreg.on + task bgreg.warp.load + active true + end + task bgreg.warp.run + active true + end + task bgreg.stack.load + active true + end + task bgreg.stack.run + active true + end +end +## or off +macro bgreg.off + task bgreg.warp.load + active false + end + task bgreg.warp.run + active false + end + task bgreg.stack.load + active false + end + task bgreg.stack.run + active false + end +end + +# Load task for warp +task bgreg.warp.load + host local + + periods -poll $LOADPOLL + periods -poll $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/bgreg.warp.load + + task.exec + if ($LABEL:n == 0) break + $run = warptool -warped -background_model 0 + if ($DB:n == 0) + option DEFAULT + else + #save the DB naem for the exit tasks + option $DB:$bgRegWarp_DB + $run = $run -dbname $DB:$bgRegWarp_DB + $bgRegWarp_DB ++ + if ($bgRegWarp_DB >= $DB:n) set bgRegWarp_DB = 0 + end + add_poll_args run + add_poll_labels run + # change the limit? + $run = $run -limit $POLL_LIMIT_BGREG + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout bgRegWarp -key warp_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook bgRegWarp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup bgRegWarp + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bgreg.warp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we can't exec, set the retry time long + periods -exec $RUNEXEC + + book npages bgRegWarp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new pages + book getpage bgRegWarp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword bgRegWarp $pageName pantaskState RUN + book getword bgRegWarp $pageName warp_id -var STAGE_ID + book getword bgRegWarp $pageName skycell_id -var SKYCELL_ID + book getword bgRegWarp $pageName camera -var CAMERA + book getword bgRegWarp $pageName dbname -var DBNAME + + set.host.for.skycell $SKYCELL_ID + + stdout $LOGDIR/bgreg.warp.log + stderr $LOGDIR/bgreg.warp.log + + $run = regenerate_background.pl --stage warp --stage_id $STAGE_ID --skycell_id $SKYCELL_ID --camera $CAMERA + + add_standard_args run + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit bgRegWarp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword bgRegWarp $options:0 pantasksState CRASH + end + + # operation timed out + task.exit timeout + showcommand timeout + book setword bgRegWarp $options:0 pantaskState TIMEOUT + end +end + +# Load task for stack +task bgreg.stack.load + host local + + periods -poll $LOADPOLL + periods -poll $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/bgreg.stack.load + + task.exec + if ($LABEL:n == 0) break + $run = stacktool -tobkg + if ($DB:n == 0) + option DEFAULT + else + #save the DB naem for the exit tasks + option $DB:$bgRegStack_DB + $run = $run -dbname $DB:$bgRegStack_DB + $bgRegStack_DB ++ + if ($bgRegStack_DB >= $DB:n) set bgRegStack_DB = 0 + end + add_poll_args run + add_poll_labels run + # change the limit? + $run = $run -limit $POLL_LIMIT_BGREG + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout bgRegStack -key stack_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook bgRegStack + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup bgRegStack + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bgreg.stack.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we can't exec, set the retry time long + periods -exec $RUNEXEC + + book npages bgRegStack -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new pages + book getpage bgRegStack 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword bgRegStack $pageName pantaskState RUN + book getword bgRegStack $pageName stack_id -var STAGE_ID + book getword bgRegStack $pageName skycell_id -var SKYCELL_ID + book getword bgRegStack $pageName camera -var CAMERA + book getword bgRegStack $pageName dbname -var DBNAME + set.host.for.skycell $SKYCELL_ID + + stdout $LOGDIR/bgreg.stack.log + stderr $LOGDIR/bgreg.stack.log + + $run = regenerate_background.pl --stage stack --stage_id $STAGE_ID --skycell_id $SKYCELL_ID --camera $CAMERA + + add_standard_args run + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit bgRegStack $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword bgRegStack $options:0 pantasksState CRASH + end + + # operation timed out + task.exit timeout + showcommand timeout + book setword bgRegStack $options:0 pantaskState TIMEOUT + end +end + + + + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/burntool.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/burntool.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/burntool.pro (revision 41106) @@ -0,0 +1,572 @@ +## burntool.pro : -*- sh -*- +## This task is designed to only be used when re-burntooling data. It is not necessary for standard registration. +check.globals + +macro burntool.it.on +# bt.initday.on + bt.registration.on + bt.burntool.on + bt.chips.on +end + +macro burntool.it.off +# bt.initday.off + bt.registration.off + bt.burntool.off + bt.chips.off +end + +macro bt.on + burntool.it.on +end + +macro bt.off + burntool.it.off +end + +macro bt.initday.on + task bt.initday.load + active true + end +end + +macro bt.registration.on + task bt.registration.load + active true + end +end + +macro bt.burntool.on + task bt.burntool.load + active true + end + task bt.burntool.run + active true + end +end + +macro bt.chips.on + task bt.chips.load + active true + end +end + +macro bt.initday.off + task bt.initday.load + active false + end +end + +macro bt.registration.off + task bt.registration.load + active false + end +end + +macro bt.burntool.off + task bt.burntool.load + active false + end + task bt.burntool.run + active false + end +end + +macro bt.chips.off + task bt.chips.load + active false + end +end + +$bt_regPAGE = 0 +$bt_burnPAGE = 0 +$bt_RburnPAGE = 0 +$bt_RburnCELL = 0 +$bt_chipPAGE = 0 +$bt_RchipPAGE = 0 + +book init btData +book init btBurntool +# +# Macros to control the book. +# +macro bt.add.date + book newpage btData $1 + book setword btData $1 nsState NEW +end + +macro bt.show.dates + book npages btData -var Npages + for i 0 $Npages + book getpage btData $i -var date + book getword btData $date nsState -var state + echo $date $state + end + book npages btBurntool -var Npages +end + +macro bt.del.date + book delpage btData $1 +end + +macro bt.set.date + book setword btData $1 nsState $2 +end + + + +# +# Start a new date to work on +# +task bt.initday.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + active false + npending 1 + + task.exec + stdout $LOGDIR/bt.initday.log + stderr $LOGDIR/bt.initday.log + $today = `date +%Y-%m-%d` + book newpage btData $today + book setword btData $today nsState NEW + + command echo $today + end + + task.exit 0 + # nothing to do here? + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if registration is complete +# +task bt.registration.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + active false + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/bt.registration.log + + book getpage btData $bt_regPAGE -var date + book getword btData $date nsState -var bt_STATE + book npages btData -var Npages + + if ($VERBOSE > 5) + echo "bt.registration.load: " $bt_regPAGE $date $bt_STATE $Npages + end + + $bt_regPAGE ++ + if ($bt_regPAGE >= $Npages) set bt_regPAGE = 0 + option $date + + if ("$bt_STATE" != "NEW") break + $run = automate_stacks.pl --check_registration --date $date + command $run + end + + # success + task.exit 0 +# convert 'stdout' to book format + book delpage btData $options:0 + ipptool2book stdout btData -uniq -key date + + if ($VERBOSE > 2) + book listbook btData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the chips have been properly burntooled and are ready for queueing +# +task bt.chips.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + active false + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/bt.chips.log + + book getpage btData $bt_chipPAGE -var date + book getword btData $date nsState -var bt_STATE + book npages btData -var Npages + + if ($VERBOSE > 5) + echo "bt.chips.load: " $bt_chipPAGE $date $bt_STATE $Npages + end + + $bt_chipPAGE ++ + if ($bt_chipPAGE >= $Npages) set bt_chipPAGE = 0 + option $date + + if (("$bt_STATE" != "REGISTERED")&&("$bt_STATE" != "BURNING")) break + $run = automate_stacks.pl --check_chips --date $date + + if ("$bt_STATE" == "BURNING") + $run = $run --isburning + end + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage btData $options:0 + ipptool2book stdout btData -uniq -key date + + # remove the burntool page if we're done with it. + book getword btData $options:0 nsState -var bt_STATE + if ("$bt_STATE" == "QUEUECHIPS") + book delpage btBurntool $options:0 + book delpage btData $options:0 + end + + if ($VERBOSE > 2) + book listbook btData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Generate a list of date ranges and chips that need to be processed with burntool +# +task bt.burntool.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + active false + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/bt.burntool.log + + book getpage btData $bt_burnPAGE -var date + book getword btData $date nsState -var bt_STATE + book npages btData -var Npages + + if ($VERBOSE > 5) + echo "bt.burntool.load: " $bt_burnPAGE $date $bt_STATE $Npages + end + + $bt_burnPAGE ++ + if ($bt_burnPAGE >= $Npages) set bt_burnPAGE = 0 + option $date + + if ("$bt_STATE" != "NEEDSBURNING") break + + $run = automate_stacks.pl --define_burntool --date $date + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format +# book delpage btBurntool $options:0 + ipptool2book stdout btBurntool -uniq -key date + book setword btData $options:0 nsState "QUEUEBURNING" + + if ($VERBOSE > 2) + book listbook btData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Magically run burntool on the data, based on the information stored in our book. +# +task bt.burntool.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + active false + npending 30 + task.exec + periods -exec $RUNEXEC + stdout NULL + stderr $LOGDIR/bt.burntool.log + + book getpage btData $bt_RburnPAGE -var date + book getword btData $date nsState -var bt_STATE + book npages btData -var Npages + + if ($VERBOSE > 5) + echo "bt.burntool.run: " $bt_RburnPAGE $date $bt_STATE $Npages + end + + $bt_RburnPAGE ++ + if ($bt_RburnPAGE >= $Npages) set bt_RburnPAGE = 0 + if (("$bt_STATE" != "QUEUEBURNING")&&("$bt_STATE" != "BURNING")) break + $BURNTOOLING = 1 + # Find out where in the list of jobs we are + book getword btBurntool $date btN -var btN + book getword btBurntool $date btNCounter -var btNcounter + + if ($VERBOSE > 5) + echo "bt.burntool.run: Status: " $btNcounter $date + end + + if ("$bt_STATE" == "QUEUEBURNING") + $new_state = "QUEUEBURNING" + end + if ("$bt_STATE" == "BURNING") + $new_state = "BURNING" + end + if ($btNcounter + 1 > $btN) + $new_state = "BURNING" + end + if ($btNcounter > $btN) + $btNcounter = 0 + end + if ($VERBOSE > 5) + echo "bt.burntool.run: Status: " $btNcounter $new_state + end + + # Increment the counter in the book for the next job. + $counter_update = $btNcounter + 1 + book setword btBurntool $date btNCounter $counter_update +# book setword btData $options:0 nsState $options:1 + book setword btData $date nsState $new_state + + # Get the current status of this job, and skip if it doesn't need to process. + sprintf status_label "bt%dStatus" $btNcounter + book getword btBurntool $date $status_label -var status + + if ($VERBOSE > 5) + echo "bt.burntool.run: Status: " $btNcounter $status $date $status_label + end + if (("$status" == "FINISHED")||("$status" == "RUN")) break + + # Continue loading information to process this job + sprintf start_date_label "bt%dBegin" $btNcounter + sprintf end_date_label "bt%dEnd" $btNcounter + sprintf class_count_label "bt%dClass" $btNcounter + + book getword btBurntool $date $start_date_label -var start_date + book getword btBurntool $date $end_date_label -var end_date + book getword btBurntool $date $class_count_label -var chip_counter + + # Lookup class_id/host pairs + list word -split $hostmatch:$chip_counter + $class_id = $word:0 + $host = $word:1 + host $host +# set.host.for.camera GPC1 $class_id + $logfile = "/data/$host.0/burntool_logs/$class_id.$start_date.log" + + $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile + + echo "bt.burntool.run: " $date $btN $btNcounter $start_date $end_date $chip_counter $class_id $host $logfile $run + echo "bt.burntool.run: " $date $btN $btNcounter $chip_counter $new_state + + book setword btBurntool $date $status_label RUN + option $date $new_state $status_label + + command $run + periods -exec 0.05 +# command /bin/true + end + # success + task.exit 0 +# convert 'stdout' to book format + # Set data state based on if we're queueing or waiting + # Set the job state for success. + book setword btBurntool $options:0 $options:2 FINISHED + if ($VERBOSE > 2) + book listbook btData + end + end + + # locked list + task.exit default + book setword btBurntool $options:0 $options:2 FAIL + showcommand failure + end + task.exit crash + book setword btBurntool $options:0 $options:2 FAIL + showcommand crash + end + #operation times out? + task.exit timeout + book setword btBurntool $options:0 $option:2 FAIL + showcommand timeout + end +end + + +# +# This is all burntool related stuff. +# + +macro burntool + if ($0 != 3) + echo "USAGE: burntool (dateobs_begin) (dateobs_end)" + break + end + + for i 0 $hostmatch:n + job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile + end +end +macro loadhosts + for i 0 $allhosts:n + controller host add $allhosts:$i + end +end + +# sorry this list is messy, it's supposed to be that way to try and keep to burntools from running on the same host at the same time. + +list hostmatch + XY01 ipp014 + XY03 ipp038 + XY05 ipp023 + XY10 ipp039 + XY12 ipp024 + XY15 ipp040 + XY17 ipp020 + XY21 ipp041 + XY23 ipp042 + XY25 ipp043 + XY27 ipp028 + XY31 ipp044 + XY33 ipp029 + XY35 ipp045 + XY37 ipp030 + XY41 ipp046 + XY43 ipp031 + XY45 ipp047 + XY47 ipp032 + XY51 ipp048 + XY53 ipp033 + XY55 ipp049 + XY57 ipp034 + XY61 ipp050 + XY63 ipp035 + XY65 ipp051 + XY67 ipp036 + XY72 ipp052 + XY74 ipp015 + XY76 ipp053 + XY02 ipp014 + XY04 ipp038 + XY06 ipp023 + XY11 ipp039 + XY13 ipp024 + XY14 ipp040 + XY16 ipp020 + XY20 ipp041 + XY22 ipp042 + XY24 ipp043 + XY26 ipp028 + XY30 ipp044 + XY32 ipp029 + XY34 ipp045 + XY36 ipp030 + XY40 ipp046 + XY42 ipp031 + XY44 ipp047 + XY46 ipp032 + XY50 ipp048 + XY52 ipp033 + XY54 ipp049 + XY56 ipp034 + XY60 ipp050 + XY62 ipp035 + XY64 ipp051 + XY66 ipp036 + XY71 ipp052 + XY73 ipp015 + XY75 ipp053 +end + +list allhosts + ipp043 + ipp014 + ipp015 + ipp023 + ipp024 + ipp020 + ipp028 + ipp029 + ipp030 + ipp031 + ipp032 + ipp033 + ipp034 + ipp035 + ipp036 + ipp038 + ipp039 + ipp040 + ipp041 + ipp042 + ipp044 + ipp045 + ipp046 + ipp047 + ipp048 + ipp049 + ipp050 + ipp051 + ipp052 + ipp053 +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/calibration.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/calibration.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/calibration.pro (revision 41106) @@ -0,0 +1,182 @@ +## calibration.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the calibration stage +## these tasks use the book 'calBook' + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/calibration +mkdir $LOGSUBDIR + +book init calBook + +macro calibration.reset + book init calBook +end + +macro calibration.status + book listbook calBook +end + +macro calibration.on + task calibration.init + active true + end +end + +macro calibration.off + task calibration.init + active false + end +end + +# these variables will cycle through the known ippdb database names +$calInit_DB = 0 + +# we have three steps here: +# 1) get the list of dvo databases: caltool -dbs +# 2) define a new calibration run for each of the dvo dbs +# caltool + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task calibration.load + host local + + # check the list of available dvo dbs regularly + periods -poll 10 + periods -exec 900 + periods -timeout 60 + + # this is a strange construction; what is the purpose? + if ($DB:n == 0) + npending $DB:n + else + npending 1 + end + + # define the command (does not depend on previous queries) + if ($DB:n != 0) + command caltool -dbs -active true + else + # save the DB name for the exit tasks + # note that this DB name refers to the ippdb, not the dvodb + option $DB:$calInit_DB + command caltool -dbs -dbname $DB:$calInit_DB + $calInit_DB ++ + if ($calInit_DB >= $DB:n) set calInit_DB = 0 + end + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + # XXX have ippTools report the dbname? + ipptool2book stdout calBook -key dbname:cal_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook calBook + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup calBook + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - one pass for each db +# can we include information that the db has been updated without a recent cal analysis? +task calibration.resort + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 60 + # trange Hourly@00:00 Hourly@00:10 + # every hour on the hour + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + task.exec + book npages calBook -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in calBook + # the sequencing in this task set is by the pantasksState (see notes.txt) + book getpage calBook 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # get the current ST: + $RAs = 15*($ST - 1) + $RAe = 15*($ST - 2) + $DECs = -90.0 + $DECe = +90.0 + + $REGION = "$RAs,$RAe:$DECs,$DECe" + + book setword calBook $pageName pantaskState RUN + book setword calBook $pageName region $REGION + + # XXX probably need to set the output / log based on WORKDIR... + book getword calBook $pageName cal_id -var ID + book getword calBook $pageName dvodb -var DVODB + book getword calBook $pageName dbname -var DBNAME + + # specify choice of remote host + # set a specific DVO host here + if ($PARALLEL) + host anyhost + else + host local + end + + # save the pageName for future reference below + options $pageName + + # XXX do we modify relphot to loop over the filters? + $run = calibrate_dvo.pl --cal_id $ID --dvodb $DVODB --region $REGION + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit calBook $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword calBook $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword calBook $options:0 pantaskState TIMEOUT + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/camera.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/camera.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/camera.pro (revision 41106) @@ -0,0 +1,268 @@ +## camera.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the camera analysis stage +## these tasks use the book camPendingExp + +# test for required global variables +check.globals + +# camera.pro should have a more restricted polling limit (to avoid stress with getstar) +if ($?POLL_LIMIT_CAMERA == 0) set POLL_LIMIT_CAMERA = 10 + +macro set.camera.poll + if ($0 != 2) + echo "USAGE:set.camera.poll (value)" + break + end + + $POLL_LIMIT_CAMERA = $1 +end + +macro get.camera.poll + echo "camera poll limit: $POLL_LIMIT_CAMERA" +end + +book init camPendingExp + +macro camera.status + book listbook camPendingExp +end + +macro camera.reset + book init camPendingExp +end + +macro camera.on + task camera.exp.load + active true + end + task camera.exp.run + active true + end + task camera.revert + active true + end +end + +macro camera.off + task camera.exp.load + active false + end + task camera.exp.run + active false + end + task camera.revert + active false + end +end + +macro camera.revert.on + task camera.revert + active true + end +end + +macro camera.revert.off + task camera.revert + active false + end +end + +# this variable will cycle through the known database names +$camera_DB = 0 +$camera_revert_DB = 0 + +# select images ready for camera analysis +# new entries are added to camPendingExp +# skip already-present entries +task camera.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/camera.exp.log + + task.exec + if ($LABEL:n == 0) break + $run = camtool -pendingexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$camera_DB + $run = $run -dbname $DB:$camera_DB + $camera_DB ++ + if ($camera_DB >= $DB:n) set camera_DB = 0 + end + $run = $run -limit $POLL_LIMIT_CAMERA + # NOTE: we do not want to overload the dvo db machine with getstar queries, + # so we will limit the camera stage to a smaller number than the other stages + # add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout camPendingExp -key cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook camPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup camPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the cameraexposure script on pending images +task camera.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10 + + task.exec + book npages camPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new images in camPendingExp (pantaskState == INIT) + book getpage camPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword camPendingExp $pageName pantaskState RUN + book getword camPendingExp $pageName camera -var CAMERA + book getword camPendingExp $pageName exp_tag -var EXP_TAG + book getword camPendingExp $pageName cam_id -var CAM_ID + book getword camPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword camPendingExp $pageName path_base -var PATH_BASE + book getword camPendingExp $pageName dvodb -var DVODB + book getword camPendingExp $pageName dbname -var DBNAME + book getword camPendingExp $pageName reduction -var REDUCTION + book getword camPendingExp $pageName state -var RUN_STATE + + # specify choice of remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + if ("$PATH_BASE" == "NULL") + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID + else + $outroot = $PATH_BASE + end + + stdout $LOGDIR/camera.exp.log + stderr $LOGDIR/camera.exp.log + + $run = camera_exp.pl --exp_tag $EXP_TAG --cam_id $CAM_ID --camera $CAMERA --outroot $outroot --redirect-output --run-state $RUN_STATE + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$DVODB" != "NULL") + $run = $run --dvodb $DVODB + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit camPendingExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword camPendingExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword camPendingExp $options:0 pantaskState TIMEOUT + end +end + +task camera.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = camtool -revertprocessedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$camera_revert_DB + $run = $run -dbname $DB:$camera_revert_DB + $camera_revert_DB ++ + if ($camera_revert_DB >= $DB:n) set camera_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/chip.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/chip.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/chip.pro (revision 41106) @@ -0,0 +1,330 @@ +## chip.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the phase 0 stage +## these tasks use the book chipPendingImfile + +# test for required global variables +check.globals + +book init chipPendingImfile + +macro chip.status + book listbook chipPendingImfile +end + +macro chip.reset + book init chipPendingImfile +end + +macro chip.on + task chip.imfile.load + active true + end + task chip.imfile.run + active true + end + task chip.advanceexp + active true + end + task chip.revert + active true + end +end + +macro chip.off + task chip.imfile.load + active false + end + task chip.imfile.run + active false + end + task chip.advanceexp + active false + end + task chip.revert + active false + end +end + +macro chip.revert.on + task chip.revert + active true + end +end + +macro chip.revert.off + task chip.revert + active false + end +end + +# this variable will cycle through the known database names +$chip_DB = 0 +$chip_revert_DB = 0 + +# select images ready for chip analysis +# new entries are added to chipPendingImfile +# skip already-present entries +task chip.imfile.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/chip.imfile.log + + task.exec + if ($LABEL:n == 0) break + $run = chiptool -pendingimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$chip_DB + $run = $run -dbname $DB:$chip_DB + $chip_DB ++ + if ($chip_DB >= $DB:n) set chip_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipPendingImfile -key chip_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup chipPendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the chip_imfile.pl script on pending images +task chip.imfile.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages chipPendingImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new images in chipPendingImfile (pantaskState == INIT) + book getpage chipPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipPendingImfile $pageName pantaskState RUN + book getword chipPendingImfile $pageName camera -var CAMERA + book getword chipPendingImfile $pageName exp_id -var EXP_ID + book getword chipPendingImfile $pageName exp_tag -var EXP_TAG + book getword chipPendingImfile $pageName raw_magicked -var RAW_MAGICKED + book getword chipPendingImfile $pageName deburned -var DEBURNED + book getword chipPendingImfile $pageName chip_id -var CHIP_ID + book getword chipPendingImfile $pageName chip_imfile_id -var CHIP_IMFILE_ID + book getword chipPendingImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword chipPendingImfile $pageName class_id -var CLASS_ID + book getword chipPendingImfile $pageName uri -var URI + book getword chipPendingImfile $pageName dbname -var DBNAME + book getword chipPendingImfile $pageName reduction -var REDUCTION + book getword chipPendingImfile $pageName state -var RUN_STATE + book getword chipPendingImfile $pageName path_base -var PATH_BASE + book getword chipPendingImfile $pageName update_mode -var UPDATE_MODE + + if ($RAW_MAGICKED > 0) + $MAGICKED_ARG = "--magicked $RAW_MAGICKED" + else + $MAGICKED_ARG = "" + end + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + if ("$PATH_BASE" == "NULL") + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.ch.%s" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID + else + $outroot = $PATH_BASE + end + + stdout $LOGDIR/chip.imfile.log + stderr $LOGDIR/chip.imfile.log + + $run = chip_imfile.pl --threads @MAX_THREADS@ --exp_id $EXP_ID --chip_id $CHIP_ID --chip_imfile_id $CHIP_IMFILE_ID --class_id $CLASS_ID --uri $URI --camera $CAMERA --run-state $RUN_STATE $MAGICKED_ARG --deburned $DEBURNED --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ($UPDATE_MODE) + $run = $run --update-mode $UPDATE_MODE + end + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit chipPendingImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + ### Getting a lot of chip crashes (no idea why), so remove verbosity for now + #showcommand crash + #echo "hostname: $JOB_HOSTNAME" + + # Set a fault code in the database + exec chiptool -addprocessedimfile -dbname $DBNAME -chip_id $CHIP_ID -class_id $CLASS_ID -fault $EXIT_CRASH_ERR + process_exit chipPendingImfile $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipPendingImfile $options:0 pantaskState TIMEOUT + end +end + +# this variable will cycle through the known database names +$chip_advance_DB = 0 + +# advance exposures for which all imfiles have completed processing +# sets the exposure state to full and queues warp processing if requested +task chip.advanceexp + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/chip.advanceexp.log + + task.exec + if ($LABEL:n == 0) break + $run = chiptool -advanceexp -limit 10 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$chip_advance_DB + $run = $run -dbname $DB:$chip_advance_DB + $chip_advance_DB ++ + if ($chip_advance_DB >= $DB:n) set chip_advance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task chip.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = chiptool -revertprocessedimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$chip_revert_DB + $run = $run -dbname $DB:$chip_revert_DB + $chip_revert_DB ++ + if ($chip_revert_DB >= $DB:n) set chip_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/configure.ac =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/configure.ac (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/configure.ac (revision 41106) @@ -0,0 +1,14 @@ +AC_PREREQ(2.59) + +AC_INIT([ippTasks], [1.1.0], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([pantasks.pro]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_MAINTAINER_MODE + +AC_PROG_INSTALL + +AC_CONFIG_FILES([ + Makefile +]) +AC_OUTPUT Index: /branches/ccl_branches/ipponly-20191108/ippTasks/destreak.cleanup.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/destreak.cleanup.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/destreak.cleanup.pro (revision 41106) @@ -0,0 +1,170 @@ +## destreak.cleanup.pro : tasks for the cleaning up destreak stage : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/destreak +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicDSToCleanup + +### indexes into Database lists +$magicDSToCleanup_DB = 0 + +### Check status of tasks +macro destreak.cleanup.status + book listbook magicDSToCleanup +end + +### Reset tasks +macro destreak.cleanup.reset + book init magicDSToCleanup +end + +### Turn tasks on +macro destreak.cleanup.on + task destreak.cleanup.load + active true + end + task destreak.cleanup.run + active true + end +end + +### Turn tasks off +macro destreak.cleanup.off + task destreak.cleanup.load + active false + end + task destreak.cleanup.run + active false + end +end + +task destreak.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 20 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/destreak.cleanup.load.log + + task.exec + $run = magicdstool -tocleanup + if ($DB:n == 0) + option DEFAULT + else + + # save the DB name for the exit tasks + option $DB:$magicDSToCleanup_DB + $run = $run -dbname $DB:$magicDSToCleanup_DB + + # only bump database number after we have gone through all of the stages + $magicDSToCleanup_DB ++ + if ($magicDSToCleanup_DB >= $DB:n) set magicDSToCleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicDSToCleanup -key magic_ds_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicDSToCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicDSToCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + stdout $LOGSUBDIR/destreak.cleanup.run.log + stderr $LOGSUBDIR/destreak.cleanup.run.log + + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages magicDSToCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicDSToCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicDSToCleanup $pageName pantaskState RUN + book getword magicDSToCleanup $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicDSToCleanup $pageName camera -var CAMERA + book getword magicDSToCleanup $pageName stage -var STAGE + book getword magicDSToCleanup $pageName outroot -var OUTROOT + book getword magicDSToCleanup $pageName dbname -var DBNAME + + sprintf logfile "%s/mds.%s.cleanup.log" $OUTROOT $MAGIC_DS_ID + + host anyhost + + $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --stage $STAGE --camera $CAMERA --logfile $logfile + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are ready to run, drop the retry timeout low so we fill up the queue + periods -exec 0.05 + command $run + end + + # default exit status + task.exit 0 + process_exit magicDSToCleanup $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicDSToCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicDSToCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicDSToCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/destreak.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/destreak.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/destreak.pro (revision 41106) @@ -0,0 +1,527 @@ +## destreak.pro : support for the streak removal : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/destreak +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicToDS +book init magicDSToRevert + +### indexes into Database lists +$magicToDS_DB = 0 +$magicDSAdvance_DB = 0 +$magicDSToRevert_DB = 0 +$magicDSCompletedRevert_DB = 0 +$magicDSToCleanup_DB = 0 + +#list of stages +$DS_STAGE:n = 0 +list DS_STAGE -add "raw" +list DS_STAGE -add "chip" +list DS_STAGE -add "camera" +list DS_STAGE -add "warp" +list DS_STAGE -add "diff" + +$magicDSStage = 0 +$magicDSRevertStage = 0 + +macro destreak.show.stages + echo $DS_STAGE:n stages enabled for destreak processing + for i 0 $DS_STAGE:n + echo $i $DS_STAGE:$i + end + echo current magicDSStage = $magicDSStage + echo current magicDSRevertStage = $magicDSRevertStage +end + +### Check status of tasks +macro destreak.status + book listbook magicToDS + book listbook magicDSToRevert +end + +### Reset tasks +macro destreak.reset + book init magicToDS + book init magicDSToRevert +end + +### Turn tasks on +macro destreak.on + task destreak.load + active true + end + task destreak.run + active true + end + task destreak.advance + active true + end +end + +macro destreak.revert.on + task destreak.revert.load + active true + end + task destreak.revert.run + active true + end + task destreak.completed.revert + active true + end +end + +### Turn tasks off +macro destreak.off + task destreak.load + active false + end + task destreak.run + active false + end + task destreak.advance + active false + end +end + +macro destreak.revert.off + task destreak.revert.load + active false + end + task destreak.revert.run + active false + end + task destreak.completed.revert + active false + end +end + +task destreak.load + host local + + periods -poll $LOADPOLL + # this query can take a long time (XXX: the long time should be fixed now) + periods -exec 10 + periods -timeout 120 + npending 1 + +# stdout NULL +# stderr $LOGSUBDIR/destreak.load.log + + task.exec + # we check twice in case an entry has been removed from DS_STAGE + if ($magicDSStage >= $DS_STAGE:n) set magicDSStage = 0 + + $run = magicdstool -todestreak -stage $DS_STAGE:$magicDSStage + $magicDSStage ++ + if ($magicDSStage >= $DS_STAGE:n) set magicDSStage = 0 + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicToDS_DB + $run = $run -dbname $DB:$magicToDS_DB + + # only bump database number after we have gone through all of the stages + if ($magicDSStage == 0) + $magicToDS_DB ++ + if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 + end + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToDS + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToDS + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + stdout $LOGSUBDIR/destreak.run.log + stderr $LOGSUBDIR/destreak.run.log + + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + if ($NETWORK == 0) break + book npages magicToDS -var N + if ($N == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicToDS 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicToDS $pageName pantaskState RUN + book getword magicToDS $pageName exp_id -var EXP_ID + book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicToDS $pageName state -var RUN_STATE + book getword magicToDS $pageName camera -var CAMERA + book getword magicToDS $pageName streaks_uri -var STREAKS + book getword magicToDS $pageName streaks_path_base -var STREAKS_PATH_BASE + book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS + book getword magicToDS $pageName inv_streaks_path_base -var INV_STREAKS_PATH_BASE + book getword magicToDS $pageName stage -var STAGE + book getword magicToDS $pageName stage_id -var STAGE_ID + book getword magicToDS $pageName component -var COMPONENT + book getword magicToDS $pageName uri -var URI + book getword magicToDS $pageName path_base -var PATH_BASE + book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE + book getword magicToDS $pageName cam_reduction -var CAM_REDUCTION + book getword magicToDS $pageName outroot -var OUTROOT + book getword magicToDS $pageName recoveryroot -var RECROOT + book getword magicToDS $pageName re_place -var REPLACE + book getword magicToDS $pageName magicked -var MAGICKED + book getword magicToDS $pageName dbname -var DBNAME + book getword magicToDS $pageName diff_tess_id -var DIFF_TESS_ID + book getword magicToDS $pageName mismatched_tess -var MISMATCHED_TESS + + substr $COMPONENT 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + set.host.for.skycell $COMPONENT + set.workdir.by.skycell $COMPONENT $OUTROOT $default_host WORKDIR + else + # assume component is a class_id, if not we will default to anyhost + set.host.for.camera $CAMERA $COMPONENT + set.workdir.by.camera $CAMERA $COMPONENT $OUTROOT $default_host WORKDIR + end + + sprintf logfile "%s/%s.mds.%s.%s.%s.log" $WORKDIR $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT + + # TODO: do not add recoveryroot or replace if they are null or zero + + $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $WORKDIR --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED --run-state $RUN_STATE + + if ($MISMATCHED_TESS) + book getword magicToDS $pageName diff_tess_id -var DIFF_TESS_ID + $run = $run --mismatched_tess --diff_tess_id $DIFF_TESS_ID + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + + # if we are ready to run, drop the retry timeout low so we fill up the queue + periods -exec 0.05 + command $run + end + + # default exit status + task.exit 0 + process_exit magicToDS $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicToDS $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicToDS $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicToDS $options:0 pantaskState TIMEOUT + end +end + +task destreak.advance + # task to finish processing for magicDSRuns + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 300 + npending 1 + +# stdout NULL +# stderr $LOGSUBDIR/destreak.advance.log + + task.exec + $run = magicdstool -advancerun + if ($DB:n != 0) + + $run = $run -dbname $DB:$magicDSAdvance_DB + + $magicDSAdvance_DB ++ + if ($magicDSAdvance_DB >= $DB:n) set magicDSAdvance_DB = 0 + end + add_poll_args run + add_poll_labels run + # since this command can be expensive, reduce the limit + $run = $run -limit 24 + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.revert.load + host local + periods -poll 5 + periods -exec 120 + periods -timeout 120 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.revert.log + + task.exec + $run = magicdstool -torevert -stage $DS_STAGE:$magicDSRevertStage + $magicDSRevertStage ++ + if ($magicDSRevertStage >= $DS_STAGE:n) set magicDSRevertStage = 0 + + if ($DB:n == 0) + option DEFAULT + else + + # save the DB name for the exit tasks + option $DB:$magicDSToRevert_DB + $run = $run -dbname $DB:$magicDSToRevert_DB + + # only bump database number after we have gone through all of the stages + if ($magicDSRevertStage == 0) + $magicDSToRevert_DB ++ + if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0 + end + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicDSToRevert + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicDSToRevert + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.revert.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + stdout $LOGSUBDIR/destreak.revert.run.log + stderr $LOGSUBDIR/destreak.revert.run.log + + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages magicDSToRevert -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicDSToRevert $pageName pantaskState RUN + book getword magicDSToRevert $pageName exp_id -var EXP_ID + book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicDSToRevert $pageName camera -var CAMERA + book getword magicDSToRevert $pageName stage -var STAGE + book getword magicDSToRevert $pageName state -var RUN_STATE + book getword magicDSToRevert $pageName stage_id -var STAGE_ID + book getword magicDSToRevert $pageName component -var COMPONENT + book getword magicDSToRevert $pageName path_base -var PATH_BASE + book getword magicDSToRevert $pageName recovery_path_base -var RECOVERY_PATH_BASE + book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE + book getword magicDSToRevert $pageName cam_reduction -var CAM_REDUCTION + book getword magicDSToRevert $pageName outroot -var OUTROOT + book getword magciDSToRevert $pageName bytes -var BYTES + book getword magciDSToRevert $pageName md5sum -var md5sum +# book getword magicDSToRevert $pageName recoveryroot -var RECROOT + book getword magicDSToRevert $pageName re_place -var REPLACE + book getword magicDSToRevert $pageName bothways -var BOTHWAYS + book getword magicDSToRevert $pageName magicked -var MAGICKED + book getword magicDSToRevert $pageName dbname -var DBNAME + + substr $COMPONENT 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + set.host.for.skycell $COMPONENT + set.workdir.by.skycell $COMPONENT $OUTROOT $default_host WORKDIR + else + # assume component is a class_id, if not we will default to anyhost + set.host.for.camera $CAMERA $COMPONENT + set.workdir.by.camera $CAMERA $COMPONENT $OUTROOT $default_host WORKDIR + end + + if (("$RUN_STATE" == "goto_restored") && ("$STAGE" == "camera")) + + sprintf logfile "%s.dsrestore.log" $PATH_BASE + + $run = destreak_restore_camera.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --cam_id $STAGE_ID --path_base $PATH_BASE --dbname $DBNAME --logfile $logfile + + else + sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $WORKDIR $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT + $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $WORKDIR --logfile $logfile --replace $REPLACE --bothways $BOTHWAYS --magicked $MAGICKED --run-state $RUN_STATE --recovery_path_base $RECOVERY_PATH_BASE + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are ready to run, drop the retry timeout low so we fill up the queue + periods -exec 0.05 + command $run + end + + # default exit status + task.exit 0 + process_exit magicDSToRevert $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicDSToRevert $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicDSToRevert $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicDSToRevert $options:0 pantaskState TIMEOUT + end +end + +task destreak.completed.revert + # task to finish processing for magicDSRuns being reverted or restored + host local + + periods -poll $LOADPOLL + #periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 20 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.completed.revert.log + + task.exec + $run = magicdstool -completedrevert + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicDSCompletedRevert_DB + + $run = $run -dbname $DB:$magicDSCompletedRevert_DB + + $magicDSCompletedRevert_DB ++ + if ($magicDSCompletedRevert_DB >= $DB:n) set magicDSCompletedRevert_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.cleanup.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.cleanup.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.cleanup.pro (revision 41106) @@ -0,0 +1,834 @@ +## detrend.process.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingProcessedImfile and detPendingProcessedExp + +# test for required global variables +check.globals + +book init detCleanupProcessed +book init detCleanupResid + +book init detCleanupStackedImfile +book init detCleanupNormStatImfile +book init detCleanupNormImfile +book init detCleanupNormExp + + +macro detclean.reset + book init detCleanupProcessed + book init detCleanupResid + + book init detCleanupStackedImfile + book init detCleanupNormStatImfile + book init detCleanupNormImfile + book init detCleanupNormExp +end + +macro detclean.status + book listbook detCleanupProcessed + book listbook detCleanupResid + + book listbook detCleanupStackedImfile + book listbook detCleanupNormStatImfile + book listbook detCleanupNormImfile + book listbook detCleanupNormExp + +end + +macro detclean.on + task detrend.cleanup.processed.load + active true + end + task detrend.cleanup.processed.run + active true + end + + task detrend.cleanup.resid.load + active true + end + task detrend.cleanup.resid.run + active true + end + +# task detrend.cleanup.stack.load +# active true +# end +# task detrend.cleanup.stack.run +# active true +# end +# task detrend.cleanup.norm.load +# active true +# end +# task detrend.cleanup.norm.run +# active true +# end +# task detrend.cleanup.normexp.load +# active true +# end +# task detrend.cleanup.normexp.run +# active true +# end +# task detrend.cleanup.normstat.load +# active true +# end +# task detrend.cleanup.normstat.run +# active true +# end +end + +macro detclean.off + task detrend.cleanup.processed.load + active false + end + task detrend.cleanup.processed.run + active false + end + + task detrend.cleanup.resid.load + active false + end + task detrend.cleanup.resid.run + active false + end + + task detrend.cleanup.stack.load + active false + end + task detrend.cleanup.stack.run + active false + end + task detrend.cleanup.norm.load + active false + end + task detrend.cleanup.norm.run + active false + end + task detrend.cleanup.normexp.load + active false + end + task detrend.cleanup.normexp.run + active false + end + task detrend.cleanup.normstat.load + active false + end + task detrend.cleanup.normstat.run + active false + end + +end + + +# these variables will cycle through the known database names +$detCleanupProcessed_DB = 0 +$detCleanupResid_DB = 0 + +$detCleanupStackedImfile_DB = 0 +$detCleanupNormStatImfile_DB = 0 +$detCleanupNormImfile_DB = 0 +$detCleanupNormExp_DB = 0 + +######## cleanup processed imfile/exp ######## +task detrend.cleanup.processed.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.processed.log + + task.exec + $run = dettool -pendingcleanup_processedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupProcessed_DB + $run = $run -dbname $DB:$detCleanupProcessed_DB + $detCleanupProcessed_DB ++ + if ($detCleanupProcessed_DB >= $DB:n) set detCleanupProcessed_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupProcessed -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupProcessed + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupProcessed + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.processed.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupProcessed -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupProcessed (pantaskState == INIT) + book getpage detCleanupProcessed 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupProcessed $pageName pantaskState RUN + book getword detCleanupProcessed $pageName det_id -var DET_ID + book getword detCleanupProcessed $pageName exp_id -var EXP_ID + book getword detCleanupProcessed $pageName camera -var CAMERA + book getword detCleanupProcessed $pageName data_state -var CLEANUP_MODE + book getword detCleanupProcessed $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and diff (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.processed.log + stderr $LOGDIR/detrend.cleanup.processed.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.processed --stage_id $DET_ID.$EXP_ID --camera $CAMERA --mode $CLEANUP_MODE --dbname $DBNAME + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupProcessed $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detCleanupProcessed $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupProcessed $options:0 pantaskState TIMEOUT + end +end + + + +######## cleanup resid imfile/exp ######## +task detrend.cleanup.resid.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/detrend.cleanup.resid.log + + task.exec + $run = dettool -pendingcleanup_residexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupResid_DB + $run = $run -dbname $DB:$detCleanupResid_DB + $detCleanupResid_DB ++ + if ($detCleanupResid_DB >= $DB:n) set detCleanupResid_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupResid -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupResid + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupResid + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.resid.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupResid -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupResid (pantaskState == INIT) + book getpage detCleanupResid 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupResid $pageName pantaskState RUN + book getword detCleanupResid $pageName det_id -var DET_ID + book getword detCleanupResid $pageName exp_id -var EXP_ID + book getword detCleanupResid $pageName iteration -var ITERATION + book getword detCleanupResid $pageName camera -var CAMERA + book getword detCleanupResid $pageName data_state -var CLEANUP_MODE + book getword detCleanupResid $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and diff (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.resid.log + stderr $LOGDIR/detrend.cleanup.resid.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.resid --stage_id $DET_ID.$EXP_ID --camera $CAMERA --mode $CLEANUP_MODE --dbname $DBNAME + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupResid $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detCleanupResid $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupResid $options:0 pantaskState TIMEOUT + end +end + +########## cleanup stack ########### +task detrend.cleanup.stack.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/detrend.cleanup.stack.log + + task.exec + $run = dettool -pendingcleanup_stacked + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupStackedImfile_DB + $run = $run -dbname $DB:$detCleanupStackedImfile_DB + $detCleanupStackedImfile_DB ++ + if ($detCleanupStackedImfile_DB >= $DB:n) set detCleanupStackedImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupStackedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupStackedImfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.stack.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active true + + task.exec + book npages detCleanupStackedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupStackedImfile (pantaskState == INIT) + book getpage detCleanupStackedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupStackedImfile $pageName pantaskState RUN + book getword detCleanupStackedImfile $pageName det_id -var DET_ID + book getword detCleanupStackedImfile $pageName iteration -var ITERATION + book getword detCleanupStackedImfile $pageName class_id -var CLASS_ID + book getword detCleanupStackedImfile $pageName camera -var CAMERA + book getword detCleanupStackedImfile $pageName data_state -var CLEANUP_MODE + book getword detCleanupStackedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.stack.log + stderr $LOGDIR/detrend.cleanup.stack.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupStackedImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detCleanupStackedImfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupStackedImfile $options:0 pantaskState TIMEOUT + end +end + + +########## cleanup normstat ########### +task detrend.cleanup.normstat.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/detrend.cleanup.normstat.log + + task.exec + $run = dettool -pendingcleanup_normalizedstat + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormStatImfile_DB + $run = $run -dbname $DB:$detCleanupNormStatImfile_DB + $detCleanupNormStatImfile_DB ++ + if ($detCleanupNormStatImfile_DB >= $DB:n) set detCleanupNormStatImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormStatImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormStatImfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.normstat.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages detCleanupNormStatImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormStatImfile (pantaskState == INIT) + book getpage detCleanupNormStatImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormStatImfile $pageName pantaskState RUN + book getword detCleanupNormStatImfile $pageName det_id -var DET_ID + book getword detCleanupNormStatImfile $pageName iteration -var ITERATION + book getword detCleanupNormStatImfile $pageName camera -var CAMERA + book getword detCleanupNormStatImfile $pageName data_state -var CLEANUP_MODE + book getword detCleanupNormStatImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.normstat.log + stderr $LOGDIR/detrend.cleanup.normstat.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormStatImfile $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword detCleanupNormStatImfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormStatImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup norm (normalized.imfile) ########### +task detrend.cleanup.norm.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/detrend.cleanup.norm.log + + task.exec + $run = dettool -pendingcleanup_normalizedimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormImfile_DB + $run = $run -dbname $DB:$detCleanupNormImfile_DB + $detCleanupNormImfile_DB ++ + if ($detCleanupNormImfile_DB >= $DB:n) set detCleanupNormImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormImfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.norm.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages detCleanupNormImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormImfile (pantaskState == INIT) + book getpage detCleanupNormImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormImfile $pageName pantaskState RUN + book getword detCleanupNormImfile $pageName det_id -var DET_ID + book getword detCleanupNormImfile $pageName iteration -var ITERATION + book getword detCleanupNormImfile $pageName class_id -var CLASS_ID + book getword detCleanupNormImfile $pageName camera -var CAMERA + book getword detCleanupNormImfile $pageName data_state -var CLEANUP_MODE + book getword detCleanupNormImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.norm.log + stderr $LOGDIR/detrend.cleanup.norm.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormImfile $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword detCleanupNormImfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormImfile $options:0 pantaskState TIMEOUT + end +end + +########## cleanup normexp ########### +task detrend.cleanup.normexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/detrend.normexp.cleanup.log + + task.exec + $run = dettool -pendingcleanup_normalizedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detCleanupNormExp_DB + $run = $run -dbname $DB:$detCleanupNormExp_DB + $detCleanupNormExp_DB ++ + if ($detCleanupNormExp_DB >= $DB:n) set detCleanupNormExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detCleanupNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detCleanupNormExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detCleanupNormExp + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task detrend.cleanup.normexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages detCleanupNormExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detCleanupNormExp (pantaskState == INIT) + book getpage detCleanupNormExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detCleanupNormExp $pageName pantaskState RUN + book getword detCleanupNormExp $pageName det_id -var DET_ID + book getword detCleanupNormExp $pageName iteration -var ITERATION + book getword detCleanupNormExp $pageName camera -var CAMERA + book getword detCleanupNormExp $pageName data_state -var CLEANUP_MODE + book getword detCleanupNormExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/detrend.cleanup.normexp.log + stderr $LOGDIR/detrend.cleanup.normexp.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detCleanupNormExp $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword detCleanupNormExp $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword detCleanupNormExp $options:0 pantaskState TIMEOUT + end +end + + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.correct.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.correct.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.correct.pro (revision 41106) @@ -0,0 +1,164 @@ +## detrend.correct.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend correction stage +## these tasks use the books detPendingCorrectImfile + +# test for required global variables +check.globals + +book init detPendingCorrectImfile + +macro detcorr.reset + book init detPendingCorrectImfile +end + +macro detcorr.status + echo detPendingCorrectImfile + book listbook detPendingCorrectImfile +end + +macro detcorr.on + task detrend.correct.load + active true + end + task detrend.correct.run + active true + end +end + +macro detcorr.off + task detrend.correct.load + active false + end + task detrend.correct.run + active false + end +end + + +# these variables will cycle through the known database names +$detPendingCorrectImfile_DB = 0 + +# select images ready for copy +# new entries are added to detPendingCorrectImfile +# compare the new list with the ones already selected +task detrend.correct.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detcorr.load.log + + task.exec + $run = dettool -tocorrectimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingCorrectImfile_DB + $run = $run -dbname $DB:$detPendingCorrectImfile_DB + $detPendingCorrectImfile_DB ++ + if ($detPendingCorrectImfile_DB >= $DB:n) set detPendingCorrectImfile_DB = 0 + end + # add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingCorrectImfile -key det_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingCorrectImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingCorrectImfile + end + + # error + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run detrend_correct_imfile.pl on pending images +task detrend.correct.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingCorrectImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingCorrectImfile + book getpage detPendingCorrectImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingCorrectImfile $pageName pantaskState RUN + book getword detPendingCorrectImfile $pageName det_id -var DET_ID + book getword detPendingCorrectImfile $pageName class_id -var CLASS_ID + book getword detPendingCorrectImfile $pageName det_type -var DET_TYPE + book getword detPendingCorrectImfile $pageName camera -var CAMERA + book getword detPendingCorrectImfile $pageName uri -var URI + book getword detPendingCorrectImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingCorrectImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID + + stdout $LOGDIR/detcorr.run.log + stderr $LOGDIR/detcorr.run.log + + $run = detrend_correct_imfile.pl --det_id $DET_ID --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA --outroot $outroot --redirect-output --verbose + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingCorrectImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detPendingCorrectImfile $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingCorrectImfile $options:0 pantaskState TIMEOUT + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.norm.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.norm.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.norm.pro (revision 41106) @@ -0,0 +1,607 @@ +## detrend.norm.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend normalization stages +## these tasks use the books detPendingNormStatImfile detPendingNormImfile detPendingNormExp + +# test for required global variables +check.globals + +book init detPendingNormStatImfile +book init detPendingNormImfile +book init detPendingNormExp + +macro detnorm.reset + book init detPendingNormStatImfile + book init detPendingNormImfile + book init detPendingNormExp +end + +macro detnorm.status + book listbook detPendingNormStatImfile + book listbook detPendingNormImfile + book listbook detPendingNormExp +end + +macro detnorm.on + task detrend.norm.load + active true + end + task detrend.norm.run + active true + end + task detrend.normexp.load + active true + end + task detrend.normexp.run + active true + end + task detrend.normstat.load + active true + end + task detrend.normstat.run + active true + end +end + +macro detnorm.off + task detrend.norm.load + active false + end + task detrend.norm.run + active false + end + task detrend.normexp.load + active false + end + task detrend.normexp.run + active false + end + task detrend.normstat.load + active false + end + task detrend.normstat.run + active false + end +end + +macro detnorm.revert.off + task detrend.norm.revert + active false + end + task detrend.normexp.revert + active false + end + task detrend.normstat.revert + active false + end +end + +macro detnorm.revert.on + task detrend.norm.revert + active true + end + task detrend.normexp.revert + active true + end + task detrend.normstat.revert + active true + end +end + + +# these variables will cycle through the known database names +$detPendingNormStatImfile_DB = 0 +$detPendingNormImfile_DB = 0 +$detPendingNormExp_DB = 0 +$detPendingNormStatImfile_DB_revert = 0 +$detPendingNormImfile_DB_revert = 0 +$detPendingNormExp_DB_revert = 0 + +# select images ready for copy +# new entries are added to detPendingNormStatImfile +# compare the new list with the ones already selected +task detrend.normstat.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.normstat.log + + task.exec + $run = dettool -tonormalizedstat + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormStatImfile_DB + $run = $run -dbname $DB:$detPendingNormStatImfile_DB + $detPendingNormStatImfile_DB ++ + if ($detPendingNormStatImfile_DB >= $DB:n) set detPendingNormStatImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormStatImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormStatImfile + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.normstat.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingNormStatImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingNormStatImfile + book getpage detPendingNormStatImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormStatImfile $pageName pantaskState RUN + book getword detPendingNormStatImfile $pageName det_id -var DET_ID + book getword detPendingNormStatImfile $pageName det_type -var DET_TYPE + book getword detPendingNormStatImfile $pageName iteration -var ITERATION + book getword detPendingNormStatImfile $pageName camera -var CAMERA + book getword detPendingNormStatImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingNormStatImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.normstat.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.normstat.log + stderr $LOGDIR/detrend.normstat.log + + $run = detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --outroot $outroot --redirect-output + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormStatImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detPendingNormStatImfile $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormStatImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingNormImfile +# compare the new list with the ones already selected +task detrend.norm.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.norm.log + + task.exec + $run = dettool -tonormalize + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormImfile_DB + $run = $run -dbname $DB:$detPendingNormImfile_DB + $detPendingNormImfile_DB ++ + if ($detPendingNormImfile_DB >= $DB:n) set detPendingNormImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormImfile + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.norm.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingNormImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingNormImfile + book getpage detPendingNormImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormImfile $pageName pantaskState RUN + book getword detPendingNormImfile $pageName det_type -var DET_TYPE + book getword detPendingNormImfile $pageName camera -var CAMERA + book getword detPendingNormImfile $pageName uri -var URI + book getword detPendingNormImfile $pageName det_id -var DET_ID + book getword detPendingNormImfile $pageName iteration -var ITERATION + book getword detPendingNormImfile $pageName class_id -var CLASS_ID + book getword detPendingNormImfile $pageName norm -var NORM + book getword detPendingNormImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingNormImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this stack + sprintf outroot "%s/%s.%s.%s/%s.%s.norm.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.norm.log + stderr $LOGDIR/detrend.norm.log + + $run = detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE --outroot $outroot --redirect-output --verbose + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detPendingNormImfile $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingNormExp +# compare the new list with the ones already selected +task detrend.normexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.normexp.log + + task.exec + $run = dettool -tonormalizedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormExp_DB + $run = $run -dbname $DB:$detPendingNormExp_DB + $detPendingNormExp_DB ++ + if ($detPendingNormExp_DB >= $DB:n) set detPendingNormExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.normexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingNormExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingNormExp + book getpage detPendingNormExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormExp $pageName pantaskState RUN + book getword detPendingNormExp $pageName det_id -var DET_ID + book getword detPendingNormExp $pageName iteration -var ITERATION + book getword detPendingNormExp $pageName det_type -var DET_TYPE + book getword detPendingNormExp $pageName camera -var CAMERA + book getword detPendingNormExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingNormExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.normexp.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.normexp.log + stderr $LOGDIR/detrend.normexp.log + + $run = detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detPendingNormExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormExp $options:0 pantaskState TIMEOUT + end +end + +task detrend.norm.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertnormalizedimfile -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormImfile_DB_revert + $run = $run -dbname $DB:$detPendingNormImfile_DB_revert + $detPendingNormImfile_DB_revert ++ + if ($detPendingNormImfile_DB_revert >= $DB:n) set detPendingNormImfile_DB_revert = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task detrend.normexp.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertnormalizedexp -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormExp_DB_revert + $run = $run -dbname $DB:$detPendingNormExp_DB_revert + $detPendingNormExp_DB_revert ++ + if ($detPendingNormExp_DB_revert >= $DB:n) set detPendingNormExp_DB_revert = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task detrend.normstat.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertnormalizedstat -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormStatImfile_DB_revert + $run = $run -dbname $DB:$detPendingNormStatImfile_DB_revert + $detPendingNormStatImfile_DB_revert ++ + if ($detPendingNormStatImfile_DB_revert >= $DB:n) set detPendingNormStatImfile_DB_revert = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.process.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.process.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.process.pro (revision 41106) @@ -0,0 +1,425 @@ +## detrend.process.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingProcessedImfile and detPendingProcessedExp + +# test for required global variables +check.globals + +book init detPendingProcessedImfile +book init detPendingProcessedExp + +macro detproc.reset + book init detPendingProcessedImfile + book init detPendingProcessedExp +end + +macro detproc.status + echo detPendingProcessedImfile + book listbook detPendingProcessedImfile + echo detPendingProcessedExp + book listbook detPendingProcessedExp +end + +macro detproc.on + task detrend.process.load + active true + end + task detrend.process.run + active true + end + task detrend.processexp.load + active true + end + task detrend.processexp.run + active true + end +end + +macro detproc.off + task detrend.process.load + active false + end + task detrend.process.run + active false + end + task detrend.processexp.load + active false + end + task detrend.processexp.run + active false + end +end + +macro detproc.revert.off + task detrend.process.revert + active false + end + task detrend.processexp.revert + active false + end +end + +macro detproc.revert.on + task detrend.process.revert + active true + end + task detrend.processexp.revert + active true + end +end + +# these variables will cycle through the known database names +$detPendingProcessedImfile_DB = 0 +$detPendingProcessedExp_DB = 0 +$detPendingProcessedImfile_revert_DB = 0 +$detPendingProcessedExp_revert_DB = 0 + +# select images ready for copy +# new entries are added to detPendingProcessedImfile +# compare the new list with the ones already selected +task detrend.process.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.process.imfile.log + + task.exec + $run = dettool -toprocessedimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedImfile_DB + $run = $run -dbname $DB:$detPendingProcessedImfile_DB + $detPendingProcessedImfile_DB ++ + if ($detPendingProcessedImfile_DB >= $DB:n) set detPendingProcessedImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingProcessedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingProcessedImfile + end + + # error + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run detrend_process_imfile.ps on pending images +task detrend.process.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingProcessedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingProcessedImfile + book getpage detPendingProcessedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingProcessedImfile $pageName pantaskState RUN + book getword detPendingProcessedImfile $pageName det_id -var DET_ID + book getword detPendingProcessedImfile $pageName det_type -var DET_TYPE + book getword detPendingProcessedImfile $pageName exp_id -var EXP_ID + book getword detPendingProcessedImfile $pageName class_id -var CLASS_ID + book getword detPendingProcessedImfile $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedImfile $pageName uri -var URI + book getword detPendingProcessedImfile $pageName camera -var CAMERA + book getword detPendingProcessedImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingProcessedImfile $pageName dbname -var DBNAME + book getword detPendingProcessedImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # see chip.pro for examples + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + stdout $LOGDIR/detrend.process.imfile.log + stderr $LOGDIR/detrend.process.imfile.log + + $run = detrend_process_imfile.pl --threads @MAX_THREADS@ --det_id $DET_ID --exp_id $EXP_ID --det_type $DET_TYPE --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingProcessedImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detPendingProcessedImfile $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingProcessedExp +# compare the new list with the ones already selected +task detrend.processexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.process.exp.log + + task.exec + $run = dettool -toprocessedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedExp_DB + $run = $run -dbname $DB:$detPendingProcessedExp_DB + $detPendingProcessedExp_DB ++ + if ($detPendingProcessedExp_DB >= $DB:n) set detPendingProcessedExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingProcessedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingProcessedExp + end + + # error + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.processexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingProcessedExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new exposures in detPendingProcessedExp + book getpage detPendingProcessedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingProcessedExp $pageName pantaskState RUN + book getword detPendingProcessedExp $pageName det_id -var DET_ID + book getword detPendingProcessedExp $pageName exp_id -var EXP_ID + book getword detPendingProcessedExp $pageName det_type -var DET_TYPE + book getword detPendingProcessedExp $pageName camera -var CAMERA + book getword detPendingProcessedExp $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingProcessedExp $pageName dbname -var DBNAME + book getword detPendingProcessedExp $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID + + stdout $LOGDIR/detrend.process.exp.log + stderr $LOGDIR/detrend.process.exp.log + + $run = detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit detPendingProcessedExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detPendingProcessedExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedExp $options:0 pantaskState TIMEOUT + end +end + +task detrend.process.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertprocessedimfile -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedImfile_revert_DB + $run = $run -dbname $DB:$detPendingProcessedImfile_revert_DB + $detPendingProcessedImfile_revert_DB ++ + if ($detPendingProcessedImfile_revert_DB >= $DB:n) set detPendingProcessedImfile_revert_DB = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task detrend.processexp.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertprocessedexp -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedExp_revert_DB + $run = $run -dbname $DB:$detPendingProcessedExp_revert_DB + $detPendingProcessedExp_revert_DB ++ + if ($detPendingProcessedExp_revert_DB >= $DB:n) set detPendingProcessedExp_revert_DB = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.reject.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.reject.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.reject.pro (revision 41106) @@ -0,0 +1,161 @@ +## detrend.reject.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend processing stage +## these tasks use the book detRejectExp + +# test for required global variables +check.globals + +book init detRejectExp + +macro detreject.reset + book init detRejectExp +end + +macro detreject.status + book listbook detRejectExp +end + +macro detreject.on + task detrend.reject.load + active true + end + task detrend.reject.run + active true + end +end + +macro detreject.off + task detrend.reject.load + active false + end + task detrend.reject.run + active false + end +end + +# this variable will cycle through the known database names +$detRejectExp_DB = 0 + +# select images ready for copy +# new entries are added to detRejectExp +# compare the new list with the ones already selected +task detrend.reject.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.reject.log + + task.exec + $run = dettool -todetrunsummary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detRejectExp_DB + $run = $run -dbname $DB:$detRejectExp_DB + $detRejectExp_DB ++ + if ($detRejectExp_DB >= $DB:n) set detRejectExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detRejectExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detRejectExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detRejectExp + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.reject.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detRejectExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detRejectExp + book getpage detRejectExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detRejectExp $pageName pantaskState RUN + book getword detRejectExp $pageName det_id -var DET_ID + book getword detRejectExp $pageName iteration -var ITERATION + book getword detRejectExp $pageName det_type -var DET_TYPE + book getword detRejectExp $pageName mode -var MODE + book getword detRejectExp $pageName camera -var CAMERA + book getword detRejectExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detRejectExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s.%s.%s.%s.detreject" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.reject.log + stderr $LOGDIR/detrend.reject.log + + $run = detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detRejectExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detRejectExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand failure + book setword detRejectExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.resid.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.resid.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.resid.pro (revision 41106) @@ -0,0 +1,433 @@ +## detrend.resid.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingResidImfile and detPendingResidExp + +# test for required global variables +check.globals + +book init detPendingResidImfile +book init detPendingResidExp + +macro detresid.reset + book init detPendingResidImfile + book init detPendingResidExp +end + +macro detresid.status + book listbook detPendingResidImfile + book listbook detPendingResidExp +end + +macro detresid.on + task detrend.resid.load + active true + end + task detrend.resid.run + active true + end + task detrend.residexp.load + active true + end + task detrend.residexp.run + active true + end +end + +macro detresid.off + task detrend.resid.load + active false + end + task detrend.resid.run + active false + end + task detrend.residexp.load + active false + end + task detrend.residexp.run + active false + end +end + +macro detresid.revert.on + task detrend.resid.revert + active true + end + task detrend.residexp.revert + active true + end +end + +macro detresid.revert.off + task detrend.resid.revert + active false + end + task detrend.residexp.revert + active false + end +end + +# these variables will cycle through the known database names +$detPendingResidImfile_DB = 0 +$detPendingResidExp_DB = 0 +$detPendingResidImfile_revert_DB = 0 +$detPendingResidExp_revert_DB = 0 + +# select images ready for copy +# new entries are added to detPendingResidImfile +# compare the new list with the ones already selected +task detrend.resid.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.resid.imfile.log + + task.exec + $run = dettool -toresidimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingResidImfile_DB + $run = $run -dbname $DB:$detPendingResidImfile_DB + $detPendingResidImfile_DB ++ + if ($detPendingResidImfile_DB >= $DB:n) set detPendingResidImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingResidImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingResidImfile + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.resid.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + periods -exec $RUNEXEC + + book npages detPendingResidImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingResidImfile + book getpage detPendingResidImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingResidImfile $pageName pantaskState RUN + book getword detPendingResidImfile $pageName det_id -var DET_ID + book getword detPendingResidImfile $pageName exp_id -var EXP_ID + book getword detPendingResidImfile $pageName iteration -var ITERATION + book getword detPendingResidImfile $pageName det_type -var DET_TYPE + book getword detPendingResidImfile $pageName mode -var MODE + book getword detPendingResidImfile $pageName exp_tag -var EXP_TAG + book getword detPendingResidImfile $pageName class_id -var CLASS_ID + book getword detPendingResidImfile $pageName uri -var URI + book getword detPendingResidImfile $pageName det_uri -var DET_URI + book getword detPendingResidImfile $pageName ref_det_id -var REF_DET_ID + book getword detPendingResidImfile $pageName ref_iter -var REF_ITER + book getword detPendingResidImfile $pageName camera -var CAMERA + book getword detPendingResidImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingResidImfile $pageName dbname -var DBNAME + book getword detPendingResidImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION + + stdout $LOGDIR/detrend.resid.imfile.log + stderr $LOGDIR/detrend.resid.imfile.log + + $run = detrend_resid_imfile.pl --threads @MAX_THREADS@ --det_id $DET_ID --iteration $ITERATION --ref_det_id $REF_DET_ID --ref_iter $REF_ITER --exp_id $EXP_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE --outroot $outroot --redirect-output --verbose + + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create command + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit detPendingResidImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detPendingResidImfile $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingResidExp +# compare the new list with the ones already selected +task detrend.residexp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.resid.exp.log + + task.exec + $run = dettool -toresidexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingResidExp_DB + $run = $run -dbname $DB:$detPendingResidExp_DB + $detPendingResidExp_DB ++ + if ($detPendingResidExp_DB >= $DB:n) set detPendingResidExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingResidExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingResidExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.residexp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingResidExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingResidExp + book getpage detPendingResidExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingResidExp $pageName pantaskState RUN + book getword detPendingResidExp $pageName det_id -var DET_ID + book getword detPendingResidExp $pageName exp_id -var EXP_ID + book getword detPendingResidExp $pageName iteration -var ITERATION + book getword detPendingResidExp $pageName det_type -var DET_TYPE + book getword detPendingResidExp $pageName mode -var MODE + book getword detPendingResidExp $pageName exp_tag -var EXP_TAG + book getword detPendingResidExp $pageName include -var INCLUDE + book getword detPendingResidExp $pageName camera -var CAMERA + book getword detPendingResidExp $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingResidExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION + + stdout $LOGDIR/detrend.resid.exp.log + stderr $LOGDIR/detrend.resid.exp.log + + $run = detrend_resid_exp.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_mode $MODE --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingResidExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detPendingResidExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidExp $options:0 pantaskState TIMEOUT + end +end + +task detrend.resid.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertresidimfile -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingResidImfile_revert_DB + $run = $run -dbname $DB:$detPendingResidImfile_revert_DB + $detPendingResidImfile_revert_DB ++ + if ($detPendingResidImfile_revert_DB >= $DB:n) set detPendingResidImfile_revert_DB = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task detrend.residexp.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + + $run = dettool -revertresidexp -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingResidExp_revert_DB + $run = $run -dbname $DB:$detPendingResidExp_revert_DB + $detPendingResidExp_revert_DB ++ + if ($detPendingResidExp_revert_DB >= $DB:n) set detPendingResidExp_revert_DB = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.stack.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.stack.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/detrend.stack.pro (revision 41106) @@ -0,0 +1,226 @@ +## detrend.stack.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the detrend stacking stage +## these tasks use the book detPendingStackedImfile + +# test for required global variables +check.globals + +book init detPendingStackedImfile + +macro detstack.reset + book init detPendingStackedImfile +end + +macro detstack.status + echo detPendingStackedImfile + book listbook detPendingStackedImfile +end + +macro detstack.on + task detrend.stack.load + active true + end + task detrend.stack.run + active true + end +end + +macro detstack.off + task detrend.stack.load + active false + end + task detrend.stack.run + active false + end +end + +macro detstack.revert.off + task detrend.stack.revert + active false + end +end + +macro detstack.revert.on + task detrend.stack.revert + active true + end +end + +# this variable will cycle through the known database names +$detPendingStackedImfile_DB = 0 +$detPendingStackedImfile_revert_DB = 0 + +# select images ready for detrend_stack.pl +# new entries are added to detPendingStackedImfile +# compare the new list with the ones already selected +task detrend.stack.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/detrend.stack.log + + task.exec + $run = dettool -tostacked + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingStackedImfile_DB + $run = $run -dbname $DB:$detPendingStackedImfile_DB + $detPendingStackedImfile_DB ++ + if ($detPendingStackedImfile_DB >= $DB:n) set detPendingStackedImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingStackedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingStackedImfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.stack.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages detPendingStackedImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in detPendingStackedImfile + book getpage detPendingStackedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingStackedImfile $pageName pantaskState RUN + book getword detPendingStackedImfile $pageName det_id -var DET_ID + book getword detPendingStackedImfile $pageName iteration -var ITERATION + book getword detPendingStackedImfile $pageName det_type -var DET_TYPE + book getword detPendingStackedImfile $pageName class_id -var CLASS_ID + book getword detPendingStackedImfile $pageName camera -var CAMERA + book getword detPendingStackedImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword detPendingStackedImfile $pageName dbname -var DBNAME + book getword detPendingStackedImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA $CLASS_ID + + # set workdir (interpolate host; see camera.pro for examples) + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure + sprintf outroot "%s/%s.%s.%s/%s.%s.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION + + stdout $LOGDIR/detrend.stack.log + stderr $LOGDIR/detrend.stack.log + + $run = detrend_stack.pl --threads @MAX_THREADS@ --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingStackedImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword detPendingStackedImfile $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingStackedImfile $options:0 pantaskState TIMEOUT + end +end + + +task detrend.stack.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertstacked -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingStackedImfile_revert_DB + $run = $run -dbname $DB:$detPendingStackedImfile_revert_DB + $detPendingStackedImfile_revert_DB ++ + if ($detPendingStackedImfile_revert_DB >= $DB:n) set detPendingStackedImfile_revert_DB = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/diff.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/diff.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/diff.pro (revision 41106) @@ -0,0 +1,656 @@ +## diff.pro : image difference analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the image differencing. +## After a difference (with associated diff_id) is defined, the difference is performed +## (tasks in diffSkyfile). + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init diffSkyfile +#book init diffCleanup +book init diffPendingSummary + +### Database lists +$diffSkycell_DB = 0 +$diffAdvance_DB = 0 +$diff_revert_DB = 0 +$diffSummary_DB = 0 +#$diffCleanup_DB = 0 + +### Check status of diffing tasks +macro diff.status + book listbook diffSkyfile +# book listbook diffCleanup +end + +### Reset diffing tasks +macro diff.reset + book init diffSkyfile + book init diffPendingSummary +# book init diffCleanup +end + +### Turn diffing tasks on +macro diff.on + task diff.skycell.load + active true + end + task diff.skycell.run + active true + end + task diff.advance + active true + end + task diff.revert + active true + end +end + +### Turn diffing tasks off +macro diff.off + task diff.skycell.load + active false + end + task diff.skycell.run + active false + end + task diff.advance + active false + end + task diff.revert + active false + end +end + +macro diff.summary.on + task diff.summary.load + active true + end + task diff.summary.run + active true + end +end + +macro diff.summary.off + task diff.summary.load + active false + end + task diff.summary.run + active false + end +end + +macro diff.revert.on + task diff.revert + active true + end +end + +macro diff.revert.off + task diff.revert + active false + end +end + +# macro diff.cleanup.on +# task diff.cleanup.load +# active true +# end +# task diff.cleanup.run +# active true +# end +# end + +# macro diff.cleanup.off +# task diff.cleanup.load +# active false +# end +# task diff.cleanup.run +# active false +# end +# end + + +### Load tasks for doing the differences +### Tasks are loaded into diffSkyfile. + + +task diff.skycell.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/diff.skycell.log + + task.exec + if ($LABEL:n == 0) break + $run = difftool -todiffskyfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffSkycell_DB + $run = $run -dbname $DB:$diffSkycell_DB + $diffSkycell_DB ++ + if ($diffSkycell_DB >= $DB:n) set diffSkycell_DB = 0 + end + add_poll_args run + add_poll_labels run + # increase the poll limit for warp over the default to + # help it keep up with chip processing + # NOTE : it is not a problem for difftool to have multiple + # -limit entries: the last one is used + $run = $run -limit {$POLL_LIMIT * 2} + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout diffSkyfile -key diff_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffSkyfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffSkyfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the diff overlaps +### Tasks are taken from diffSkyfile. +task diff.skycell.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages diffSkyfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new images in diffSkyfile (pantaskState == INIT) + book getpage diffSkyfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffSkyfile $pageName pantaskState RUN + book getword diffSkyfile $pageName diff_id -var DIFF_ID + book getword diffSkyfile $pageName diff_skyfile_id -var DIFF_SKYFILE_ID + book getword diffSkyfile $pageName tess_id -var TESS_DIR + book getword diffSkyfile $pageName skycell_id -var SKYCELL_ID + book getword diffSkyfile $pageName camera -var CAMERA + book getword diffSkyfile $pageName bothways -var BOTHWAYS + book getword diffSkyfile $pageName workdir -var WORKDIR_TEMPLATE + book getword diffSkyfile $pageName state -var RUN_STATE + book getword diffSkyfile $pageName dbname -var DBNAME + book getword diffSkyfile $pageName reduction -var REDUCTION + book getword diffSkyfile $pageName diff_mode -var DIFF_MODE + book getword diffSkyfile $pageName path_base -var PATH_BASE + + # set the host and workdir based on the skycell hash + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # XXX old code: + # host anyhost + # $WORKDIR = $WORKDIR_TEMPLATE + if (($DIFF_MODE == 1)||("$DIFF_MODE" == "NULL")) + $DIFF_TAG = "" + end + if ($DIFF_MODE == 2) + $DIFF_TAG = "WS." + end + if ($DIFF_MODE == 3) + $DIFF_TAG = "SW." + end + if ($DIFF_MODE == 4) + $DIFF_TAG = "SS." + end + + basename $TESS_DIR -var TESS_ID + + if ("$PATH_BASE" == "NULL") + sprintf outroot "%s/%s/%s/%s.%s.%sdif.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $DIFF_TAG $DIFF_ID + else + $outroot = $PATH_BASE + end + + stdout $LOGDIR/diff.skycell.log + stderr $LOGDIR/diff.skycell.log + + $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --skycell_id $SKYCELL_ID --diff_skyfile_id $DIFF_SKYFILE_ID --outroot $outroot --redirect-output --run-state $RUN_STATE + if ("$BOTHWAYS" == "T") + $run = $run --inverse + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit diffSkyfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword diffSkyfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffSkyfile $options:0 pantaskState TIMEOUT + end +end + + +# Advance exposures which have completed +task diff.advance + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/diff.advance.log + + task.exec + if ($LABEL:n == 0) break + $run = difftool -advance + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffAdvance_DB + $run = $run -dbname $DB:$diffAdvance_DB + $diffAdvance_DB ++ + if ($diffAdvance_DB >= $DB:n) set diffAdvance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + +# # select images ready for diff analysis +# # new entries are added to diffPendingImfile +# # skip already-present entries +# task diff.cleanup.load +# host local + +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 30 +# npending 1 +# active false + +# stdout NULL +# stderr $LOGDIR/diff.cleanup.log + +# task.exec +# if ($LABEL:n == 0) break +# $run = difftool -pendingcleanuprun +# if ($DB:n == 0) +# option DEFAULT +# else +# # save the DB name for the exit tasks +# option $DB:$diffCleanup_DB +# $run = $run -dbname $DB:$diffCleanup_DB +# $diffCleanup_DB ++ +# if ($diffCleanup_DB >= $DB:n) set diffCleanup_DB = 0 +# end +# add_poll_args run +# add_poll_labels run +# command $run +# end + +# # success +# task.exit 0 +# # convert 'stdout' to book format +# ipptool2book stdout diffCleanup -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT +# if ($VERBOSE > 2) +# book listbook diffCleanup +# end + +# # delete existing entries in the appropriate pantaskStates +# process_cleanup diffCleanup +# end + +# # locked list +# task.exit default +# showcommand failure +# end + +# task.exit crash +# showcommand crash +# end + +# # operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # run the ipp_cleanup.pl script on pending images +# task diff.cleanup.run +# periods -poll $RUNPOLL +# periods -exec $RUNEXEC +# periods -timeout 60 +# active false + +# task.exec +# book npages diffCleanup -var N +# if ($N == 0) break +# if ($NETWORK == 0) break + +# # look for new images in diffCleanup (pantaskState == INIT) +# book getpage diffCleanup 0 -var pageName -key pantaskState INIT +# if ("$pageName" == "NULL") break + +# book setword diffCleanup $pageName pantaskState RUN +# book getword diffCleanup $pageName camera -var CAMERA +# book getword diffCleanup $pageName state -var CLEANUP_MODE +# book getword diffCleanup $pageName diff_id -var DIFF_ID +# book getword diffCleanup $pageName dbname -var DBNAME + +# # specify choice of local or remote host based on camera and diff (class_id) +# set.host.for.camera $CAMERA FPA + +# stdout $LOGDIR/diff.cleanup.log +# stderr $LOGDIR/diff.cleanup.log + +# # XXX is everything listed here needed? +# $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE +# add_standard_args run + +# # save the pageName for future reference below +# options $pageName + +# # create the command line +# if ($VERBOSE > 1) +# echo command $run +# end +# command $run +# end + +# # default exit status +# task.exit default +# process_exit diffCleanup $options:0 $JOB_STATUS +# end + +# task.exit crash +# showcommand crash +# book setword diffCleanup $options:0 pantaskState CRASH +# end + +# # operation timed out? +# task.exit timeout +# showcommand timeout +# book setword diffCleanup $options:0 pantaskState TIMEOUT +# end +# end + +task diff.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + # XXX: actually we are getting enough jobs with fault 4 that work after reverting + # that we are going to remove that cut for now + # + #$run = difftool -revertdiffskyfile -fault 2 + $run = difftool -revertdiffskyfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diff_revert_DB + $run = $run -dbname $DB:$diff_revert_DB + $diff_revert_DB ++ + if ($diff_revert_DB >= $DB:n) set diff_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Load tasks for doing the diffs +### Tasks are loaded into diffPendingSkyCell. +task diff.summary.load + host local + active false + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1200 + npending 1 + + stdout NULL + stderr $LOGDIR/diff.summary.log + + task.exec + if ($LABEL:n == 0) break + $run = difftool -tosummary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffSummary_DB + $run = $run -dbname $DB:$diffSummary_DB + $diffSummary_DB ++ + if ($diffSummary_DB >= $DB:n) set diffSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + # XXX change tess_id to tess_dir when db is updated + ipptool2book stdout diffPendingSummary -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffPendingSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffPendingSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the diff overlaps +### Tasks are taken from diffPendingSkyCell. +task diff.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages diffPendingSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in diffPendingSkyCell (pantaskState == INIT) + book getpage diffPendingSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffPendingSummary $pageName pantaskState RUN + book getword diffPendingSummary $pageName diff_id -var DIFF_ID + book getword diffPendingSummary $pageName camera -var CAMERA + book getword diffPendingSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword diffPendingSummary $pageName dbname -var DBNAME + book getword diffPendingSummary $pageName tess_id -var TESS_DIR + book getword diffPendingSummary $pageName diff_mode -var DIFF_MODE + book getword diffPendingSummary $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + host anyhost + strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR +# set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + +# if ("$PATH_BASE" == "NULL") + + if (($DIFF_MODE == 1)||("$DIFF_MODE" == "NULL")) + $DIFF_TAG = "" + end + if ($DIFF_MODE == 2) + $DIFF_TAG = "WS." + end + if ($DIFF_MODE == 3) + $DIFF_TAG = "SW." + end + if ($DIFF_MODE == 4) + $DIFF_TAG = "SS." + end + + basename $TESS_DIR -var TESS_ID + + ## generate outroot specific to this exposure + sprintf outroot "%s/%s/%s.%sdif.%s.summary" $WORKDIR $TESS_ID $TESS_ID $DIFF_TAG $DIFF_ID + + + stdout $LOGDIR/diff.summary.log + stderr $LOGDIR/diff.summary.log + + $run = skycell_jpeg.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit diffPendingSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword diffPendingSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffPendingSummary $options:0 pantaskState TIMEOUT + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/diffphot.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/diffphot.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/diffphot.pro (revision 41106) @@ -0,0 +1,296 @@ +## diffphot.pro: (re-)photometry of diffs : -*- sh -*- + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init diffphotRuns + +### Database lists +$diffphotLoad_DB = 0 +$diffphotAdvance_DB = 0 +$diffphotRevert_DB = 0 + +### Check status of diffing tasks +macro diffphot.status + book listbook diffphotRuns +end + +### Reset diffing tasks +macro diffphot.reset + book init diffphotRuns +end + +### Turn diffing tasks on +macro diffphot.on + task diffphot.load + active true + end + task diffphot.run + active true + end + task diffphot.advance + active true + end + task diffphot.revert + active false + end +end + +### Turn diffing tasks off +macro diffphot.off + task diffphot.load + active false + end + task diffphot.run + active false + end + task diffphot.advance + active false + end + task diffphot.revert + active false + end +end + +macro diffphot.revert.on + task diffphot.revert + active true + end +end + +macro diffphot.revert.off + task diffphot.revert + active false + end +end + + +# Load processing jobs +task diffphot.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/diffphot.load.log + + task.exec + if ($LABEL:n == 0) break + $run = diffphottool -pending + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffphotLoad_DB + $run = $run -dbname $DB:$diffphotLoad_DB + $diffphotLoad_DB ++ + if ($diffphotLoad_DB >= $DB:n) set diffphotLoad_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout diffphotRuns -key diff_phot_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffphotRuns + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffphotRuns + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the diff overlaps +### Tasks are taken from diffSkyfile. +task diffphot.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages diffphotRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + book getpage diffphotRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffphotRuns $pageName pantaskState RUN + book getword diffphotRuns $pageName diff_phot_id -var DIFF_PHOT_ID + book getword diffphotRuns $pageName skycell_id -var SKYCELL_ID + book getword diffphotRuns $pageName tess_id -var TESS_DIR + book getword diffphotRuns $pageName workdir -var WORKDIR_TEMPLATE + book getword diffphotRuns $pageName state -var RUN_STATE + book getword diffphotRuns $pageName dbname -var DBNAME + book getword diffphotRuns $pageName reduction -var REDUCTION + + # set the host and workdir based on the skycell hash + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.dp.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $DIFF_PHOT_ID + + stdout $LOGDIR/diffphot.run.log + stderr $LOGDIR/diffphot.run.log + + $run = diffphot.pl --threads @MAX_THREADS@ --diff_phot_id $DIFF_PHOT_ID --skycell_id $SKYCELL_ID --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit diffphotRuns $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword diffphotRuns $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffphotRuns $options:0 pantaskState TIMEOUT + end +end + + +# Advance runs which have completed +task diffphot.advance + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/diffphot.advance.log + + task.exec + if ($LABEL:n == 0) break + $run = diffphottool -advance + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffphotAdvance_DB + $run = $run -dbname $DB:$diffphotAdvance_DB + $diffphotAdvance_DB ++ + if ($diffphotAdvance_DB >= $DB:n) set diffphotAdvance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +# Revert runs that failed +task diffphot.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/diffphot.revert.log + + task.exec + if ($LABEL:n == 0) break + $run = diffphottool -revert + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffphotRevert_DB + $run = $run -dbname $DB:$diffphotRevert_DB + $diffphotRevert_DB ++ + if ($diffphotRevert_DB >= $DB:n) set diffphotRevert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/diskbalance.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/diskbalance.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/diskbalance.pro (revision 41106) @@ -0,0 +1,466 @@ +## diskbalance.pro : tasks for data replication : -*- sh -*- +## this file contains the tasks for maintaining duplicates as needed in Nebulous +## these tasks use the books balancePending + +# test for required global variables +check.globals + +# load in the local configuration (NEB_USER, etc.) +module nebulous.site.pro + +$LOGSUBDIR = $LOGDIR/balance +mkdir $LOGSUBDIR + +book init balancePending +book init balanceControl + + +book init targetPending +book init targetControl + +macro balance.reset + book init balancePending +end + +macro balance.status + book listbook balancePending +end + +macro balance.on + task balance.load + active true + end + task balance.run + active true + end +end + +macro balance.off + task balance.load + active false + end + task balance.run + active false + end +end + +macro target.reset + book init targetPending +end + +macro target.status + book listbook targetPending +end + +macro target.on + task target.load + active true + end + task target.run + active true + end +end + +macro target.off + task target.load + active false + end + task target.run + active false + end +end + +macro set.host.for.balance + if ($0 != 2) + echo "USAGE: set.host.for.balance (hostname)" + break + end + + if (not($PARALLEL)) + host local + return + end + +# parse volume name + + if ("$1" == "NULL") + host anyhost + else + host $1 + end +end + +macro set.balance.numbers + if ($0 != 3) + echo "USAGE: set.balance.numbers (N_sources) (M_destinations)" + break + end + + book newpage balanceControl control + book setword balanceControl control nSources $1 + book setword balanceControl control mDestinations $2 +end + +macro set.balance.range + if ($0 != 3) + echo "USAGE: set.balance.range (SO_ID_START) (SO_ID_RANGE)" + break + end + + $BALANCE_SO_ID_START = $1 + $BALANCE_SO_ID_RANGE = $2 +end + + +macro show.balance.numbers + book listpage balanceControl control + echo "balance so_ids: $BALANCE_SO_ID_START $BALANCE_SO_ID_RANGE" + echo "targetting: $BALANCE_TARGET" +end + +macro show.balancePending + book npages balancePending -var N + echo "$N pages in book" + book listbook balancePending +end + +macro set.balance.target + if ($0 != 2) + echo "USAGE: set.balance.target (DESTINATION|SOURCE)" + break + end + $BALANCE_TARGET = $1 +end + +macro set.target.range + if ($0 != 3) + echo "USAGE: set.target.range (SO_ID_START) (SO_ID_RANGE)" + break + end + + $TARGET_SO_ID_START = $1 + $TARGET_SO_ID_RANGE = $2 +end + + +macro show.target.numbers + echo "target so_ids: $TARGET_SO_ID_START $TARGET_SO_ID_RANGE" +end + +macro show.targetPending + book npages targetPending -var N + echo "$N pages in book" + book listbook targetPending +end + +# the balance process interacts with only the single Nebulous server + +# Each 'pendingbalance query is limited to a finite number of so_id values. +# Each time we call balance.load, we increment the so_id start by the range value +# If the pendingbalance query exits with exit status 10, we start over at 0 + +# This value will need to be incremented periodically. It looks like we grow by about 5e7 so_ids per month. +$BALANCE_BIG_SO_ID = 6e8 +$BALANCE_SO_ID_START = int($BALANCE_BIG_SO_ID * rnd(0)) +$BALANCE_SO_ID_RANGE = 500000 +$BALANCE_LIMIT = 500 +$BALANCE_OFFSET = 0 +$BALANCE_TARGET = SOURCE +# Select Nebulous objects which should be shuffled +task balance.load + host local + + periods -poll 60 + periods -exec 60 + periods -timeout 1500 + npending 1 + + # logs + stdout NULL + stderr $LOGSUBDIR/balance.log + + task.exec + book npages balancePending -var N + if ($N > 2000) + process_cleanup balancePending + break + end + + book getword balanceControl control nSources -var BALANCE_N + book getword balanceControl control mDestinations -var BALANCE_M + + command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingbalance --limit $BALANCE_LIMIT --offset $BALANCE_OFFSET --so_id_start $BALANCE_SO_ID_START --so_id_range $BALANCE_SO_ID_RANGE --balancesources $BALANCE_N --balancedestination $BALANCE_M + + end + + # success : 0 -- we did not hit the limit, advance so_id counter + task.exit 0 + # advance the so_id counter + $BALANCE_SO_ID_START = $BALANCE_SO_ID_START + $BALANCE_SO_ID_RANGE + $BALANCE_OFFSET = 0 + # convert 'stdout' to book format + ipptool2book stdout balancePending -key key -uniq -setword pantaskState INIT + + if ($VERBOSE > 2) + book listbook balancePending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup balancePending + end + + # success : 1 -- we DID hit the limit, do NOT advance so_id counter + task.exit 1 + # convert 'stdout' to book format + ipptool2book stdout balancePending -key key -uniq -setword pantaskState INIT + $BALANCE_OFFSET = $BALANCE_OFFSET + $BALANCE_LIMIT + if ($VERBOSE > 2) + book listbook balancePending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup balancePending + end + + # out of so_id range, reset + task.exit 10 + # advance the so_id counter + $BALANCE_SO_ID_START = 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# task to execute shuffles +task balance.run + periods -poll 0.5 + periods -exec 5 + periods -timeout 5 + npending 200 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec 5 + + book npages balancePending -var N + if ($NETWORK == 0) break + if ($N == 0) break + + # look for new objects in balancePending + book getpage balancePending 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword balancePending $pageName pantaskState RUN + + book getword balancePending $pageName key -var KEY + book getword balancePending $pageName source_name -var SOURCE + book getword balancePending $pageName source_host -var SOURCE_HOST + book getword balancePending $pageName destination_name -var DESTINATION + book getword balancePending $pageName destination_host -var DEST_HOST + # Fix this with multihost restriction when possible. + if ("$BALANCE_TARGET" == "DESTINATION") + set.host.for.balance $DEST_HOST + else + set.host.for.balance $SOURCE_HOST + end + + + stdout NULL + stderr $LOGSUBDIR/balance.log + + $run = neb-shift --volume $DESTINATION $KEY $SOURCE + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit balancePending $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword balancePending $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword balancePending $options:0 pantaskState TIMEOUT + end +end + + +# This value will need to be incremented periodically. It looks like we grow by about 5e7 so_ids per month. +$TARGET_BIG_SO_ID = 6e8 +$TARGET_SO_ID_START = int($TARGET_BIG_SO_ID * rnd(0)) +$TARGET_SO_ID_RANGE = 500000 +$TARGET_LIMIT = 500 +$TARGET_OFFSET = 0 +$TARGET_TARGET = SOURCE +# Select Nebulous objects which should be shuffled +task target.load + host local + + periods -poll 60 + periods -exec 60 + periods -timeout 1500 + npending 1 + + # logs + stdout NULL + stderr $LOGSUBDIR/target.log + + task.exec + book npages targetPending -var N + if ($N > 2000) + process_cleanup targetPending + break + end + + command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingtarget --limit $TARGET_LIMIT --offset $TARGET_OFFSET --so_id_start $TARGET_SO_ID_START --so_id_range $TARGET_SO_ID_RANGE + end + + # success : 0 -- we did not hit the limit, advance so_id counter + task.exit 0 + # advance the so_id counter + $TARGET_SO_ID_START = $TARGET_SO_ID_START + $TARGET_SO_ID_RANGE + $TARGET_OFFSET = 0 + # convert 'stdout' to book format + ipptool2book stdout targetPending -key key -uniq -setword pantaskState INIT + + if ($VERBOSE > 2) + book listbook targetPending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup targetPending + end + + # success : 1 -- we DID hit the limit, do NOT advance so_id counter + task.exit 1 + # convert 'stdout' to book format + ipptool2book stdout targetPending -key key -uniq -setword pantaskState INIT + $TARGET_OFFSET = $TARGET_OFFSET + $TARGET_LIMIT + if ($VERBOSE > 2) + book listbook targetPending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup targetPending + end + + # out of so_id range, reset + task.exit 10 + # advance the so_id counter + $TARGET_SO_ID_START = 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# task to execute shuffles +task target.run + periods -poll 0.5 + periods -exec 5 + periods -timeout 5 + npending 200 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec 5 + + book npages targetPending -var N + if ($NETWORK == 0) break + if ($N == 0) break + + # look for new objects in targetPending + book getpage targetPending 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword targetPending $pageName pantaskState RUN + + book getword targetPending $pageName key -var KEY + book getword targetPending $pageName source_name -var SOURCE + book getword targetPending $pageName source_host -var SOURCE_HOST + book getword targetPending $pageName destination_name -var DESTINATION + book getword targetPending $pageName destination_host -var DEST_HOST + book getword targetPending $pageName destination_uri -var DEST_URI + book getword targetPending $pageName ins_id -var INS_ID + + # Fix this with multihost restriction when possible. + if ("$TARGET_TARGET" == "DESTINATION") + set.host.for.balance $DEST_HOST + else + set.host.for.balance $SOURCE_HOST + end + + + stdout NULL + stderr $LOGSUBDIR/target.log + + $run = neb-insedit --key $KEY --ins_id $INS_ID --uri $DEST_URI --volume $DESTINATION + $run = $run --db $NEB_DB --host $NEB_HOST --user $NEB_USER --pass $NEB_PASS + $run = $run --neb_host http://ippc04/nebulous + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit targetPending $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword targetPending $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword targetPending $options:0 pantaskState TIMEOUT + end +end + + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/dist.cleanup.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/dist.cleanup.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/dist.cleanup.pro (revision 41106) @@ -0,0 +1,156 @@ +## dist.cleanup.pro : -*- sh -*- + +check.globals + +## dist.cleanup.pro : -*- sh -*- + +book init distPendingCleanup + +macro dist.cleanup.status + book listbook distPendingCleanup +end + +macro dist.cleanup.reset + book init distPendingCleanup +end + +macro dist.cleanup.on + task dist.cleanup.load + active true + end + task dist.cleanup.run + active true + end +end + +macro dist.cleanup.off + task dist.cleanup.load + active false + end + task dist.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$dist_cleanup_DB = 0 + +# select distRuns ready for cleanup + +task dist.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/dist.cleanup.load.log + + task.exec + if ($LABEL:n == 0) break + $run = disttool -pendingcleanup + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$dist_cleanup_DB + $run = $run -dbname $DB:$dist_cleanup_DB + $dist_cleanup_DB ++ + if ($dist_cleanup_DB >= $DB:n) set dist_cleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout distPendingCleanup -key dist_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook distPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup distPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task dist.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 600 + + host anyhost + + task.exec + book npages distPendingCleanup -var N + periods -exec $RUNEXEC + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in distPendingCleanup (pantaskState == INIT) + book getpage distPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + + book setword distPendingCleanup $pageName pantaskState RUN + book getword distPendingCleanup $pageName dist_id -var DIST_ID + book getword distPendingCleanup $pageName stage -var STAGE + book getword distPendingCleanup $pageName stage_id -var STAGE_ID + book getword distPendingCleanup $pageName outdir -var OUTDIR + book getword distPendingCleanup $pageName dbname -var DBNAME + + sprintf logfile "%s/dist.cleanup.%s.log" $OUTDIR $DIST_ID + + stdout $LOGDIR/dist.cleanup.log + stderr $LOGDIR/dist.cleanup.log + + $run = dist_cleanup.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outdir $OUTDIR --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit distPendingCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword distPendingCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword distPendingCleanup $options:0 pantaskState TIMEOUT + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/dist.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/dist.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/dist.pro (revision 41106) @@ -0,0 +1,520 @@ +## dist.pro : support for the production of distribution bundles : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/dist +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init distToProcess +book init distToAdvance + +### Database lists +$distToProcess_DB = 0 +$distToAdvance_DB = 0 +$distQueue_DB = 0 +$dist_revert_DB = 0 + +### list of stages +#list of stages +$DIST_STAGE:n = 0 +list DIST_STAGE -add "raw" +list DIST_STAGE -add "chip" +list DIST_STAGE -add "camera" +list DIST_STAGE -add "fake" +list DIST_STAGE -add "warp" +list DIST_STAGE -add "diff" +list DIST_STAGE -add "stack" +list DIST_STAGE -add "SSdiff" +list DIST_STAGE -add "chip_bg" +list DIST_STAGE -add "warp_bg" +list DIST_STAGE -add "sky" +list DIST_STAGE -add "skycal" +list DIST_STAGE -add "ff" + +$currentStage = 0 + +### Check status of tasks +macro dist.status + book listbook distToProcess + book listbook distToAdvance +end + +### Reset tasks +macro dist.reset + book init distToProcess + book init distToAdvance +end + +### Turn tasks on +macro dist.on + task dist.process.load + active true + end + task dist.process.run + active true + end + task dist.advance.load + active true + end + task dist.advance.run + active true + end + task dist.revert + active true + end +end + +macro dist.off + task dist.process.load + active false + end + task dist.process.run + active false + end + task dist.advance.load + active false + end + task dist.advance.run + active false + end + task dist.revert + active false + end +end + +macro dist.revert.on + task dist.revert + active true + end +end + +macro dist.revert.off + task dist.revert + active false + end +end + +# This is now only used if we are not using a nebulous outroot +macro get.host.for.component + if ($0 != 3) + echo "USAGE: get.host.for.component (componentID) (varname)" + break + end + local component host varname + $component = $1 + $varname = $2 + substr $component 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + book getword ipphosts dist_skycell count -var count + local j myValue skyhash + list word -splitbychar . $component + $skyhash = 0 + for j 0 $word:n + inthash $word:$j $count -var myValue + $skyhash = $skyhash + $myValue + end + inthash $skyhash $count -var skyhash + sprintf skyname "sky%02d" $skyhash + book getword ipphosts dist_skycell $skyname -var host + else + book getword ipphosts dist_chip $component -var host + end + $$varname = $host +end + +# This is now only used if we are not using a nebulous outroot +macro set.dist.workdir.by.component + if ($0 != 5) + echo "USAGE: set.workdir.by.component (stage_id) (componentID) (template) (varname)" + break + end + local host stage_id component default template varname length start count selector component_id random_number + + $stage_id = $1 + $component = $2 + $template = $3 + $varname = $4 + if ("$template" == "NULL") + echo "ERROR: WORKDIR template not set." + break + end + book getword ipphosts distribution count -var count + if ("$count" == "NULL") + echo "ERROR: distribution hosts list is empty" + break + end + $host = "foo" + if ("$component" == "exposure") + # take last two letters of selector + # treat it as an integer and use modulous of length of distribution hosts + # to compute an index into the host table + strlen $stage_id length + if ($length >= 2) + $start = $length - 2 + substr $stage_id $start 2 index + else + # stage_id < 10 caused a very annoying couple of hours for bills to debug + substr $stage_id 0 1 index + end + $component_id = $index % $count + book getword ipphosts distribution $component_id -var myhost + else + get.host.for.component $component myhost + end + + if ("$myhost" == "NULL") + echo "ERROR: failed to find host for $component" + break + end + + strsub $template @HOST@ $myhost -var $varname + + #echo template is $template + #echo outdir is $$varname +end + + +task dist.process.load + host local + + periods -exec 20 + periods -poll $LOADPOLL + periods -timeout 300 + npending 1 + + + task.exec + # stdout NULL + # stderr $LOGSUBDIR/dist.process.load.log + + # check current stage in case it is out of range because entries + # have been removed + if ($currentStage >= $DIST_STAGE:n) set currentStage = 0 + + $run = disttool -pendingcomponent -stage $DIST_STAGE:$currentStage + $currentStage ++ + if ($currentStage >= $DIST_STAGE:n) set currentStage = 0 + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$distToProcess_DB + $run = $run -dbname $DB:$distToProcess_DB + + # only increment the database number after we have gone through all of + # the stages + if ($currentStage == 0) + $distToProcess_DB ++ + if ($distToProcess_DB >= $DB:n) set distToProcess_DB = 0 + end + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout distToProcess -key dist_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook distToProcess + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup distToProcess + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task dist.process.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + periods -exec 10 + + book npages distToProcess -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new components to process (pantaskState == INIT) + book getpage distToProcess 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + +# echo running $pageName + + book setword distToProcess $pageName pantaskState RUN + book getword distToProcess $pageName dist_id -var DIST_ID + book getword distToProcess $pageName camera -var CAMERA + book getword distToProcess $pageName stage -var DIST_STAGE + book getword distToProcess $pageName stage_id -var DIST_STAGE_ID + book getword distToProcess $pageName clean -var CLEAN + book getword distToProcess $pageName component -var COMPONENT + book getword distToProcess $pageName path_base -var PATH_BASE + book getword distToProcess $pageName chip_path_base -var CHIP_PATH_BASE + book getword distToProcess $pageName state -var STATE + book getword distToProcess $pageName data_state -var DATA_STATE + book getword distToProcess $pageName quality -var QUALITY + book getword distToProcess $pageName no_magic -var NO_MAGIC + book getword distToProcess $pageName magicked -var MAGICKED + book getword distToProcess $pageName alt_path_base -var ALT_PATH_BASE + book getword distToProcess $pageName outdir -var OUTDIR_TEMPLATE + book getword distToProcess $pageName exp_type -var EXP_TYPE + book getword distToProcess $pageName dbname -var DBNAME + + $EXTRA_ARGS = "" + if ("$CLEAN" == "T") + $EXTRA_ARGS = --clean + end + # magicked is output as integer due to the union in the sql + if ($MAGICKED) + $EXTRA_ARGS = $EXTRA_ARGS --magicked + end + # is this right for stack and fake? + if ("$NO_MAGIC" == "T") + $EXTRA_ARGS = $EXTRA_ARGS --no_magic + end + if ($QUALITY) + $EXTRA_ARGS = $EXTRA_ARGS --poor_quality + end + if ("$ALT_PATH_BASE" != "NULL") + $EXTRA_ARGS = $EXTRA_ARGS --alt_path_base $ALT_PATH_BASE + end + + substr $COMPONENT 0 3 COMP_HEAD + if ("$COMP_HEAD" == "sky") + set.host.for.skycell $COMPONENT + set.workdir.by.skycell $COMPONENT $OUTDIR_TEMPLATE $default_host OUTDIR + else + # assume component is a class_id, if it is not we will default to + # host "anyhost" + # and volume "any" + set.host.for.camera $CAMERA $COMPONENT + set.workdir.by.camera $CAMERA $COMPONENT $OUTDIR_TEMPLATE $default_host OUTDIR + end + + if ("$OUTDIR" == "NULL") + echo ERROR failed to set workdir for $COMPONENT + break + end + + sprintf logfile "%s/dist.%s.%s.log" $OUTDIR $DIST_ID $COMPONENT + + $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $DIST_STAGE --stage_id $DIST_STAGE_ID --component $COMPONENT --exp_type $EXP_TYPE --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit distToProcess $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword distToProcess $options:0 pantaskState TIMEOUT + end +end + +task dist.advance.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 3000 + npending 1 + + task.exec + # stderr $LOGSUBDIR/dist.advance.load.log + + $run = disttool -toadvance + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$distToAdvance_DB + $run = $run -dbname $DB:$distToAdvance_DB + $distToAdvance_DB ++ + if ($distToAdvance_DB >= $DB:n) set distToAdvance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout distToAdvance -key dist_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook distToAdvance + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup distToAdvance + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +task dist.advance.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages distToAdvance -var N + if ($NETWORK == 0) break + if ($N == 0) + periods -exec 10 + break + end + periods -exec .05 + + # look for new components to process (pantaskState == INIT) + book getpage distToAdvance 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # XXX the sql for disttool -toadvance does not include the camera: + # hack around this for now + $CAMERA = GPC1 + + book setword distToAdvance $pageName pantaskState RUN + book getword distToAdvance $pageName dist_id -var DIST_ID + book getword distToAdvance $pageName stage -var STAGE + book getword distToAdvance $pageName stage_id -var STAGE_ID + book getword distToAdvance $pageName outdir -var OUTDIR_TEMPLATE + book getword distToAdvance $pageName clean -var CLEAN + book getword distToAdvance $pageName dbname -var DBNAME + $EXTRA_ARGS = "" + if ("$CLEAN" == "T") + $EXTRA_ARGS = --clean + end + + substr $OUTDIR_TEMPLATE 0 3 SCHEME + if ("$SCHEME" == "neb") + # XXX This does not work if workdir is not in nebulous + # and $default_host == "any" because /data/any does not exist + # use "dummy" as component. Result will be 'any' + # set.workdir.by.camera $CAMERA dummy $OUTDIR_TEMPLATE $default_host OUTDIR + set.workdir.by.camera GPC1 dummy $OUTDIR_TEMPLATE $default_host OUTDIR + else + # using $DIST_ID as the "component" works fine here since we only look + # at the last two digits. But make sure that there 2 digits + $fake_component = $STAGE_ID + 10 + set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR + end + + if ("$OUTDIR" == "NULL") + echo ERROR failed to set workdir for $DIST_ID + break + end + host anyhost + + sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID + + $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --camera $CAMERA --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit distToAdvance $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword distToAdvance $options:0 pantaskState TIMEOUT + end +end + +task dist.revert + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = disttool -revertcomponent + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$dist_revert_DB + $run = $run -dbname $DB:$dist_revert_DB + $dist_revert_DB ++ + if ($dist_revert_DB >= $DB:n) set dist_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/dqstats.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/dqstats.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/dqstats.pro (revision 41106) @@ -0,0 +1,238 @@ +## dqstats.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the dqstats analysis stage +## these tasks use the book dqstatsPendingBundle + +# test for required global variables +check.globals + +book init dqstatsPendingBundle + +macro dqstats.status + book listbook dqstatsPendingBundle +end + +macro dqstats.reset + book init dqstatsPendingBundle +end + +macro dqstats.on + task dqstats.load + active true + end + task dqstats.run + active true + end + task dqstats.revert + active true + end +end + +macro dqstats.off + task dqstats.load + active false + end + task dqstats.run + active false + end + task dqstats.revert + active false + end +end + +macro dqstats.revert.on + task dqstats.revert + active true + end +end + +macro dqstats.revert.off + task dqstats.revert + active true + end +end + +# this variable will cycle through the known database names +$dqstats_DB = 0 +$dqstats_revert_DB = 0 + +# select images ready for dqstats analysis +# new entries are added to dqstatsPendingBundle +# skip already-present entries +task dqstats.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/dqstats.log + + task.exec + if ($LABEL:n == 0) break + $run = dqstatstool -pendingbundle + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$dqstats_DB + $run = $run -dbname $DB:$dqstats_DB + $dqstats_DB ++ + if ($dqstats_DB >= $DB:n) set dqstats_DB = 0 + end +# add_poll_args run +# add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout dqstatsPendingBundle -key dqstats_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook dqstatsPendingBundle + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup dqstatsPendingBundle + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the dqstats script on pending exposures +task dqstats.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 600 + + ## we want only a single outstanding dqstats job. + npending 1 + + task.exec + book npages dqstatsPendingBundle -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in dqstatsPendingBundle (pantaskState == INIT) + book getpage dqstatsPendingBundle 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword dqstatsPendingBundle $pageName pantaskState RUN + book getword dqstatsPendingBundle $pageName camera -var CAMERA + book getword dqstatsPendingBundle $pageName dqstats_id -var DQSTATS_ID + book getword dqstatsPendingBundle $pageName dbname -var DBNAME + + # specify choice of remote host based on camera and chip (class_id) + # set.host.for.camera $CAMERA FPA + + # set the WORKDIR variable +# set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + # Uri's need to be standardized when the run is created. +# sprintf outroot "%s/%s/%s.dqstats.%s" $WORKDIR $EXP_TAG $EXP_TAG $DQSTATS_ID + + stdout $LOGDIR/dqstats.log + stderr $LOGDIR/dqstats.log + + $run = dqstats_bundle.pl --dqstats_id $DQSTATS_ID --camera $CAMERA --redirect-output + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit dqstatsPendingBundle $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword dqstatsPendingBundle $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword dqstatsPendingBundle $options:0 pantaskState TIMEOUT + end +end + +task dqstats.revert + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = dqstatstool -revertrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$dqstats_revert_DB + $run = $run -dbname $DB:$dqstats_revert_DB + $dqstats_revert_DB ++ + if ($dqstats_revert_DB >= $DB:n) set dqstats_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/fake.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/fake.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/fake.pro (revision 41106) @@ -0,0 +1,293 @@ +## fake.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the phase 0 stage +## these tasks use the book fakePendingImfile + +# test for required global variables +check.globals + +book init fakePendingImfile + +macro fake.status + book listbook fakePendingImfile +end + +macro fake.reset + book init fakePendingImfile +end + +macro fake.on + task fake.imfile.load + active true + end + task fake.imfile.run + active true + end + task fake.advanceexp + active true + end + task fake.revert + active true + end +end + +macro fake.off + task fake.imfile.load + active false + end + task fake.imfile.run + active false + end + task fake.advanceexp + active false + end + task fake.revert + active false + end +end +macro fake.revert.off + task fake.revert + active false + end +end +macro fake.revert.on + task fake.revert + active true + end +end + +# this variable will cycle through the known database names +$fakeImfile_DB = 0 +$fake_revert_DB = 0 + +# select images ready for fake analysis +# new entries are added to fakePendingImfile +# skip already-present entries +task fake.imfile.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/fake.imfile.log + + task.exec + if ($LABEL:n == 0) break + $run = faketool -pendingimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fakeImfile_DB + $run = $run -dbname $DB:$fakeImfile_DB + $fakeImfile_DB ++ + if ($fakeImfile_DB >= $DB:n) set fakeImfile_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fakePendingImfile -key fake_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fakePendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fakePendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the fake_imfile.pl script on pending images +task fake.imfile.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + # fake is fast, so generate a bunch of commands in a row + book npages fakePendingImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in fakePendingImfile (pantaskState == INIT) + book getpage fakePendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fakePendingImfile $pageName pantaskState RUN + book getword fakePendingImfile $pageName camera -var CAMERA + book getword fakePendingImfile $pageName exp_id -var EXP_ID + book getword fakePendingImfile $pageName exp_tag -var EXP_TAG + book getword fakePendingImfile $pageName fake_id -var FAKE_ID + book getword fakePendingImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword fakePendingImfile $pageName class_id -var CLASS_ID + book getword fakePendingImfile $pageName chip_path_base -var CHIPROOT + book getword fakePendingImfile $pageName cam_path_base -var CAMROOT + book getword fakePendingImfile $pageName dbname -var DBNAME + book getword fakePendingImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and fake (class_id) + # XXX: this is not worth the trouble. use anyhost + # set.host.for.camera $CAMERA $CLASS_ID + host anyhost + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.fk.%s" $WORKDIR $EXP_TAG $EXP_TAG $FAKE_ID + + stderr $LOGDIR/fake.imfile.log + stderr $LOGDIR/fake.imfile.log + + $run = fake_imfile.pl --exp_id $EXP_ID --fake_id $FAKE_ID --class_id $CLASS_ID --chiproot=$CHIPROOT --camroot=$CAMROOT --camera $CAMERA --outroot $outroot + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are able to run the 'exec', use a short retry time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit fakePendingImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword fakePendingImfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fakePendingImfile $options:0 pantaskState TIMEOUT + end +end + +# advance exposures for which all imfiles have completed processing +# sets the exposure state to full and queues warp processing if requested +$fake_advance_DB = 0 +task fake.advanceexp + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/fake.advanceexp.log + + task.exec + if ($LABEL:n == 0) break + $run = faketool -advanceexp -limit 10 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fake_advance_DB + $run = $run -dbname $DB:$fake_advance_DB + $fake_advance_DB ++ + if ($fake_advance_DB >= $DB:n) set fake_advance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task fake.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = faketool -revertprocessedimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fake_revert_DB + $run = $run -dbname $DB:$fake_revert_DB + $fake_revert_DB ++ + if ($fake_revert_DB >= $DB:n) set fake_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/flatcorr.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/flatcorr.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/flatcorr.pro (revision 41106) @@ -0,0 +1,323 @@ +## flatcorr.pro : tasks to run flat-field correction : -*- sh -*- +## this file contains the tasks for running the flatcorr stage +## these tasks use the book 'flatcorrBook' + +# a flat-field correction run is initiated (manually) with a command like: +# flatcorr -definebyquery -filter r ... +# the result is a flatcorrRun entry, a set of chipRun entries, a linking list in flatcorrChipList. +# the chipRun entries are all set to stop at the "chip" stage. + +# we also can define a run with: +# flatcorr -definerun +# flatcorr -addchip ... + +# as the chip analysis progresses, we need to occasionally migrate the +# completed chips to the camera stage by calling: +# flatcorr -advancecamera + +# as the camera analysis progresses, we need to occasionally migrate the +# completed exposures to the addstar stage by calling: +# flatcorr -advanceaddstar + +# we wait for the completed addstar jobs and search for +# completed processing analysis with: +# flatcorr -pendingprocess + +# these define arguments to an analysis run. successful completion of +# the analysis is marked with: +# flatcorr -addprocess +# and failures with: +# flatcorr -addprocess -fault N + +# test for required global variables +check.globals + +book init flatcorrBook + +macro flatcorr.reset + book init flatcorrBook +end + +macro flatcorr.status + book listbook flatcorrBook +end + +macro flatcorr.on + task flatcorr.advancecamera + active true + end + task flatcorr.advanceaddstar + active true + end + task flatcorr.load + active true + end + task flatcorr.run + active true + end +end + +macro flatcorr.off + task flatcorr.advancecamera + active false + end + task flatcorr.advanceaddstar + active false + end + task flatcorr.load + active false + end + task flatcorr.run + active false + end +end + +macro flatcorr.proc.on + task flatcorr.load + active true + end + task flatcorr.run + active true + end +end + +macro flatcorr.proc.off + task flatcorr.load + active false + end + task flatcorr.run + active false + end +end + +# these variables will cycle through the known ippdb database names +$flatcorr_advancecamera_DB = 0 +$flatcorr_advanceaddstar_DB = 0 +$flatcorr_pendingprocess_DB = 0 + +# migrate complete flatcorr chips to the camera stage analysis +task flatcorr.advancecamera + host local + + # check the list of pending flatcorr runs + periods -poll 5 + periods -exec 30 + periods -timeout 60 + npending 1 + + task.exec + # define the command (does not depend on previous queries) + if ($DB:n == 0) + command flatcorr -advancecamera + else + # save the DB name for the exit tasks + # note that this DB name refers to the ippdb, not the dvodb + option $DB:$flatcorr_advancecamera_DB + command flatcorr -advancecamera -dbname $DB:$flatcorr_advancecamera_DB + $flatcorr_advancecamera_DB ++ + if ($flatcorr_advancecamera_DB >= $DB:n) set flatcorr_advancecamera_DB = 0 + end + end + + # silently drop stdout + stdout NULL + stderr $LOGDIR/flatcorr.log + + # success (no action required) + task.exit $EXIT_SUCCESS + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# migrate complete flatcorr exposures to the addstar process +task flatcorr.advanceaddstar + host local + + # check the list of pending flatcorr runs + periods -poll 5 + periods -exec 30 + periods -timeout 60 + npending 1 + + task.exec + # define the command (does not depend on previous queries) + if ($DB:n == 0) + command flatcorr -advanceaddstar + else + # save the DB name for the exit tasks + # note that this DB name refers to the ippdb, not the dvodb + option $DB:$flatcorr_advanceaddstar_DB + command flatcorr -advanceaddstar -dbname $DB:$flatcorr_advanceaddstar_DB + $flatcorr_advanceaddstar_DB ++ + if ($flatcorr_advanceaddstar_DB >= $DB:n) set flatcorr_advanceaddstar_DB = 0 + end + end + + # silently drop stdout + stdout NULL + stderr $LOGDIR/flatcorr.log + + # success (no action required) + task.exit $EXIT_SUCCESS + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create new flatcorr entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task flatcorr.load + host local + + # check the list of pending flatcorr runs + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 60 + npending 1 + + # define the command (does not depend on previous queries) + task.exec + if ($DB:n == 0) + $run = flatcorr -pendingprocess + else + $run = flatcorr -pendingprocess + # save the DB name for the exit tasks + option $DB:$flatcorr_pendingprocess_DB + $run = $run -dbname $DB:$flatcorr_pendingprocess_DB + $flatcorr_pendingprocess_DB ++ + if ($flatcorr_pendingprocess_DB >= $DB:n) set flatcorr_pendingprocess_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # silently drop stdout + stdout NULL + stderr $LOGDIR/flatcorr.log + + # success + task.exit $EXIT_SUCCESS + ipptool2book stdout flatcorrBook -key corr_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook flatcorrBook + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup flatcorrBook + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# check for flatcorr runs which are ready to go +task flatcorr.run + host local + npending 1 + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 3600 + + # silently drop stdout + stdout $LOGDIR/flatcorr.log + stderr $LOGDIR/flatcorr.log + + task.exec + book npages flatcorrBook -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in flatcorrBook + book getpage flatcorrBook 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword flatcorrBook $pageName pantaskState RUN + + # XXX probably need to set the output / log based on WORKDIR... + book getword flatcorrBook $pageName corr_id -var CORR_ID + book getword flatcorrBook $pageName det_type -var DET_TYPE + book getword flatcorrBook $pageName dvodb -var DVODB + book getword flatcorrBook $pageName camera -var CAMERA + book getword flatcorrBook $pageName region -var REGION + book getword flatcorrBook $pageName filter -var FILTER + book getword flatcorrBook $pageName make_corr -var MAKE_CORRECTION + book getword flatcorrBook $pageName dbname -var DBNAME + book getword flatcorrBook $pageName workdir -var WORKDIR_TEMPLATE + + # save the pageName for future reference below + options $pageName + + # see chip.pro for examples + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s.flatcorr.%s" $WORKDIR $CAMERA $CORR_ID + + # XXX get these arguments right + $run = flatcorr_proc.pl --corr_id $CORR_ID --det_type $DET_TYPE --dvodb $DVODB --camera $CAMERA --region $REGION --filter $FILTER --workdir $outroot + if ("$MAKE_CORRECTION" == "T") + $run = $run --make_correction + end + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit flatcorrBook $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword flatcorrBook $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword flatcorrBook $options:0 pantaskState TIMEOUT + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/fullforce.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/fullforce.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/fullforce.pro (revision 41106) @@ -0,0 +1,458 @@ +## fullforce.pro : tasks for static sky calibration analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the fullforce analysis. + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init fullForceRun +book init fullForceSummary + +### Database lists +$fullForce_DB = 0 +$fullForce_revert_DB = 0 +$fullForceSummary_DB = 0 +$fullForceSummary_revert_DB = 0 + +### Check status of fullForce tasks +macro fullforce.status + book listbook fullForceRun +end + +### Reset fullForce tasks +macro fullforce.reset + book init fullForceRun +end +macro fullforce.summary.status + book listbook fullForceSummary +end + +### Reset fullForce tasks +macro fullforce.summary.reset + book init fullForceSummary +end + +### Turn fullForce tasks on +macro fullforce.on + task fullforce.load + active true + end + task fullforce.run + active true + end + task fullforce.summary.load + active true + end + task fullforce.summary.run + active true + end +end + +### Turn fullForce tasks off +macro fullforce.off + task fullforce.load + active false + end + task fullforce.run + active false + end + task fullforce.summary.load + active false + end + task fullforce.summary.run + active false + end +end + +macro fullforce.revert.on + task fullforce.revert + active true + end + task fullforce.summary.revert + active true + end +end + +macro fullforce.revert.off + task fullforce.revert + active false + end + task fullforce.summary.revert + active false + end +end + +### Load tasks for fullforce +### Tasks are loaded into fullForceRun. +task fullforce.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/fullforce.log + + task.exec + if ($LABEL:n == 0) break + $run = fftool -todo + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fullForce_DB + $run = $run -dbname $DB:$fullForce_DB + $fullForce_DB ++ + if ($fullForce_DB >= $DB:n) set fullForce_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fullForceRun -key ff_id:warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fullForceRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fullForceRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the fullforce analysis +### Tasks are taken from fullForceRun. +task fullforce.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run use "long" exectime + periods -exec $RUNEXEC + book npages fullForceRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + + # look for new entries in fullForceRun (pantaskState == INIT) + book getpage fullForceRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fullForceRun $pageName pantaskState RUN + book getword fullForceRun $pageName ff_id -var FF_ID + book getword fullForceRun $pageName warp_id -var WARP_ID + book getword fullForceRun $pageName sources_path_base -var SOURCES_PATH_BASE + book getword fullForceRun $pageName tess_id -var TESS_DIR + book getword fullForceRun $pageName skycell_id -var SKYCELL_ID + book getword fullForceRun $pageName camera -var CAMERA + book getword fullForceRun $pageName workdir -var WORKDIR_TEMPLATE + book getword fullForceRun $pageName warp_path_base -var WARP_PATH_BASE + book getword fullForceRun $pageName reduction -var REDUCTION + book getword fullForceRun $pageName dbname -var DBNAME + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.wrp.%s.ff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $WARP_ID $FF_ID + + stdout $LOGDIR/fullforce.log + stderr $LOGDIR/fullforce.log + + $run = psphot_fullforce_warp.pl --ff_id $FF_ID --warp_id $WARP_ID --outroot $outroot --redirect-output --camera $CAMERA --warp_path_base $WARP_PATH_BASE --sources_path_base $SOURCES_PATH_BASE --skycell_id $SKYCELL_ID --threads @MAX_THREADS@ + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # since we have work to do shorten exec time + periods -exec .05 + command $run + end + + # default exit status + task.exit default + process_exit fullForceRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword fullForceRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fullForceRun $options:0 pantaskState TIMEOUT + end +end + +task fullforce.revert + host local + + periods -poll 10.0 + periods -exec 1200.0 + periods -timeout 120.0 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/fullforce.revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = fftool -revert -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fullForce_revert_DB + $run = $run -dbname $DB:$fullForce_revert_DB + $fullForce_revert_DB ++ + if ($fullForce_revert_DB >= $DB:n) set fullForce_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Load tasks for fullForceSummary +### Tasks are loaded into fullForceSummary. +task fullforce.summary.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/fullforce.summary.log + + task.exec + if ($LABEL:n == 0) break + $run = fftool -toadvance + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fullForceSummary_DB + $run = $run -dbname $DB:$fullForceSummary_DB + $fullForceSummary_DB ++ + if ($fullForceSummary_DB >= $DB:n) set fullForceSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fullForceSummary -key ff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fullForceSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fullForceSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the fullForceSummary analyasis +### Tasks are taken from fullForceSummary. +task fullforce.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run use "long" exectime + periods -exec $RUNEXEC + book npages fullForceSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + + # look for new entries in fullForceSummary (pantaskState == INIT) + book getpage fullForceSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fullForceSummary $pageName pantaskState RUN + book getword fullForceSummary $pageName ff_id -var FF_ID + book getword fullForceSummary $pageName tess_id -var TESS_DIR + book getword fullForceSummary $pageName skycell_id -var SKYCELL_ID + book getword fullForceSummary $pageName camera -var CAMERA + book getword fullForceSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword fullForceSummary $pageName reduction -var REDUCTION + book getword fullForceSummary $pageName dbname -var DBNAME + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.ff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $FF_ID + + stdout $LOGDIR/fullforce.summary.log + stderr $LOGDIR/fullforce.summary.log + + $run = psphot_fullforce_summary.pl --ff_id $FF_ID --outroot $outroot --redirect-output --camera $CAMERA --threads @MAX_THREADS@ + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # since we have work to do shorten exec time + periods -exec .05 + command $run + end + + # default exit status + task.exit default + process_exit fullForceSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword fullForceSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fullForceSummary $options:0 pantaskState TIMEOUT + end +end + +task fullforce.summary.revert + host local + + periods -poll 10.0 + periods -exec 1200.0 + periods -timeout 120.0 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/fullforce.summary.revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = fftool -revertsummary -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fullForceSummary_revert_DB + $run = $run -dbname $DB:$fullForceSummary_revert_DB + $fullForceSummary_revert_DB ++ + if ($fullForceSummary_revert_DB >= $DB:n) set fullForceSummary_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/ipphosts.manoa.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/ipphosts.manoa.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/ipphosts.manoa.config (revision 41106) @@ -0,0 +1,19 @@ +## basic chiphost table: use the defaults for everything + +chiphosts MULTI + +chiphosts METADATA + camera STR MEGACAM +END + +chiphosts METADATA + camera STR CFH12K +END + +chiphosts METADATA + camera STR GPC1 +END + +chiphosts METADATA + camera STR gpc1 +END Index: /branches/ccl_branches/ipponly-20191108/ippTasks/ipphosts.mhpcc.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/ipphosts.mhpcc.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/ipphosts.mhpcc.config (revision 41106) @@ -0,0 +1,410 @@ +ipphosts MULTI + +# In this file, we assign all storage hosts to chip and skycells any +# hosts which are having problems are removed as active volumes from +# nebulous. data targeted to those machines are simply re-distributed +# randomly. the only exception is ipp004, which is the DVO db host + +# I've allocated hosts based on disk size. +# 40TB => 2 units of skycells +# 20TB => 1 unit of skycells +# Each host gets one OTA/chip of data. +# These hosts are skipped: +# ipp004 : dvo +# ipp005 : dvo +# ipp026 : crashes +# ipp029 : crashes +# ipp045 : reference cat +## +## PSNSC -- above is way out of date -- update 20170112 in anticipation of +## nodes 2 units of skycells +# 20TB => 1 unit of skycells +# Each host gets one OTA/chip of data. +# These hosts are skipped: +# ipp004 : dvo +# ipp005 : dvo +# ipp026 : crashes +# ipp029 : crashes +# ipp045 : reference cat +## +## now focus on new storage nodes only 12/18/14 +## -- focus on lower used nodes as well, limit 067-081 since warp,stack,diff permament +## -- MD also typically x1--x8, 10--80 -- so need to try and balance that.. +## -- easier if raise the count to 90, then 00-10,90-100 map the same for MD as low +ipphosts METADATA + camera STR skycell + count S32 90 + sky00 STR ipp067.0 + sky01 STR ipp068.0 + sky02 STR ipp069.0 + sky03 STR ipp070.0 + sky04 STR ipp071.0 + sky05 STR ipp072.0 + sky06 STR ipp073.0 + sky07 STR ipp074.0 + sky08 STR ipp075.0 + sky09 STR ipp076.0 + sky10 STR ipp077.0 + sky11 STR ipp078.0 + + sky12 STR ipp077.0 + sky13 STR ipp071.0 + sky14 STR ipp072.0 + sky15 STR ipp073.0 + sky16 STR ipp074.0 + sky17 STR ipp075.0 + + sky18 STR ipp079.0 + sky19 STR ipp080.0 + sky20 STR ipp081.0 + + sky21 STR ipp089.0 + sky22 STR ipp088.0 + sky23 STR ipp087.0 + sky24 STR ipp086.0 + sky25 STR ipp085.0 + sky26 STR ipp084.0 + sky27 STR ipp083.0 + sky28 STR ipp082.0 + + sky29 STR ipp067.0 + sky30 STR ipp068.0 + + sky31 STR ipp095.0 + sky32 STR ipp095.0 + sky33 STR ipp093.0 + sky34 STR ipp093.0 + sky35 STR ipp090.0 + sky36 STR ipp090.0 + sky37 STR ipp089.0 + sky38 STR ipp088.0 + + sky39 STR ipp069.0 + sky40 STR ipp070.0 + + sky41 STR ipp087.0 + sky42 STR ipp086.0 + sky43 STR ipp085.0 + sky44 STR ipp084.0 + sky45 STR ipp083.0 + sky46 STR ipp082.0 + sky47 STR ipp095.0 + sky48 STR ipp095.0 + + sky49 STR ipp071.0 + sky50 STR ipp072.0 + + sky51 STR ipp093.0 + sky52 STR ipp093.0 + sky53 STR ipp090.0 + sky54 STR ipp090.0 + sky55 STR ipp089.0 + sky56 STR ipp088.0 + sky57 STR ipp087.0 + sky58 STR ipp086.0 + + sky59 STR ipp073.0 + sky60 STR ipp074.0 + + sky61 STR ipp085.0 + sky62 STR ipp084.0 + sky63 STR ipp083.0 + sky64 STR ipp082.0 + sky65 STR ipp095.0 + sky66 STR ipp095.0 + sky67 STR ipp093.0 + sky68 STR ipp093.0 + + sky69 STR ipp075.0 + sky70 STR ipp076.0 + + sky71 STR ipp090.0 + sky72 STR ipp090.0 + sky73 STR ipp089.0 + sky74 STR ipp088.0 + sky75 STR ipp087.0 + sky76 STR ipp086.0 + sky77 STR ipp085.0 + sky78 STR ipp084.0 + + sky79 STR ipp077.0 + sky80 STR ipp078.0 + sky81 STR ipp079.0 + + sky82 STR ipp076.0 + sky83 STR ipp077.0 + sky84 STR ipp078.0 + sky85 STR ipp079.0 + sky86 STR ipp080.0 + sky87 STR ipp081.0 + + sky88 STR ipp080.0 + sky89 STR ipp081.0 +END + +# These are ordered oddly to prevent excessive targetting shuffles. +## now focus on new storage nodes only 12/18/14 +## -- no limit loading on lower 067-081 since chips transient +ipphosts METADATA + camera STR GPC1 + XY01 STR ipp095 + XY02 STR ipp095 + XY03 STR ipp093 + XY04 STR ipp093 + XY05 STR ipp090 + XY06 STR ipp090 + XY10 STR ipp089 + XY11 STR ipp088 + XY12 STR ipp087 + XY13 STR ipp086 + XY14 STR ipp085 + XY15 STR ipp084 + XY16 STR ipp083 + XY17 STR ipp082 + XY20 STR ipp081 + XY21 STR ipp080 + XY22 STR ipp079 + XY23 STR ipp078 + XY24 STR ipp077 + XY25 STR ipp076 + XY26 STR ipp075 + XY27 STR ipp074 + XY30 STR ipp073 + XY31 STR ipp072 + XY32 STR ipp071 + XY33 STR ipp070 + XY34 STR ipp069 + XY35 STR ipp068 + XY36 STR ipp067 + + XY37 STR ipp095 + XY40 STR ipp095 + XY41 STR ipp093 + XY42 STR ipp093 + XY43 STR ipp090 + XY44 STR ipp090 + XY45 STR ipp089 + XY46 STR ipp088 + XY47 STR ipp087 + XY50 STR ipp086 + XY51 STR ipp085 + XY52 STR ipp084 + XY53 STR ipp083 + XY54 STR ipp082 + XY55 STR ipp081 + XY56 STR ipp080 + XY57 STR ipp079 + XY60 STR ipp078 + XY61 STR ipp077 + XY62 STR ipp076 + XY63 STR ipp075 + XY64 STR ipp074 + XY65 STR ipp073 + XY66 STR ipp072 + XY67 STR ipp071 + XY71 STR ipp070 + XY72 STR ipp069 + XY73 STR ipp068 + XY74 STR ipp067 + + XY75 STR ipp087 + XY76 STR ipp084 +END + +## focus nightly processing ota on disks with more space 12/18/14 +## -- ipp066 and lower just raw ota products until full +## -- shouldn't be used for reprocessing? +ipphosts METADATA + camera STR gpc1 + ota01 STR ipp067 + ota02 STR ipp068 + ota03 STR ipp069 + ota04 STR ipp070 + ota05 STR ipp071 + ota06 STR ipp072 + ota10 STR ipp073 + ota11 STR ipp074 + ota12 STR ipp075 + ota13 STR ipp076 + ota14 STR ipp077 + ota15 STR ipp078 + ota16 STR ipp079 + ota17 STR ipp080 + ota20 STR ipp081 + ota21 STR ipp082 + ota22 STR ipp083 + ota23 STR ipp084 + ota24 STR ipp085 + ota25 STR ipp086 + ota26 STR ipp087 + ota27 STR ipp088 + ota30 STR ipp089 + ota31 STR ipp090 + ota32 STR ipp090 + ota33 STR ipp093 + ota34 STR ipp093 + ota35 STR ipp095 + ota36 STR ipp095 + + ota37 STR ipp067 + ota40 STR ipp068 + ota41 STR ipp069 + ota42 STR ipp070 + ota43 STR ipp071 + ota44 STR ipp072 + ota45 STR ipp073 + ota46 STR ipp074 + ota47 STR ipp075 + ota50 STR ipp076 + ota51 STR ipp077 + ota52 STR ipp078 + ota53 STR ipp079 + ota54 STR ipp080 + ota55 STR ipp081 + ota56 STR ipp082 + ota57 STR ipp083 + ota60 STR ipp084 + ota61 STR ipp085 + ota62 STR ipp086 + ota63 STR ipp087 + ota64 STR ipp088 + ota65 STR ipp089 + ota66 STR ipp090 + ota67 STR ipp090 + ota71 STR ipp093 + ota72 STR ipp093 + ota73 STR ipp095 + ota74 STR ipp095 + + ota75 STR ipp060 + ota76 STR ipp060 +END + +ipphosts METADATA + camera STR distribution + count S32 26 + 0 STR ipp021 + 1 STR ipp023 + 2 STR ipp024 + 3 STR ipp026 + 4 STR ipp028 + 5 STR ipp029 + 6 STR ipp030 + 7 STR ipp031 + 8 STR ipp032 + 9 STR ipp033 + 10 STR ipp034 + 11 STR ipp035 + 12 STR ipp036 + 13 STR ipp038 + 14 STR ipp039 + 15 STR ipp040 + 16 STR ipp041 + 17 STR ipp043 + 18 STR ipp044 + 19 STR ipp045 + 20 STR ipp046 + 21 STR ipp047 + 22 STR ipp048 + 23 STR ipp050 + 24 STR ipp051 + 25 STR ipp052 +END + +ipphosts METADATA + camera STR dist_chip + count S32 60 + + XY01 STR ipp010 + XY02 STR ipp011 + XY03 STR ipp006 + XY04 STR ipp054 + + XY05 STR ipp007 + XY06 STR ipp055 + XY10 STR ipp008 + XY11 STR ipp056 + + XY12 STR ipp009 + XY13 STR ipp057 + XY14 STR ipp058 + XY15 STR ipp059 + + XY16 STR ipp012 + XY17 STR ipp060 + XY20 STR ipp013 + XY21 STR ipp014 + + XY22 STR ipp061 + XY23 STR ipp015 + XY24 STR ipp016 + XY25 STR ipp062 + + XY26 STR ipp064 + XY27 STR ipp065 + XY30 STR ipp066 + XY31 STR ipp020 + + XY32 STR ipp063 + XY33 STR ipp021 + XY34 STR ipp023 + XY35 STR ipp013 + + XY36 STR ipp024 + XY37 STR ipp019 + XY40 STR ipp025 + XY41 STR ipp018 + + XY42 STR ipp017 + XY43 STR ipp015 + XY44 STR ipp027 + XY45 STR ipp028 + + XY46 STR ipp029 + XY47 STR ipp030 + XY50 STR ipp031 + XY51 STR ipp032 + + XY52 STR ipp033 + XY53 STR ipp034 + XY54 STR ipp035 + XY55 STR ipp036 + + XY56 STR ipp037 + XY57 STR ipp038 + XY60 STR ipp039 + XY61 STR ipp040 + + XY62 STR ipp041 + XY63 STR ipp042 + XY64 STR ipp043 + XY65 STR ipp044 + + XY66 STR ipp046 + XY67 STR ipp047 + XY71 STR ipp048 + XY72 STR ipp049 + + XY73 STR ipp050 + XY74 STR ipp051 + XY75 STR ipp052 + XY76 STR ipp053 +END + +ipphosts METADATA + camera STR dist_skycell + count S32 86 + sky00 STR ipp006 + sky01 STR ipp006 + sky02 STR ipp007 + sky03 STR ipp007 + sky04 STR ipp008 + sky05 STR ipp008 + sky06 STR ipp009 + sky07 STR ipp009 + sky08 STR ipp010 + sky09 STR ipp010 + sky10 STR ipp011 + sky11 STR ipp011 + sky12 STR ipp012 + sky13 STR ipp012 + sky14 STR ipp013 + sky15 STR ipp013 + sky16 STR ipp014 + sky17 STR ipp014 + sky18 STR ipp015 + sky19 STR ipp015 + sky20 STR ipp016 + sky21 STR ipp016 + sky22 STR ipp017 + sky23 STR ipp017 + sky24 STR ipp018 + sky25 STR ipp018 + sky26 STR ipp019 + sky27 STR ipp019 + sky28 STR ipp020 + sky29 STR ipp020 + sky30 STR ipp021 + sky31 STR ipp021 + sky32 STR ipp023 + sky33 STR ipp024 + sky34 STR ipp025 + sky35 STR ipp027 + sky36 STR ipp028 + sky37 STR ipp030 + sky38 STR ipp031 + sky39 STR ipp032 + sky40 STR ipp033 + sky41 STR ipp034 + sky42 STR ipp035 + sky43 STR ipp036 + sky44 STR ipp037 + sky45 STR ipp038 + sky46 STR ipp039 + sky47 STR ipp040 + sky48 STR ipp041 + sky49 STR ipp042 + sky50 STR ipp043 + sky51 STR ipp044 + sky52 STR ipp046 + sky53 STR ipp047 + sky54 STR ipp048 + sky55 STR ipp049 + sky56 STR ipp050 + sky57 STR ipp051 + sky58 STR ipp052 + sky59 STR ipp053 + sky60 STR ipp054 + sky61 STR ipp054 + sky62 STR ipp055 + sky63 STR ipp055 + sky64 STR ipp056 + sky65 STR ipp056 + sky66 STR ipp057 + sky67 STR ipp057 + sky68 STR ipp058 + sky69 STR ipp058 + sky70 STR ipp059 + sky71 STR ipp059 + sky72 STR ipp060 + sky73 STR ipp060 + sky74 STR ipp061 + sky75 STR ipp061 + sky76 STR ipp062 + sky77 STR ipp062 + sky78 STR ipp063 + sky79 STR ipp063 + sky80 STR ipp064 + sky81 STR ipp064 + sky82 STR ipp065 + sky83 STR ipp065 + sky84 STR ipp066 + sky85 STR ipp066 +END + +ipphosts METADATA + camera STR STARE + + ota01 STR stare00 + ota02 STR stare01 + ota03 STR stare02 + ota04 STR stare04 + + ota05 STR stare00 + ota06 STR stare01 + ota10 STR stare02 + ota11 STR stare04 + + ota12 STR stare00 + ota13 STR stare01 + ota14 STR stare02 + ota15 STR stare04 + + ota16 STR stare00 + ota17 STR stare01 + ota20 STR stare02 + ota21 STR stare04 + + ota22 STR stare00 + ota23 STR stare01 + ota24 STR stare02 + ota25 STR stare04 + + ota26 STR stare00 + ota27 STR stare01 + ota30 STR stare02 + ota31 STR stare04 + + ota32 STR stare00 + ota33 STR stare01 + ota34 STR stare02 + ota35 STR stare04 + + ota36 STR stare00 + ota37 STR stare01 + ota40 STR stare02 + ota41 STR stare04 + + ota42 STR stare00 + ota43 STR stare01 + ota44 STR stare02 + ota45 STR stare04 + + ota46 STR stare00 + ota47 STR stare01 + ota50 STR stare02 + ota51 STR stare04 + + ota52 STR stare00 + ota53 STR stare01 + ota54 STR stare02 + ota55 STR stare04 + + ota56 STR stare00 + ota57 STR stare01 + ota60 STR stare02 + ota61 STR stare04 + + ota62 STR stare00 + ota63 STR stare01 + ota64 STR stare02 + ota65 STR stare04 + + ota66 STR stare00 + ota67 STR stare01 + ota71 STR stare02 + ota72 STR stare04 + + ota73 STR stare00 + ota74 STR stare01 + ota75 STR stare02 + ota76 STR stare04 +END + + +# Backup of old stare allocation setup. +# ipphosts METADATA +# camera STR STARE + +# ota01 STR ipp010 +# ota02 STR ipp010 +# ota03 STR ipp010 +# ota04 STR ipp010 + +# ota05 STR ipp011 +# ota06 STR ipp011 +# ota10 STR ipp011 +# ota11 STR ipp011 + +# ota12 STR ipp012 +# ota13 STR ipp012 +# ota14 STR ipp012 +# ota15 STR ipp012 + +# ota16 STR ipp013 +# ota17 STR ipp013 +# ota20 STR ipp013 +# ota21 STR ipp013 + +# ota22 STR ipp014 +# ota23 STR ipp014 +# ota24 STR ipp014 +# ota25 STR ipp014 + +# ota26 STR ipp015 +# ota27 STR ipp015 +# ota30 STR ipp015 +# ota31 STR ipp015 + +# ota32 STR ipp016 +# ota33 STR ipp016 +# ota34 STR ipp016 +# ota35 STR ipp016 + +# ota36 STR ipp017 +# ota37 STR ipp017 +# ota40 STR ipp017 +# ota41 STR ipp017 + +# ota42 STR ipp018 +# ota43 STR ipp018 +# ota44 STR ipp018 +# ota45 STR ipp018 + +# ota46 STR ipp019 +# ota47 STR ipp019 +# ota50 STR ipp019 +# ota51 STR ipp019 + +# ota52 STR ipp020 +# ota53 STR ipp020 +# ota54 STR ipp020 +# ota55 STR ipp020 + +# ota56 STR ipp021 +# ota57 STR ipp021 +# ota60 STR ipp021 +# ota61 STR ipp021 + +# ota62 STR ipp010 +# ota63 STR ipp011 +# ota64 STR ipp012 +# ota65 STR ipp013 + +# ota66 STR ipp014 +# ota67 STR ipp015 +# ota71 STR ipp016 +# ota72 STR ipp017 + +# ota73 STR ipp018 +# ota74 STR ipp019 +# ota75 STR ipp020 +# ota76 STR ipp021 +# END Index: /branches/ccl_branches/ipponly-20191108/ippTasks/isp.det =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/isp.det (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/isp.det (revision 41106) @@ -0,0 +1,91 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/bias -det_type bias -select_exp_type bias -select_dateobs_begin 2007-01-13T00:00:00" + dbname STR isp + pantaskState STR DONE + detRunID STR 1 +END + +detRunDef METADATA + detRunName STR DARK-warm + prereq STR BIAS + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/dark -det_type dark -select_exp_type dark -select_exp_time_min 0.5 -select_ccd_temp_min -27.5 -select_ccd_temp_max -24.0 -ccd_temp_min -27.5 -ccd_temp_max -24.0" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK-cool + prereq STR BIAS + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/dark -det_type dark -select_exp_type dark -select_exp_time_min 0.5 -select_ccd_temp_max -27.5 -ccd_temp_max -27.5" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR "DARK-warm DARK-cool" + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/shutter -det_type shutter -select_exp_type skyflat -select_exp_time_min 0.05 -select_filter g -select_dateobs_begin 2007-01-14T04:00:00 -select_dateobs_end 2007-01-14T05:00:00 -select_sat_pixel_frac_max 0.1" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-g + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/g -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter g -filter g -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/r -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter r -filter r -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/i -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter i -filter i -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-z + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/z -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter z -filter z -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-y + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/y -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter y -filter y -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FRINGE-z + prereq STR FLAT-z + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/fringe/z -det_type fringe -select_exp_type object -select_exp_time_min 0.05 -select_filter z -filter z -select_dateobs_begin 2007-01-14T09:19:00 -select_dateobs_end 2007-01-18" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FRINGE-y + prereq STR FLAT-y + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/fringe/y -det_type fringe -select_exp_type object -select_exp_time_min 0.05 -select_filter y -filter y -select_dateobs_begin 2007-01-13 -select_dateobs_end 2007-01-18" + dbname STR isp + pantaskState STR NULL +END Index: /branches/ccl_branches/ipponly-20191108/ippTasks/lap.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/lap.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/lap.pro (revision 41106) @@ -0,0 +1,520 @@ +## lap.pro : -*- sh -*- + +check.globals + +$lapSeq:n = 0 +$lap_N = 0 +$lap_DB = 0 + +$lap_NewPage = 0 +$lap_RunPage = 0 +$lap_DonePage = 0 + +$LAP_QUEUE = NULL + +book init lapNewRuns +book init lapRunRuns +book init lapFullRuns +book init lapDoneRuns + +macro lap.show.books + echo "lapNewRuns" + book listbook lapNewRuns + echo "lapRunRuns" + book listbook lapRunRuns + echo "lapDoneRuns" + book listbook lapDoneRuns + echo "lapFullRuns" + book listbook lapFullRuns +end + +macro lap.clear.books + book init lapNewRuns + book init lapRunRuns + book init lapFullRuns + book init lapDoneRuns +end + +macro lap.on + task lap.initial.load + active true + end + task lap.initial.run + active true + end + task lap.monitor.load + active true + end + task lap.monitor.run + active true + end +end + +macro lap.off + task lap.initial.load + active false + end + task lap.initial.run + active false + end + task lap.monitor.load + active false + end + task lap.monitor.run + active false + end +end + +macro lap.debug.mode + task lap.initial.load + active true + end + task lap.initial.run + active false + end + task lap.monitor.load + active true + end + task lap.monitor.run + active false + end +end + +macro lap.cleanup.on + task lap.cleanup.load + active true + end + task lap.cleanup.run + active true + end +end + +macro lap.cleanup.off + task lap.cleanup.load + active false + end + task lap.cleanup.run + active false + end +end + +macro lap.initial.on + task lap.initial.load + active true + end + task lap.initial.run + active true + end +end + +macro lap.initial.off + task lap.initial.load + active false + end + task lap.initial.run + active false + end +end + +macro lap.add.sequence + if ($0 != 2) + echo "USAGE: lap.add.sequence (seq_id)" + break + end + if ($?lapSeq:n == 0) + list lapSeq -add $1 + return + end + + local found + $found = 0 + for i 0 $lapSeq:n + if ($lapSeq:$i == $1) + $found = 1 + echo "$lapSeq:$i set" + last + end + end + + if ($found == 0) + list lapSeq -add $1 + end +end + +macro lap.del.sequence + if ($0 != 2) + echo "USAGE: lap.del.sequence (seq_id)" + break + end + if ($?lapSeq:n == 0) + return + end + + list lapSeq -del $1 +end + + +macro lap.define.queue + $LAP_QUEUE = $1 +end + + +task lap.initial.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/lap.load.log + + $run = laptool -pendingrun -state new + + if ($lapSeq:n == 0) + break + else + option $lapSeq:$lap_N + $run = $run -seq_id $lapSeq:$lap_N + $lap_N ++ + if ($lap_N >= $lapSeq:n) set lap_N = 0 + end + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$lap_DB + $run = $run -dbname $DB:$lap_DB + $lap_DB ++ + if ($lap_DB >= $DB:n) set lap_DB = 0 + end + + command $run + end + # success + task.exit 0 + ipptool2book stdout lapNewRuns -uniq -key lap_id -setword dbname $options:0 -setword pantaskState INIT + + process_cleanup lapNewRuns + + if ($VERBOSE > 2) + book listbook lapNewRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task lap.initial.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 3600 + active false +# This can probably be increased and spread over hosts in the future. + npending 1 + + task.exec + stdout $LOGDIR/lap.initial.log + stderr $LOGDIR/lap.initial.log + + book npages lapNewRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + + + book getpage lapNewRuns 0 -var lapNewPageName -key pantaskState INIT + + $lap_NewPage ++ + if ($lap_NewPage >= $N) set lap_NewPage = 0 + + if ("$lapNewPageName" == "NULL") break + + + + book setword lapNewRuns $lapNewPageName pantaskState RUN + book getword lapNewRuns $lapNewPageName lap_id -var LAP_ID + book getword lapNewRuns $lapNewPageName dbname -var DBNAME + + option $LAP_ID + + $run = lap_science.pl --chip_mode --dbname $DBNAME --lap_id $LAP_ID + + command $run + + end + + # success + task.exit 0 + process_exit lapNewRuns $options:0 0 + if ($VERBOSE > 2) + book listbook lapNewRuns + end + end + # locked list + task.exit default + process_exit lapNewRuns $options:0 0 + showcommand failure + end + task.exit crash + process_exit lapNewRuns $options:0 0 + showcommand crash + end + #operation times out? + task.exit timeout + process_exit lapNewRuns $options:0 0 + showcommand timeout + end +end + + + +task lap.monitor.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/lap.load.log + + $run = laptool -pendingrun -state run + + if ($lapSeq:n == 0) + break + else + option $lapSeq:$lap_N + $run = $run -seq_id $lapSeq:$lap_N + $lap_N ++ + if ($lap_N >= $lapSeq:n) set lap_N = 0 + end + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$lap_DB + $run = $run -dbname $DB:$lap_DB + $lap_DB ++ + if ($lap_DB >= $DB:n) set lap_DB = 0 + end + + add_poll_labels run + command $run + end + # success + task.exit 0 + ipptool2book stdout lapRunRuns -uniq -key lap_id -setword dbname $options:0 -setword pantaskState INIT + + process_cleanup lapRunRuns + + if ($VERBOSE > 2) + book listbook lapRunRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task lap.monitor.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 3600 + active false +# This can probably be increased and spread over hosts in the future. + npending 4 + + task.exec + stdout $LOGDIR/lap.monitor.log + stderr $LOGDIR/lap.monitor.log + + book npages lapRunRuns -var N + + if ($N == 0) break + if ($NETWORK == 0) break + + + book getpage lapRunRuns 0 -var lapRunPageName -key pantaskState INIT + + $lap_RunPage ++ + if ($lap_RunPage >= $N) set lap_RunPage = 0 + + if ("$lapRunPageName" == "NULL") break + + book setword lapRunRuns $lapRunPageName pantaskState RUN + book getword lapRunRuns $lapRunPageName lap_id -var LAP_ID + book getword lapRunRuns $lapRunPageName dbname -var DBNAME + + option $LAP_ID + + $run = lap_science.pl --monitor_mode --dbname $DBNAME --lap_id $LAP_ID + + command $run + + end + + # success + task.exit 0 + process_exit lapRunRuns $options:0 0 + if ($VERBOSE > 2) + + book listbook lapRunRuns + end + end + # locked list + task.exit default + process_exit lapRunRuns $options:0 0 + showcommand failure + end + task.exit crash + process_exit lapRunRuns $options:0 0 + showcommand crash + end + #operation times out? + task.exit timeout + process_exit lapRunRuns $options:0 0 + showcommand timeout + end +end + + + +task lap.cleanup.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + active false + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/lap.load.log + + $run = laptool -pendingrun -state full + + if ($lapSeq:n != 0) + option $lapSeq:$lap_N + $run = $run -seq_id $lapSeq:$lap_N + $lap_N ++ + if ($lap_N >= $lapSeq:n) set lap_N = 0 + end + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$lap_DB + $run = $run -dbname $DB:$lap_DB + $lap_DB ++ + if ($lap_DB >= $DB:n) set lap_DB = 0 + end + + add_poll_labels run + + command $run + end + # success + task.exit 0 + ipptool2book stdout lapDoneRuns -uniq -key lap_id -setword dbname $options:0 -setword pantaskState INIT + + process_cleanup lapDoneRuns + if ($VERBOSE > 2) + book listbook lapRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task lap.cleanup.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 3600 + active false +# This can probably be increased and spread over hosts in the future. + npending 1 + + task.exec + stdout $LOGDIR/lap.cleanup.log + stderr $LOGDIR/lap.cleanup.log + + book npages lapDoneRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + + + book getpage lapDoneRuns 0 -var lapDonePageName -key pantaskState INIT + + $lap_DonePage ++ + if ($lap_DonePage >= $N) set lap_DonePage = 0 + + if ("$lapDonePageName" == "NULL") break + + book setword lapDoneRuns $lapDonePageName pantaskState RUN + book getword lapDoneRuns $lapDonePageName lap_id -var LAP_ID + book getword lapDoneRuns $lapDonePageName dbname -var DBNAME + + option $LAP_ID + $run = lap_science.pl --cleanup_mode --dbname $DBNAME --lap_id $LAP_ID + if ("$LAP_QUEUE" != "NULL") + $run = $run --queue_list $LAP_QUEUE + end + + + command $run + + end + + # success + task.exit 0 + process_exit lapDoneRuns $options:0 0 + if ($VERBOSE > 2) + book listbook lapDoneRuns + end + end + # locked list + task.exit default + process_exit lapDoneRuns $options:0 0 + showcommand failure + end + task.exit crash + process_exit lapDoneRuns $options:0 0 + showcommand crash + end + #operation times out? + task.exit timeout + process_exit lapDoneRuns $options:0 0 + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/lapgroup.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/lapgroup.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/lapgroup.pro (revision 41106) @@ -0,0 +1,161 @@ +## lapgroup.pro : tasks for lap group management : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/lapgroup +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init pendinglapGroup + +### Database lists +$lapgroup_DB = 0 + +### Check status of lapgroup tasks +macro lapgroup.status + book listbook pendinglapGroup +end + +### Reset lapgroup tasks +macro lapgroup.reset + book init pendinglapGroup +end + +### Turn lapgroup tasks on +macro lapgroup.on + task lapgroup.load + active true + end + task lapgroup.run + active true + end +end + +### Turn lapgroup tasks off +macro lapgroup.off + task lapgroup.load + active false + end + task lapgroup.run + active false + end +end + +### Load jobs for lapGroup +### Tasks are loaded into pendinglapGroup. +task lapgroup.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/lapgroup.log + + task.exec + if ($LABEL:n == 0) break + $run = laptool -pendinggroup + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$lapgroup_DB + $run = $run -dbname $DB:$lapgroup_DB + $lapgroup_DB ++ + if ($lapgroup_DB >= $DB:n) set lapgroup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pendinglapGroup -key seq_id:tess_id:projection_cell -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook pendinglapGroup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pendinglapGroup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task lapgroup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 120 + npending 1 + + task.exec + book npages pendinglapGroup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in pendinglapGroup (pantaskState == INIT) + book getpage pendinglapGroup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pendinglapGroup $pageName pantaskState RUN + book getword pendinglapGroup $pageName seq_id -var SEQ_ID + book getword pendinglapGroup $pageName tess_id -var TESS_ID + book getword pendinglapGroup $pageName projection_cell -var PROJECTION_CELL + book getword pendinglapGroup $pageName label -var LABEL + book getword pendinglapGroup $pageName dist_group -var DIST_GROUP + book getword pendinglapGroup $pageName dbname -var DBNAME + + stdout $LOGSUBDIR/lapgroup.log + stderr $LOGSUBDIR/lapgroup.log + + host anyhost + + $run = queuestaticsky.pl --seq_id $SEQ_ID --tess_id $TESS_ID --projection_cell $PROJECTION_CELL --label $LABEL --dist_group $DIST_GROUP + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pendinglapGroup $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword pendinglapGroup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pendinglapGroup $options:0 pantaskState TIMEOUT + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/lossy_compress.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/lossy_compress.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/lossy_compress.pro (revision 41106) @@ -0,0 +1,355 @@ +## lossy_compress.pro : tasks to lossy compress old raw images that have negligible science value : -*- sh -*- +## use the books compPendingCompress, compPendingClean and compPendingExp + +# test for required global variables +check.globals + +book init compPendingCompress +book init compPendingClean +book init compPendingExp + +macro lossycomp.reset + book init compPendingCompress + book init compPendingClean + book init compPendingExp +end + +macro lossycomp.status + book list + book listbook compPendingCompress + book listbook compPendingClean + book listbook compPendingExp +end + +macro lossycomp.on + task lossycomp.compress.load + active true + end + task lossycomp.compress.run + active true + end + task lossycomp.clean.load + active true + end + task lossycomp.clean.run + active true + end + task lossycomp.exp.finish + active true + end +end + +macro lossycomp.off + task lossycomp.compress.load + active false + end + task lossycomp.compress.run + active false + end + task lossycomp.clean.load + active false + end + task lossycomp.clean.run + active false + end + task lossycomp.exp.finish + active false + end +end + +# these variables will cycle through the known database names +$compPendingCompress_DB = 0 +$compPendingClean_DB = 0 +$comp_finish_DB = 0 + +# select images ready to be compressed +task lossycomp.compress.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/lossycomp.compress.load.log + + #select entries from the current DB, cycle to the next DB if possible + task.exec + $run = regtool -pendingcompressimfile -compress + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$compPendingCompress_DB + $run = $run -dbname $DB:$compPendingCompress_DB + $compPendingCompress_DB ++ + if ($compPendingCompress_DB >= $DB:n) set compPendingCompress_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout compPendingCompress -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle compPendingCompress + if ($VERBOSE > 2) + book listbook compPendingCompress + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup compPendingCompress + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +# run the lossy_compress_imfile.pl script on the pending images +task lossycomp.compress.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + periods -exec $RUNEXEC + + book npages compPendingCompress -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images + book getpage compPendingCompress 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword compPendingCompress $pageName pantaskState RUN + book getword compPendingCompress $pageName exp_id -var EXP_ID + book getword compPendingCompress $pageName exp_name -var EXP_NAME + book getword compPendingCompress $pageName exp_tag -var EXP_TAG + book getword compPendingCompress $pageName tmp_class_id -var TMP_CLASS_ID + book getword compPendingCompress $pageName class_id -var CLASS_ID + book getword compPendingCompress $pageName camera -var CAMERA + book getword compPendingCompress $pageName uri -var URI + book getword compPendingCompress $pageName bytes -var BYTES + book getword compPendingCompress $pageName md5sum -var MD5SUM + book getword compPendingCompress $pageName workdir -var WORKDIR + book getword compPendingCompress $pageName data_state -var STATE + book getword compPendingCompress $pageName dbname -var DBNAME + + # specify choice of remote host + set.host.for.camera $CAMERA $CLASS_ID + + # set logfile name + if ("$WORKDIR" == "NULL") + sprintf logfile "compress_log/%s.%d.lossycomp.%s.log" $EXP_NAME $EXP_ID $TMP_CLASS_ID + else + sprintf logfile "%s/%s/%s.lossycomp.%s.log" $WORKDIR $EXP_TAG $EXP_TAG $TMP_CLASS_ID + end + + stderr $LOGDIR/lossycomp.compress.run.log + stdout $LOGDIR/lossycomp.compress.run.log + + $run = lossy_compress_imfile.pl --exp_id $EXP_ID --class_id $CLASS_ID --exp_name $EXP_NAME --uri $URI --camera $CAMERA --state $STATE --logfile $logfile --bytes $BYTES --md5sum $MD5SUM + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit compPendingCompress $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword compPendingCompress $options:0 pantaskState CRASH + end + task.exit timeout + showcommand timeout + book setword compPendingCompress $options:0 pantaskState TIMEOUT + end +end +# select images ready to be compressed +task lossycomp.clean.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/lossycomp.clean.load.log + + #select entries from the current DB, cycle to the next DB if possible + task.exec + $run = regtool -pendingcompressimfile -clean + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$compPendingClean_DB + $run = $run -dbname $DB:$compPendingClean_DB + $compPendingClean_DB ++ + if ($compPendingClean_DB >= $DB:n) set compPendingClean_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout compPendingClean -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle compPendingClean + if ($VERBOSE > 2) + book listbook compPendingClean + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup compPendingClean + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +# run the lossy_compress_imfile.pl script on the pending images +task lossycomp.clean.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + periods -exec $RUNEXEC + + book npages compPendingClean -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images + book getpage compPendingClean 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword compPendingClean $pageName pantaskState RUN + book getword compPendingClean $pageName exp_id -var EXP_ID + book getword compPendingClean $pageName exp_name -var EXP_NAME + book getword compPendingClean $pageName exp_tag -var EXP_TAG + book getword compPendingClean $pageName tmp_class_id -var TMP_CLASS_ID + book getword compPendingClean $pageName class_id -var CLASS_ID + book getword compPendingClean $pageName camera -var CAMERA + book getword compPendingClean $pageName uri -var URI + book getword compPendingClean $pageName bytes -var BYTES + book getword compPendingClean $pageName md5sum -var MD5SUM + book getword compPendingClean $pageName workdir -var WORKDIR + book getword compPendingClean $pageName data_state -var STATE + book getword compPendingClean $pageName dbname -var DBNAME + + # specify choice of remote host + set.host.for.camera $CAMERA $CLASS_ID + + # set logfile name + if ("$WORKDIR" == "NULL") + sprintf logfile "compress_log/%s.%d.lossycomp.%s.log" $EXP_NAME $EXP_ID $TMP_CLASS_ID + else + sprintf logfile "%s/%s/%s.lossycomp.%s.log" $WORKDIR $EXP_TAG $EXP_TAG $TMP_CLASS_ID + end + + stderr $LOGDIR/lossycomp.clean.run.log + stdout $LOGDIR/lossycomp.clean.run.log + + $run = lossy_compress_imfile.pl --exp_id $EXP_ID --class_id $CLASS_ID --exp_name $EXP_NAME --uri $URI --camera $CAMERA --state $STATE --logfile $logfile --bytes $BYTES --md5sum $MD5SUM + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit compPendingClean $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword compPendingClean $options:0 pantaskState CRASH + end + task.exit timeout + showcommand timeout + book setword compPendingClean $options:0 pantaskState TIMEOUT + end +end + +# select exposures ready for lossy_compress_exp.pl +task lossycomp.exp.finish + host local + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/lossycomp.exp.load.log + + task.exec + $run = regtool -finishcompressexp -limit 10 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$comp_finish_DB + $run = $run -dbname $DB:$comp_finish_DB + $comp_finish_DB ++ + if ($comp_finish_DB >= $DB:n) set comp_finish_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/magic.cleanup.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/magic.cleanup.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/magic.cleanup.pro (revision 41106) @@ -0,0 +1,219 @@ +## magic.cleanup.pro : support for the cleaning magic temporary files : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/magic +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicToCleanup + +### Database lists +$magicToCleanup_DB = 0 + +### Check status of tasks +macro magic.cleanup.status + book listbook magicToCleanup +end + +### Reset tasks +macro magic.cleanup.reset + book init magicToCleanup +end + +### Turn tasks on +macro magic.cleanup.revert.on + task magic.cleanup.revert + active true + end +end + +macro magic.cleanup.on + task magic.cleanup.load + active true + end + task magic.cleanup.run + active true + end +end + +macro magic.cleanup.revert.off + task magic.cleanup.revert + active false + end +end + +### Turn tasks off +macro magic.cleanup.off + task magic.cleanup.load + active false + end + task magic.cleanup.run + active false + end +end + +task magic.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/magic.cleanup.load.log + + task.exec + $run = magictool -tocleanup + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicToCleanup_DB + $run = $run -dbname $DB:$magicToCleanup_DB + $magicToCleanup_DB ++ + if ($magicToCleanup_DB >= $DB:n) set magicToCleanup_DB = 0 + end + add_poll_args run + # XXX: add this back in once our inital cleanup is done +# add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToCleanup -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task magic.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + + task.exec + periods -exec $RUNEXEC + + book npages magicToCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + book getpage magicToCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicToCleanup $pageName pantaskState RUN + book getword magicToCleanup $pageName magic_id -var MAGIC_ID + book getword magicToCleanup $pageName exp_id -var EXP_ID + book getword magicToCleanup $pageName camera -var CAMERA + book getword magicToCleanup $pageName workdir -var WORKDIR + book getword magicToCleanup $pageName dbname -var DBNAME + + host anyhost + + sprintf outroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID + + sprintf logfile "%s.cleanup.log" $outroot + + stdout $logfile + stderr $logfile + + $run = magic_cleanup.pl --magic_id $MAGIC_ID --exp_id $EXP_ID --camera $CAMERA --outroot $outroot --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit magicToCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicToCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicToCleanup $options:0 pantaskState TIMEOUT + end +end + +task magic.cleanup.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/revertnode.log + + task.exec + if ($LABEL:n == 0) break + $run = magictool -revertnode + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicRevertNode_DB + $run = $run -dbname $DB:$magicRevertNode_DB + $magicRevertNode_DB ++ + if ($magicRevertNode_DB >= $DB:n) set magicRevertNode_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/magic.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/magic.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/magic.pro (revision 41106) @@ -0,0 +1,451 @@ +## magic.pro : support for the streak removal : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/magic +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicToTree +book init magicToProcess + +### Database lists +$magicToTree_DB = 0 +$magicToProcess_DB = 0 +$magicRevertTree_DB = 0 +$magicRevertNode_DB = 0 + +### Check status of tasks +macro magic.status + book listbook magicToTree + book listbook magicToProcess +end + +### Reset tasks +macro magic.reset + book init magicToTree + book init magicToProcess +end + +### Turn tasks on +macro magic.revert.on + task magic.revert.node + active true + end + task magic.revert.tree + active true + end +end + +macro magic.on + task magic.tree.load + active true + end + task magic.tree.run + active true + end + task magic.process.load + active true + end + task magic.process.run + active true + end + magic.revert.on +end + +macro magic.revert.off + task magic.revert.node + active false + end + task magic.revert.tree + active false + end +end + +### Turn tasks off +macro magic.off + task magic.tree.load + active false + end + task magic.tree.run + active false + end + task magic.process.load + active false + end + task magic.process.run + active false + end + magic.revert.off +end + +task magic.tree.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/magic.tree.log + + task.exec + $run = magictool -totree + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicToTree_DB + $run = $run -dbname $DB:$magicToTree_DB + $magicToTree_DB ++ + if ($magicToTree_DB >= $DB:n) set magicToTree_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToTree -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToTree + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToTree + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task magic.tree.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages magicToTree -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicToTree 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicToTree $pageName pantaskState RUN + book getword magicToTree $pageName magic_id -var MAGIC_ID + book getword magicToTree $pageName exp_id -var EXP_ID + book getword magicToTree $pageName camera -var CAMERA + book getword magicToTree $pageName workdir -var WORKDIR_TEMPLATE + book getword magicToTree $pageName dbname -var DBNAME + book getword magicToTree $pageName tess_id -var TESS_DIR + book getword magicToTree $pageName ra -var RA + book getword magicToTree $pageName decl -var DEC + +# set.host.for.camera $CAMERA $MAGIC_ID +# set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR + host anyhost + $WORKDIR = $WORKDIR_TEMPLATE + + sprintf outroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID + + ## generate output log based on filerule (convert the URI to a PATH) + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot` + if ("$logfile" == "") + echo "WARNING: logfile not defined in magic.tree.run" + break + end + + stdout $logfile + stderr $logfile + dirname $logfile -var outpath + mkdir $outpath + + $run = magic_tree.pl --magic_id $MAGIC_ID --camera $CAMERA --tess_id $TESS_DIR --ra $RA --dec $DEC --outroot $outroot --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit magicToTree $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicToTree $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicToTree $options:0 pantaskState TIMEOUT + end +end + +task magic.process.load + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 300 + npending 1 + + stdout NULL + stderr $LOGDIR/magic.process.log + + task.exec + $run = magictool -toprocess + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicToProcess_DB + $run = $run -dbname $DB:$magicToProcess_DB + $magicToProcess_DB ++ + if ($magicToProcess_DB >= $DB:n) set magicToProcess_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToProcess -key magic_id:node -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToProcess + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToProcess + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task magic.process.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + periods -exec 10 + + book npages magicToProcess -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicToProcess 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicToProcess $pageName pantaskState RUN + book getword magicToProcess $pageName magic_id -var MAGIC_ID + book getword magicToProcess $pageName exp_id -var EXP_ID + book getword magicToProcess $pageName node -var NODE + book getword magicToProcess $pageName camera -var CAMERA + book getword magicToProcess $pageName workdir -var WORKDIR_TEMPLATE + book getword magicToProcess $pageName raw_workdir -var RAW_TEMPLATE + book getword magicToProcess $pageName exp_name -var EXP_NAME + book getword magicToProcess $pageName dbname -var DBNAME + +# XXX: we need new macros that select the host and workdir based on the NODE +# we also need to make the magic scripts able to deal with the workdir being spread +# around. That should not be too hard +# For now select nodes by skycell + + substr $NODE 0 3 NODE_HEAD + if ("$NODE_HEAD" == "sky") + set.host.for.skycell $NODE + else + host anyhost + end + +# currently DetectStreaks expects that the workdir is not in nebulous +# set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + $WORKDIR = $WORKDIR_TEMPLATE + + sprintf baseroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID + + if ("$NODE" == "root") + # This is the root node, put the streaks file into the rawExps workdir + # which is presumably in nebulous and thus will be replicatable + set.workdir.by.camera $CAMERA FPA $RAW_TEMPLATE $default_host RAW_WORKDIR + sprintf final_outroot "%s/%s.%s/%s.%s.mgc.%s" $RAW_WORKDIR $EXP_NAME $EXP_ID $EXP_NAME $EXP_ID $MAGIC_ID + + $EXTRA_ARGS = --final-outroot $final_outroot + else + $EXTRA_ARGS = "" + end + + $logfile = $baseroot.$NODE.log + if ("$logfile" == "") + echo "WARNING: logfile not defined in magic.process.run" + break + end + + stdout $logfile + stderr $logfile + dirname $logfile -var outpath + mkdir $outpath + + $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile $EXTRA_ARGS + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit magicToProcess $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicToProcess $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicToProcess $options:0 pantaskState TIMEOUT + end +end + +task magic.revert.node + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/revertnode.log + + task.exec + if ($LABEL:n == 0) break + $run = magictool -revertnode + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicRevertNode_DB + $run = $run -dbname $DB:$magicRevertNode_DB + $magicRevertNode_DB ++ + if ($magicRevertNode_DB >= $DB:n) set magicRevertNode_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end +task magic.revert.tree + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/reverttree.log + + task.exec + if ($LABEL:n == 0) break + $run = magictool -reverttree + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$magicRevertTree_DB + $run = $run -dbname $DB:$magicRevertTree_DB + $magicRevertTree_DB ++ + if ($magicRevertTree_DB >= $DB:n) set magicRevertTree_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/mergedvodb.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/mergedvodb.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/mergedvodb.pro (revision 41106) @@ -0,0 +1,226 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + + +#There is a book for each task, because I don't use labels. + +if (not($?havemergedvodbBooks)) + book create MERGEDVODB_MERGE + $havemergedvodbBooks = TRUE +end + +$MERGEDVODB_DB = 0 +$MERGEDVODB_MERGE_DB = 0 + +book init mergedvodbMergelist + +macro mergedvodb.merge.status + book listbook mergedvodbMergelist +end + +macro mergedvodb.merge.reset + book init mergedvodbMergelist +end + +macro mergedvodb.merge.on + task mergedvodb.merge.load + active true + end + task mergedvodb.merge.run + active true + end +end +macro mergedvodb.merge.off + task mergedvodb.merge.load + active false + end + task mergedvodb.merge.run + active false + end +end + +## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. +macro add.mergedvodb + if ($0 != 2) + echo "USAGE: add.mergedvodb (mergedvodb)" + break + end + #merge + book newpage MERGEDVODB_MERGE $1 + book setword MERGEDVODB_MERGE $1 MERGEDVODB $1 + book setword MERGEDVODB_MERGE $1 STATE PENDING +end + +macro del.mergedvodb + if ($0 != 2) + echo "USAGE: del.mergedvodb (mergedvodb)" + break + end + book delpage MERGEDVODB_MERGE $1 +end + +macro show.mergedvodb + if ($0 != 1) + echo "USAGE: show.mergedvodb" + break + end + echo "mergedvodb merge" + book listbook MERGEDVODB_MERGE + echo "n pages" + book npages MERGEDVODB_MERGE -var N + echo $N +end + + +task mergedvodb.merge.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/mergedvodb.merge.load.log + + task.exec + book npages MERGEDVODB_MERGE -var N + if ($N == 0) + echo "No labels for processing" + break + endif + book getpage MERGEDVODB_MERGE 0 -var mergedvodb -key STATE NEW + if ("$mergedvodb" == "NULL") + # All labels have been done --- reset + for i 0 $N + book getpage MERGEDVODB_MERGE $i -var mergedvodb + book setword MERGEDVODB_MERGE $mergedvodb STATE NEW + end + book getpage MERGEDVODB_MERGE 0 -var mergedvodb -key STATE NEW + + # Select different database + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + book setword MERGEDVODB_MERGE $mergedvodb STATE DONE + + $run = mergetool -pendingmerge -limit 1 + $run = $run -mergedvodb $mergedvodb + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MERGEDVODB_DB + $run = $run -dbname $DB:$MERGEDVODB_DB + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + #add_poll_args run + #echo $run + command $run + end + + # success + task.exit 0 + #echo "success" + # convert 'stdout' to book format + ipptool2book stdout mergedvodbMergelist -key merge_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook mergedvodbMergelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup mergedvodbMergelist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task mergedvodb.merge.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60000 + + #we only want one running at a time + + host local + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + book npages mergedvodbMergelist -var N + if ($N == 0) break + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage mergedvodbMergelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + book setword mergedvodbMergelist $pageName pantaskState RUN + book getword mergedvodbMergelist $pageName merge_id -var MERGE_ID + book getword mergedvodbMergelist $pageName mergedvodb -var MERGEDVODB + book getword mergedvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH + book getword mergedvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword mergedvodbMergelist $pageName state -var STATE + stdout $LOGDIR/mergedvodb.merge.run.log + stderr $LOGDIR/mergedvodb.merge.run.log + $run = mergedvodb_merge.pl --camera GPC1 + $run = $run --mergedvodb_path $MERGEDVODB_PATH + $run = $run --minidvodb_path $MINIDVODB_PATH + $run = $run --merge_id $MERGE_ID --mergedvodb $MERGEDVODB + echo $run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MERGEDVODB_DB + $run = $run --dbname $DB:$MERGEDVODB_DB + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit mergedvodbMergelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + + echo "hostname: $JOB_HOSTNAME" + + # Set a fault code in the database + + process_exit mergedvodbMergelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword mergedvodbMergelist $options:0 pantaskState TIMEOUT + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/minidvodb.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/minidvodb.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/minidvodb.pro (revision 41106) @@ -0,0 +1,739 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + + +#There is a book for each task, because I don't use labels. + +if (not($?haveminidvodbBooks)) + book create MINIDVODB + book create MINIDVODB_PREMERGE + book create MINIDVODB_CREATE + book create MINIDVODB_ACTIVE + $haveminidvodbBooks = TRUE +end + +$MINIDVODB_DB = 0 + +$MINIDVODB_WAIT_DB = 0 +$MINIDVODB_PREMERGE_DB = 0 +$MINIDVODB_CREATE_DB = 0 +$MINIDVODB_ACTIVE_DB = 0 + +book init minidvodbWaitlist +book init minidvodbPreMergelist +book init minidvodbCreatelist +book init minidvodbActivelist + +macro minidvodb.create.status + book listbook minidvodbCreatelist +end + +macro minidvodb.create.reset + book init minidvodbCreatelist +end + +macro minidvodb.active.status + book listbook minidvodbActivelist +end + +macro minidvodb.active.reset + book init minidvodbActivelist +end + + +macro minidvodb.wait.status + book listbook minidvodbWaitlist +end + +macro minidvodb.wait.reset + book init minidvodbWaitlist +end + +macro minidvodb.premerge.status + book listbook minidvodbPreMergelist +end + +macro minidvodb.premerge.reset + book init minidvodbPreMergelist +end + +#this is the create task, there is no load, it just runs +macro minidvodb.create.on + task minidvodb.create + active true + end +end +macro minidvodb.create.off + task minidvodb.create + active false + end +end +##these are the tasks that flip from new -> active +macro minidvodb.active.on + task minidvodb.active.load + active true + end + task minidvodb.active.run + active true + end +end +macro minidvodb.active.off + task minidvodb.active.load + active false + end + task minidvodb.active.run + active false + end +end + +##these are the tasks that check to see if addRun processing is finished +##for a minidvodb in state wait +macro minidvodb.wait.on + task minidvodb.wait.load + active true + end + task minidvodb.wait.run + active true + end +end +macro minidvodb.wait.off + task minidvodb.wait.load + active false + end + task minidvodb.wait.run + active false + end +end + +##these merge the dbs + +macro minidvodb.premerge.on + task minidvodb.premerge.load + active true + end + task minidvodb.premerge.run + active true + end +end +macro minidvodb.premerge.off + task minidvodb.premerge.load + active false + end + task minidvodb.premerge.run + active false + end +end + +## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. +macro add.minidvodb + if ($0 != 7) + echo "USAGE: add.minidvodb (minidvodb_group) (minidvodb) (dvodb) (interval days) (interval addruns) (camera)" + break + end + + #wait - shoudl be renamed MINIDVODB_WAIT + book newpage MINIDVODB $1 + book setword MINIDVODB $1 MINIDVODB_GROUP $1 + book setword MINIDVODB $1 DVODB $3 + book setword MINIDVODB $1 STATE PENDING + #merge + book newpage MINIDVODB_PREMERGE $1 + book setword MINIDVODB_PREMERGE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_PREMERGE $1 DVODB $3 + book setword MINIDVODB_PREMERGE $1 CAMERA $5 + book setword MINIDVODB_PREMERGE $1 STATE PENDING + #active + book newpage MINIDVODB_ACTIVE $1 + book setword MINIDVODB_ACTIVE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_ACTIVE $1 DVODB $3 + book setword MINIDVODB_ACTIVE $1 STATE PENDING + + #create note that camera should be GPC1 for it to work. I couldn't figure out how to easily get this out. + book newpage MINIDVODB_CREATE $1 + book setword MINIDVODB_CREATE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_CREATE $1 MINIDVODB $2 + book setword MINIDVODB_CREATE $1 DVODB $3 + book setword MINIDVODB_CREATE $1 DVODB_DAYS $4 + book setword MINIDVODB_CREATE $1 DVODB_NUM $5 + book setword MINIDVODB_CREATE $1 CAMERA $6 + book setword MINIDVODB_CREATE $1 STATE PENDING +end + +macro del.minidvodb + if ($0 != 2) + echo "USAGE: del.minidvodb (minidvodb)" + break + end + book delpage MINIDVODB $1 + book delpage MINIDVODB_PREMERGE $1 + book delpage MINIDVODB_CREATE $1 + book delpage MINIDVODB_ACTIVE $1 +end + +macro show.minidvodb + if ($0 != 1) + echo "USAGE: show.minidvodb" + break + end + echo "minidvodb wait" + book listbook MINIDVODB + echo "minidvodb premerge" + book listbook MINIDVODB_PREMERGE + echo "minidvodb create" + book listbook MINIDVODB_CREATE + echo "minidvodb_active" + book listbook MINIDVODB_ACTIVE +end + + +$LOADEXEC_MDB = 300 + + + + + +task minidvodb.wait.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC_MDB + periods -timeout 300 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.wait.load.log + + task.exec + book npages MINIDVODB -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB $i -var minidvodb_group + book setword MINIDVODB $minidvodb_group STATE NEW + end + book getpage MINIDVODB 0 -var minidvodb_group -key STATE NEW + + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end +#using check as opposed to list because it sees if it is done with the addRun state yet. + book setword MINIDVODB $minidvodb_group STATE DONE + $run = addtool -checkminidvodbrunaddrun -state waiting + $run = $run -minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + add_poll_args run + + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout minidvodbWaitlist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbWaitlist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbWaitlist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +task minidvodb.wait.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbWaitlist -var N + if ($N == 0) break + + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage minidvodbWaitlist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword minidvodbWaitlist $pageName pantaskState RUN + book getword minidvodbWaitlist $pageName minidvodb_id -var MINIDVODB_ID + book getword minidvodbWaitlist $pageName state -var STATE + stdout $LOGDIR/minidvodb.wait.run.log + stderr $LOGDIR/minidvodb.wait.run.log + + $run = addtool -updateminidvodbrun -minidvodb_id $MINIDVODB_ID -set_state to_be_merged + + +if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit minidvodbWaitlist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + echo "hostname: $JOB_HOSTNAME" + process_exit minidvodbWaitlist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbWaitlist $options:0 pantaskState TIMEOUT + end +end + + + +task minidvodb.premerge.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC_MDB + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.premerge.load.log + + task.exec + book npages MINIDVODB_PREMERGE -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_PREMERGE $i -var minidvodb_group + book setword MINIDVODB_PREMERGE $minidvodb_group STATE NEW + end + book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW + + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + #finds the minidvodbs in a state of 'to_be_merged' + book setword MINIDVODB_PREMERGE $minidvodb_group STATE DONE + $run = addtool -listminidvodbrun -state to_be_merged -limit 1 + $run = $run -minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + add_poll_args run + + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout minidvodbPreMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbPreMergelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbPreMergelist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +task minidvodb.premerge.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60000 + + #we only want one running at a time + + host local + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbPreMergelist -var N + if ($N == 0) break + + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage minidvodbPreMergelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword minidvodbPreMergelist $pageName pantaskState RUN + book getword minidvodbPreMergelist $pageName minidvodb_id -var MINIDVODB_ID + book getword minidvodbPreMergelist $pageName minidvodb_group -var MINIDVODB_GROUP + book getword minidvodbPreMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword minidvodbPreMergelist $pageName camera -var CAMERA + book getword minidvodbPreMergelist $pageName state -var STATE + stdout $LOGDIR/minidvodb.premerge.run.log + stderr $LOGDIR/minidvodb.premerge.run.log + + #still buggy + + $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run --dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit minidvodbPreMergelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + + echo "hostname: $JOB_HOSTNAME" + + # Set a fault code in the database + + process_exit minidvodbPreMergelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbPreMergelist $options:0 pantaskState TIMEOUT + end +end + + +#this is the complicated script - it creates the minidvodbRuns +#since none may exist by default, I decided not to have a load.task +#it calls a perl script, which creates a new one if: +#there is none in an active state (true if there is none at all) +#the current one is older than the dvodb_age +#there are none in new (we have a task that moves it to 'active') +#the current active one has > 30000 add_ids in it + +#also confusing: it succeeds if it doesn't create it (if it doesn't want to create one) +#and it succeeds if it creates one. (otherwise there would be a lot of false failures, +# like in replication) +#it fails if it has problems with one of the addtool commands in the script. + +task minidvodb.create + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC_MDB + periods -timeout 240 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.create.log + + task.exec + + periods -poll 60 +#wait a bit before trying again + + book npages MINIDVODB_CREATE -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_CREATE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_CREATE $i -var minidvodb_group + book setword MINIDVODB_CREATE $minidvodb_group STATE NEW + end + book getpage MINIDVODB_CREATE 0 -var minidvodb_group -key STATE NEW + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + + book setword MINIDVODB_CREATE $minidvodb_group STATE DONE + book getword MINIDVODB_CREATE $minidvodb_group MINIDVODB -var minidvodb + book getword MINIDVODB_CREATE $minidvodb_group DVODB -var dvodb + book getword MINIDVODB_CREATE $minidvodb_group DVODB_DAYS -var dvodb_days + book getword MINIDVODB_CREATE $minidvodb_group DVODB_NUM -var dvodb_num + book getword MINIDVODB_CREATE $minidvodb_group CAMERA -var camera + $run = minidvodb_createdb.pl --camera $camera --outroot $dvodb --dvodb $dvodb --minidvodb $minidvodb --minidvodb_group $minidvodb_group --interval $dvodb_days --num $dvodb_num + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run --dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + command $run + end + + # success + task.exit 0 + if ($VERBOSE > 2) + showcommand + end + end + + # locked list + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + +task minidvodb.active.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC_MDB + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.active.load.log + + task.exec + book npages MINIDVODB_ACTIVE -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_ACTIVE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_ACTIVE $i -var minidvodb_group + book setword MINIDVODB_ACTIVE $minidvodb_group STATE NEW + end + book getpage MINIDVODB_ACTIVE 0 -var minidvodb_group -key STATE NEW + + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + book setword MINIDVODB_ACTIVE $minidvodb_group STATE DONE + + $run = addtool -listminidvodbrun -state new + $run = $run -minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + add_poll_args run + + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout minidvodbActivelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbActivelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbActivelist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +task minidvodb.active.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbActivelist -var N + if ($N == 0) break + + # look for new images in minidvodbActivelist (pantaskState == INIT) + book getpage minidvodbActivelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword minidvodbActivelist $pageName pantaskState RUN + book getword minidvodbActivelist $pageName minidvodb_group -var MINIDVODB_GROUP + stdout $LOGDIR/minidvodb.active.run.log + stderr $LOGDIR/minidvodb.active.run.log + + $run = addtool -flipminidvodbrun -minidvodb_group $MINIDVODB_GROUP + + +if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end +# save the pageName for future reference below + options $pageName + + + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end +# default exit status + task.exit default + process_exit minidvodbActivelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + #showcommand crash + echo "hostname: $JOB_HOSTNAME" + # Set a fault code in the database + + process_exit minidvodbActivelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbActivelist $options:0 pantaskState TIMEOUT + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/minidvodbcopy.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/minidvodbcopy.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/minidvodbcopy.pro (revision 41106) @@ -0,0 +1,435 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + + +#There is a book for each task, because I don't use labels. + +if (not($?haveminidvodbCopyBooks)) + book create MINIDVODB_COPY + book create MINIDVODB_HOST + $haveminidvodbCopyBooks = TRUE +end + +$MINIDVODBCOPY_DB = 0 +$MINIDVODB_HOSTS_DB = 0 + +book init minidvodbCopylist + +macro minidvodb.copy.status + book listbook minidvodbCopylist +end + +macro minidvodb.copy.reset + book init minidvodbCopylist +end + +#this is the task that manages the copying of minidvodbs to another machine +macro minidvodb.copy.on + task minidvodb.copy.load + active true + end + task minidvodb.copy.run + active true + end + task minidvodb.copy.queue + active true + end +end + +macro minidvodb.copy.queue.on + task minidvodb.copy.queue + active true + end +end + +macro minidvodb.copy.queue.off + task minidvodb.copy.queue + active false + end +end + +macro minidvodb.copy.load.on + task minidvodb.copy.load + active true + end +end + +macro minidvodb.copy.load.off + task minidvodb.copy.load + active false + end +end + +macro minidvodb.copy.run.on + task minidvodb.copy.run + active true + end +end + +macro minidvodb.copy.run.off + task minidvodb.copy.run + active false + end +end + +macro minidvodb.copy.off + task minidvodb.copy.load + active false + end + task minidvodb.copy.run + active false + end + task minidvodb.copy.queue + active false + end +end + + +## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. +macro add.minidvodb.copy + if ($0 != 4) + echo "USAGE: add.minidvodb.copy (minidvodb_group) (host) (minidvodbcopy_path)" + break + end + $hostandgroup = $2.$1 + #wait - shoudl be renamed MINIDVODB_WAIT + book newpage MINIDVODB_COPY $hostandgroup + book setword MINIDVODB_COPY $hostandgroup MINIDVODBHOSTANDGROUP $hostandgroup + book setword MINIDVODB_COPY $hostandgroup MINIDVODB_GROUP $1 + book setword MINIDVODB_COPY $hostandgroup HOST $2 + book setword MINIDVODB_COPY $hostandgroup MINIDVODB_COPY_PATH $3 + book setword MINIDVODB_COPY $hostandgroup STATE PENDING + book setword MINIDVODB_COPY $hostandgroup QUEUE_STATE PENDING +end + +macro del.minidvodb.copy + if ($0 != 3) + echo "USAGE: del.minidvodb.copy (host) (minidvodb_group)" + break + end + book delpage MINIDVODB_COPY $1.$2 +end + +macro show.minidvodb.copy + if ($0 != 1) + echo "USAGE: show.minidvodb.copy" + break + end + echo "minidvodb copy" + book listbook MINIDVODB_COPY +end + +macro add.minidvodb.host + if ($0 != 2) + echo "USAGE: add.minidvodb.host (host)" + break + end + #wait - shoudl be renamed MINIDVODB_WAIT + book newpage MINIDVODB_HOST $1 + book setword MINIDVODB_HOST $1 HOST $1 + +end + +macro del.minidvodb.host + if ($0 != 2) + echo "USAGE: del.minidvodb.host (host)" + break + end + book delpage MINIDVODB_HOST $1 +end + +macro show.minidvodb.host + if ($0 != 1) + echo "USAGE: show.minidvodb.host" + break + end + echo "minidvodb hosts" + book listbook MINIDVODB_HOST +end + +task minidvodb.copy.queue + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.copy.queue.log + + task.exec + book npages MINIDVODB_COPY -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_COPY 0 -var minidvodbhostandgroup -key QUEUE_STATE NEW + if ("$minidvodbhostandgroup" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_COPY $i -var minidvodbhostandgroup + book setword MINIDVODB_COPY $minidvodbhostandgroup QUEUE_STATE NEW + end + book getpage MINIDVODB_COPY 0 -var minidvodbhostandgroup -key QUEUE_STATE NEW + book getword MINIDVODB_COPY $minidvodbhostandgroup HOST -var host + book getword MINIDVODB_COPY $minidvodbhostandgroup MINIDVODB_GROUP -var minidvodb_group + book getword MINIDVODB_COPY $minidvodbhostandgroup MINIDVODB_COPY_PATH -var minidvodb_copy_path + + + # echo $minidvodb_group $minidvodb_copy_path + # book getword MINIDVODB_COPY 0 minidvodb_copy_path -var MINIDVODB_COPY_PATH + #this needs work - it is setting everything to null + #also, I want the database (I think?) + + # Select different database + $MINIDVODB_COPY_DB ++ + if ($MINIDVODB_COPY_DB >= $DB:n) set MINIDVODB_COPY_DB = 0 + end +#using check as opposed to list because it sees if it is done with the addRun state yet. + book setword MINIDVODB_COPY $minidvodbhostandgroup QUEUE_STATE DONE + + book npages MINIDVODB_HOST -var NUM + if ($N ==0) + break + endif + + book getpage MINIDVODB_HOST $host -var host2 -key HOST $host + if ($host2 != $host) + break + end + + #echo $host $host2 + + + $run = minidvodbtool -definebyquery -set_destination_host $host -set_minidvodb_rsync_path $minidvodb_copy_path + $run = $run -minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_COPY_DB + $run = $run -dbname $DB:$MINIDVODB_COPY_DB + $MINIDVODB_COPY_DB ++ + if ($MINIDVODB_COPY_DB >= $DB:n) set MINIDVODB_COPY_DB = 0 + end + #add_poll_args run + #echo $run + command $run + end + # success + task.exit 0 + # convert 'stdout' to book format + #ipptool2book stdout minidvodCopylist -key minidvodbcopy_id -uniq -setword dbname $options:0 -setword pantaskState INIT + #if ($VERBOSE > 2) + # book listbook minidvodbCopylist + #end + # delete existing entries in the appropriate pantaskStates + #process_cleanup minidvodbCopylist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + + + + + + + +task minidvodb.copy.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.copy.load.log + + task.exec + book npages MINIDVODB_COPY -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_COPY 0 -var minidvodbhostandgroup -key STATE NEW + if ("$minidvodbhostandgroup" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_COPY $i -var minidvodbhostandgroup + book setword MINIDVODB_COPY $minidvodbhostandgroup STATE NEW + end + book getpage MINIDVODB_COPY 0 -var minidvodbhostandgroup -key STATE NEW + book getword MINIDVODB_COPY $minidvodbhostandgroup HOST -var host + book getword MINIDVODB_COPY $minidvodbhostandgroup MINIDVODB_GROUP -var minidvodb_group + + + # book getword MINIDVODB_COPY 0 minidvodb_copy_path -var MINIDVODB_COPY_PATH + #this needs work - it is setting everything to null + #also, I want the database (I think?) + + # Select different database + $MINIDVODB_COPY_DB ++ + if ($MINIDVODB_COPY_DB >= $DB:n) set MINIDVODB_COPY_DB = 0 + end +#using check as opposed to list because it sees if it is done with the addRun state yet. + book setword MINIDVODB_COPY $minidvodbhostandgroup STATE DONE + + book npages MINIDVODB_HOST -var NUM + if ($N ==0) + break + endif + + book getpage MINIDVODB_HOST $host -var host2 -key HOST $host + if ($host2 != $host) + break + end + + #echo $host $host2 + + + $run = minidvodbtool -listminidvodbcopy -destination_host $host + $run = $run -minidvodb_group $minidvodb_group -pending + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_COPY_DB + $run = $run -dbname $DB:$MINIDVODB_COPY_DB + $MINIDVODB_COPY_DB ++ + if ($MINIDVODB_COPY_DB >= $DB:n) set MINIDVODB_COPY_DB = 0 + end + add_poll_args run + #echo $run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout minidvodbCopylist -key minidvodbcopy_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbCopylist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbCopylist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +task minidvodb.copy.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 17280 + #if rsync is slow I want to make sure it doesn't time out + + ## we want only a single outstanding rsync job. + host local + npending 1 + + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbCopylist -var N + if ($N == 0) break + + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage minidvodbCopylist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + echo "got here" + book setword minidvodbCopylist $pageName pantaskState RUN + book getword minidvodbCopylist $pageName minidvodbcopy_id -var MINIDVODB_COPY_ID + book getword minidvodbCopylist $pageName minidvodb_path -var MINIDVODBRUN_PATH + book getword minidvodbCopylist $pageName minidvodb_id -var MINIDVODB_ID + book getword minidvodbCopylist $pageName minidvodb_rsync_path -var MINIDVODB_RSYNC_PATH + book getword minidvodbCopylist $pageName destination_host -var DESTINATION_HOST + book getword minidvodbCopylist $pageName state -var STATE + stdout $LOGDIR/minidvodb.copy.run.log + stderr $LOGDIR/minidvodb.copy.run.log + + $run = minidvodb_copy.pl --minidvodbcopy_id $MINIDVODB_COPY_ID --minidvodb_id $MINIDVODB_ID +# and more things + $run = $run --minidvodbrun_path $MINIDVODBRUN_PATH --minidvodb_rsync_path $MINIDVODB_RSYNC_PATH + $run = $run --destination_host $DESTINATION_HOST + +if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_COPY_DB + $run = $run --dbname $DB:$MINIDVODB_COPY_DB + $MINIDVODB_COPY_DB ++ + if ($MINIDVODB_COPY_DB >= $DB:n) set MINIDVODB_COPY_DB = 0 + end + # save the pageName for future reference below + options $pageName + echo command $run + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit minidvodbCopylist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + echo "hostname: $JOB_HOSTNAME" + process_exit minidvodbCopylist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbCopylist $options:0 pantaskState TIMEOUT + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/mkdetrend.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/mkdetrend.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/mkdetrend.config (revision 41106) @@ -0,0 +1,43 @@ + +detRunDef MULTI + +# add in the min number of input images and a time range +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK + prereq STR BIAS + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR DARK + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" + dbname STR eamtest + pantaskState STR NULL +END Index: /branches/ccl_branches/ipponly-20191108/ippTasks/mkdetrend.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/mkdetrend.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/mkdetrend.pro (revision 41106) @@ -0,0 +1,217 @@ +## mkdetrend.pro : autorun the detrend analysis : -*- sh -*- + +# detRun METADATA +# det_id STR 3 +# iteration S32 0 +# det_type STR SHUTTER +# mode STR master +# state STR run +# filelevel STR FPA +# END + +macro load.detRunDefs + if ($0 != 2) + echo "USAGE: load.detRunDefs (filename)" + break + end + queueload detruns -x "cat $MODULES:0/$1" + ipptool2book detruns detRunDef -key detRunName -uniq -setword pantaskState INIT + queuedelete detruns +end + +task detrun.launch + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + # each outstanding detrend run blocks the next one. only launch the next + # if the current one is done + task.exec + local i j N found missed detRunName prereq prereqList prereqState cmdOptions dbname + + # how many detRunDefs are waiting to be started? + book npages detRunDef -var N -key pantaskState INIT + # echo "detrun.launch N: $N" + if ($N == 0) break + + $found = 0 + + # search the detRunDef book for an entry which is unstarted (state NULL) + # and for which the dependencies are done + for i 0 $N + book getpage detRunDef $i -var detRunName -key pantaskState INIT + # echo "detrun.launch detRunName: $detRunName" + if ("$detRunName" == "NULL") break + + book getword detRunDef $detRunName prereq -var prereqList + if (("$prereqList" == "NULL") || ("$prereqList" == "NONE")) + $found = 1 + last + end + + list prereq -split $prereqList + $missed = 0 + for j 0 $prereq:n + # prereq must exist and be in state of DONE + book getword detRunDef $prereq:$j pantaskState -var prereqState + + # echo "detrun.launch prereqState: $prereqState" + # echo "detrun.launch detRunName: $detRunName" + if ("$prereqState" != "DONE") + $missed = 1 + last + end + end + if ($missed == 0) + $found = 1 + last + end + + if ($found == 0) + if ($VERBOSE >= 2) + echo "$detRunName not ready" + end + else + if ($VERBOSE >= 2) + echo "$detRunName ready" + end + end + end + + if ($found == 0) break + + # echo "detrun.launch detRunName: $detRunName" + + book getword detRunDef $detRunName options -var cmdOptions + book getword detRunDef $detRunName dbname -var dbname + book setword detRunDef $detRunName pantaskState INIT + + if ($VERBOSE >= 1) + echo "starting detrend $detRunName :" + echo "command dettool $cmdOptions -dbname $dbname" + end + + options $dbname $detRunName + command dettool $cmdOptions -dbname $dbname + end + + task.exit $EXIT_SUCCESS + local detRunID + + $dbname = $options:0 + $detRunName = $options:1 + + # convert 'stdout' to book format + ipptool2book stdout detRun -key det_id -uniq + if ($VERBOSE > 2) + book listbook detRun + end + + # we should have launched only a single detRun + book getpage detRun 0 -var detRunID + if ("$detRunID" == "NULL") + echo "ERROR:detRun not defined" + break + end + + # we need to track the detID generated + book setword detRunDef $detRunName detRunID $detRunID + book setword detRunDef $detRunName pantaskState RUN + + # drop the detRun book after we've grabbed the state + book delete detRun + end +end + +$detRunNcheck = 0 + +# loop over the outstanding detRuns and update state +task detrun.check + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + # each outstanding detrend run blocks the next one. only launch the next + # if the current one is done + task.exec + local i N found detRunName detRunID dbname + + book npages detRunDef -var N -key pantaskState RUN + if ($N == 0) break + if ($detRunNcheck >= $N) + $detRunNcheck = 0 + end + + # try the next detRun entry which is in state RUN + book getpage detRunDef $detRunNcheck -var detRunName -key pantaskState RUN + $detRunNcheck ++ + + if ("$detRunName" == "NULL") + if ($VERBOSE >= 3) + echo "no more active detruns (Ncheck = $detRunNcheck)" + end + break + end + + # info needed to check on current state + book getword detRunDef $detRunName detRunID -var detRunID + book getword detRunDef $detRunName dbname -var dbname + + if ($VERBOSE >= 2) + echo " checking on $detRunName" + echo "command dettool -runs -det_id $detRunID -dbname $dbname" + end + + # we need a dettool command which takes a specific det_id and returns + # the current state + options $dbname $detRunName + command dettool -runs -det_id $detRunID -dbname $dbname + end + + task.exit $EXIT_SUCCESS + local detRunID state + + $dbname = $options:0 + $detRunName = $options:1 + + # convert 'stdout' to book format + ipptool2book stdout detRun -key det_id -uniq + if ($VERBOSE > 2) + book listbook detRun + end + + # we should have launched only a single detRun + book getpage detRun 0 -var detRunID + if ("$detRunID" == "NULL") + echo "ERROR:detRun not defined" + break + end + + # what is the current state of the detRun? + book getword detRun $detRunID state -var state + + # use any other state? + if ("$state" == "stop") + # update the state for this detRunName, drop detRun page + book setword detRunDef $detRunName pantaskState DONE + if ($VERBOSE > 1) + echo "$detRunName is done" + end + else + if ($VERBOSE > 1) + echo "$detRunName is still running" + end + end + + # drop this book when completed + book delete detRun + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/nebulous.site.pro.in =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/nebulous.site.pro.in (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/nebulous.site.pro.in (revision 41106) @@ -0,0 +1,7 @@ +# customize these variables for your site +# and copy the file to share/pantasks/modules directory in your installation + +$NEB_DB = your_nebulous_database_name +$NEB_HOST = host_for_nebulous_mysql_database +$NEB_USER = XXX +$NEB_PASS = XXX Index: /branches/ccl_branches/ipponly-20191108/ippTasks/nightly_science.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/nightly_science.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/nightly_science.pro (revision 41106) @@ -0,0 +1,738 @@ +## nightly_science.pro : -*- sh -*- + +check.globals + +macro nightly.science.on + ns.initday.on +# ns.detrends.off + ns.dqstats.on + ns.stacks.on + ns.diffs.on +end + +macro nightly.science.off + ns.initday.off +# ns.detrends.off + ns.dqstats.off + ns.stacks.off + ns.diffs.off +end + +macro ns.on + nightly.science.on +end + +macro ns.off + nightly.science.off +end + +macro ns.initday.on + task ns.initday.load + active true + end +end + + +macro ns.dqstats.on + task ns.dqstats.load + active true + end +end + +macro ns.stacks.on + task ns.stacks.run + active true + end +end + +macro ns.diffs.on + task ns.diffs.run + active true + end +end + +macro ns.initday.off + task ns.initday.load + active false + end +end + +macro ns.dqstats.off + task ns.dqstats.load + active false + end +end + +macro ns.stacks.off + task ns.stacks.run + active false + end +end + +macro ns.diffs.off + task ns.diffs.run + active false + end +end + + +$ns_stackPAGE = 0 +$ns_RstackPAGE = 0 +$ns_CstackPAGE = 0 +$ns_diffPAGE = 0 +$ns_RdiffPAGE = 0 + +book init nsData +book init nsDiffs +book init nsStacks + +# Hard coded values +$camera_map:gpc1 = "GPC1" +$camera_map:gpc2 = "GPC2" + +# Macro to manage the camera maps +macro ns.add.camera_map + if ($0 != 3) + echo "USAGE: ns.add.camera_map (dbname) (camera)" + break + end + + $camera_map:$1 = $2 +end + +# +# Macros to control the book. +# +macro ns.add.date + if ($0 != 4) + echo "USAGE: ns.add.date (YYYY-MM-DD) (dbname) (camera)" + break + end + + if ($?camera_map:$2) + if ($camera_map:$2 != "$3") + echo "camera $3 does not match expected camera $camera_map:$2" + break + end + else + echo "dbname $2 does not have a camera_map entry" + break + end + + # build key from date and database name + sprintf key "%s-%s" $1 $2 + book newpage nsData $key + book setword nsData $key nsState NEW + book setword nsData $key date $1 + book setword nsData $key dbname $2 + book setword nsData $key camera $3 + + book newpage nsDiffs $key + book setword nsDiffs $key date $1 + book setword nsDiffs $key nsDiffState TOWARP + book setword nsDiffs $key nsObservingState UNKNOWN + + book newpage nsStacks $key + book setword nsStacks $key date $1 + book setword nsStacks $key nsStackState TOWARP + +end + +macro ns.show.dates + book npages nsData -var Npages + for i 0 $Npages + book getpage nsData $i -var key + book getword nsData $key date -var date + book getword nsData $key dbname -var dbname + book getword nsData $key camera -var camera + book getword nsData $key nsState -var state + book getword nsStacks $key nsStackState -var Sstate + book getword nsDiffs $key nsDiffState -var Dstate + book getword nsDiffs $key nsObservingState -var Ostate +# I've moved dbname to end as czartool reads information from here, and it may be fixed length + echo $key $date $state $Sstate $Dstate $Ostate $dbname $camera + + end +end + +macro ns.del.date + book delpage nsData $1 +end + +macro ns.set.date + book setword nsData $1 nsState $2 +end + + +# +# Start a new date to work on +# +$ns_init_DB = 0 + +task ns.initday.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + trange 0:00:00 1:00:00 -nmax 2 + npending 1 + + task.exec + stdout $LOGDIR/ns.initday.log + stderr $LOGDIR/ns.initday.log + $today = `date -u +%Y-%m-%d` + # loop over all databases +# for i 0 $DB:n + $thisDB = $DB:$ns_init_DB + + sprintf key "%s-%s" $today $thisDB + book newpage nsData $key + book setword nsData $key date $today + book setword nsData $key nsState NEW + book setword nsData $key dbname $thisDB + book setword nsData $key camera $camera_map:$thisDB + + book newpage nsStacks $key + book setword nsStacks $key date $today + book setword nsStacks $key nsStackState TOWARP + book newpage nsDiffs $key + book setword nsDiffs $key date $today + book setword nsDiffs $key nsDiffState TOWARP + + $ns_init_DB ++ + if ($ns_init_DB >= $DB:n) set ns_init_DB = 0 + + #command echo $today + $run = nightly_science.pl --clean_old --date $today --dbname $thisDB --camera $camera_map:$thisDB + command $run +# end + end + + task.exit 0 + # nothing to do here? + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Queue dqstats runs +# + +# CZW: 2015-05-05 One day we will stop making these, but that day is not today. +# According to Ken, we do not need to implement this for gpc2, as nothing is +# written on the OTIS/scheduler side to accept these products. We should +# continue to generate these for gpc1, as the machinery does exist to check +# for them, and not generating them may cause that to fail (even if nothing +# is done with the results). + +task ns.dqstats.load + host local + periods -poll 20 + periods -exec $LOADEXEC + periods -timeout 300 + trange 22:00:00 23:59:59 + trange 00:00:00 10:00:00 + npending 1 + + task.exec + stdout $LOGDIR/ns.dqstats.log + stderr $LOGDIR/ns.dqstats.log + $today = `date -u +%Y-%m-%d` + + $run = nightly_science.pl --queue_dqstats --date $today --dbname gpc1 --camera GPC1 +# add_standard_args $run + command $run + + end + + task.exit 0 + # nothing to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the warps are finished and so we can make stacks. +# + +task ns.stacks.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1000 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_RstackPAGE -var key + book getword nsStacks $key date -var date + book getword nsStacks $key nsStackState -var ns_STATE + book getword nsDiffs $key nsDiffState -var ns_diff_STATE + book getword nsData $key dbname -var DBNAME + book getword nsData $key camera -var CAMERA + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.run: " $ns_RstackPAGE $key $date $ns_STATE $Npages + end + + $ns_RstackPAGE ++ + if ($ns_RstackPAGE >= $Npages) set ns_RstackPAGE = 0 + option $key + + if (("$ns_STATE" != "STACKING")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break + $run = nightly_science.pl --queue_stacks --date $date --dbname $DBNAME --camera $CAMERA +# add_standard_args $run + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsStacks $options:0 +# We've queued up stacking. We're done with this date, so don't reload the page. + ipptool2book stdout nsStacks -uniq -key key + + book getword nsStacks $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsStacks $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + +# +# Check to see if the warps are finished and so we can make diffs. +# + +task ns.diffs.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 900 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.diffs.log + + book getpage nsData $ns_RdiffPAGE -var key + book getword nsDiffs $key date -var date + book getword nsDiffs $key nsDiffState -var ns_STATE + book getword nsStacks $key nsStackState -var ns_stack_STATE + book getword nsData $key dbname -var DBNAME + book getword nsData $key camera -var CAMERA + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.diffs.run: " $ns_RdiffPAGE $date $ns_STATE $Npages + end + + $ns_RdiffPAGE ++ + if ($ns_RdiffPAGE >= $Npages) set ns_RdiffPAGE = 0 + option $key + + if (("$ns_STATE" != "DIFFING")&&("$ns_STATE" != "QUEUEDIFFS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break + $run = nightly_science.pl --queue_diffs --date $date --dbname $DBNAME --camera $CAMERA +# add_standard_args $run + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsDiffs $options:0 +# We've queued up diffing. We're done with this date, so don't reload the page. + ipptool2book stdout nsDiffs -uniq -key key + + book getword nsDiffs $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsDiffs $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + + +# This is the graveyard of tasks that we've defined in the past, but either never use, or never let go active. + +# if (0) + + +# macro ns.detrends.on +# task ns.detrends.load +# active true +# end +# end + + +# macro ns.detrends.off +# task ns.detrends.load +# active false +# end +# end + +# macro ns.stacks.confirm.on +# task ns.stacks.confirm +# active true +# end +# end + +# macro ns.stacks.confirm.off +# task ns.stacks.confirm +# active false +# end +# end + + + + + +# # +# # Queue off any possible detrend verification runs +# # +# task ns.detrends.load +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 30 +# trange 23:00:00 23:59:59 -nmax 1 +# npending 1 + +# task.exec +# stdout $LOGDIR/ns.detrends.log +# stdout $LOGDIR/ns.detrends.log +# $today = `date -u +%Y-%m-%d` + +# $run = nightly_science.pl --queue_detrends --date $today --dbname $DB:0 +# # add_standard_args $run +# command $run +# end + +# task.exit 0 +# # nothign to do here +# end +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# # operation times out +# task.exit timeout +# showcommand timeout +# end +# end + + +# # +# # Check to see if the warps are finished +# # +# task ns.stacks.load +# active false +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 120 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.stacks.log + +# book getpage nsData $ns_stackPAGE -var key +# book getword nsStacks $key date -var date +# book getword nsStacks $key nsStackState -var ns_STATE +# book getword nsDiffs $key nsDiffState -var ns_diff_STATE +# book getword nsData $key dbname -var DBNAME +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.stacks.load: " $ns_stackPAGE $date $DBNAME $ns_STATE $Npages +# end + +# $ns_stackPAGE ++ +# if ($ns_stackPAGE >= $Npages) set ns_stackPAGE = 0 +# option $key + +# if ("$ns_STATE" != "TOWARP") break + +# $run = nightly_science.pl --check_stacks --date $date --dbname $DBNAME +# # add_standard_args $run +# command $run + +# end + +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsStacks $options:0 +# ipptool2book stdout nsStacks -uniq -key date + +# book getword nsStacks $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsStacks $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # +# # Confirm that all the stacks that can be built have been built, or at least attempted. +# # +# task ns.stacks.confirm +# active false +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 120 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.stacks.log + +# book getpage nsData $ns_CstackPAGE -var key +# book getword nsStacks $key date -var date +# book getword nsStacks $key nsStackState -var ns_STATE +# book getword nsData $key dbname -var DBNAME +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.stacks.confirm: " $ns_CstackPAGE $key $date $DBNAME $ns_STATE $Npages +# end + +# $ns_CstackPAGE ++ +# if ($ns_CstackPAGE >= $Npages) set ns_CstackPAGE = 0 +# option $key + +# if ("$ns_STATE" != "STACKING") break +# $run = nightly_science.pl --confirm_stacks --date $date --dbname $DBNAME +# # add_standard_args $run +# command $run +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsStacks $options:0 +# ipptool2book stdout nsStacks -uniq -key date + +# book getword nsStacks $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsStacks $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + + + + +# # +# # Check to see if the warps are finished +# # +# task ns.diffs.load +# active false +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 120 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.diffs.log + +# book getpage nsData $ns_diffPAGE -var key +# book getword nsDiffs $key date -var date +# book getword nsDiffs $key nsDiffState -var ns_STATE +# book getword nsStacks $key nsStackState -var ns_stack_STATE +# book getword nsData $key dbname -var DBNAME +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.diffs.load: " $ns_diffPAGE $key $date $DBNAME $ns_STATE $Npages +# end + +# $ns_diffPAGE ++ +# if ($ns_diffPAGE >= $Npages) set ns_diffPAGE = 0 +# option $key + +# if ("$ns_STATE" != "TOWARP") break +# $run = nightly_science.pl --check_diffs --date $date --dbname $DBNAME +# # add_standard_args $run +# command $run +# end + +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsDiffs $options:0 +# ipptool2book stdout nsDiffs -uniq -key date + +# book getword nsDiffs $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsDiffs $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + + +# # END GRAVEYARD +# end + + +# # +# # Queue sweetspot runs +# # +# task ns.sweetspot.load +# host local +# periods -poll 3600 +# periods -exec $LOADEXEC +# periods -timeout 30 +# trange 02:00:00 02:59:59 -nmax 1 +# npending 1 + +# task.exec +# stdout $LOGDIR/ns.sweetspot.log +# stderr $LOGDIR/ns.sweetspot.log +# $today = `date -u +%Y-%m-%d` + +# command nightly_science.pl --queue_sweetspot --date $today +# end + +# task.exit 0 +# # nothing to do here +# end +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# # operation times out +# task.exit timeout +# showcommand timeout +# end +# end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/nightly_stacks.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/nightly_stacks.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/nightly_stacks.pro (revision 41106) @@ -0,0 +1,1053 @@ +## nightly_stacks.pro : -*- sh -*- + +check.globals + +macro nightly.stacks.on + ns.initday.on + ns.detrends.off + ns.dqstats.on +# ns.sweetspot.on +# ns.registration.on +# ns.burntool.on +# ns.chips.on + ns.stacks.on + ns.diffs.on +end + +macro nightly.stacks.off + ns.initday.off + ns.detrends.off + ns.dqstats.off +# ns.sweetspot.off +# ns.registration.off +# ns.burntool.off +# ns.chips.off + ns.stacks.off + ns.diffs.off +end + +macro ns.on + nightly.stacks.on +end + +macro ns.off + nightly.stacks.off +end + +macro ns.initday.on + task ns.initday.load + active true + end +end + +macro ns.detrends.on + task ns.detrends.load + active true + end +end + +macro ns.dqstats.on + task ns.dqstats.load + active true + end +end + +# macro ns.sweetspot.on +# task ns.sweetspot.load +# active true +# end +# end + +# macro ns.registration.on +# task ns.registration.load +# active true +# end +# end + +# macro ns.burntool.on +# task ns.burntool.load +# active true +# end +# task ns.burntool.run +# active true +# end +# end + +# macro ns.chips.on +# task ns.chips.load +# active true +# end +# task ns.chips.run +# active true +# end +# end + +macro ns.stacks.on + task ns.stacks.load + active false + end + task ns.stacks.run + active true + end + task ns.stacks.confirm + active true + end +end + +macro ns.diffs.on + task ns.diffs.load + active false + end + task ns.diffs.run + active true + end +end + +macro ns.initday.off + task ns.initday.load + active false + end +end + +macro ns.detrends.off + task ns.detrends.load + active false + end +end + +macro ns.dqstats.off + task ns.dqstats.load + active false + end +end + +# macro ns.sweetspot.off +# task ns.sweetspot.load +# active false +# end +# end + +# macro ns.registration.off +# task ns.registration.load +# active false +# end +# end + +# macro ns.burntool.off +# task ns.burntool.load +# active false +# end +# task ns.burntool.run +# active false +# end +# end + +# macro ns.chips.off +# task ns.chips.load +# active false +# end +# task ns.chips.run +# active false +# end +# end + +macro ns.stacks.off + task ns.stacks.load + active false + end + task ns.stacks.run + active false + end +end + +macro ns.diffs.off + task ns.diffs.load + active false + end + task ns.diffs.run + active false + end +end + +# $ns_regPAGE = 0 +# $ns_burnPAGE = 0 +# $ns_RburnPAGE = 0 +# $ns_RburnCELL = 0 +# $ns_chipPAGE = 0 +# $ns_RchipPAGE = 0 +$ns_stackPAGE = 0 +$ns_RstackPAGE = 0 +$ns_CstackPAGE = 0 +$ns_diffPAGE = 0 +$ns_RdiffPAGE = 0 + +book init nsData +# book init nsBurntool +# +# Macros to control the book. +# +macro ns.add.date + book newpage nsData $1 + book setword nsData $1 nsState NEW +end + +macro ns.show.dates + book npages nsData -var Npages + for i 0 $Npages + book getpage nsData $i -var date + book getword nsData $date nsState -var state + echo $date $state + end +# book npages nsBurntool -var Npages +end + +macro ns.del.date + book delpage nsData $1 +end + +macro ns.set.date + book setword nsData $1 nsState $2 +end + + + +# +# Start a new date to work on +# +task ns.initday.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + trange 0:00:00 1:00:00 -nmax 1 + npending 1 + + task.exec + stdout $LOGDIR/ns.initday.log + stderr $LOGDIR/ns.initday.log + $today = `date -u +%Y-%m-%d` + book newpage nsData $today + book setword nsData $today nsState NEW + +# command echo $today + command automate_stacks.pl --clean_old --date $today + end + + task.exit 0 + # nothing to do here? + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Queue off any possible detrend verification runs +# +task ns.detrends.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + trange 23:00:00 23:59:59 -nmax 1 + npending 1 + + task.exec + stdout $LOGDIR/ns.detrends.log + stdout $LOGDIR/ns.detrends.log + $today = `date -u +%Y-%m-%d` + + command automate_stacks.pl --queue_detrends --date $today + end + + task.exit 0 + # nothign to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Queue dqstats runs +# +task ns.dqstats.load + host local + periods -poll 10 + periods -exec $LOADEXEC + periods -timeout 600 + trange 22:00:00 23:59:59 + trange 00:00:00 10:00:00 + npending 1 + + task.exec + stdout $LOGDIR/ns.dqstats.log + stderr $LOGDIR/ns.dqstats.log + $today = `date -u +%Y-%m-%d` + + command automate_stacks.pl --queue_dqstats --date $today + end + + task.exit 0 + # nothing to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Queue sweetspot runs +# +task ns.sweetspot.load + host local + periods -poll 3600 + periods -exec $LOADEXEC + periods -timeout 30 + trange 02:00:00 02:59:59 -nmax 1 + npending 1 + + task.exec + stdout $LOGDIR/ns.sweetspot.log + stderr $LOGDIR/ns.sweetspot.log + $today = `date -u +%Y-%m-%d` + + command automate_stacks.pl --queue_sweetspot --date $today + end + + task.exit 0 + # nothing to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if registration is complete +# +# task ns.registration.load +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 30 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.registration.log + +# book getpage nsData $ns_regPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.registration.load: " $ns_regPAGE $date $ns_STATE $Npages +# end + +# $ns_regPAGE ++ +# if ($ns_regPAGE >= $Npages) set ns_regPAGE = 0 +# option $date + +# if ("$ns_STATE" != "NEW") break +# $run = automate_stacks.pl --check_registration --date $date +# command $run +# end + +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsData $options:0 +# ipptool2book stdout nsData -uniq -key date + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # +# # Check to see if the chips have been properly burntooled and are ready for queueing +# # +# task ns.chips.load +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 300 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.chips.log + +# book getpage nsData $ns_chipPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.chips.load: " $ns_chipPAGE $date $ns_STATE $Npages +# end + +# $ns_chipPAGE ++ +# if ($ns_chipPAGE >= $Npages) set ns_chipPAGE = 0 +# option $date + +# if (("$ns_STATE" != "REGISTERED")&&("$ns_STATE" != "BURNING")) break +# $run = automate_stacks.pl --check_chips --date $date + +# if ("$ns_STATE" == "BURNING") +# $run = $run --isburning +# end +# command $run +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsData $options:0 +# ipptool2book stdout nsData -uniq -key date + +# # remove the burntool page if we're done with it. +# book getword nsData $options:0 nsState -var ns_STATE +# if ("$ns_STATE" == "QUEUECHIPS") +# book delpage nsBurntool $options:0 +# end + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # +# # Check to see if the chips have been properly burntooled and are ready for queueing +# # +# task ns.chips.run +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 120 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.chips.log + +# book getpage nsData $ns_RchipPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.chips.run: " $ns_RchipPAGE $date $ns_STATE $Npages +# end + +# $ns_RchipPAGE ++ +# if ($ns_RchipPAGE >= $Npages) set ns_RchipPAGE = 0 +# option $date + +# if ("$ns_STATE" != "QUEUECHIPS") break +# $BURNTOOLING = 0 +# $run = automate_stacks.pl --queue_chips --date $date +# command $run +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsData $options:0 +# ipptool2book stdout nsData -uniq -key date + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + + +# +# Check to see if the warps are finished +# +task ns.stacks.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_stackPAGE -var date + book getword nsData $date nsState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.load: " $ns_stackPAGE $date $ns_STATE $Npages + end + + $ns_stackPAGE ++ + if ($ns_stackPAGE >= $Npages) set ns_stackPAGE = 0 + option $date + + if ("$ns_STATE" != "TOWARP") break + $run = automate_stacks.pl --check_stacks --date $date + command $run + end + + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the warps are finished and so we can make stacks. +# + +task ns.stacks.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 900 # this task can be quite slow + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_RstackPAGE -var date + book getword nsData $date nsState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.run: " $ns_RstackPAGE $date $ns_STATE $Npages + end + + $ns_RstackPAGE ++ + if ($ns_RstackPAGE >= $Npages) set ns_RstackPAGE = 0 + option $date + + if (("$ns_STATE" != "QUEUESTACKS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break + $run = automate_stacks.pl --queue_stacks --date $date + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 +# We've queued up stacking. We're done with this date, so don't reload the page. + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Confirm that all the stacks that can be built have been built, or at least attempted. +# +task ns.stacks.confirm + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_CstackPAGE -var date + book getword nsData $date nsState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.confirm: " $ns_CstackPAGE $date $ns_STATE $Npages + end + + $ns_CstackPAGE ++ + if ($ns_CstackPAGE >= $Npages) set ns_CstackPAGE = 0 + option $date + + if ("$ns_STATE" != "STACKING") break + $run = automate_stacks.pl --confirm_stacks --date $date + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# # +# # Generate a list of date ranges and chips that need to be processed with burntool +# # +# task ns.burntool.load +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 30 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.burntool.log + +# book getpage nsData $ns_burnPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.burntool.load: " $ns_burnPAGE $date $ns_STATE $Npages +# end + +# $ns_burnPAGE ++ +# if ($ns_burnPAGE >= $Npages) set ns_burnPAGE = 0 +# option $date + +# if ("$ns_STATE" != "NEEDSBURNING") break + +# $run = automate_stacks.pl --define_burntool --date $date +# command $run +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# # book delpage nsBurntool $options:0 +# ipptool2book stdout nsBurntool -uniq -key date +# book setword nsData $options:0 nsState "QUEUEBURNING" + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # +# # Magically run burntool on the data, based on the information stored in our book. +# # +# task ns.burntool.run +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 10800 +# npending 30 +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.burntool.log + +# book getpage nsData $ns_RburnPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.burntool.run: " $ns_RburnPAGE $date $ns_STATE $Npages +# end + +# $ns_RburnPAGE ++ +# if ($ns_RburnPAGE >= $Npages) set ns_RburnPAGE = 0 +# if (("$ns_STATE" != "QUEUEBURNING")&&("$ns_STATE" != "BURNING")) break +# $BURNTOOLING = 1 +# # Find out where in the list of jobs we are +# book getword nsBurntool $date btN -var btN +# book getword nsBurntool $date btNCounter -var btNcounter + +# if ($VERBOSE > 5) +# echo "ns.burntool.run: Status: " $btNcounter $date +# end + +# if ("$ns_STATE" == "QUEUEBURNING") +# $new_state = "QUEUEBURNING" +# end +# if ("$ns_STATE" == "BURNING") +# $new_state = "BURNING" +# end +# if ($btNcounter + 1 > $btN) +# $new_state = "BURNING" +# end +# if ($btNcounter > $btN) +# $btNcounter = 0 +# end +# if ($VERBOSE > 5) +# echo "ns.burntool.run: Status: " $btNcounter $new_state +# end + +# # Increment the counter in the book for the next job. +# $counter_update = $btNcounter + 1 +# book setword nsBurntool $date btNCounter $counter_update +# # book setword nsData $options:0 nsState $options:1 +# book setword nsData $date nsState $new_state + +# # Get the current status of this job, and skip if it doesn't need to process. +# sprintf status_label "bt%dStatus" $btNcounter +# book getword nsBurntool $date $status_label -var status + +# if ($VERBOSE > 5) +# echo "ns.burntool.run: Status: " $btNcounter $status $date $status_label +# end +# if (("$status" == "FINISHED")||("$status" == "RUN")) break + +# # Continue loading information to process this job +# sprintf start_date_label "bt%dBegin" $btNcounter +# sprintf end_date_label "bt%dEnd" $btNcounter +# sprintf class_count_label "bt%dClass" $btNcounter + +# book getword nsBurntool $date $start_date_label -var start_date +# book getword nsBurntool $date $end_date_label -var end_date +# book getword nsBurntool $date $class_count_label -var chip_counter + +# # Lookup class_id/host pairs +# list word -split $hostmatch:$chip_counter +# $class_id = $word:0 +# $host = $word:1 +# host $host +# # set.host.for.camera GPC1 $class_id +# $logfile = "/data/$host.0/burntool_logs/$class_id.$start_date.log" + +# $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile + +# echo "ns.burntool.run: " $date $btN $btNcounter $start_date $end_date $chip_counter $class_id $host $logfile $run +# echo "ns.burntool.run: " $date $btN $btNcounter $chip_counter $new_state + +# book setword nsBurntool $date $status_label RUN +# option $date $new_state $status_label + +# command $run +# # command /bin/true +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# # Set data state based on if we're queueing or waiting +# # Set the job state for success. +# book setword nsBurntool $options:0 $options:2 FINISHED + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# book setword nsBurntool $options:0 $options:2 FAIL +# showcommand failure +# end +# task.exit crash +# book setword nsBurntool $options:0 $options:2 FAIL +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# book setword nsBurntool $options:0 $options:2 FAIL +# showcommand timeout +# end +# end + + +# # +# # This is all burntool related stuff. +# # + +# macro burntool +# if ($0 != 3) +# echo "USAGE: burntool (dateobs_begin) (dateobs_end)" +# break +# end + +# for i 0 $hostmatch:n +# job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile +# end +# end +# macro loadhosts +# for i 0 $allhosts:n +# controller host add $allhosts:$i +# end +# end + +# # sorry this list is messy, it's supposed to be that way to try and keep to burntools from running on the same host at the same time. + +# list hostmatch +# XY01 ipp014 +# XY03 ipp038 +# XY05 ipp023 +# XY10 ipp039 +# XY12 ipp024 +# XY15 ipp040 +# XY17 ipp020 +# XY21 ipp041 +# XY23 ipp042 +# XY25 ipp043 +# XY27 ipp028 +# XY31 ipp044 +# XY33 ipp029 +# XY35 ipp045 +# XY37 ipp030 +# XY41 ipp046 +# XY43 ipp031 +# XY45 ipp047 +# XY47 ipp032 +# XY51 ipp048 +# XY53 ipp033 +# XY55 ipp049 +# XY57 ipp034 +# XY61 ipp050 +# XY63 ipp035 +# XY65 ipp051 +# XY67 ipp036 +# XY72 ipp052 +# XY74 ipp015 +# XY76 ipp053 +# XY02 ipp014 +# XY04 ipp038 +# XY06 ipp023 +# XY11 ipp039 +# XY13 ipp024 +# XY14 ipp040 +# XY16 ipp020 +# XY20 ipp041 +# XY22 ipp042 +# XY24 ipp043 +# XY26 ipp028 +# XY30 ipp044 +# XY32 ipp029 +# XY34 ipp045 +# XY36 ipp030 +# XY40 ipp046 +# XY42 ipp031 +# XY44 ipp047 +# XY46 ipp032 +# XY50 ipp048 +# XY52 ipp033 +# XY54 ipp049 +# XY56 ipp034 +# XY60 ipp050 +# XY62 ipp035 +# XY64 ipp051 +# XY66 ipp036 +# XY71 ipp052 +# XY73 ipp015 +# XY75 ipp053 +# end + +# list allhosts +# ipp043 +# ipp014 +# ipp015 +# ipp023 +# ipp024 +# ipp020 +# ipp028 +# ipp029 +# ipp030 +# ipp031 +# ipp032 +# ipp033 +# ipp034 +# ipp035 +# ipp036 +# ipp038 +# ipp039 +# ipp040 +# ipp041 +# ipp042 +# ipp044 +# ipp045 +# ipp046 +# ipp047 +# ipp048 +# ipp049 +# ipp050 +# ipp051 +# ipp052 +# ipp053 +# end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/notes.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/notes.txt (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/notes.txt (revision 41106) @@ -0,0 +1,373 @@ + +2008.08.31 + + * flatcorr modes: + * definebyquery : select the matching rawExp entries and add them to chip table. this is functionally identical to chiptool -definebyquery, + except that it also adds entries in the flatcorr tables to track the processing + + + + +2008.08.22 + + issues related to creating and applying the flat-field correction in a simtest context + + * generating the fake data: + * ppSim must be called with a recipe in which the SCATTER.FRACTION is non-zero + * simtest thus needs to accept a ppsim_recipe and supply it to ppSimSequence + * ppSimSequence needs to accept the ppsim_recipe and supply it to the generated ppSim calls + + * first stage processing: + * we need to build flats which are not corrected (use the standard 'flat' type?) + * we need to apply flats which are not corrected + ** these can both be accomplished with the current DEFAULT reduction class + + * second stage processing: + * we need to apply flats which are corrected + ** define a reduction class of type FLATCORR which: + * modifies ppImage.config:DETREND.CONTRAINTS:FLAT:DETTYPE = FLAT_CORRECT + + ** adding a new detrend type: + + * add an entry to detrend_stack.pl FILERULES + * add an entry to detrend_stack.pl STATRECIPES + * add an entry to ppMerge/src/ppMergeArguments.c + * add an entry to ippconfig/recipes/ppMerge.config + + * add an entry to ippconfig/recipes/rejections.config + * add a section to ippconfig/recipes/reductionClasses.mdc + + * ippScripts/scripts/detrend_norm_apply.pl : entry in DETTYPE table + * ippScripts/scripts/detrend_norm_calc.pl : entry in NORMALIZE table + * ippScripts/scripts/detrend_resid_imfile.pl : entry in DETRENDS table + * ippScripts/scripts/detrend_stack.pl : entry in FILERULES and STATRECIPES tables + * ippScripts/scripts/detrend_resid_imfile.pl : entry in DETRENDS table + + * need a block in simtest.auto (or equiv) to check for when the chip processing / cam processing is complete + + * need to modify the detselect -inst arguments in simtest.auto (use @INST@ or equiv) + + * reductions currently being used: + ** CHIP -- chip-level analysis + ** JPEG.BIN1, JPEG.BIN2 -- camera jpeg terms + ** JPEG_BIN1_IMAGE_(dettype), JPEG_BIN2_IMAGE_(dettype) + ** JPEG_BIN1_RESID_(dettype), JPEG_BIN2_RESID_(dettype) + ** (dettype)_STACK, + ** (dettype)_PROCESS, + ** FAKEPHOT + +2008.01.22 + + proposal to clean scripts/tasks up somewhat wrt names: + + * scripts do NOT use workdir to change the workdir in the db + * thus, what is supplied is actually the outputRoot + * require the output root for the scripts + * have the calling tasks construct the outputRoot + ** replace -workdir with -outroot + + some filename rules : + + * the db entry WORKDIR is the most abstract version: + path://GPC-@HOST@/20080130 + file://data/@HOST@.0/gpc1/20080130 + neb:///@HOST@/gpc1/20080130 + neb://gpc1/20080130 + XXX have nebulous distinguish between neb:///host/path and + neb://path? + + * the db WORKDIR entry is carried from step to step unless + over-ridden with a -queue command + + * psLib-based binaries (ppImage, etc) may be passed URIs, but not + files with the @HOST@ abstraction + + * IPP perl scripts may be passed URIs, but not files with the @HOST@ + abstraction + + * nebulous : convert neb:///@HOST@/path/name + + *** examples: + + * this tells ppImage to use the ipp004 version for both input and output: + ppImage -file neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01.fits neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01 + + * we already know the hostname at this point; include it in the call + chip_imfile.pl --uri neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01.fits --workdir neb:///ipp004-v0/gpc1/20080130/ + + + + + +2008.01.21 + + I'm working on the summit.copy.pro interaction with nebulous (and + those of the other ippTasks) and the assignment of jobs and data + locations to hosts on the basis of the chip/host relationship. I + have defined a table which relates the chip (class_id) to the + desired host name. There are a few places, and a few slightly + different ways, in which this gets used: + + * choice of host for command execution: + + every job which operates on a specific chip must be told the chip + host so it can run the job on that machine. the information is + supplied in pantasks to the job with the 'host' command. I've + made a pantasks macro 'set.host.by.camera' which takes the camera + and class_id, looks up the match in the chip/host table, and calls + the host command with the desired host. If there is no match + (camera not listed in chip/host table or chip not listed in + table), the host is set to 'anyhost', which randomly chooses one. + I have used this command for all tasks which generate a pcontrol + command. For the tasks which are performed at the camera level, + these are set to anyhost. For the tasks which operate on stacks, + we can use the same structure to associate the skycell with a + host. + + * specific output filenames + + In both ippTasks and ippScripts, there are references to files + which must correctly located on the host. + + * workdir choice of output directory. + + the + + * dsget output target: + + dsget is called by pantasks in the task 'dsget'. currently, dsget + accepts a --filename argument which specifies the output filename + to use. in the nebulous context, this is of the form + neb://top/next/stuff, without any host-specific information. To + target the file instance to a specific host, we pass dsget the + --volume (name) argument. The volumes refer to the specific + devices (potentially more than one per host), and the names can + then be chosen to include the host. + +2007.10.11 + + The calibration suite uses a database table of known/registered DVO + databases to be calibrated. The suite operates on a segment of the + sky in each run, nominally a section ~1h west of the current ST. + + caltool -pendingdbs returns a list of entries (mdc format), one for + each DVO database in the table. + + caltool -databases [-active] : returns the list of dvo databases + caltool -addcalrun : inserts the stats for a single calibration run + caltool -setstate (active/inactive) : modify the state of a dvo + database + caltool -adddatabase : insert a new database + + XXX a potential source of collisions. the books are marked by + unique keys in the db tables, but a single pantasks run may be + addressing multiple databases. IDs are not unique across + databases. we can solve this by including in the key the dbname + + the calibration suite runs the following tasks, in sequences on each + of the databases: + + * resort, relphot, uniphot, relastro.objects, relastro.images + + ** detrend correction: any detrend images which need to be corrected + get inserted in the detCorrectedExp table. + + dettool commands needed: + * add a detrend exposure (det_id, exp_id) to the detCorrectedExp table + * add a detrend imfile (det_id, exp_id, class_id) to the + detCorrectedImfile table + * select pending detCorrectedExp + * select pending detCorrectedImfiles + + +2007.01.08 + + I have updated pantasks to used named tags in named groups to make + life much easier when changing the schema of the db tables. The + function ipptool2book convernts the output from an ippTool query in + the form of a set of metadata structures (saved in stdout) into a + book format. + + dettool -tonormalizedstat DetrendNormStat + dettool -tonormalize DetrendNorm + dettool -tonormalizedexp DetrendNormExp + dettool -toprocessedimfile DetrendProcessImfiles + dettool -toprocessedexp DetrendProcessExposures + dettool -residdetrun DetrendRejectExp + dettool -toresidimfile DetrendResidImfiles + dettool -toresidexp DetrendResidExposures + dettool -tostacked DetrendStackClass + p0tool -pendingimfile Phase0Imfiles + p0tool -pendingexp Phase0Exposures + p2tool -pendingimfile Phase2Imfiles + p3tool -pendingexp Phase3Exposures + +2006.08.28 + +- ipptools need to use expID:camera:telescope as unique key (ie, not expID alone) +- p0tool -pendingimfile -simple needs to report the expKey (ie, not expID) +- + + +This directory contains the PanTasks scripts which define the steps +performed by the IPP. These scripts use the ippTools to decide what +images need to be processed, and to update the Metadata Database with +the current state of the analysis. The analysis is performed using +IPP programs which are distributed to the cluster computers as needed. + +1. The Detrend Creation Scripts + +The detrend creation process is among the more complex elements of the +IPP. There are N major steps which must be performed: + +1.1. Define New Detrend Runs + +The script 'detrend.mkruns.pro' defines tasks which initiate new +Detrend Runs. There is a separate task for each camera automatically +processed by the system (mkdetruns.CAMERA). These tasks are executed +on a regular, but infrequent basis. The tasks call external helper +programs which actually run the command dettools -define. The helper +programs are thus responsible for identifying all of the types of +detrend images to be run for a given camera, as well as the correct +set of filters to apply. These tasks are run locally and only affect +the database. PanTasks does not track the detrend runs which are +created. + +- we may need to define these within the PanTasks scripts +- how frequently are these tasks run (1/day?) +- if no raw images are selected by the query, no new detrun should be + created. +- we may need different schedules for the dark (bias/dark) and the + light (flat/fringe) data. And yet another schedule for sky data + (skyflat/skyfringe). + +1.2. Process new raw imfiles + +The script 'detrend.process.pro' defines tasks which processes the new +raw imfiles for any detrend run. This script defines two tasks, +'dettool.raw.load' and 'dettool.raw.process'. The first script +occasionally polls the Metadata Database to select raw imfiles which +have been identified as part of a detrend run. The identified raw +imfiles are added to an internal queue, DetrendImfilesToProcess. Any +raw imfile already in the list is ignored. The imfiles are added to +the queue with a state of NEW. The second task examines the queue for +imfiles with state NEW and defines a ppImage analysis job for that +imfile. The job is sent to pcontrol for processing on the distributed +cluster. At this point, the corresponding queue entry is set to the +state RUN. Upon success, the queue entry is set to DONE and the +database is notified of the successful operation. The DONE entries +are occasionally pruned from the queue; this must be done in the same +task as the initial query and the update of the list based on the +database results. + +- we need to define the columns in the output of dettool -raw +- we need a method to associate an imfile with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs +- we need a method to identify the correct recipe (function of type?) + +1.3. Stack new processed imfiles + +The script 'detrend.stack.pro' defines tasks which stack the newly +processed imfiles for any detrend run iteration. This script defines +two tasks, 'dettool.stack.load and 'dettool.stack.process. The first +script occasionally polls the Metadata Database to select class IDs +for detrend runs for which the input imfiles have all been processed. +The identified detRun/classID combination is added to an internal +queue, DetrendClassIDtoStack. The entries are added to the queue with +a state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppMerge analysis job for that +detID/classID. The helper script selects the corresponding input +imfiles and passes them to ppMerge. The job is sent to pcontrol for +processing on the distributed cluster. At this point, the +corresponding queue entry is set to the state RUN. Upon success, the +queue entry is set to DONE and the database is notified of the +successful operation. The DONE entries are occasionally pruned from +the queue; this must be done in the same task as the initial query and +the update of the list based on the database results. + +- we need to define the columns in the output of dettool -chip -processed +- we need a method to associate a classID with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) + + +1.4. Normalize stacked masters + +The script 'detrend.norm.pro' defines tasks which normalize the newly +processed stacks for any detrend run iteration. This script defines +two tasks, 'dettool.norm.load and 'dettool.norm.process. The first +script occasionally polls the Metadata Database to select detrun IDs +for which the classes have all been stacked. The identified +detRunID/iteration combination is added to an internal queue, +DetrendRunToNormalize. The entries are added to the queue with a +state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppNorm analysis job for that +detID/iteration. The job is sent to pcontrol for processing on the +distributed cluster. At this point, the corresponding queue entry is +set to the state RUN. Upon success, the queue entry is set to DONE and +the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need to define the ppNorm operation inputs and outputs better + + +1.5. Generate Resid Imfiles + +The script 'detrend.resid.pro' defines tasks which apply the newly +generated detrend image to the corresponding input images to generate +the residual images. This script defines two tasks, +'dettool.resid.load and 'dettool.resid.process. The first script +occasionally polls the Metadata Database to select imfiles for which +the master detrend images has been normalized. The identified imfiles +are added to an internal queue, DetrendResidImfiles. The entries are +added to the queue with a state of NEW. The second task examines the +queue for entries with a state of NEW. It defines a ppImage analysis +job for that imfile. The job is sent to pcontrol for processing on +the distributed cluster. At this point, the corresponding queue entry +is set to the state RUN. Upon success, the queue entry is set to DONE +and the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- can we select the imfiles here without apriori knowledge that the + master has been normalized? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need a method to pass the matching detrend image to ppImage in + the correct context (ie, a bias needs to be supplied to the -bias element...) + +1.6. Assess Detrend Results + +The script 'detrend.assess.pro' defines tasks which assess the +residual statistics for completed detrend runs. This script defines +two tasks, 'dettool.assess.load and 'dettool.assess.process. The +first script occasionally polls the Metadata Database to select +detrunIDs/iterations for which the resid imfiles have all been +generated. The identified detruns are added to an internal queue, +DetrendRunToAssess. The entries are added to the queue with a state +of NEW. The second task examines the queue for entries with a state +of NEW. It defines an analysis job for that collection. The job is +sent to pcontrol for processing on the distributed cluster. At this +point, the corresponding queue entry is set to the state RUN. Upon +success, the queue entry is set to DONE and the database is notified +of the successful operation. The DONE entries are occasionally pruned +from the queue; this must be done in the same task as the initial +query and the update of the list based on the database results. + +- the assess tool needs to trigger a new run, if needed. +- what is the command to run on the resid statistics? +- we need to define the columns in the output of dettool +- can we select the detrend runs here? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) Index: /branches/ccl_branches/ipponly-20191108/ippTasks/pantasks.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/pantasks.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/pantasks.pro (revision 41106) @@ -0,0 +1,779 @@ +## pantasks.pro : globals and support macros : -*- sh -*- + +# globals that may be modified by the user -- only init if not set +if ($?NEBULOUS == 0) set NEBULOUS = 0 +if ($?NETWORK == 0) set NETWORK = 1 +if ($?BURNTOOLING == 0) set BURNTOOLING = 0 +if ($?PARALLEL == 0) set PARALLEL = 1 +if ($?VERBOSE == 0) set VERBOSE = 1 +if ($?LABEL:n == 0) set LABEL:n = 0 +if ($?POLL_LIMIT == 0) set POLL_LIMIT = 32 +if ($?KEEP_FAILURES == 0) set KEEP_FAILURES = 0 + +if ($?LOGDIR == 0) + $LOGDIR = `pwd` + $LOGDIR = $LOGDIR/pantasks_logs + mkdir $LOGDIR +end + +# globals used to control system behavior : these should be in uppercase +$LOADPOLL = 1.0 +$LOADEXEC = 5.0 + +$RUNPOLL = 0.5 +$RUNEXEC = 2.5 + +$EXIT_SUCCESS = 0 +$EXIT_UNKNOWN_ERR = 1 +$EXIT_SYS_ERR = 2 +$EXIT_CONFIG_ERR = 3 +$EXIT_PROG_ERR = 4 +$EXIT_DATA_ERR = 5 +$EXIT_CRASH_ERR = 256 + +# DB lists the database names in use; the default ipprc database +# is only used if DB:n is 0 +$DB:n = 0 + +# very basic values: set these with init.copy.mhpcc +$default_host = any +$workdir_template = `pwd` + +# user functions to manage databases +macro add.database + if ($0 != 2) + echo "USAGE: add.database (db)" + break + end + if ($?DB:n == 0) + list DB -add $1 + return + end + + local found + $found = 0 + for i 0 $DB:n + if ($DB:$i == $1) + $found = 1 + echo "$DB:$i set" + last + end + end + + if ($found == 0) + list DB -add $1 + end +end + +macro del.database + if ($0 != 2) + echo "USAGE: del.database (db)" + break + end + if ($?DB:n == 0) + return + end + + list DB -del $1 +end + +macro show.databases + if ($0 != 1) + echo "USAGE: show.databases" + break + end + if ($?DB:n == 0) + echo "no databases defined" + end + if ($DB:n == 0) + echo "no databases defined" + end + + local i + for i 0 $DB:n + echo $DB:$i + end +end + +# user functions to manipulate labels +macro add.label + if ($0 != 2) + echo "USAGE: add.label (label)" + break + end + if ($?LABEL:n == 0) + list LABEL -add $1 + return + end + + local found + $found = 0 + for i 0 $LABEL:n + if ($LABEL:$i == $1) + $found = 1 + echo "$LABEL:$i set" + last + end + end + + if ($found == 0) + list LABEL -add $1 + end +end + +macro del.label + if ($0 != 2) + echo "USAGE: del.label (label)" + break + end + if ($?LABEL:n == 0) + return + end + + list LABEL -del $1 +end + +macro show.labels + if ($0 != 1) + echo "USAGE: show.labels" + break + end + if ($?LABEL:n == 0) + echo "no labels defined" + end + if ($LABEL:n == 0) + echo "no labels defined" + end + + local i + for i 0 $LABEL:n + echo $LABEL:$i + end +end + +macro save.labels + if ($0 != 1) + echo "USAGE: save.labels" + break + end + if ($?LABEL:n == 0) + echo "no labels defined" + end + if ($LABEL:n == 0) + echo "no labels defined" + end + + exec rm -f labels.list + local i + for i 0 $LABEL:n + exec echo $LABEL:$i >> labels.list + end +end + +macro load.labels + if ($0 != 1) + echo "USAGE: load.labels" + break + end + + file labels.list found + if (not($found)) + echo "no saved labels in labels.list" + return + end + + list mylabels -x "cat labels.list" + local i + for i 0 $mylabels:n + add.label $mylabels:$i + end +end + +macro init.isp + list DB -add isp +end + +macro init.essence + list DB -add essence_v2 +end + +macro init.simtest.local + list DB -add simtest + $PARALLEL = 0 + controller exit true +end + +macro init.simtest.default.db + $host = `hostname` + if ($PARALLEL) + controller exit true + controller host add $host + end +end + +macro init.simtest + list DB -add simtest + + $host = `hostname` + if ($PARALLEL) + controller exit true + controller host add $host + end +end + +# need the ability to activate or deactivate specific tasks... + +macro detrend.on + detproc.on + detstack.on + detnorm.on + detresid.on + detreject.on + detcorr.on +end + +macro detrend.off + detproc.off + detstack.off + detnorm.off + detresid.off + detreject.off + detcorr.off +end + +macro detrend.revert.off + detproc.revert.off + detstack.revert.off + detnorm.revert.off + detresid.revert.off +end + +macro detrend.revert.on + detproc.revert.on + detstack.revert.on + detnorm.revert.on + detresid.revert.on +end + +macro detrend.reset + detproc.reset + detstack.reset + detnorm.reset + detresid.reset + detreject.reset +end + +macro all.on + detrend.on + flatcorr.on + chip.on + camera.on + addstar.on + fake.on + warp.on + diff.on + stack.on +end + +macro all.off + detrend.off + flatcorr.off + chip.off + camera.off + addstar.off + fake.off + warp.off + diff.off + stack.off +end + +macro module.tasks + module register.pro + module detrend.process.pro + module detrend.stack.pro + module detrend.norm.pro + module detrend.resid.pro + module detrend.reject.pro + module detrend.correct.pro + module flatcorr.pro + module chip.pro + module camera.pro + module addstar.pro + module fake.pro + module warp.pro + module diff.pro + module stack.pro +end + +macro detrend.modules + module detrend.process.pro + module detrend.stack.pro + module detrend.norm.pro + module detrend.resid.pro + module detrend.reject.pro + module detrend.correct.pro +end + +macro showcommand + local n + + if ($VERBOSE) + $command = $taskarg:0 + for n 1 $taskarg:n + $command = $command $taskarg:$n + end + date -var exitdate + if ($0 == 2) + echo "" + echo "$1 for: $command" + echo "job exit status: $JOB_STATUS" + echo "job host: $JOB_HOSTNAME" + echo "job dtime: $JOB_DTIME" + echo "job exit date: $exitdate" + else + echo "command: $command" + end + end +end + +# Add standard arguments to the poll functions +macro add_poll_args + if ($0 != 2) + echo "Must pass in the command of interest, and this function will supplement" + stop + end + + local command i + $command = $$1 -limit $POLL_LIMIT + + $$1 = $command +end + +macro add_poll_labels + if ($0 != 2) + echo "Must pass in the command of interest, and this function will supplement" + stop + end + + local command i + + $command = $$1 + + # Only process the data with the specified label. + for i 0 $LABEL:n + $command = $command -label $LABEL:$i + end + + $$1 = $command +end + +macro add_standard_args + if ($0 != 2) + echo "Must pass in the command of interest, and this function will supplement" + stop + end + + local command + $command = $$1 + + if ($?DBNAME && "$DBNAME" != "DEFAULT") + $command = $command --dbname $DBNAME + end + if ($?NOOP != 0) + if ($NOOP != 0) + $command = $command --no-op + end + end + if ($?NOUPDATE != 0) + if ($NOUPDATE != 0) + $command = $command --no-update + end + end + $$1 = $command --verbose +end + + +macro process_exit + if ($0 != 4) + echo "USAGE: process_exit (bookName) (pageName) (exitCode)" + break + end + + $bookName = $1 + $pageName = $2 + $exitCode = $3 + + if ($VERBOSE > 4) + echo "*** exit status ***" + echo "JOB_STATUS: $JOB_STATUS" + echo "JOB_HOSTNAME: $JOB_HOSTNAME" + echo "JOB_DTIME: $JOB_DTIME" + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + + # success + if ($exitCode == $EXIT_SUCCESS) + # the handler scripts update DB the tables; here we just update the page + book setword $bookName $pageName pantaskState DONE + return + end + + # failure related to the data files + # jobs which result in DATAERR must have db pantaskState updated + if ($exitCode == $EXIT_DATA_ERR) + showcommand failure + book setword $bookName $pageName pantaskState DATA_ERR + return + end + + if ($VERBOSE) + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + + # failure related to the data files + if ($exitCode == $EXIT_SYS_ERR) + # stop + showcommand "system failure" + book setword $bookName $pageName pantaskState SYS_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_CONFIG_ERR) + # stop + showcommand "config error" + book setword $bookName $pageName pantaskState CONFIG_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_PROG_ERR) + # stop + showcommand "programming error" + book setword $bookName $pageName pantaskState PROG_ERR + return + end + + # failure related to pantasks + if ($exitCode == $EXIT_CRASH_ERR) + # stop + showcommand "crash error" + book setword $bookName $pageName pantaskState CRASH_ERR + return + end + + # any other exit status + showcommand failure + book setword $bookName $pageName pantaskState UNKNOWN_ERR +end + +## XXX for the moment, remove all errors +macro process_cleanup + if ($0 != 2) + echo "USAGE: process_cleanup (bookname)" + break + end + + book delpage $1 -key pantaskState DONE + + if (not($KEEP_FAILURES)) + book delpage $1 -key pantaskState SYS_ERR + book delpage $1 -key pantaskState DATA_ERR + book delpage $1 -key pantaskState PROG_ERR + book delpage $1 -key pantaskState CONFIG_ERR + book delpage $1 -key pantaskState UNKNOWN_ERR + book delpage $1 -key pantaskState CRASH_ERR + book delpage $1 -key pantaskState CRASH + book delpage $1 -key pantaskState TIMEOUT + end +end + +macro set.poll + if ($0 != 2) + echo "USAGE:set.poll (value)" + break + end + + $POLL_LIMIT = $1 +end + +macro get.poll + echo "poll limit: $POLL_LIMIT" +end + +macro set.keep.failures + if ($0 != 2) + echo "USAGE:set.keep.failures (value)" + break + end + + $KEEP_FAILURES = $1 +end + +macro get.keep.failures + echo "KEEP_FAILURES: $KEEP_FAILURES" +end + + + + +macro set.verbosity + if ($0 != 2) + echo "USAGE: set.verbosity (level)" + break + end + $VERBOSE = $1 +end + +macro get.verbosity + echo "verbosity: $VERBOSE" +end + +macro show.variables + ?? +end + +macro set.workdir.by.camera + if ($0 != 6) + echo "USAGE: set.workdir.by.camera (camera) (class_id) (template) (default) (varname)" + break + end + + local host default template camera classID varname + $camera = $1 + $classID = $2 + $template = $3 + $default = $4 + $varname = $5 + + if ("$template" == "NULL") + $$varname = $workdir_template + echo "WARNING: WORKDIR template not set. Defaulting to $workdir_template" + return + end + + # missing camera and/or ipphosts table results in host = NULL + book getword ipphosts $camera $classID -var host + + if ("$host" == "NULL") + # we are modifying something like /data/@HOST@/foo/bar, + # but some implementations have multiple entries for the same host, + # like /data/@HOST@.0/foo/bar. + # try a couple of these first: + strsub $template @HOST@.0 $default -var $varname + strsub $$varname @HOST@.1 $default -var $varname + strsub $$varname @HOST@ $default -var $varname + else + strsub $template @HOST@ $host -var $varname + end + + strsub $$varname .0.0/ .0/ -var $varname + strsub $$varname .1.0/ .1/ -var $varname + strsub $$varname .2.0/ .2/ -var $varname +end + +macro set.workdir.by.skycell + if ($0 != 5) + echo "USAGE: set.workdir.by.skycell (skycellID) (template) (default) (varname)" + break + end + + local host default template skycellID varname count + $skycellID = $1 + $template = $2 + $default = $3 + $varname = $4 + + if ("$template" == "NULL") + $$varname = $workdir_template + echo "WARNING: WORKDIR template not set. Defaulting to $workdir_template" + return + end + + # get the folding count for this camera + book getword ipphosts skycell count -var count + if ("$count" == "NULL") + strsub $template @HOST@.0 $default -var $varname + strsub $$varname @HOST@.1 $default -var $varname + strsub $$varname @HOST@ $default -var $varname + return + end + + local j myValue skyhash + list word -splitbychar . $skycellID + $skyhash = 0 + for j 0 $word:n + inthash $word:$j $count -var myValue + $skyhash = $skyhash + $myValue + end + inthash $skyhash $count -var skyhash + sprintf skyname "sky%02d" $skyhash + + # missing ipphosts table results in host = NULL + book getword ipphosts skycell $skyname -var host + + if ("$host" == "NULL") + strsub $template @HOST@.0 $default -var $varname + strsub $$varname @HOST@.1 $default -var $varname + strsub $$varname @HOST@ $default -var $varname + else + # Replace .0 forms if we can, otherwise, use plain forms. + strsub $template @HOST@.0 $host -var $varname + strsub $$varname @HOST@ $host -var $varname + end +end + +macro set.host.for.camera + if ($0 != 3) + echo "USAGE: set.host.for.camera (camera) (class_id)" + break + end + + local host + + if (not($PARALLEL)) + host local + return + end + + # missing camera and/or ipphosts table results in host = NULL + book getword ipphosts $1 $2 -var host + + if ("$host" == "NULL") + host anyhost + else + # echo host $host + host $host + end +end + +macro set.host.for.skycell + if ($0 != 2) + echo "USAGE: set.host.for.skycell (skycellID)" + break + end + + local skycellID varname count host skyname skyhash fullhost + $skycellID = $1 + + # get the folding count for this camera + book getword ipphosts skycell count -var count + if ("$count" == "NULL") + host anyhost + return + end + + local j myValue skyhash + list word -splitbychar . $skycellID + $skyhash = 0 + for j 0 $word:n + inthash $word:$j $count -var myValue + $skyhash = $skyhash + $myValue + end + inthash $skyhash $count -var skyhash + sprintf skyname "sky%02d" $skyhash + + # missing ipphosts table results in host = NULL + book getword ipphosts skycell $skyname -var fullhost + list word -splitbychar . $fullhost + $host = $word:0 + + if ("$host" == "NULL") + host anyhost + else + # echo host $host + host $host + end +end + +macro get.host.for.camera + if ($0 != 3) + echo "USAGE: get.host.for.camera (camera) (class_id)" + break + end + + if (not($PARALLEL)) + echo local + return + end + + # missing camera and/or ipphosts table results in host = NULL + book getword ipphosts $1 $2 -var host + + if ("$host" == "NULL") + echo anyhost + else + echo $host + end +end + +macro check.globals + if ($?NETWORK == 0) + echo "NETWORK not defined: load pantasks.pro first" + break + end + if ($?PARALLEL == 0) + echo "PARALLEL not defined: load pantasks.pro first" + break + end + if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break + end + if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break + end + if ($?KEEP_FAILURES == 0) + echo "KEEP_FAILURES not defined: load pantasks.pro first" + break + end +end + +macro show.book + if ($0 != 2) + echo "USAGE: show.book (book)" + break + end + + book npages $1 -var npages + for i 0 $npages + book getpage $1 $i -var pagename + book listpage $1 $pagename + end + + echo "npages: $npages" +end + +macro del.page.from.book + if ($0 != 3) + echo "USAGE: del.page.from.book (book) (page)" + break + end + + book delpage $1 $2 +end + +macro tasks.revert.off + chip.revert.off + camera.revert.off + fake.revert.off + warp.revert.off + diff.revert.off + stack.revert.off +end +macro tasks.revert.on + chip.revert.on + camera.revert.on + fake.revert.on + warp.revert.on + diff.revert.on + stack.revert.on +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/pstamp.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/pstamp.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/pstamp.pro (revision 41106) @@ -0,0 +1,1266 @@ +# pstamp.pro : Postage Stamp Server tasks + +check.globals + +$LOGSUBDIR = $LOGDIR/pstamp +mkdir $LOGSUBDIR + +# these variables wil cycle through the known database names + +$pstampDS_DB = 0 +$pstampReq_DB = 0 +$pstampJob_DB = 0 +$pstampFin_DB = 0 +$pstampRev_DB = 0 +$pstampRevReq_DB = 0 +$pstampDep_DB = 0 +$pstampRevDep_DB = 0 +$pstampCleanup_DB = 0 +$pstampStopFaulted_DB = 0 +$pstampQCleanup_DB = 0 + +# limit number of requests in the queue to avoid blocking everything +# when jobs take an infinite amount of time to parse +$PSTAMP_PARSE_LIMIT = 20 +macro set.parse.limit + $PSTAMP_PARSE_LIMIT = $1 +end +macro get.parse.limit + echo parse limit : $PSTAMP_PARSE_LIMIT +end + +# give up on dependents with fault_count >= $PSTAMP_MAX_FAULT_COUNT +$PSTAMP_MAX_FAULT_COUNT = 3 +macro set.max.fault.count + $PSTAMP_MAX_FAULT_COUNT = $1 +end +macro get.max.fault.count + echo maximum fault count: $PSTAMP_MAX_FAULT_COUNT +end +$POLL_DEP = 500 + +macro set.dependent.poll + $POLL_DEP = $1 +end +macro get.dependent.poll + echo dependent poll limit: $POLL_DEP +end + +# set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config +# warning: no quotes around the two words. That causes the variable to get passed to pstamptool as one word +# and a tricky to debug problem ensues +# example: +# $PS_DBSERVER = -dbserver hostname + +if ($?PS_DBSERVER == 0) + $PS_DBSERVER = "" +end + +macro pstamp.reset + book init pstampRequest + book init pstampJob + book init pstampFinish + book init pstampDependent + book init pstampCleanup +end + +pstamp.reset + +macro pstamp.on + task pstamp.request.find + active true + end + task pstamp.request.load + active true + end + task pstamp.request.run + active true + end + + task request.finish.load + active true + end + task request.finish.run + active true + end + + task pstamp.job.load + active true + end + task pstamp.job.run + active true + end + task pstamp.dependent.load + active true + end +end + + +macro pstamp.off + task pstamp.request.find + active false + end + task pstamp.request.load + active false + end + task pstamp.request.run + active false + end + task request.finish.load + active false + end + task request.finish.run + active false + end + + task pstamp.job.load + active false + end + task pstamp.job.run + active false + end + task pstamp.dependent.load + active false + end + task pstamp.dependent.run + active false + end +end + +macro pstamp.revert.on + task pstamp.request.revert + active true + end + task pstamp.job.revert + active true + end + task pstamp.dependent.revert + active true + end + task pstamp.stopfaulted + active true + end +end +macro pstamp.revert.off + task pstamp.request.revert + active false + end + task pstamp.job.revert + active false + end + task pstamp.dependent.revert + active false + end + task pstamp.stopfaulted + active false + end +end +macro pstamp.cleanup.on + task pstamp.cleanup.load + active true + end + task pstamp.cleanup.run + active true + end + task pstamp.queue.cleanup + active true + end +end +macro pstamp.cleanup.off + task pstamp.cleanup.load + active false + end + task pstamp.cleanup.run + active false + end + task pstamp.queue.cleanup + active false + end +end + +macro pstamp.find.on + task pstamp.request.find + active true + end +end +macro pstamp.find.off + task pstamp.request.find + active false + end +end +macro pstamp.parse.on + task pstamp.request.load + active true + end + task pstamp.request.run + active true + end +end +macro pstamp.parse.off + task pstamp.request.load + active false + end + task pstamp.request.run + active false + end +end +macro pstamp.job.on + task pstamp.job.load + active true + end + task pstamp.job.run + active true + end +end +macro pstamp.job.off + task pstamp.job.load + active false + end + task pstamp.job.run + active false + end +end +macro pstamp.finish.on + task pstamp.finish.load + active true + end + task pstamp.finish.run + active true + end +end +macro pstamp.finish.off + task pstamp.finish.load + active false + end + task pstamp.finish.run + active false + end +end + +macro pstamp.dependent.on + task pstamp.dependent.load + active true + end + task pstamp.dependent.run + active true + end +end +macro pstamp.dependent.off + task pstamp.dependent.load + active false + end + task pstamp.dependent.run + active false + end +end + + +macro pstamp.status.on + task pstamp.save.status + active true + end +end +macro pstamp.status.off + task pstamp.save.status + active false + end +end + +macro pstamp.status.set.exec + task pstamp.save.status + periods -exec $1 + end +end + +echo DEFINING PLABEL stuff +# keep a separate list of labels for request parsing +if ($?PLABEL:n == 0) set PLABEL:n = 0 +set PLABEL:n = 0 + +macro add.parse.label + if ($0 != 2) + echo "USAGE: add.parse.label (label)" + break + end + if ($?PLABEL:n == 0) + list PLABEL -add $1 + return + end + + local found + $found = 0 + for i 0 $PLABEL:n + if ($PLABEL:$i == $1) + $found = 1 + echo "$PLABEL:$i set" + last + end + end + + if ($found == 0) + list PLABEL -add $1 + end +end + + +macro del.parse.label + if ($0 != 2) + echo "USAGE: del.parse.label (label)" + break + end + if ($?PLABEL:n == 0) + return + end + + list PLABEL -del $1 +end + +macro show.parse.labels + if ($0 != 1) + echo "USAGE: show.parse.labels" + break + end + if ($?PLABEL:n == 0) + echo "no labels defined" + end + if ($PLABEL:n == 0) + echo "no labels defined" + end + + local i + for i 0 $PLABEL:n + echo $PLABEL:$i + end +end + +macro add_parse_labels + if ($0 != 2) + echo "Must pass in the command of interest, and this function will supplement" + stop + end + + local command i + + $command = $$1 + + # Only process the data with the specified label. + for i 0 $PLABEL:n + $command = $command -label $PLABEL:$i + end + + $$1 = $command +end + +echo DONE defineing plabel stuf + +task pstamp.request.find + host local + + periods -poll $LOADPOLL + periods -exec 10 + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.request.find.log + if ($DB:n == 0) + option DEFAULT + command pstamp_queue_requests.pl --limit 5 + else + option $DB:$pstampDS_DB + command pstamp_queue_requests.pl --limit 5 --dbname $DB:$pstampDS_DB --verbose + $pstampDS_DB ++ + if ($pstampDS_DB >= $DB:n) set pstampDS_DB = 0 + end + end + + task.exit $EXIT_SUCCESS + # echo nothing to do + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.request.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.request.load.log + $run = pstamptool -pendingreq + if ($DB:n == 0) + option DEFAULT + else + option $DB:$pstampReq_DB + $run = $run $PS_DBSERVER -dbname $DB:$pstampReq_DB + $pstampReq_DB ++ + if ($pstampReq_DB >= $DB:n) set pstampReq_DB = 0 + end + add_poll_args run + # add_poll_labels run + add_parse_labels run + # limit number of requests in the queue to avoid blocking everything + # when jobs take an infinite amount of time to parse + #$run = $run -limit 10 + # except parsing is not label/prio based, high prio can get stuck behind low prio, so bump up to 20 + # -- should be a config option + $run = $run -limit $PSTAMP_PARSE_LIMIT + command $run + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampRequest -key req_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + echo starting request + book listbook pstampRequest + end + + process_cleanup pstampRequest + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.request.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.request.run.log + book npages pstampRequest -var N + if ($N == 0) break + + book getpage pstampRequest 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pstampRequest $pageName pantaskState RUN + book getword pstampRequest $pageName req_id -var REQ_ID + book getword pstampRequest $pageName dbname -var DBNAME + book getword pstampRequest $pageName uri -var URI + book getword pstampRequest $pageName ds_outProduct -var PRODUCT + book getword pstampRequest $pageName outdir -var OUTDIR + book getword pstampRequest $pageName need_magic -var NEED_MAGIC + book getword pstampRequest $pageName label -var LABEL + + host anyhost + + $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --outdir $OUTDIR --label $LABEL --redirect-output + + if ($NEED_MAGIC != 0) + $run = $run --need_magic + end + + add_standard_args run + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + + task.exit $EXIT_SUCCESS + process_exit pstampRequest $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pstampRequest $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword pstampRequest $options:0 pantaskState CRASH + end + + task.exit timeout + showcommand timeout + book setword pstampRequest $options:0 pantaskState TIMEOUT + end +end + +task request.finish.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.finish.load.log + $run = pstamptool -completedreq + if ($DB:n == 0) + option DEFAULT + else + option $DB:$pstampFin_DB + $run = $run -dbname $DB:$pstampFin_DB $PS_DBSERVER + $pstampFin_DB ++ + if ($pstampFin_DB >= $DB:n) set pstampFin_DB = 0 + end + add_poll_args run + add_poll_labels run + # limit query for finished requests to small number + # because we only run 4 at a time. Using the default queue + # depth fouls up the priority orer + # XXX: use a varaible + $run = $run -limit 8 + command $run + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampFinish -key req_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook pstampFinish + end + + process_cleanup pstampFinish + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + +task request.finish.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 2400 + host anyhost + npending 4 + + task.exec + stdout NULL + stderr $LOGSUBDIR/request.finish.run.log + book npages pstampFinish -var N + if ($N == 0) break + + book getpage pstampFinish 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pstampFinish $pageName pantaskState RUN + book getword pstampFinish $pageName req_id -var REQ_ID + book getword pstampFinish $pageName uri -var URI + book getword pstampFinish $pageName dbname -var DBNAME + book getword pstampFinish $pageName reqType -var REQ_TYPE + book getword pstampFinish $pageName name -var REQ_NAME + book getword pstampFinish $pageName outProduct -var PRODUCT + book getword pstampFinish $pageName outdir -var OUTDIR + + $run = request_finish.pl --req_id $REQ_ID --req_type $REQ_TYPE --req_file $URI --req_name $REQ_NAME --product $PRODUCT --outdir $OUTDIR --redirect-output + + add_standard_args run + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + task.exit $EXIT_SUCCESS + process_exit pstampFinish $options:0 $JOB_STATUS + end + + task.exit default + # echo request.finish.run task.exit status: $JOB_STATUS + process_exit pstampFinish $options:0 $JOB_STATUS + showcommand failure + end + + task.exit crash + showcommand crash + book setword pstampFinish $options:0 pantaskState CRASH.run + end + + task.exit timeout + showcommand timeout + book setword pstampFinish $options:0 pantaskState TIMEOUT.run + end +end + +task pstamp.job.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.job.load.log + $run = pstamptool -pendingjob + if ($DB:n == 0) + option DEFAULT + else + option $DB:$pstampJob_DB + $run = $run -dbname $DB:$pstampJob_DB $PS_DBSERVER + $pstampJob_DB ++ + if ($pstampJob_DB >= $DB:n) set pstampJob_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampJob -key job_id -uniq -setword dbname $options:0 -setword pantaskState INIT + + book npages pstampJob -var N + if ($VERBOSE > 2) + book listbook pstampJob + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pstampJob + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end + + task.exit default + showcommand failure + end + +end + +task pstamp.job.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 1200 + + task.exec + periods -exec $RUNEXEC + + stdout NULL + stderr $LOGSUBDIR/pstamp.job.run.log + book npages pstampJob -var N + if ($N == 0) break + + book getpage pstampJob 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + #echo pageName: $pageName + + book setword pstampJob $pageName pantaskState RUN + book getword pstampJob $pageName job_id -var JOB_ID + book getword pstampJob $pageName jobType -var JOB_TYPE + book getword pstampJob $pageName rownum -var ROWNUM + book getword pstampJob $pageName dbname -var DBNAME + book getword pstampJob $pageName outputBase -var OUTPUT_BASE + book getword pstampJob $pageName options -var OPTIONS + + if ($VERBOSE > 1) + book listpage pstampJob $pageName + end + + host anyhost + + $run = pstamp_job_run.pl --job_id $JOB_ID --job_type $JOB_TYPE --rownum $ROWNUM --output_base $OUTPUT_BASE --options $OPTIONS --redirect-output + add_standard_args run + + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + + task.exit $EXIT_SUCCESS + if ($VERBOSE > 1) + echo pstamp.job.run task.exit $JOB_ID status: $JOB_STATUS + end + process_exit pstampJob $options:0 $JOB_STATUS + end + task.exit default + if ($VERBOSE > 1) + echo pstamp.job.run task.exit $JOB_ID status: $JOB_STATUS + end +# showcommand failure + process_exit pstampJob $options:0 $JOB_STATUS + end + + task.exit crash + echo pstamp.job.run task.crash $JOB_ID status: $JOB_STATUS + process_exit pstampJob $options:0 $JOB_STATUS + showcommand crash + book setword pstampJob $options:0 pantaskState CRASH + end + + task.exit timeout + echo pstamp.job.run task.timeout $JOB_ID status: $JOB_STATUS + process_exit pstampJob $options:0 $JOB_STATUS + showcommand timeout + book setword pstampJob $options:0 pantaskState TIMEOUT + end +end + +task pstamp.request.revert + host local + + periods -poll $LOADPOLL + periods -exec 1200 + periods -timeout 20 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.request.revert.log + $run = pstamptool -revertreq + if ($DB:n == 0) + option DEFAULT + else + $run = $run $PS_DBSERVER -dbname $DB:$pstampRevReq_DB + $pstampRevReq_DB ++ + if ($pstampRevReq_DB >= $DB:n) set pstampRevReq_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + # nothing to do + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end +task pstamp.job.revert + host local + + periods -poll $LOADPOLL + periods -exec 300 + periods -timeout 300 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.job.revert.log + $run = pstamptool -revertjob + if ($DB:n == 0) + option DEFAULT + else + $run = $run $PS_DBSERVER -dbname $DB:$pstampRev_DB + $pstampRev_DB ++ + if ($pstampRev_DB >= $DB:n) set pstampRev_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + # nothing to do + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.dependent.load + host local + + periods -poll $LOADPOLL + # XXX: create a macro for this time + periods -exec 15 +# periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.dependent.load.log + $run = pstamptool -pendingdependent + if ($DB:n == 0) + option DEFAULT + else + option $DB:$pstampDep_DB + $run = $run -dbname $DB:$pstampDep_DB $PS_DBSERVER + $pstampDep_DB ++ + if ($pstampDep_DB >= $DB:n) set pstampDep_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run -limit $POLL_DEP + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampDependent -key dep_id -uniq -setword dbname $options:0 -setword pantaskState INIT + + book npages pstampDependent -var N + if ($VERBOSE > 2) + book listbook pstampDependent + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pstampDependent + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end + + task.exit default + showcommand failure + end + +end + +task pstamp.dependent.run + periods -poll $RUNPOLL + periods -exec 0.5 + periods -timeout 300 + + task.exec + # periods -exec $RUNEXEC + + book npages pstampDependent -var N + if ($N == 0) break + + book getpage pstampDependent 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pstampDependent $pageName pantaskState RUN + book getword pstampDependent $pageName dep_id -var DEP_ID + book getword pstampDependent $pageName state -var STATE + book getword pstampDependent $pageName stage -var STAGE + book getword pstampDependent $pageName stage_id -var STAGE_ID + book getword pstampDependent $pageName component -var COMPONENT + book getword pstampDependent $pageName imagedb -var IMAGEDB + book getword pstampDependent $pageName rlabel -var RLABEL + book getword pstampDependent $pageName label -var LABEL + book getword pstampDependent $pageName outdir -var OUTDIR + book getword pstampDependent $pageName need_magic -var NEED_MAGIC + book getword pstampDependent $pageName fault_count -var FAULT_COUNT + book getword pstampDependent $pageName dbname -var DBNAME + + if ($VERBOSE > 1) + book listpage pstampDependent $pageName + end + + host anyhost + + if ("$NEED_MAGIC" == "T") + $NEED_MAGIC="--need_magic" + else + $NEED_MAGIC="" + end + + $MYLOGFILE = $OUTDIR/checkdep.$DEP_ID.log + stdout NULL + stderr $MYLOGFILE + + $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL --label $LABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT --logfile $MYLOGFILE + + add_standard_args run + + options $pageName + + if ($VERBOSE > 1) + echo command $run + end +# periods -exec 0.05 + command $run + end + + + task.exit $EXIT_SUCCESS + if ($VERBOSE > 1) + echo pstamp.dependent.run task.exit $DEP_ID status: $JOB_STATUS + end + process_exit pstampDependent $options:0 $JOB_STATUS + end + task.exit default + if ($VERBOSE > 1) + echo pstamp.job.run task.exit $DEP_ID status: $JOB_STATUS + end +# showcommand failure + process_exit pstampDependent $options:0 $JOB_STATUS + end + + task.exit crash + echo pstamp.job.run task.crash $DEP_ID status: $JOB_STATUS + process_exit pstampDependent $options:0 $JOB_STATUS + showcommand crash + book setword pstampDependent $options:0 pantaskState CRASH + end + + task.exit timeout + echo pstamp.job.run task.timeout $DEP_ID status: $JOB_STATUS + process_exit pstampDependent $options:0 $JOB_STATUS + showcommand timeout + book setword pstampDependent $options:0 pantaskState TIMEOUT + end +end + +task pstamp.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.cleanup.load.log + $run = pstamptool -pendingcleanup + if ($DB:n == 0) + option DEFAULT + else + option $DB:$pstampCleanup_DB + $run = $run $PS_DBSERVER -dbname $DB:$pstampCleanup_DB + $pstampCleanup_DB ++ + if ($pstampCleanup_DB >= $DB:n) set pstampCleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampCleanup -key req_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + echo starting request + book listbook pstampCleanup + end + + process_cleanup pstampCleanup + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + # since everything is on one file system keep npending low to avoid + # overloading nfs + + npending 2 + + task.exec + periods -exec $RUNEXEC + + book npages pstampCleanup -var N + if ($N == 0) break + + book getpage pstampCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pstampCleanup $pageName pantaskState RUN + book getword pstampCleanup $pageName req_id -var REQ_ID + book getword pstampCleanup $pageName dbname -var DBNAME + book getword pstampCleanup $pageName name -var NAME + book getword pstampCleanup $pageName outdir -var OUTDIR + book getword pstampCleanup $pageName uri -var URI + book getword pstampCleanup $pageName reqType -var REQTYPE + book getword pstampCleanup $pageName outProduct -var PRODUCT + + # XXX: have the script set this up this + $MYLOGFILE=/data/ippc17.0/pstamp/work/logs/cleanup.$REQ_ID + stdout $MYLOGFILE + stderr $MYLOGFILE + + host anyhost + + $run = pstamp_cleanup.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --name $NAME --outdir $OUTDIR --reqType $REQTYPE + + add_standard_args run + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + + task.exit $EXIT_SUCCESS + process_exit pstampCleanup $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pstampCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword pstampCleanup $options:0 pantaskState CRASH + end + + task.exit timeout + showcommand timeout + book setword pstampCleanup $options:0 pantaskState TIMEOUT + end +end + +task pstamp.dependent.revert + host local + + periods -poll $LOADPOLL + periods -exec 900 + periods -timeout 300 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.dependent.revert.log + $run = pstamptool -revertdependent + if ($DB:n == 0) + option DEFAULT + else + $run = $run $PS_DBSERVER -dbname $DB:$pstampRevDep_DB + $pstampRevDep_DB ++ + if ($pstampRevDep_DB >= $DB:n) set pstampRevDep_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + # nothing to do + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.stopfaulted + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 300 + npending 1 + + task.exec + stdout $LOGSUBDIR/pstamp.stopfaulted.log + stderr $LOGSUBDIR/pstamp.stopfaulted.log + $run = pstampstopfaulted -fault_count $PSTAMP_MAX_FAULT_COUNT + if ($DB:n == 0) + option DEFAULT + else + $run = $run $PS_DBSERVER -dbname $DB:$pstampStopFaulted_DB + $pstampStopFaulted_DB ++ + if ($pstampStopFaulted_DB >= $DB:n) set pstampStopFaulted_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + # nothing to do + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.save.status + host local + + periods -poll $LOADPOLL + periods -exec 60 + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.save.status.log + + $run = pstamp_save_server_status.pl --update-link + command $run + end + + task.exit $EXIT_SUCCESS + # echo nothing to do + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.queue.cleanup + host local + + periods -poll $RUNPOLL + periods -exec 600 + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.queue.cleanup.log + + $run = pstamp_queue_cleanup.pl + command $run + end + + task.exit $EXIT_SUCCESS + # echo nothing to do + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.queue.update.cleanup + host local + trange 22:00:00 23:59:59 -nmax 2 + periods -poll $RUNPOLL + periods -exec 3600 + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.queue.update.cleanup.log + + $DBNAME = $DB:$pstampQCleanup_DB + $pstampQCleanup_DB++ + if ($pstampQCleanup_DB >= $DB:n) set pstampQCleanup_DB = 0 + + $run = pstamp_queue_update_cleanup.pl -$PS_DBSERVER --imagedb gpc1 --label ps_ud% + add_standard_args run + echo $run + command $run + end + + task.exit $EXIT_SUCCESS + # echo nothing to do + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/publish.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/publish.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/publish.pro (revision 41106) @@ -0,0 +1,224 @@ +## publish.pro: -*- sh -*- + +# test for required global variables +check.globals + +book init publishRun + +macro publish.status + book listbook publishRun +end + +macro publish.reset + book init publishRun +end + +macro publish.on + task publish.load + active true + end + task publish.run + active true + end + task publish.revert + active true + end +end + +macro publish.off + task publish.load + active false + end + task publish.run + active false + end + task publish.revert + active false + end +end + + +macro publish.revert.on + task publish.revert + active true + end +end + +macro publish.revert.off + task publish.revert + active false + end +end + + +# this variable will cycle through the known database names +$publish_load_DB = 0 +$publish_revert_DB = 0 + +task publish.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/publish.load.log + + task.exec + $run = pubtool -pending + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$publish_load_DB + $run = $run -dbname $DB:$publish_load_DB + $publish_load_DB ++ + if ($publish_load_DB >= $DB:n) set publish_load_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout publishRun -key pub_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook publishRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup publishRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task publish.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + + task.exec + book npages publishRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + + book getpage publishRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword publishRun $pageName pantaskState RUN + book getword publishRun $pageName pub_id -var PUB_ID + book getword publishRun $pageName camera -var CAMERA + book getword publishRun $pageName workdir -var WORKDIR_TEMPLATE + book getword publishRun $pageName product -var PRODUCT + book getword publishRun $pageName stage -var STAGE + book getword publishRun $pageName stage_id -var STAGE_ID + book getword publishRun $pageName dbname -var DBNAME + book getword publishRun $pageName output_format -var OUTPUT_FORMAT + book getword publishRun $pageName need_magic -var NEED_MAGIC + + stdout $LOGDIR/publish.run.log + stderr $LOGDIR/publish.run.log + + host anyhost + strsub $WORKDIR_TEMPLATE @HOST@ $default_host -var WORKDIR + + $run = publish_file.pl --pub_id $PUB_ID --camera $CAMERA --workdir $WORKDIR --product $PRODUCT --stage $STAGE --stage_id $STAGE_ID --output_format $OUTPUT_FORMAT --redirect-output + + if ("$NEED_MAGIC" == "T") + $run = $run --need-magic + end + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit publishRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword publishRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword publishRun $options:0 pantaskState TIMEOUT + end +end + + +task publish.revert + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = pubtool -revert + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$publish_revert_DB + $run = $run -dbname $DB:$publish_revert_DB + $publish_revert_DB ++ + if ($publish_revert_DB >= $DB:n) set publish_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/rawcheck.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/rawcheck.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/rawcheck.pro (revision 41106) @@ -0,0 +1,190 @@ +## replicate.pro : tasks for data replication : -*- sh -*- + +# test for required global variables +check.globals + +# load in the local configuration (NEB_USER, etc.) +# module nebulous.site.pro + +$LOGSUBDIR = $LOGDIR/rawcheck +mkdir $LOGSUBDIR + +book init rawcheckPending + +macro rawcheck.reset + book init checkPending +end + +macro rawcheck.status + book listbook rawcheckPending +end + +macro rawcheck.on + task rawcheck.load + active true + end + task rawcheck.run + active true + end +end + +macro rawcheck.off + task rawcheck.load + active false + end + task rawcheck.run + active false + end +end + +# this variable will cycle through the known database names +$rawcheck_DB = 0 + + +$DATEOBS_MIN = 1970-01-01T00:00:00 + +macro rawcheck.set.date + if ($0 != 2) + echo "USAGE: rawcheck.set.date (YYYY-MM-DD)" + break + end + + $DATEOBS_MIN = "$1" +end + +macro rawcheck.show.date + + echo "DATE: " + echo $DATEOBS_MIN +end + + +task rawcheck.load + host local + + # modify these after the tasks are tested + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 60 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/rawcheck.log + + task.exec + book npages rawcheckPending -var N + if ($N > 200) + process_cleanup rawcheckPending + break + end + if ($N > 1) + book getpage rawcheckPending N -var pageName + book getword rawcheckPending $pageName dateobs -var DATEOBS_MIN + end + + + $run = regtool -processedexp -state full -dateobs_begin $DATEOBS_MIN + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$rawcheck_DB + $run = $run -dbname $DB:$rawcheck_DB + $rawcheck_DB ++ + if ($rawcheck_DB >= $DB:n) set rawcheck_DB = 0 + + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout rawcheckPending -key exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook rawcheckPending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup rawcheckPending + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +task rawcheck.run + periods -poll 0.5 + periods -exec 5 + periods -timeout 1200 + host anyhost + + + task.exec + periods -exec 5 + + book npages rawcheckPending -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new objects in rawcheckPending + book getpage rawcheckPending 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword rawcheckPending $pageName pantaskState RUN + + # XXX what values do I need to get back? + book getword rawcheckPending $pageName exp_id -var EXP_ID + book getword rawcheckPending $pageName dbname -var DBNAME + +# stdout NULL + stdout $LOGSUBDIR/rawcheck.log + stderr $LOGSUBDIR/rawcheck.log + + # these operations do not require a database to be specified + $run = rawcheck.pl --dbname $DBNAME --exp_id $EXP_ID + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit rawcheckPending $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword rawcheckPending $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword rawcheckPending $options:0 pantaskState TIMEOUT + end +end + + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/rcserver.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/rcserver.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/rcserver.pro (revision 41106) @@ -0,0 +1,214 @@ +## rcserver.pro : tasks for managing the server side of the ipp data distribution system : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/rcserver +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init rcPendingFS + +### Database lists +$rcPendingFS_DB = 0 +$rcRevertFS_DB = 0 + +### Check status of tasks +macro rcserver.status + book listbook rcPendingFS +end + +### Reset tasks +macro rcserver.reset + book init rcPendingFS +end + +### Turn tasks on +macro rcserver.on + task rcserver.makefileset.load + active true + end + task rcserver.makefileset.run + active true + end +end +macro rcserver.off + task rcserver.makefileset.load + active false + end + task rcserver.makefileset.run + active false + end +end + +macro rcserver.revert.on + task rcserver.revert + active true + end +end + +macro rcserver.revert.off + task rcserver.revert + active false + end +end + +task rcserver.makefileset.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/rcserver.makefileset.load.log + + task.exec + $run = disttool -pendingfileset + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$rcPendingFS_DB + $run = $run -dbname $DB:$rcPendingFS_DB + $rcPendingFS_DB ++ + if ($rcPendingFS_DB >= $DB:n) set rcPendingFS_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout rcPendingFS -key dist_id:dest_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook rcPendingFS + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup rcPendingFS + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task rcserver.makefileset.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + npending 5 + + task.exec + periods -exec 10 + + book npages rcPendingFS -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new components to process (pantaskState == INIT) + book getpage rcPendingFS 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book getword rcPendingFS $pageName dist_id -var DIST_ID + book getword rcPendingFS $pageName target_id -var TARGET_ID + book getword rcPendingFS $pageName stage -var STAGE + book getword rcPendingFS $pageName stage_id -var STAGE_ID + book getword rcPendingFS $pageName data_group -var DATA_GROUP + book getword rcPendingFS $pageName filter -var FILTER + book getword rcPendingFS $pageName dist_dir -var DIST_DIR + book getword rcPendingFS $pageName dest_id -var DEST_ID + book getword rcPendingFS $pageName product_name -var PRODUCT_NAME + book getword rcPendingFS $pageName ds_dbhost -var DS_DBHOST + book getword rcPendingFS $pageName ds_dbname -var DS_DBNAME + book getword rcPendingFS $pageName dbname -var DBNAME + + host anyhost + + sprintf logfile "%s/makefs.%s.%s.log" $DIST_DIR $DIST_ID $DEST_ID + + book setword rcPendingFS $pageName pantaskState RUN + + $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --data_group $DATA_GROUP --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR --logfile $logfile + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit rcPendingFS $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword rcPendingFS $options:0 pantaskState TIMEOUT + end +end + + +task rcserver.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = disttool -revertfileset + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$rcRevertFS_DB + $run = $run -dbname $DB:$rcRevertFS_DB + $rcRevertFS_DB ++ + if ($rcRevertFS_DB >= $DB:n) set rcRevertFS_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/receive.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/receive.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/receive.pro (revision 41106) @@ -0,0 +1,572 @@ +## receive.pro: -*- sh -*- + +### receivetool -definesource: add new receive source (run by user for setup) +### receivetool -list: get list of sources (for pantasks setup) +### +### receive_source.pl: launched regularly by pantasks; check source for new filesets +### dsproductls: get available filesets from datastore +### receivetool -addfileset: add new fileset +### receivetool -updatelast: update last fileset for source +### +### receivetool -pendingfileset: get pending fileset (for pantasks) +### receive_fileset.pl: launched by pantasks; get list of files +### dsfilesetls: get files for fileset +### receivetool -addfiles: add file to receive +### +### receivetool -pendingfile: get pending file (for pantasks) +### receive_file.pl: launched by pantasks; download file, extract fileset, import +### dsget: get file from datastore +### receivetool -addresult: add result +### +### receivetool -revert: remove result after something goes wrong +### +### receivetool -toadvance: filesets for which all files have been downloaded and are ready +### to complete processing + +# test for required global variables +check.globals + +book init receiveSource +book init receiveFileset +book init receiveFile +book init receiveAdvance + +macro receive.status + book listbook receiveSource + book listbook receiveFileset + book listbook receiveFile + book listbook receiveAdvance +end + +macro receive.reset + book init receiveSource + book init receiveFileset + book init receiveFile + book init receiveAdvance +end + +macro receive.on + task receive.source.load + active true + end + task receive.source.run + active true + end + task receive.fileset.load + active true + end + task receive.fileset.run + active true + end + task receive.file.load + active true + end + task receive.file.run + active true + end + task receive.advance.load + active true + end + task receive.advance.run + active true + end +end + +macro receive.off + task receive.source.load + active false + end + task receive.source.run + active false + end + task receive.fileset.load + active false + end + task receive.fileset.run + active false + end + task receive.file.load + active false + end + task receive.file.run + active false + end + task receive.advance.load + active false + end + task receive.advance.run + active false + end +end + + +# this variable will cycle through the known database names +$receive_DB = 0 +$receive_Advance_DB = 0 + +# set this to skip extraction of tarfiles into their destination directories +$NO_EXTRACT = "" +macro set.no.extract + $NO_EXTRACT = "--no-extract" +end +macro clear.no.extract + $NO_EXTRACT = "" +end +macro show.no.extract + echo $NO_EXTRACT +end + +task receive.source.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/receive.source.log + + task.exec + $run = receivetool -list + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$receive_DB + $run = $run -dbname $DB:$receive_DB + $receive_DB ++ + if ($receive_DB >= $DB:n) set receive_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout receiveSource -key source_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook receiveSource + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup receiveSource + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task receive.source.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages receiveSource -var N + if ($N == 0) break + if ($NETWORK == 0) break + + book getpage receiveSource 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword receiveSource $pageName pantaskState RUN + book getword receiveSource $pageName source_id -var SOURCE_ID + book getword receiveSource $pageName source -var SOURCE + book getword receiveSource $pageName product -var PRODUCT + book getword receiveSource $pageName fileset_last -var FILESET + book getword receiveSource $pageName dbname -var DBNAME + book getword receiveSource $pageName state -var RUN_STATE + + stdout $LOGDIR/receive.source.log + stderr $LOGDIR/receive.source.log + + $run = receive_source.pl --source_id $SOURCE_ID --source $SOURCE --product $PRODUCT + if ("$FILESET" != "NULL") + $run = $run --last_fileset $FILESET + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit receiveSource $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword receiveSource $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword receiveSource $options:0 pantaskState TIMEOUT + end +end + + +task receive.fileset.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/receive.fileset.log + + task.exec + $run = receivetool -pendingfileset + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$receive_DB + $run = $run -dbname $DB:$receive_DB + $receive_DB ++ + if ($receive_DB >= $DB:n) set receive_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout receiveFileset -key fileset_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook receiveFileset + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup receiveFileset + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task receive.fileset.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages receiveFileset -var N + if ($N == 0) break + if ($NETWORK == 0) break + + book getpage receiveFileset 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword receiveFileset $pageName pantaskState RUN + book getword receiveFileset $pageName fileset_id -var FILESET_ID + book getword receiveFileset $pageName source -var SOURCE + book getword receiveFileset $pageName product -var PRODUCT + book getword receiveFileset $pageName fileset -var FILESET + book getword receiveFileset $pageName dbname -var DBNAME + book getword receiveFileset $pageName state -var RUN_STATE + + stdout $LOGDIR/receive.fileset.log + stderr $LOGDIR/receive.fileset.log + + host anyhost + + $run = receive_fileset.pl --fileset_id $FILESET_ID --source $SOURCE --product $PRODUCT --fileset $FILESET + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit receiveFileset $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword receiveFileset $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword receiveFileset $options:0 pantaskState TIMEOUT + end +end + + +task receive.file.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/receive.file.log + + task.exec + $run = receivetool -pendingfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$receive_DB + $run = $run -dbname $DB:$receive_DB + $receive_DB ++ + if ($receive_DB >= $DB:n) set receive_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout receiveFile -key file_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook receiveFile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup receiveFile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task receive.file.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 600 + + task.exec + book npages receiveFile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + book getpage receiveFile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword receiveFile $pageName pantaskState RUN + book getword receiveFile $pageName file_id -var FILE_ID + book getword receiveFile $pageName source -var SOURCE + book getword receiveFile $pageName product -var PRODUCT + book getword receiveFile $pageName fileset -var FILESET + book getword receiveFile $pageName fileset_id -var FILESET_ID + book getword receiveFile $pageName file -var FILE + book getword receiveFile $pageName bytes -var BYTES + book getword receiveFile $pageName md5sum -var MD5SUM + book getword receiveFile $pageName component -var COMPONENT + book getword receiveFile $pageName workdir -var WORKDIR + book getword receiveFile $pageName dirinfo -var DIRINFO + book getword receiveFile $pageName dbname -var DBNAME + book getword receiveFile $pageName state -var RUN_STATE + + stdout $LOGDIR/receive.file.log + stderr $LOGDIR/receive.file.log + + host anyhost + + $run = receive_file.pl --file_id $FILE_ID --source $SOURCE --product $PRODUCT --fileset $FILESET --fileset_id $FILESET_ID --file $FILE --component $COMPONENT --bytes $BYTES --md5 $MD5SUM --workdir $WORKDIR --dirinfo $DIRINFO $NO_EXTRACT + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit receiveFile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword receiveFile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword receiveFile $options:0 pantaskState TIMEOUT + end +end + +task receive.advance.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/receive.advance.log + + task.exec + $run = receivetool -toadvance + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$receive_Advance_DB + $run = $run -dbname $DB:$receive_Advance_DB + $receive_Advance_DB ++ + if ($receive_Advance_DB >= $DB:n) set receive_Advance_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout receiveAdvance -key fileset_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook receiveAdvance + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup receiveAdvance + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task receive.advance.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + book npages receiveAdvance -var N + if ($N == 0) break + if ($NETWORK == 0) break + + book getpage receiveAdvance 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword receiveAdvance $pageName pantaskState RUN + book getword receiveAdvance $pageName fileset_id -var FILESET_ID + book getword receiveAdvance $pageName fileset -var FILESET + book getword receiveAdvance $pageName dbinfo -var DBINFO + book getword receiveAdvance $pageName status_product -var STATUS_PRODUCT + book getword receiveAdvance $pageName ds_dbname -var DS_DBNAME + book getword receiveAdvance $pageName ds_dbhost -var DS_DBHOST + book getword receiveAdvance $pageName dbname -var DBNAME + + stdout $LOGDIR/receive.advance.log + stderr $LOGDIR/receive.advance.log + + host anyhost + + $run = receive_advance.pl --fileset_id $FILESET_ID --fileset $FILESET --dbinfo $DBINFO --status_product $STATUS_PRODUCT --ds_dbname $DS_DBNAME --ds_dbhost $DS_DBHOST + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit receiveAdvance $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword receiveAdvance $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword receiveAdvance $options:0 pantaskState TIMEOUT + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/register.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/register.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/register.pro (revision 41106) @@ -0,0 +1,791 @@ +## register.pro : tasks for image registration (insert into database) : -*- sh -*- +## this file contains the tasks for running the registration stage +## these tasks use the books regPendingImfile and regPendingExp + +# test for required global variables +check.globals + +book init regPendingImfile +book init regPendingExp +book init regPendingBurntoolImfile +book init regDates + +macro register.reset + book init regPendingImfile + book init regPendingExp + book init regPendingBurntoolImfile + book init regDates +end + +macro register.status + book list + book listbook regPendingImfile + book listbook regPendingExp + book listbook regPendingBurntoolImfile + book listbook regDates +end + +macro register.on + task register.imfile.load + active true + end + task register.imfile.run + active true + end + task register.imfile.revert + active false + end + task register.exp.revert + active false + end + task register.exp.load + active true + end + task register.exp.run + active true + end + task register.burntool.load + active true + end + task register.burntool.run + active true + end +end + +macro register.off + task register.imfile.load + active false + end + task register.imfile.run + active false + end + task register.imfile.revert + active false + end + task register.exp.revert + active false + end + task register.exp.load + active false + end + task register.exp.run + active false + end + task register.burntool.load + active false + end + task register.burntool.run + active false + end +end + +macro register.revert.on + task register.imfile.revert + active true + end + task register.exp.revert + active true + end +end + +macro register.revert.off + task register.imfile.revert + active false + end + task register.exp.revert + active false + end +end + +macro burntool.on + task register.burntool.load + active true + end + task register.burntool.run + active true + end +end + +macro burntool.off + task register.burntool.load + active false + end + task register.burntool.run + active false + end +end + +macro register.add.date + if ($0 != 4) + echo "USAGE: register.add.date (YYYY-MM-DD) (DBNAME) (VALID_BURNTOOL)" + break + end + + book newpage regDates $1 + book setword regDates $1 nsState NEW + book setword regDates $1 dbname $2 + book setword regDates $1 valid_burntool $3 +end + + +# $valid_burntool_value = 14 +$sunrise = 17:30:00 +$sunset = 03:30:00 + +macro replace.survey.values + if ($0 != 5) + echo "USAGE: replace.survey.values (varname) (CAMERA) (FILTER) (DATE)" + break + end + + strsub $$1 @CAMERA@ $2 -var $1 + strsub $$1 @FILTER@ $3 -var $1 + strsub $$1 @DATE@ $4 -var $1 + if ($VERBOSE > 2) + echo "adding $$1" + end +end + +# we want to set the following arguments to register_exp.pl +# --label ($myLABEL) --dvodb ($myDVODB) --workdir ($myWORKDIR) +macro set.survey.data + if ($0 != 7) + echo "USAGE: set.survey.data (surveyID) (camera) (filter) (datestr) (workdir) (cmdflags)" + break + end + + local surveyID myCAMERA myFILTER myDATE myUPDATE + + $surveyID = $1 + $myCAMERA = $2 + $myFILTER = $3 + $myDATE = $4 + + $$6 = "" + + # we will find a more elegant solution for this later + if ("$myFILTER" == "z.00000") return + if ("$myFILTER" == "w.00000") return + + $myUPDATE1 = "" + book getword surveys $surveyID label -var myLABEL + if ("$myLABEL" != "NULL") + replace.survey.values myLABEL $myCAMERA $myFILTER $myDATE + $myUPDATE1 = --label $myLABEL + end + + $myUPDATE2 = "" + book getword surveys $surveyID dvodb -var myDVODB + if ("$myDVODB" != "NULL") + replace.survey.values myDVODB $myCAMERA $myFILTER $myDATE + $myUPDATE2 = --dvodb $myDVODB + end + + $myUPDATE3 = "" + book getword surveys $surveyID end_stage -var myENDSTAGE + if ("$myENDSTAGE" != "NULL") + replace.survey.values myENDSTAGE $myCAMERA $myFILTER $myDATE + $myUPDATE3 = --end_stage $myENDSTAGE + end + + $myUPDATE4 = "" + book getword surveys $surveyID tess_id -var myTESSID + if ("$myTESSID" != "NULL") + replace.survey.values myTESSID $myCAMERA $myFILTER $myDATE + $myUPDATE4 = --tess_id $myTESSID + end + + $$6 = $myUPDATE1 $myUPDATE2 $myUPDATE3 $myUPDATE4 + + book getword surveys $surveyID worksubdir -var myWORKSUB + if ("$myWORKSUB" != "NULL") + replace.survey.values myWORKSUB $myCAMERA $myFILTER $myDATE + $$5 = $$5/$myWORKSUB + end +end + +macro load.surveys + queueload tmp -x "cat $MODULES:0/surveys.mhpcc.config" + ipptool2book tmp surveys -key survey +end + +# these variables will cycle through the known database names +$regPendingImfile_DB = 0 +$regPendingBurntoolImfile_DB = 0 +$regRevertImfile_DB = 0 +$regRevertExp_DB = 0 +$regPendingExp_DB = 0 +$reg_datePAGE = 0 +# select images ready for register analysis +# new entries are added to regPendingImfile +# compare the new list with the ones already selected +task register.imfile.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGDIR/register.imfile.load.log + + # select entries from the current DB; cycle to the next DB, if it exists + # iff the DB list is not set, use the value defined in .ipprc + task.exec + $run = regtool -pendingimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$regPendingImfile_DB + $run = $run -dbname $DB:$regPendingImfile_DB + $regPendingImfile_DB ++ + if ($regPendingImfile_DB >= $DB:n) set regPendingImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingImfile -key exp_id:tmp_class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle regPendingImfile + if ($VERBOSE > 2) + book listbook regPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup regPendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the register_imfile.pl script on pending images +task register.imfile.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + periods -exec $RUNEXEC + + book npages regPendingImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in regPendingImfile + book getpage regPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingImfile $pageName pantaskState RUN + + book getword regPendingImfile $pageName exp_id -var EXP_ID + book getword regPendingImfile $pageName tmp_exp_name -var TMP_EXP_NAME + book getword regPendingImfile $pageName tmp_camera -var TMP_CAMERA + book getword regPendingImfile $pageName tmp_class_id -var TMP_CLASS_ID + book getword regPendingImfile $pageName uri -var URI + book getword regPendingImfile $pageName bytes -var BYTES + book getword regPendingImfile $pageName md5sum -var MD5SUM + book getword regPendingImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword regPendingImfile $pageName dbname -var DBNAME + book getword regPendingImfile $pageName summit_dateobs -var SUMMIT_DATEOBS + + # EXP_TAG is used to generate the unique, but human-readable, filenames + sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID + + # specify choice of remote host + set.host.for.camera $TMP_CAMERA $TMP_CLASS_ID + + # set the WORKDIR variable + set.workdir.by.camera $TMP_CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf logfile "%s/%s/%s.reg.%s.log" $WORKDIR $EXP_TAG $EXP_TAG $TMP_CLASS_ID + + stdout $LOGDIR/register.imfile.run.log + stderr $LOGDIR/register.imfile.run.log + + # XXX register_imfile.pl differs from the standard script : it does not have an 'outroot' argument, and it does not take '--redirect' + $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI --logfile $logfile --bytes $BYTES --md5sum $MD5SUM + $run = $run --sunset $sunset --sunrise $sunrise --summit_dateobs $SUMMIT_DATEOBS + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit regPendingImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword regPendingImfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword regPendingImfile $options:0 pantaskState TIMEOUT + end +end + +task register.imfile.revert + host local + periods -poll 60.0 + periods -exec 600.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/register.imfile.revert.log + + task.exec + $run = regtool -revertprocessedimfile + if ($DB:n == 0) + option DEFAULT + else + option $DB:$regRevertImfile_DB + $run = $run -dbname $DB:$regRevertImfile_DB + $regRevertImfile_DB ++ + if ($regRevertImfile_DB >= $DB:n) set regRevertImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out + task.exit timeout + showcommand timeout + end +end + + + +task register.exp.revert + host local + periods -poll 60.0 + periods -exec 600.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/register.exp.revert.log + + task.exec +# Only revert fault 2, because other faults likely mean something horrible is happening. + $run = regtool -revertprocessedexp -fault 2 + if ($DB:n == 0) + option DEFAULT + else + option $DB:$regRevertExp_DB + $run = $run -dbname $DB:$regRevertExp_DB + $regRevertExp_DB ++ + if ($regRevertExp_DB >= $DB:n) set regRevertExp_DB = 0 + end +# add_poll_args run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out + task.exit timeout + showcommand timeout + end +end + + +# select exposures ready for register_exp.pl +task register.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/register.exp.load.log + + task.exec + $run = regtool -pendingexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$regPendingExp_DB + $run = $run -dbname $DB:$regPendingExp_DB + $regPendingExp_DB ++ + if ($regPendingExp_DB >= $DB:n) set regPendingExp_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingExp -key exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook regPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup regPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the register_exp.pl script on pending exposures +task register.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + book npages regPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in regPendingExp (pantaskState == INIT) + book getpage regPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingExp $pageName pantaskState RUN + book getword regPendingExp $pageName exp_id -var EXP_ID + book getword regPendingExp $pageName tmp_exp_name -var TMP_EXP_NAME + book getword regPendingExp $pageName tmp_camera -var TMP_CAMERA + book getword regPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword regPendingExp $pageName dbname -var DBNAME + book getword regPendingExp $pageName camera -var CAMERA + book getword regPendingExp $pageName filter -var FILTER + book getword regPendingExp $pageName dateobs -var DATEOBS + book getword regPendingExp $pageName obs_mode -var OBS_MODE + book getword regPendingExp $pageName obs_group -var OBS_GROUP + + # EXP_TAG is used to generate the unique, but human-readable, filenames + sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID + + # 2007-08-30T05:09:59Z + strlen $DATEOBS length + if ($length > 10) + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 5 2 MONTH + $datestr = "$YEAR\$MONTH" + end + + # look up label, dvodb based on survey + set.survey.data $OBS_MODE $CAMERA $FILTER $datestr WORKDIR_TEMPLATE CMDFLAGS + + # specify choice of remote host + set.host.for.camera $TMP_CAMERA FPA + + # set the WORKDIR variable + set.workdir.by.camera $TMP_CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@.0/gpc1/20080130 + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + ## generate output log based on filerule + sprintf logfile "%s/%s/%s.reg.log" $WORKDIR $EXP_TAG $EXP_TAG + + stdout $LOGDIR/register.exp.run.log + stderr $LOGDIR/register.exp.run.log + + $run = register_exp.pl --exp_id $EXP_ID --exp_tag $EXP_TAG --logfile $logfile $CMDFLAGS + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit regPendingExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword regPendingExp $options:0 pantaskState CRASH + end + + task.exit crash + showcommand crash + book setword regPendingExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword regPendingExp $options:0 pantaskState TIMEOUT + end +end + + +# select imfiles that can now be burntooled. +task register.burntool.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGDIR/register.burntool.load.log + + # select entried from the current DB; cycle to the next DB, if it exists + task.exec + book npages regDates -var Npages + if ($Npages == 0) + $today = `date -u +%Y-%m-%d` + $dbname = $DB:0 + $valid_burntool = 14 + else + book getpage regDates $reg_datePAGE -var today + book getword regDates $today dbname -var dbname + book getword regDates $today valid_burntool -var valid_burntool + $reg_datePAGE ++ + if ($reg_datePAGE >= $Npages) set reg_datePAGE = 0 + end + + $run = regtool -pendingburntoolimfile + +# debugging purposes +# $today = "2010-12-25" + $dateobs_begin = $today\T$sunset + $dateobs_end = $today\T$sunrise + $run = $run -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end -valid_burntool $valid_burntool + $run = $run -dbname $dbname + option $dbname +# if ($DB:n == 0) +# option DEFAULT +# else +# # save the DB name for the exit tasks +# option $DB:$regPendingBurntoolImfile_DB +# $run = $run -dbname $DB:$regPendingBurntoolImfile_DB +# if ("$dbname" != "$DB:$regPendingBurntoolImfile_DB") +# break +# end +# $regPendingBurntoolImfile_DB ++ +# if ($regPendingBurntoolImfile_DB >= $DB:n) set regPendingBurntoolImfile_DB = 0 +# end + +# echo $run + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingBurntoolImfile -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle regPendingBurntoolImfile + if ($VERBOSE > 2) + book listbook regPendingBurntoolImfile + end + + # delete existing entries in the appropriate pantasksStates + process_cleanup regPendingBurntoolImfile + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +# run the ipp_apply_burntool_single.pl script on the pending images +task register.burntool.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + periods -exec $RUNEXEC + + book npages regPendingBurntoolImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images to burn in regPendingBurntoolImfile + book getpage regPendingBurntoolImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingBurntoolImfile $pageName pantaskState RUN + + book getword regPendingBurntoolImfile $pageName exp_id -var EXP_ID + book getword regPendingBurntoolImfile $pageName tmp_class_id -var TMP_CLASS_ID + book getword regPendingBurntoolImfile $pageName class_id -var CLASS_ID + book getword regPendingBurntoolImfile $pageName uri -var THIS_URI + book getword regPendingBurntoolImfile $pageName previous_uri -var PREVIOUS_URI + book getword regPendingBurntoolImfile $pageName camera -var CAMERA + book getword regPendingBurntoolImfile $pageName dbname -var DBNAME + set.host.for.camera $DBNAME $TMP_CLASS_ID + + stdout $LOGDIR/register.burntool.run.log + stderr $LOGDIR/register.burntool.run.log + + $run = ipp_apply_burntool_single.pl --camera $CAMERA --exp_id $EXP_ID --class_id $CLASS_ID --this_uri $THIS_URI --continue 10 + if ("$PREVIOUS_URI" != "NULL") + $run = $run --previous_uri $PREVIOUS_URI + end + + add_standard_args run + + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit regPendingBurntoolImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword regPendingBurntoolImfile $options:0 pantaskState CRASH + end + + # operation timed out + task.exit timeout + showcommand timeout + book setword regPendingBurntoolImfile $options:0 pantaskState TIMEOUT + end +end + +task register.initday.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + trange 0:00:00 1:00:00 -nmax 1 + npending 1 + + task.exec + $today = `date -u +%Y-%m-%d` + book newpage regDates $today + book setword regDates $today nsState NEW + book setword regDates $today dbname $DB:0 + book setword regDates $today valid_burntool 14 + + command true + end + + task.exit 0 + # nothing to do here? + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + + +## XXX add a global path to output files Index: /branches/ccl_branches/ipponly-20191108/ippTasks/release.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/release.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/release.pro (revision 41106) @@ -0,0 +1,167 @@ +## release.pro : tasks for release management : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/release +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init pendingrelGroup + +### Database lists +$relgroup_DB = 0 + +### Check status of release tasks +macro release.status + book listbook pendingrelGroup +end + +### Reset release tasks +macro release.reset + book init pendingrelgroup +end + +### Turn release tasks on +macro release.on + task relgroup.load + active true + end + task relgroup.run + active true + end +end + +### Turn release tasks off +macro release.off + task relgroup.load + active false + end + task relgroup.run + active false + end +end + +### Load jobs for relGroup +### Tasks are loaded into pendingrelGroup. +task relgroup.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/relgroup.log + + task.exec + if ($LABEL:n == 0) break + $run = releasetool -pendingrelgroup + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$relgroup_DB + $run = $run -dbname $DB:$relgroup_DB + $relgroup_DB ++ + if ($relgroup_DB >= $DB:n) set relgroup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pendingrelGroup -key group_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook pendingrelGroup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pendingrelGroup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task relgroup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 120 + npending 1 + + task.exec + book npages pendingrelGroup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in pendingrelGroup (pantaskState == INIT) + book getpage pendingrelGroup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pendingrelGroup $pageName pantaskState RUN + book getword pendingrelGroup $pageName group_id -var GROUP_ID + book getword pendingrelGroup $pageName rel_id -var REL_ID + book getword pendingrelGroup $pageName group_type -var GROUP_TYPE + book getword pendingrelGroup $pageName lap_id -var LAP_ID + book getword pendingrelGroup $pageName group_name -var GROUP_NAME + book getword pendingrelGroup $pageName release_name -var RELEASE_NAME + book getword pendingrelGroup $pageName dbname -var DBNAME + + stdout $LOGSUBDIR/relgroup.log + stderr $LOGSUBDIR/relgroup.log + + host anyhost + + $run = relgroup_exp_list.pl --group_id $GROUP_ID --group_type $GROUP_TYPE --release_name $RELEASE_NAME --rebuild + if ("$GROUP_TYPE" == "lap") + $run = $run --lap_id $LAP_ID + else + $run = $run --group_name $GROUP_NAME + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pendingrelGroup $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword pendingrelGroup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pendingrelGroup $options:0 pantaskState TIMEOUT + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/remote.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/remote.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/remote.pro (revision 41106) @@ -0,0 +1,653 @@ +## remote.pro : -*- sh -*- + +check.globals + +# define chips + +book init remotePrepCompRuns +book init remotePrepRuns +book init remoteExecRuns +book init remotePollRuns + +$remote_label_iter = 0 +$remote_label_warp = 0 +$remote_stage_iter = 0 + +$remoteDefine_DB = 0 +$remoteDefineWarp_DB = 0 +$remotePrepComp_DB = 0 +$remotePrep_DB = 0 +$remoteExec_DB = 0 +$remotePoll_DB = 0 +$REMOTE_RECIPE = 0 + + +list STAGES + chip + camera +# warp + stack + staticsky + diff + ff +end + +macro remote.off + task remote.define + active false + end + task remote.define.warp + active false + end + task remote.prepcomp.load + active false + end + task remote.prepcomp.run + active false + end + task remote.prep.load + active false + end + task remote.prep.run + active false + end + task remote.exec.load + active false + end + task remote.exec.run + active false + end + task remote.poll.load + active false + end + task remote.poll.run + active false + end +end + +macro remote.on + task remote.define + active true + end + task remote.define.warp + active true + end + task remote.prepcomp.load + active true + end + task remote.prepcomp.run + active true + end + task remote.prep.load + active true + end + task remote.prep.run + active true + end + task remote.exec.load + active true + end + task remote.exec.run + active true + end + task remote.poll.load + active true + end + task remote.poll.run + active true + end +end + +macro set.remote.recipe + if ($0 != 2) + echo "USAGE: set.remote.recipe (recipe_name)" + break + end + $REMOTE_RECIPE = $1 +end + +task remote.define + host local + periods -poll $LOADPOLL + periods -exec 300 + periods -timeout 30 + active true + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.define.chip + + $label = $LABEL:$remote_label_iter + + $stage = $STAGES:$remote_stage_iter + $remote_stage_iter ++ + + # loop over all stages x all labels + if ($remote_stage_iter >= $STAGES:n) + set remote_stage_iter = 0 + $remote_label_iter ++ + if ($remote_label_iter >= $LABEL:n) set remote_label_iter = 0 + echo $remote_stage_iter $remote_label_iter $label $stage + end + + # min entry limit? + if ("$stage" == "stack") + $run = remotetool -definebyquery -label $label -stage $stage -path_base neb://@HOST@.0/remote/$label -limit 50 + else + $run = remotetool -definebyquery -label $label -stage $stage -path_base neb://@HOST@.0/remote/$label -limit 500 + end + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remoteDefine_DB + $run = $run -dbname $DB:$remoteDefine_DB + $remoteDefine_DB ++ + if ($remoteDefine_DB >= $DB:n) set remoteDefine_DB = 0 + end + + echo $run + command $run + + end + # success + task.exit 0 + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task remote.define.warp + host local + periods -poll $LOADPOLL + periods -exec 900 + periods -timeout 30 + active true + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.define.chip + + $label = $LABEL:$remote_label_warp + + if ($remote_label_warp >= $LABEL:n) set remote_label_warp = 0 + echo $remote_label_warp $label warp + + $run = remotetool -definebyquery -label $label -stage warp -path_base neb://@HOST@.0/remote/$label -limit 50 + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remoteDefineWarp_DB + $run = $run -dbname $DB:$remoteDefineWarp_DB + $remoteDefineWarp_DB ++ + if ($remoteDefineWarp_DB >= $DB:n) set remoteDefineWarp_DB = 0 + end + + echo $run + command $run + + end + + # success + task.exit 0 + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task remote.prepcomp.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + active true + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.prepcomp.load + + $run = remotetool -listcomponent -state new + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remotePrepComp_DB + $run = $run -dbname $DB:$remotePrepComp_DB + $remotePrepComp_DB ++ + if ($remotePrepComp_DB >= $DB:n) set remotePrepComp_DB = 0 + end + + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + ipptool2book stdout remotePrepCompRuns -uniq -key remote_id:stage_id -setword dbname $options:0 -setword pantaskState INIT + process_cleanup remotePrepCompRuns + + if ($VERBOSE > 2) + book listbook remotePrepCompRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the prep component operations in parallel +task remote.prepcomp.run + host anyhost + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 600 + active true + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + stdout $LOGDIR/remote.prepcomp.run + stderr $LOGDIR/remote.prepcomp.run + + book npages remotePrepCompRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($REMOTE_RECIPE == 0) break + + book getpage remotePrepCompRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword remotePrepCompRuns $pageName pantaskState RUN + book getword remotePrepCompRuns $pageName remote_id -var REMOTE_ID + book getword remotePrepCompRuns $pageName stage_id -var STAGE_ID + book getword remotePrepCompRuns $pageName stage -var STAGE + book getword remotePrepCompRuns $pageName run_path_base -var RUN_PATH_BASE + book getword remotePrepCompRuns $pageName dbname -var DBNAME + + sprintf outroot "%s/remote_%s.%s/stage_%s" $RUN_PATH_BASE $STAGE $REMOTE_ID $STAGE_ID + + if ("$STAGE" == "chip") + $command = sc_prepare_chip.pl --chip_id $STAGE_ID + end + if ("$STAGE" == "camera") + $command = sc_prepare_camera.pl --cam_id $STAGE_ID + end + if ("$STAGE" == "warp") + $command = sc_prepare_warp.pl --warp_id $STAGE_ID + end + if ("$STAGE" == "stack") + $command = sc_prepare_stack.pl --stack_id $STAGE_ID + end + if ("$STAGE" == "staticsky") + $command = sc_prepare_staticsky.pl --sky_id $STAGE_ID + end + if ("$STAGE" == "diff") + $command = sc_prepare_diff.pl --diff_id $STAGE_ID + end + if ("$STAGE" == "ff") + $command = sc_prepare_ff.pl --ff_id $STAGE_ID + end + +# Common elements + $command = $command --recipe $REMOTE_RECIPE --camera GPC1 --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME + + options $pageName + + periods -exec 0.05 + command $command + end + + # default exit status + task.exit default + process_exit remotePrepCompRuns $options:0 $JOB_STATUS + end + # locked list + task.exit crash + process_exit remotePrepCompRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword remotePrepCompRuns $options:0 pantaskState TIMEOUT + end +end + +task remote.prep.load + host local + periods -poll $LOADPOLL + periods -exec 30 + active true + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.prep.load + + $run = remotetool -doneprep -state new + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remotePrep_DB + $run = $run -dbname $DB:$remotePrep_DB + $remotePrep_DB ++ + if ($remotePrep_DB >= $DB:n) set remotePrep_DB = 0 + end + + add_poll_labels run + command $run + end + + # success + task.exit 0 + ipptool2book stdout remotePrepRuns -uniq -key remote_id -setword dbname $options:0 -setword pantaskState INIT + process_cleanup remotePrepRuns + + if ($VERBOSE > 2) + book listbook remotePrepRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task remote.prep.run + # this probably shouldn't be local + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 600000 + active true + npending 1 + + task.exec + stdout $LOGDIR/remote.prep.run + stderr $LOGDIR/remote.prep.run + + book npages remotePrepRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($REMOTE_RECIPE == 0) break + + book getpage remotePrepRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword remotePrepRuns $pageName pantaskState RUN + book getword remotePrepRuns $pageName remote_id -var REMOTE_ID + book getword remotePrepRuns $pageName stage -var STAGE + book getword remotePrepRuns $pageName path_base -var PATH_BASE +# book getword remotePrepRuns $pageName label -var LABEL + book getword remotePrepRuns $pageName dbname -var DBNAME + + sprintf outroot "%s/remote_%s.%s" $PATH_BASE $STAGE $REMOTE_ID + + $command = sc_prepare_run.pl --camera GPC1 --remote_id $REMOTE_ID --stage $STAGE --path_base $outroot --dbname $DBNAME --recipe $REMOTE_RECIPE + + options $pageName + command $command + end + + # default exit status + task.exit default + process_exit remotePrepRuns $options:0 $JOB_STATUS + end + # locked list + task.exit crash + process_exit remotePrepRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword remotePrepRuns $options:0 pantaskState TIMEOUT + end +end + +task remote.exec.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + active false + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.exec.load + +# $end_date = `date +%Y-%m-%dT%H:%M:%S` + $run = remotetool -listrun -state pending +# -poll_end $end_date + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remoteExec_DB + $run = $run -dbname $DB:$remoteExec_DB + $remoteExec_DB ++ + if ($remoteExec_DB >= $DB:n) set remoteExec_DB = 0 + end + + add_poll_labels run + command $run + end + # success + task.exit 0 + ipptool2book stdout remoteExecRuns -uniq -key remote_id -setword dbname $options:0 -setword pantaskState INIT + process_cleanup remoteExecRuns + + if ($VERBOSE > 2) + book listbook remoteExecRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task remote.exec.run + # this probably shouldn't be local + host local + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 6000000 + active false + npending 3 + + task.exec + stdout $LOGDIR/remote.exec.run + stderr $LOGDIR/remote.exec.run + + book npages remoteExecRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($REMOTE_RECIPE == 0) break + + book getpage remoteExecRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword remoteExecRuns $pageName pantaskState RUN + book getword remoteExecRuns $pageName remote_id -var REMOTE_ID + book getword remoteExecRuns $pageName stage -var STAGE + book getword remoteExecRuns $pageName path_base -var PATH_BASE +# book getword remoteExecRuns $pageName label -var LABEL + book getword remoteExecRuns $pageName job_id -var JOB_ID + book getword remoteExecRuns $pageName dbname -var DBNAME + + sprintf outroot "%s/remote_%s.%s" $PATH_BASE $STAGE $REMOTE_ID + + $command = sc_remote_exec.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 --recipe $REMOTE_RECIPE + + options $pageName + command $command + end + + # default exit status + task.exit default + process_exit remoteExecRuns $options:0 $JOB_STATUS + end + # locked list + task.exit crash + process_exit remoteExecRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword remoteExecRuns $options:0 pantaskState TIMEOUT + end +end + +task remote.poll.load + host local + periods -poll $LOADPOLL + periods -exec 300 + active false + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/remote.poll.load + + # $end_date = `date +%Y-%m-%dT%H:%M:%S -d '1 hour ago'` + # $run = remotetool -listrun -state run -poll_end $end_date + $run = remotetool -listrun -state run + + if ($DB:n == 0) + option DEFAULT + else + option $DB:$remotePoll_DB + $run = $run -dbname $DB:$remotePoll_DB + $remotePoll_DB ++ + if ($remotePoll_DB >= $DB:n) set remotePoll_DB = 0 + end + + add_poll_labels run + command $run + end + + # success + task.exit 0 + ipptool2book stdout remotePollRuns -uniq -key remote_id -setword dbname $options:0 -setword pantaskState INIT + process_cleanup remotePollRuns + + if ($VERBOSE > 2) + book listbook remotePollRuns + end + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +task remote.poll.run + # this probably shouldn't be local + host local + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 6000000 + active false + npending 10 + + task.exec + stdout $LOGDIR/remote.poll.run + stderr $LOGDIR/remote.poll.run + + book npages remotePollRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($REMOTE_RECIPE == 0) break + + book getpage remotePollRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword remotePollRuns $pageName pantaskState RUN + book getword remotePollRuns $pageName remote_id -var REMOTE_ID + book getword remotePollRuns $pageName stage -var STAGE + book getword remotePollRuns $pageName path_base -var PATH_BASE +# book getword remotePollRuns $pageName label -var LABEL + book getword remotePollRuns $pageName job_id -var JOB_ID + book getword remotePollRuns $pageName dbname -var DBNAME + + sprintf outroot "%s/remote_%s.%s" $PATH_BASE $STAGE $REMOTE_ID + + # This can't have an invalid job_id + if ($JOB_ID == -1) break + $command = sc_remote_poll.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 --job_id $JOB_ID --recipe $REMOTE_RECIPE + + options $pageName + command $command + end + + # default exit status + task.exit default + process_exit remotePollRuns $options:0 $JOB_STATUS + end + # locked list + task.exit crash + process_exit remotePollRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword remotePollRuns $options:0 pantaskState TIMEOUT + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/replicate.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/replicate.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/replicate.pro (revision 41106) @@ -0,0 +1,210 @@ +## replicate.pro : tasks for data replication : -*- sh -*- +## this file contains the tasks for maintaining duplicates as needed in Nebulous +## these tasks use the books replicatePending + +# test for required global variables +check.globals + +# load in the local configuration (NEB_USER, etc.) +module nebulous.site.pro + +$LOGSUBDIR = $LOGDIR/replicate +mkdir $LOGSUBDIR + +book init replicatePending + +macro replicate.reset + book init replicatePending +end + +macro replicate.status + book listbook replicatePending +end + +macro replicate.on + task replicate.load + active true + end + task replicate.run + active true + end +end + +macro replicate.off + task replicate.load + active false + end + task replicate.run + active false + end +end + +macro set.host.for.replicate + if ($0 != 2) + echo "USAGE: set.host.for.replicate (hostname)" + break + end + + if (not($PARALLEL)) + host local + return + end + +# parse volume name + + if ("$1" == "NULL") + host anyhost + else + host $1 + end +end + +# the replicate process interacts with only the single Nebulous server + +# Each 'pendingreplicate' query is limited to a finite number of so_id values. +# Each time we call replicate.load, we increment SO_ID_START by the range value. +# If the pendingreplicate query exits with exit status 10, we start over at 0 + +$SO_ID_START = 0 +$SO_ID_RANGE = 500000 + +# select Nebulous objects which desire additional copies +task replicate.load + host local + + # modify these after the tasks are tested + periods -poll 0.5 + periods -exec 5 + periods -timeout 1500 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/replicate.log + + task.exec + book npages replicatePending -var N + if ($N > 2000) + process_cleanup replicatePending + break + end + + # command does not need to be dynamic, but having it so allows us to adjust the periods + # so that we dont have to wait 10 minutes for things to start up + # XXX smaller limited? 7500 will be a huge book of things to do... + command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingreplicate --limit 500 --so_id_start $SO_ID_START --so_id_range $SO_ID_RANGE + # periods -exec 1800 + end + + # success : 0 -- we did not hit the limit, advance so_id counter + task.exit 0 + # advance the so_id counter + $SO_ID_START = $SO_ID_START + $SO_ID_RANGE + + # convert 'stdout' to book format + ipptool2book stdout replicatePending -key key -uniq -setword pantaskState INIT + + if ($VERBOSE > 2) + book listbook replicatePending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup replicatePending + end + + # success : 1 -- we DID hit the limit, do NOT advance so_id counter + task.exit 1 + # convert 'stdout' to book format + ipptool2book stdout replicatePending -key key -uniq -setword pantaskState INIT + + if ($VERBOSE > 2) + book listbook replicatePending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup replicatePending + end + + # out of so_id range, reset + task.exit 10 + # advance the so_id counter + $SO_ID_START = 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create the desired replicas +task replicate.run + periods -poll 0.5 + periods -exec 5 + periods -timeout 30 + + task.exec + periods -exec 5 + + book npages replicatePending -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new objects in replicatePending + book getpage replicatePending 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword replicatePending $pageName pantaskState RUN + + # XXX what values do I need to get back? + book getword replicatePending $pageName key -var KEY + book getword replicatePending $pageName need_copies -var NEED_COPIES + book getword replicatePending $pageName volume_name -var VOLUME_NAME + book getword replicatePending $pageName volume_host -var VOLUME_HOST + book getword replicatePending $pageName command -var COMMAND + + set.host.for.replicate $VOLUME_HOST + + stdout NULL + stderr $LOGSUBDIR/replicate.log + + # these operations do not require a database to be specified + $run = $COMMAND $KEY + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit replicatePending $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword replicatePending $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword replicatePending $options:0 pantaskState TIMEOUT + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/science.cleanup.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/science.cleanup.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/science.cleanup.pro (revision 41106) @@ -0,0 +1,1291 @@ +## science.cleanup.pro : -*- sh -*- + +macro science.cleanup.on + chip.cleanup.on + camera.cleanup.on + fake.cleanup.on + warp.cleanup.on + diff.cleanup.on + stack.cleanup.on + chip.bg.cleanup.on + warp.bg.cleanup.on +end + +check.globals + +# if set tell cleanup script to check for files from all components regardless of the data_state +$CHECK_ALL_COMPONENTS = 0 +macro set.check.all.components + $CHECK_ALL_COMPONENTS = 1 +end +macro clear.check.all.components + $CHECK_ALL_COMPONENTS = 0 +end +macro get.check.all.components + echo CHECK_ALL_COMPONENTS = $CHECK_ALL_COMPONENTS +end + + +## chip.cleanup.pro : -*- sh -*- + +book init chipPendingCleanup + +macro chip.cleanup.status + book listbook chipPendingCleanup +end + +macro chip.cleanup.reset + book init chipPendingCleanup +end + +macro chip.cleanup.on + task chip.cleanup.load + active true + end + task chip.cleanup.run + active true + end +end + +macro chip.cleanup.off + task chip.cleanup.load + active false + end + task chip.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$chip_cleanup_DB = 0 + +# select images ready for chip analysis +# new entries are added to chipPendingImfile +# skip already-present entries +task chip.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/chip.cleanup.log + + task.exec + if ($LABEL:n == 0) break + $run = chiptool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$chip_cleanup_DB + $run = $run -dbname $DB:$chip_cleanup_DB + $chip_cleanup_DB ++ + if ($chip_cleanup_DB >= $DB:n) set chip_cleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipPendingCleanup -key chip_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup chipPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task chip.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + + book npages chipPendingCleanup -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + if ($NETWORK == 0) break + + # look for new images in chipPendingCleanup (pantaskState == INIT) + book getpage chipPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipPendingCleanup $pageName pantaskState RUN + book getword chipPendingCleanup $pageName camera -var CAMERA + book getword chipPendingCleanup $pageName state -var CLEANUP_MODE + book getword chipPendingCleanup $pageName chip_id -var CHIP_ID + book getword chipPendingCleanup $pageName exp_tag -var EXP_TAG + book getword chipPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE + book getword chipPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (in this case FPA) + set.host.for.camera $CAMERA FPA + + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf LOGFILE "%s/%s/%s.ch.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID + + stdout $LOGDIR/chip.cleanup.log + stderr $LOGDIR/chip.cleanup.log + + $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE + if ($CHECK_ALL_COMPONENTS) + $run = $run --check-all + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit chipPendingCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword chipPendingCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipPendingCleanup $options:0 pantaskState TIMEOUT + end +end + +## camera.cleanup.pro + +book init camPendingCleanup + +macro camera.cleanup.status + book listbook camPendingCleanup +end + +macro camera.cleanup.reset + book init camPendingCleanup +end + +macro camera.cleanup.on + task camera.cleanup.load + active true + end + task camera.cleanup.run + active true + end +end + +macro camera.cleanup.off + task camera.cleanup.load + active false + end + task camera.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$camera_cleanup_DB = 0 + +# select images ready for cam analysis +# new entries are added to camPendingImfile +# skip already-present entries +task camera.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/camera.cleanup.log + + task.exec + if ($LABEL:n == 0) break + $run = camtool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$camera_cleanup_DB + $run = $run -dbname $DB:$camera_cleanup_DB + $camera_cleanup_DB ++ + if ($camera_cleanup_DB >= $DB:n) set camera_cleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout camPendingCleanup -key cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook camPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup camPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task camera.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages camPendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in camPendingCleanup (pantaskState == INIT) + book getpage camPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword camPendingCleanup $pageName pantaskState RUN + book getword camPendingCleanup $pageName camera -var CAMERA + book getword camPendingCleanup $pageName state -var CLEANUP_MODE + book getword camPendingCleanup $pageName cam_id -var CAM_ID + book getword camPendingCleanup $pageName exp_tag -var EXP_TAG + book getword camPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE + book getword camPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and cam (class_id) + set.host.for.camera $CAMERA FPA + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate LOGFILE specific to this exposure (& cam_id) + sprintf LOGFILE "%s/%s/%s.cm.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID + + stdout $LOGDIR/camera.cleanup.log + stderr $LOGDIR/camera.cleanup.log + + $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit camPendingCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword camPendingCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword camPendingCleanup $options:0 pantaskState TIMEOUT + end +end + + +## fake.cleanup.pro + +book init fakePendingCleanup + +macro fake.cleanup.status + book listbook fakePendingCleanup +end + +macro fake.cleanup.reset + book init fakePendingCleanup +end + +macro fake.cleanup.on + task fake.cleanup.load + active true + end + task fake.cleanup.run + active true + end +end + +macro fake.cleanup.off + task fake.cleanup.load + active false + end + task fake.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$fake_cleanup_DB = 0 + +# select images ready for fake analysis +# new entries are added to fakePendingImfile +# skip already-present entries +task fake.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/fake.cleanup.log + + task.exec + if ($LABEL:n == 0) break + $run = faketool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + command + else + # save the DB name for the exit tasks + option $DB:$fake_cleanup_DB + $run = $run -dbname $DB:$fake_cleanup_DB + $fake_cleanup_DB ++ + if ($fake_cleanup_DB >= $DB:n) set fake_cleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fakePendingCleanup -key fake_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fakePendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fakePendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task fake.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages fakePendingCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in fakePendingCleanup (pantaskState == INIT) + book getpage fakePendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fakePendingCleanup $pageName pantaskState RUN + book getword fakePendingCleanup $pageName camera -var CAMERA + book getword fakePendingCleanup $pageName state -var CLEANUP_MODE + book getword fakePendingCleanup $pageName fake_id -var FAKE_ID + book getword fakePendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and fake (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/fake.cleanup.log + stderr $LOGDIR/fake.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage fake --stage_id $FAKE_ID --camera $CAMERA --mode $CLEANUP_MODE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit fakePendingCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword fakePendingCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fakePendingCleanup $options:0 pantaskState TIMEOUT + end +end + + +## warp.cleanup.pro + + +book init warpPendingCleanup + +$warpCleanup_DB = 0 + +macro warp.cleanup.status + book listbook warpPendingCleanup +end + +macro warp.cleanup.reset + book init warpPendingCleanup +end + + +macro warp.cleanup.on + task warp.cleanup.load + active true + end + task warp.cleanup.run + active true + end +end +macro warp.cleanup.off + task warp.cleanup.load + active false + end + task warp.cleanup.run + active false + end +end + +# select images ready for warp analysis +# new entries are added to warpPendingImfile +# skip already-present entries +task warp.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/warp.cleanup.log + + task.exec + if ($LABEL:n == 0) break + $run = warptool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warpCleanup_DB + $run = $run -dbname $DB:$warpCleanup_DB + $warpCleanup_DB ++ + if ($warpCleanup_DB >= $DB:n) set warpCleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpPendingCleanup -key warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task warp.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + + book npages warpPendingCleanup -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + if ($NETWORK == 0) break + + # look for new images in warpPendingCleanup (pantaskState == INIT) + book getpage warpPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingCleanup $pageName pantaskState RUN + book getword warpPendingCleanup $pageName camera -var CAMERA + book getword warpPendingCleanup $pageName state -var CLEANUP_MODE + book getword warpPendingCleanup $pageName warp_id -var WARP_ID + book getword warpPendingCleanup $pageName exp_tag -var EXP_TAG + book getword warpPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE + book getword warpPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and warp (class_id) + set.host.for.camera $CAMERA FPA + set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf LOGFILE "%s/%s/%s.wrp.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID + + stdout $LOGDIR/warp.cleanup.log + stderr $LOGDIR/warp.cleanup.log + + $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE + if ($CHECK_ALL_COMPONENTS) + $run = $run --check-all + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit warpPendingCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword warpPendingCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingCleanup $options:0 pantaskState TIMEOUT + end +end + + + +## diff.cleanup.pro + +book init diffCleanup + +$diffCleanup_DB = 0 + +macro diff.cleanup.status + book listbook diffCleanup +end + +macro diff.cleanup.reset + book init diffCleanup +end + +macro diff.cleanup.on + task diff.cleanup.load + active true + end + task diff.cleanup.run + active true + end +end + +macro diff.cleanup.off + task diff.cleanup.load + active false + end + task diff.cleanup.run + active false + end +end + +# select images ready for diff analysis +# new entries are added to diffPendingImfile +# skip already-present entries +task diff.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/diff.cleanup.log + + task.exec + if ($LABEL:n == 0) break + $run = difftool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$diffCleanup_DB + $run = $run -dbname $DB:$diffCleanup_DB + $diffCleanup_DB ++ + if ($diffCleanup_DB >= $DB:n) set diffCleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout diffCleanup -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task diff.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + + book npages diffCleanup -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + if ($NETWORK == 0) break + + # look for new images in diffCleanup (pantaskState == INIT) + book getpage diffCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffCleanup $pageName pantaskState RUN + book getword diffCleanup $pageName camera -var CAMERA + book getword diffCleanup $pageName state -var CLEANUP_MODE + book getword diffCleanup $pageName diff_id -var DIFF_ID + book getword diffCleanup $pageName tess_id -var TESS_ID + book getword diffCleanup $pageName workdir -var WORKDIR_TEMPLATE + book getword diffCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and diff (class_id) + set.host.for.camera $CAMERA FPA + set.workdir.by.camera $CAMERA $DIFF_ID $WORKDIR_TEMPLATE $default_host WORKDIR + sprintf LOGFILE "%s/%s/%s.dif.%s.cleanup.log" $WORKDIR $TESS_ID $TESS_ID $DIFF_ID + + + stdout $LOGDIR/diff.cleanup.log + stderr $LOGDIR/diff.cleanup.log + + $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE + if ($CHECK_ALL_COMPONENTS) + $run = $run --check-all + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit diffCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword diffCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffCleanup $options:0 pantaskState TIMEOUT + end +end + +## stack.cleanup.pro + +book init stackCleanup + +$stackCleanup_DB = 0 + +macro stack.cleanup.status + book listbook stackCleanup +end + +macro stack.cleanup.reset + book init stackCleanup +end + +macro stack.cleanup.on + task stack.cleanup.load + active true + end + task stack.cleanup.run + active true + end +end + +macro stack.cleanup.off + task stack.cleanup.load + active false + end + task stack.cleanup.run + active false + end +end + +# select images ready for stack analysis +# new entries are added to stackCleanup +# skip already-present entries +task stack.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/stack.cleanup.log + + task.exec + if ($LABEL:n == 0) break + $run = stacktool -pendingcleanuprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$stackCleanup_DB + $run = $run -dbname $DB:$stackCleanup_DB + $stackCleanup_DB ++ + if ($stackCleanup_DB >= $DB:n) set stackCleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackCleanup -key stack_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task stack.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + book npages stackCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in stackCleanup (pantaskState == INIT) + book getpage stackCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackCleanup $pageName pantaskState RUN + book getword stackCleanup $pageName camera -var CAMERA + book getword stackCleanup $pageName state -var CLEANUP_MODE + book getword stackCleanup $pageName stack_id -var STACK_ID + book getword stackCleanup $pageName path_base -var PATH_BASE + book getword stackCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and stack (class_id) + set.host.for.camera $CAMERA FPA + + stdout $LOGDIR/stack.cleanup.log + stderr $LOGDIR/stack.cleanup.log + + sprintf LOGFILE "%s.cleanup.log" $PATH_BASE + $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit stackCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword stackCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackCleanup $options:0 pantaskState TIMEOUT + end +end + + +book init chipBGPendingCleanup + +macro chip.bg.cleanup.status + book listbook chipBGPendingCleanup +end + +macro chip.bg.cleanup.reset + book init chipBGPendingCleanup +end + +macro chip.bg.cleanup.on + task chip.bg.cleanup.load + active true + end + task chip.bg.cleanup.run + active true + end +end + +macro chip.bg.cleanup.off + task chip.bg.cleanup.load + active false + end + task chip.bg.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$chip_bg_cleanup_DB = 0 + +task chip.bg.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/chip.bg.cleanup.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -pendingcleanupchiprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$chip_bg_cleanup_DB + $run = $run -dbname $DB:$chip_bg_cleanup_DB + $chip_bg_cleanup_DB ++ + if ($chip_bg_cleanup_DB >= $DB:n) set chip_bg_cleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipBGPendingCleanup -key chip_bg_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipBGPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup chipBGPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task chip.bg.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + active false + + task.exec + + book npages chipBGPendingCleanup -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + if ($NETWORK == 0) break + + # look for new images in chipBGPendingCleanup (pantaskState == INIT) + book getpage chipBGPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipBGPendingCleanup $pageName pantaskState RUN + book getword chipBGPendingCleanup $pageName camera -var CAMERA + book getword chipBGPendingCleanup $pageName state -var CLEANUP_MODE + book getword chipBGPendingCleanup $pageName chip_bg_id -var CHIP_BG_ID + book getword chipBGPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE + book getword chipBGPendingCleanup $pageName exp_tag -var EXP_TAG + book getword chipBGPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + sprintf logfile "%s/%s/%s.bgc.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_BG_ID + + stdout $LOGDIR/chip.bg.cleanup.log + stderr $LOGDIR/chip.bg.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage chip_bg --stage_id $CHIP_BG_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit chipBGPendingCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword chipBGPendingCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipBGPendingCleanup $options:0 pantaskState TIMEOUT + end +end + +book init warpBGPendingCleanup + +macro warp.bg.cleanup.status + book listbook warpBGPendingCleanup +end + +macro warp.bg.cleanup.reset + book init warpBGPendingCleanup +end + +macro warp.bg.cleanup.on + task warp.bg.cleanup.load + active true + end + task warp.bg.cleanup.run + active true + end +end + +macro warp.bg.cleanup.off + task warp.bg.cleanup.load + active false + end + task warp.bg.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$warp_bg_cleanup_DB = 0 + +task warp.bg.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/warp.bg.cleanup.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -pendingcleanupwarprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warp_bg_cleanup_DB + $run = $run -dbname $DB:$warp_bg_cleanup_DB + $warp_bg_cleanup_DB ++ + if ($warp_bg_cleanup_DB >= $DB:n) set warp_bg_cleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpBGPendingCleanup -key warp_bg_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpBGPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpBGPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task warp.bg.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + active false + + task.exec + book npages warpBGPendingCleanup -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + if ($NETWORK == 0) break + + # look for new images in warpBGPendingCleanup (pantaskState == INIT) + book getpage warpBGPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpBGPendingCleanup $pageName pantaskState RUN + book getword warpBGPendingCleanup $pageName camera -var CAMERA + book getword warpBGPendingCleanup $pageName state -var CLEANUP_MODE + book getword warpBGPendingCleanup $pageName warp_bg_id -var WARP_BG_ID + book getword warpBGPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE + book getword warpBGPendingCleanup $pageName exp_tag -var EXP_TAG + book getword warpBGPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and warp + set.host.for.camera $CAMERA FPA + set.workdir.by.camera $CAMERA $WARP_BG_ID $WORKDIR_TEMPLATE $default_host WORKDIR + sprintf logfile "%s/%s/%s.bgw.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_BG_ID + + stdout $LOGDIR/warp.bg.cleanup.log + stderr $LOGDIR/warp.bg.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage warp_bg --stage_id $WARP_BG_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit warpBGPendingCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword warpBGPendingCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpBGPendingCleanup $options:0 pantaskState TIMEOUT + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.basic.auto =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.basic.auto (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.basic.auto (revision 41106) @@ -0,0 +1,116 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@" +END + +automate METADATA + name STR OBJECT-i + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc -set_data_group simtest_i -set_dist_group simtest" + block STR NONE +END + +automate METADATA + name STR OBJECT-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc -set_data_group simtest_r -set_dist_group simtest" + block STR NONE +END + +### Stack automation +automate METADATA + name STR STACK + regular STR "stacktool -definebyquery -set_label proc -set_data_group simtest_r -set_dist_group simtest -set_workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -min_num 4 -dbname @DBNAME@" +END + +### Diff automation +automate METADATA + name STR DIFF + regular STR "difftool -definewarpstack -set_label proc -set_workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@" +END + +#### Magic automation??? +#automate METADATA +# name STR MAGIC +# regular STR "magictool -definebyquery -workdir file://@CWD@/magic -good_frac 0.2 -dbname @DBNAME@" +#END + +### automate METADATA +### name STR OBJECT-r +### check STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter r" +### launch STR "chiptool -unblock -label simtest-r -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-r -dbname @DBNAME@" +### END + +### automate METADATA +### name STR OBJECT-i +### check STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i" +### launch STR "chiptool -unblock -label simtest-i -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-i -dbname @DBNAME@" +### END + +### there is a weakness in the label / block business: the labels are +### generic, and the blocks are against those fairly generic words. +### that makes it difficult to block and unblock science exposures +### based on different detrend types that are available. For example, +### to block the r-band against the absence of the desired r-band +### flat, we would need to specify a label specific to the r-band +### images and remove that block when ready. + +### a better approach might be to modify the labels rather than the +### blocks. we can define, eg, chiptool -set_label -definebyquery to +### turn assign the label names based on various properties of the +### interesting images. + +### here is what the automate element might look like for such a circumstance: + +### automate METADATA +### name STR OBJECT-i +### block STR "chiptool -dbname @DBNAME@ -unmasked -label wait -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### launch STR "chiptool -dbname @DBNAME@ -set_label proc -label wait -dbname @DBNAME@ -time_begin 2008/1/1 -time_end 2008/1/2" +### END + +### XXX still not quite there.... +### the 'checks' or maybe the 'launch' needs to include a '-n_exp_min' ? +### Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.basic.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.basic.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.basic.config (revision 41106) @@ -0,0 +1,112 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 1 + DITHER.ND S32 1 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,r,i,i + @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 + @SKYMAGS F32 20.86,20.86,20.15,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.ctemask.auto =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.ctemask.auto (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.ctemask.auto (revision 41106) @@ -0,0 +1,39 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 5 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +# XXX let's build the necessary data, then see how to run the cte mask generation (input is flattened flats) Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.ctemask.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.ctemask.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.ctemask.config (revision 41106) @@ -0,0 +1,38 @@ + +# very basic test to generate flats and test the CTE masking process +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 3, 3 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r + @EXPTIMES F32 20.0 + + NSETUP S32 7 +END + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.det =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.det (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.det (revision 41106) @@ -0,0 +1,42 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK + prereq STR BIAS + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR DARK + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" + dbname STR eamtest + pantaskState STR NULL +END Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.detverify.auto =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.detverify.auto (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.detverify.auto (revision 41106) @@ -0,0 +1,78 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR BIAS_VERIFY + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR DARK_VERIFY + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER_VERIFY + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -mode master -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT_VERIFY-r + check STR "detselect -search -inst @CAMERA@ -det_type FLAT -filter r -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -mode verify -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT_VERIFY-i + check STR "detselect -search -inst @CAMERA@ -det_type FLAT -filter i -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -mode verify -ref_det_id @det_id@ -ref_iter @iteration@ -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -mode verify -dbname @DBNAME@" +END + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.detverify.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.detverify.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.detverify.config (revision 41106) @@ -0,0 +1,37 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.flatcorr.auto =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.flatcorr.auto (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.flatcorr.auto (revision 41106) @@ -0,0 +1,76 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# this is a simplified automate script which only models BIAS and FLAT. +# generate appropriate simulated data with the FLATCORR recipe + +# wait until 20 bias frames are registered, then define a new bias run +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 10 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +# wait until the master bias frame is registered, then define a new dark run +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +# wait until the master dark frame is registered, then define a new shutter run +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +# wait until the master bias frame is registered, then define a new flat-r run +automate METADATA + name STR RAWFLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT_RAW -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT_RAW -filter r -dbname @DBNAME@" +END + +# wait until the master flat-r frame is registered, then initiate a flat-field correction analysis +automate METADATA + name STR FLATCORR-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT_RAW" + launch STR "flatcorr -definebyquery -dbname @DBNAME@ -det_type FLATCORR -inst @CAMERA@ -exp_type OBJECT -filter r -set_workdir file://@CWD@/detwork -set_reduction FLATCORR -set_dvodb @CWD@/catdir.flatcorr -set_region 270.6,270.9:-23.8,-23.5 -set_filter r" + block STR "dettool -runs -active -det_type FLATCORR -filter r -dbname @DBNAME@" +END + +# wait until the flat-r correction frame is registered, then define the flat-fields to be corrected +# XXX for the moment, the det_id here is hard-wired +automate METADATA + name STR FLATFIX-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLATCORR" + launch STR "dettool -makecorrection -dbname @DBNAME@ -det_id 4 -iteration 0 -det_type FLAT -workdir file://@CWD@/detwork" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +# wait until the corrected master flat-r frame is registered, then initiate a second flat-field correction analysis +automate METADATA + name STR FLATTEST-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" + launch STR "flatcorr -definebyquery -dbname @DBNAME@ -det_type FLATTEST -inst @CAMERA@ -exp_type OBJECT -filter r -set_workdir file://@CWD@/detwork -set_reduction FLATTEST -set_dvodb @CWD@/catdir.flattest -set_region 270.6,270.9:-23.8,-23.5 -set_filter r" + block STR "dettool -runs -active -det_type FLATCORR -filter r -dbname @DBNAME@" +END + +## # wait until the master flat-r frame is registered, then unblock the r-band data for chip-level processing +## automate METADATA +## name STR OBJECT-i +## check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" +## launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" +## block STR NONE +## END +## Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.flatcorr.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.flatcorr.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.flatcorr.config (revision 41106) @@ -0,0 +1,189 @@ + +# generate a simple fake data set with 2x2 dithers at offset of +# 50, 100, 200 arcsec (camera is 530 arcsec on a side) + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 10 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 5, 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r + @EXPTIMES F32 0.1,20.0 + + NSETUP S32 3 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +# a single image at the field center +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # the simtest camera is 2k x 2k, 530 arcsec on a side + # XXX note that we cannot define the offsets in terms of the camera dimensions... + + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 1 + DITHER.ND S32 1 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r + @EXPTIMES F32 10.00 + @SKYMAGS F32 20.86 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + +# dither set @ scale 1 = 60 arcsec +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # XXX note that we cannot define the offsets in terms of the camera dimensions... + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r + @EXPTIMES F32 10.00 + @SKYMAGS F32 20.86 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + +# dither set @ scale 2 = 120 arcsec +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # XXX note that we cannot define the offsets in terms of the camera dimensions... + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 120.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 120.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r + @EXPTIMES F32 10.00 + @SKYMAGS F32 20.86 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + +# dither set @ scale 3 = 240 arcsec +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + + ### Carina (modestly high density) + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 + + # XXX note that we cannot define the offsets in terms of the camera dimensions... + OFFSET.RA F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 0.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 240.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 240.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + # XXX is DVODB used? + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r + @EXPTIMES F32 10.00 + @SKYMAGS F32 20.86 + + # modest IQ range here + IQ_MIN F32 0.70 + IQ_MAX F32 0.90 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.large.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.large.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.large.config (revision 41106) @@ -0,0 +1,73 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +SEQUENCE METADATA + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 300.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 300.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 3 + OFFSET.ND S32 3 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 3 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r + @EXPTIMES F32 30.00 + @SKYMAGS F32 20.86 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.megacam.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.megacam.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.megacam.config (revision 41106) @@ -0,0 +1,67 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR MEGACAM + + BIAS.LEVEL F32 100.0 + BIAS.RANGE F32 10.0 + + NIMAGE S32 15 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR MEGACAM + + DARK.MIN F32 100.0 + DARK.MAX F32 200.0 + + NIMAGE S32 15 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR MEGACAM + + FILTERS STR g,r,i + @EXPTIMES F32 60.0,30.0,30.0 + + NSETUP S32 15 +END + +SEQUENCE_SKIP METADATA + OBSTYPE STR OBJECT + CAMERA STR MEGACAM + + CENTER.RA F32 10.0 + CENTER.DEC F32 60.0 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 5 + OFFSET.ND S32 5 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 2 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.grizy + FILTERS STR g,r,i + @EXPTIMES F32 60.0,30.0,30.0 + @SKYMAGS F32 21.0,20.5,20.0 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.5 + IQ_MAX F32 0.8 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.nebulous.basic.auto =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.nebulous.basic.auto (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.nebulous.basic.auto (revision 41106) @@ -0,0 +1,116 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@" +END + +automate METADATA + name STR OBJECT-i + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" + block STR NONE +END + +automate METADATA + name STR OBJECT-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc" + block STR NONE +END + +### Stack automation??? +automate METADATA + name STR STACK + regular STR "stacktool -definebyquery -all -label proc -workdir neb://anyhost/simtest.@USER@/stack -min_new 4 -min_frac 2 -min_num 4 -select_good_frac_min 0.2 -dbname @DBNAME@" +END + +### Diff automation??? +automate METADATA + name STR DIFF + regular STR "difftool -definewarpstack -label proc -workdir neb://anyhost/simtest.@USER@/diff -good_frac 0.2 -dbname @DBNAME@" +END + +#### Magic automation??? +#automate METADATA +# name STR MAGIC +# regular STR "magictool -definebyquery -workdir neb://anyhost/simtest.@USER@/magic -good_frac 0.2 -dbname @DBNAME@" +#END + +### automate METADATA +### name STR OBJECT-r +### check STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter r" +### launch STR "chiptool -unblock -label simtest-r -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-r -dbname @DBNAME@" +### END + +### automate METADATA +### name STR OBJECT-i +### check STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i" +### launch STR "chiptool -unblock -label simtest-i -dbname @DBNAME@" +### block STR "chiptool -unmasked -label simtest-i -dbname @DBNAME@" +### END + +### there is a weakness in the label / block business: the labels are +### generic, and the blocks are against those fairly generic words. +### that makes it difficult to block and unblock science exposures +### based on different detrend types that are available. For example, +### to block the r-band against the absence of the desired r-band +### flat, we would need to specify a label specific to the r-band +### images and remove that block when ready. + +### a better approach might be to modify the labels rather than the +### blocks. we can define, eg, chiptool -set_label -definebyquery to +### turn assign the label names based on various properties of the +### interesting images. + +### here is what the automate element might look like for such a circumstance: + +### automate METADATA +### name STR OBJECT-i +### block STR "chiptool -dbname @DBNAME@ -unmasked -label wait -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i -time_begin 2008/1/1 -time_end 2008/1/2" +### launch STR "chiptool -dbname @DBNAME@ -set_label proc -label wait -dbname @DBNAME@ -time_begin 2008/1/1 -time_end 2008/1/2" +### END + +### XXX still not quite there.... +### the 'checks' or maybe the 'launch' needs to include a '-n_exp_min' ? +### Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.pro (revision 41106) @@ -0,0 +1,160 @@ +## simtest.pro : automatic full-suite simulated IPP test : -*- sh -*- + +## there are four features which define a simtest run: +## the camera : any camera can be used, the sim... cameras are defined to be quick tests +## the sequence : this file defines the set of observations generated +## the auto : the automation file defines the sequence of analysis performed +## the recipe : the ppsim recipe defines the features which are included in the generate data + +## these are not completely independent: a certain sequence may be +## required to yield a sensible result for a specific auto, and a +## specific recipe may also be needed + +if ($?PPSIM_RECIPE == 0) + $PPSIM_RECIPE = default +end +if ($?SIMTEST_CAMERA == 0) + $SIMTEST_CAMERA = SIMTEST +end +if ($?SIMTEST_SEQUENCE == 0) + $SIMTEST_SEQUENCE = simtest.basic.config +end +if ($?SIMTEST_AUTO == 0) + $SIMTEST_AUTO = simtest.basic.auto +end +if ($?SIMTEST_THREADS == 0) + $SIMTEST_THREADS = 0 +end + +$SIMTEST_RAWDIR = raw +$SIMTEST_WORKDIR = work + + +macro simtest + if ($0 != 4) + echo "USAGE: simtest (dbname) (hostname) (init)" + echo " if (init) == run : just run the analysis tasks" + echo " if (init) == inject : (re)create the database and (re)inject the images" + echo " if (init) == new : restart completely from scratch" + echo "" + echo "there are additional options which may be specified by setting the following variable names:" + echo " PPSIM_RECIPE : define the recipe to use when generating the fake data" + echo " SIMTEST_CAMERA : define an alternate camera (otherwise uses entry in sequence file)" + echo " SIMTEST_SEQUENCE : define the set of observations generated (simtest.basic.config)" + echo " SIMTEST_AUTO : define the analysis steps to perform (simtest.basic.auto)" + echo " SIMTEST_THREADS : set the number of threads for the processing node (0)" + echo "" + echo "the following macros can be used to set up specific simtest suites:" + echo " simtest.setup.basic : run standard simtest suite" + echo " simtest.setup.detverify : run detrend creation and detrend verification" + echo " simtest.setup.flatcorr : run a flat-field correction demonstration" + echo " simtest.setup.ctemask : run a ctemask demonstration" + break + end + + $dbname = $1 + $hostname = $2 + $init = $3 + if (("$init" != "run") && ("$init" != "inject") && ("$init" != "new")) + echo "USAGE: simtest (dbname) (hostname) (init)" + echo " if (init) == run : just run the analysis tasks" + echo " if (init) == inject : (re)create the database and (re)inject the images" + echo " if (init) == new : restart completely from scratch" + break + end + + if (("$init" == "new") || ("$init" == "inject")) + # XXX this will fail and exit the script if the db does not exist or is old... + exec pxadmin -delete -dbname $dbname + + # XXX this gives warnings if the db exists... + exec pxadmin -create -dbname $dbname + + if ("$init" == "new") + # the labels "wait" and "proc" are special names used in automate.pro + + $ppsim = "ppSimSequence $MODULES:0/$SIMTEST_SEQUENCE simtest.mkimages simtest.inject -path $SIMTEST_RAWDIR -workdir $SIMTEST_WORKDIR -dbname $dbname -label wait -dvodb DVO -tess_id TESS" + if ("$PPSIM_RECIPE" != "default") + $ppsim = $ppsim -ppsim_recipe $PPSIM_RECIPE + end + $ppsim = $ppsim -camera $SIMTEST_CAMERA + + exec $ppsim + + exec source simtest.mkimages + + ## XXX this will need to be optional as well + ## we could use the book to find OBSTYPE = OBJECT, get CENTER.RA, CENTER.DEC + ## or else we could have ppSimSequence generate an appropriate skycells call + file TESS/Images.dat found + if ($found == 0) + # this command is approriate only to the dither pattern and camera define in simtest.basic.config + exec skycells -mode local -scale 0.2 -center 270.75 -23.70 -size 0.3 0.3 -nx 3 -ny 3 -D CATDIR TESS + end + end + + exec chiptool -dbname $dbname -block -label wait + exec source simtest.inject + end + + module pantasks.pro + module automate.pro + + module.tasks + + add.label proc + + if ($SIMTEST_THREADS == 0) + controller host add $hostname + else + controller host add $hostname -threads $SIMTEST_THREADS + end + + add.database $dbname + + automate.load $SIMTEST_AUTO $SIMTEST_CAMERA $dbname + + tasks.revert.off + + run +end + +# XXX add a test function to pxadmin to check for db existence + +## currently defined options for sequence, etc: + +# sequence: simtest.basic.config, simtest.flatcorr.config +# auto: simtest.basic.auto, simtest.flatcorr.auto + +macro simtest.setup.basic + $PPSIM_RECIPE = default + $SIMTEST_SEQUENCE = simtest.basic.config + $SIMTEST_AUTO = simtest.basic.auto +end + +macro simtest.setup.nebulous.basic + $PPSIM_RECIPE = default + $SIMTEST_SEQUENCE = simtest.basic.config + $SIMTEST_AUTO = simtest.nebulous.basic.auto + $SIMTEST_USER = `whoami` + $SIMTEST_RAWDIR = neb://anyhost/simtest.$SIMTEST_USER/raw + $SIMTEST_WORKDIR = neb://anyhost/simtest.$SIMTEST_USER/work +end + +macro simtest.setup.detverify + $PPSIM_RECIPE = default + $SIMTEST_SEQUENCE = simtest.detverify.config + $SIMTEST_AUTO = simtest.detverify.auto +end + +macro simtest.setup.ctemask + $PPSIM_RECIPE = BADCTE.TEST + $SIMTEST_SEQUENCE = simtest.ctemask.config + $SIMTEST_AUTO = simtest.ctemask.auto +end + +macro simtest.setup.flatcorr + $PPSIM_RECIPE = FLATCORR + $SIMTEST_SEQUENCE = simtest.flatcorr.config + $SIMTEST_AUTO = simtest.flatcorr.auto +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.simmosaic.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.simmosaic.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.simmosaic.config (revision 41106) @@ -0,0 +1,114 @@ + +CAMERA STR SIMMOSAIC + +FILERULE MULTI +FILERULE STR Chip00.fits +FILERULE STR Chip01.fits +FILERULE STR Chip10.fits +FILERULE STR Chip11.fits + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + + FILTERS STR r,r,i + @EXPTIMES F32 0.1,20.0,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + +SEQUENCE METADATA + OBSTYPE STR OBJECT + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 60.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 2 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,i + @EXPTIMES F32 10.00,10.00 + @SKYMAGS F32 20.86,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + + +SEQUENCE METADATA + OBSTYPE STR OBJECT + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 30.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 1 + DITHER.ND S32 1 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,r,i,i + @EXPTIMES F32 5.0, 240.0, 5.0, 240.0 + @SKYMAGS F32 20.86,20.86,20.15,20.15 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 0.55 + IQ_MAX F32 1.25 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.stack.auto =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.stack.auto (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.stack.auto (revision 41106) @@ -0,0 +1,65 @@ + +automate MULTI + +# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK +# success on CHECK -> LAUNCH +# failure on BLOCK -> CHECK + +# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis + +automate METADATA + name STR BIAS + check STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@" + ncheck S32 20 + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@" + block STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@" +END + +automate METADATA + name STR DARK + check STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@" + block STR "dettool -runs -active -det_type DARK -dbname @DBNAME@" +END + +automate METADATA + name STR SHUTTER + check STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-r + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type SKYFLAT -select_filter r -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@" +END + +automate METADATA + name STR FLAT-i + check STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@" + launch STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type SKYFLAT -select_filter i -dbname @DBNAME@" + block STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@" +END + +automate METADATA + name STR OBJECT-i + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter i -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc" + block STR NONE +END + +automate METADATA + name STR OBJECT-r + check STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -filter r -det_type FLAT" + launch STR "chiptool -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc" + block STR NONE +END + +### Stack automation??? +automate METADATA + name STR STACK + regular STR "stacktool -definebyquery -label proc -workdir file://@CWD@/stack -min_new 4 -min_num 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" +END + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.stack.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.stack.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/simtest.stack.config (revision 41106) @@ -0,0 +1,76 @@ + +# define the simulated data to be generated +SEQUENCE MULTI + +SEQUENCE METADATA + OBSTYPE STR BIAS + CAMERA STR SIMTEST + NIMAGES S32 20 +END + +SEQUENCE METADATA + OBSTYPE STR DARK + CAMERA STR SIMTEST + @EXPTIMES F32 30.0, 300.0 + @NIMAGES S32 10, 10 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r + @EXPTIMES F32 0.1,20.0 + + NSETUP S32 5 +END + +SEQUENCE METADATA + OBSTYPE STR FLAT + CAMERA STR SIMTEST + + FILTERS STR r,r,r,r,r + @EXPTIMES F32 0.5,1.0,2.0,5.0,10.0 + + NSETUP S32 1 +END + + +SEQUENCE METADATA +### We want both short and long exposures, and dark and bright sky + OBSTYPE STR OBJECT + CAMERA STR SIMTEST + +### Carina + CENTER.RA F32 270.75 + CENTER.DEC F32 -23.7 +### COSMOS field +# CENTER.RA F32 150.119167 +# CENTER.DEC F32 2.205833 + + OFFSET.RA F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction) + OFFSET.NR S32 1 + OFFSET.ND S32 1 + + DITHER.RA F32 10.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.DEC F32 10.0 # linear offset in arcsec (do not include cos(DEC) correction) + DITHER.NR S32 3 + DITHER.ND S32 3 + + DVODB STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest + FILTERS STR r,r,r + @EXPTIMES F32 10,300,300 + @SKYMAGS F32 21,21,18 + + # XXX use a more realistic IQ distribution... + IQ_MIN F32 1.00 + IQ_MAX F32 1.00 + + # rotation sequence + POS_MIN F32 0.0 + POS_MAX F32 0.0 + POS_DELTA F32 1.0 + +END + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/site.manoa.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/site.manoa.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/site.manoa.pro (revision 41106) @@ -0,0 +1,52 @@ +## site.mhpcc.pro : example configuration script for the MHPCC IPP cluster : -*- sh -*- + +## a site configuration needs to define the following pieces of information: +## 1) the machines used for processing +## 2) the default_host and workdir_template variables +## 3) the ipphost table + +macro init.cluster.po + $PARALLEL = 1 + controller exit true + + # po05 -- using for pantasks + # po23 -- broken perl + + controller host add po02 + controller host add po03 + controller host add po04 +# controller host add po05 + controller host add po06 + controller host add po07 +# controller host add po08 +# controller host add po09 +# controller host add po10 +# controller host add po11 +# controller host add po12 +# controller host add po13 +# controller host add po14 +# controller host add po15 +# controller host add po16 +# controller host add po17 +# controller host add po18 +# controller host add po19 +# controller host add po20 +# controller host add po21 +# controller host add po22 +# controller host add po23 +# controller host add po24 +end + +macro init.cluster.sn + $PARALLEL = 1 + controller exit true + controller host add sn2 + controller host add sn3 + controller host add sn4 + controller host add sn5 +end + +macro init.site + queueload tmp -x "cat $MODULES:0/ipphosts.manoa.config" + ipptool2book tmp ipphosts -key camera +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/site.mhpcc.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/site.mhpcc.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/site.mhpcc.pro (revision 41106) @@ -0,0 +1,95 @@ +## site.mhpcc.pro : example configuration script for the MHPCC IPP cluster : -*- sh -*- + +## a site configuration needs to define the following pieces of information: +## 1) the machines used for processing +## 2) the default_host and workdir_template variables +## 3) the ipphost table + +macro init.cluster.mhpcc + $PARALLEL = 1 + controller exit true + +# controller host add ipp014 + controller host add ipp021 + controller host add ipp015 + controller host add ipp023 + controller host add ipp024 + controller host add ipp025 + controller host add ipp026 + controller host add ipp028 + controller host add ipp029 + controller host add ipp030 + controller host add ipp031 + controller host add ipp032 + controller host add ipp033 + controller host add ipp034 + controller host add ipp035 + controller host add ipp036 + controller host add ipp038 + controller host add ipp039 + controller host add ipp040 + controller host add ipp041 + controller host add ipp042 + controller host add ipp043 + controller host add ipp044 + controller host add ipp045 + controller host add ipp046 + controller host add ipp047 + controller host add ipp048 + controller host add ipp049 + controller host add ipp050 + controller host add ipp051 + controller host add ipp052 +# controller host add ipp053 +end + +## override the basic inits set in pantasks.pro +macro init.copy.mhpcc + if ($0 != 2) + echo "USAGE: init.copy.mhpcc (nebulous)" + echo "nebulous may be 'on' or 'off'" + break + end + + if (("$1" != "on") && ("$1" != "off")) + echo "USAGE: init.copy.mhpcc (nebulous)" + echo "nebulous may be 'on' or 'off'" + break + end + + # XXX this is only used by summit.copy.pro. move this as a check into summit.copy.pro? + $COMPRESS = 1 + + # the templates are used if we have a class_id/host relationship; + # if none is found, the default values are used + # XXX not sure how to handle the .N value if we need to use more than one + + if ("$1" == "on") + $NEBULOUS = 1 + $default_host = any + $workdir_template = neb://@HOST@.0 + else + $NEBULOUS = 0 + $default_host = ipp023 + $workdir_template = /data/@HOST@.0 + end +end + +macro init.site + init.cluster.mhpcc + init.copy.mhpcc on + + queueload tmp -x "cat $MODULES:0/ipphosts.mhpcc.config" + ipptool2book tmp ipphosts -key camera + + # set autoprocessing for chip + queueload tmp -x "cat $MODULES:0/surveys.mhpcc.config" + ipptool2book tmp surveys -key survey +end + +macro init.site.nohosts + init.copy.mhpcc on + + queueload tmp -x "cat $MODULES:0/ipphosts.mhpcc.config" + ipptool2book tmp ipphosts -key camera +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/skycal.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/skycal.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/skycal.pro (revision 41106) @@ -0,0 +1,271 @@ +## skycal.pro : tasks for static sky calibration analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the +## skycal analysis. After a skycalRun entry (with +## associated skycal_id) is defined, psastro is performed (via script +## skycalibration.pl) (tasks in skycalRun). + +# test for required global variables +check.globals + +# skcal.pro should have a more restricted polling limit (to avoid stress with getstar) +# XXX: is this necessary anymore? +if ($?POLL_LIMIT_SKYCAL == 0) set POLL_LIMIT_SKYCAL = 64 + +macro set.skycal.poll + if ($0 != 2) + echo "USAGE:set.skycal.poll (value)" + break + end + + $POLL_LIMIT_SKYCAL = $1 +end + +macro get.skycal.poll + echo "skycal poll limit: $POLL_LIMIT_SKYCAL" +end + +### Initialise the books containing the tasks to do +book init skycalRun + +### Database lists +$skycal_DB = 0 +$skycal_revert_DB = 0 + +### Check status of skycal tasks +macro skycal.status + book listbook skycalRun +end + +### Reset skycal tasks +macro skycal.reset + book init skycalRun +end + +### Turn skycal tasks on +macro skycal.on + task skycal.load + active true + end + task skycal.run + active true + end +end + +### Turn skycal tasks off +macro skycal.off + task skycal.load + active false + end + task skycal.run + active false + end +end + +macro skycal.revert.on + task skycal.revert + active true + end +end + +macro skycal.revert.off + task skycal.revert + active false + end +end + +### Load tasks for skycal +### Tasks are loaded into skycalRun. +task skycal.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/skycal.log + + task.exec + if ($LABEL:n == 0) break + $run = staticskytool -pendingskycalrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$skycal_DB + $run = $run -dbname $DB:$skycal_DB + $skycal_DB ++ + if ($skycal_DB >= $DB:n) set skycal_DB = 0 + end + add_poll_args run + add_poll_labels run + $run = $run -limit $POLL_LIMIT_SKYCAL + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout skycalRun -key skycal_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook skycalRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup skycalRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the skycal analysis +### Tasks are taken from skycalRun. +task skycal.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run use "long" exectime + periods -exec $RUNEXEC + book npages skycalRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + + # look for new entries in skycalRun (pantaskState == INIT) + book getpage skycalRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword skycalRun $pageName pantaskState RUN + book getword skycalRun $pageName skycal_id -var SKYCAL_ID + book getword skycalRun $pageName stack_id -var STACK_ID + book getword skycalRun $pageName tess_id -var TESS_DIR + book getword skycalRun $pageName skycell_id -var SKYCELL_ID + book getword skycalRun $pageName camera -var CAMERA + book getword skycalRun $pageName filter -var FILTER + book getword skycalRun $pageName state -var RUN_STATE + book getword skycalRun $pageName workdir -var WORKDIR_TEMPLATE + book getword skycalRun $pageName path_base -var PATH_BASE + book getword skycalRun $pageName reduction -var REDUCTION + book getword skycalRun $pageName singlefilter -var SINGLEFILTER + book getword skycalRun $pageName dbname -var DBNAME + book getword skycalRun $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.stk.%s.skycal.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $STACK_ID $SKYCAL_ID + + stdout $LOGDIR/skycal.log + stderr $LOGDIR/skycal.log + + $run = skycalibration.pl --skycal_id $SKYCAL_ID --outroot $outroot --redirect-output --camera $CAMERA --path_base $PATH_BASE --stack_id $STACK_ID --filter $FILTER --run-state $RUN_STATE + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$SINGLEFILTER" != "0") + $run = $run --singlefilter + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # since we have work to do shorten exec time + periods -exec .05 + command $run + end + + # default exit status + task.exit default + process_exit skycalRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword skycalRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword skycalRun $options:0 pantaskState TIMEOUT + end +end + +task skycal.revert + host local + + periods -poll 10.0 + periods -exec 1200.0 + periods -timeout 120.0 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = staticskytool -revertskycal -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$skycal_revert_DB + $run = $run -dbname $DB:$skycal_revert_DB + $skycal_revert_DB ++ + if ($skycal_revert_DB >= $DB:n) set skycal_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/stack.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/stack.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/stack.pro (revision 41106) @@ -0,0 +1,427 @@ +## stack.pro : tasks for image stacking : -*- sh -*- + +## This file contains panTasks definitions for performing the image stacking. +## After a stack (with associated stack_id) is defined, the stack is performed +## (tasks in stackSumSkyfile). + +# test for required global variables +check.globals + +if ($?POLL_LIMIT_STACK == 0) set POLL_LIMIT_STACK = 40 + +macro set.stack.poll + if ($0 != 2) + echo "USAGE:set.stack.poll (value)" + break + end + + $POLL_LIMIT_STACK = $1 +end +macro get.stack.poll + echo $POLL_LIMIT_STACK +end + +### Initialise the books containing the tasks to do +book init stackSumSkyfile +book init stackPendingSummary + +### Database lists +$stackSkycell_DB = 0 +$stack_revert_DB = 0 +$stackSummary_DB = 0 + +### Check status of stacking tasks +macro stack.status + book listbook stackSumSkyfile +end + +### Reset stacking tasks +macro stack.reset + book init stackSumSkyfile + book init stackPendingSummary +end + +### Turn stacking tasks on +macro stack.on + task stack.skycell.load + active true + end + task stack.skycell.run + active true + end + task stack.revert + active false + end +end + +### Turn stacking tasks off +macro stack.off + task stack.skycell.load + active false + end + task stack.skycell.run + active false + end + task stack.revert + active false + end +end + +# Same for summary stages. +macro stack.summary.on + task stack.summary.load + active true + end + task stack.summary.run + active true + end +end + +macro stack.summary.off + task stack.summary.load + active false + end + task stack.summary.run + active false + end +end + +macro stack.revert.on + task stack.revert + active true + end +end + +macro stack.revert.off + task stack.revert + active false + end +end + + +### Load tasks for doing the stack +### Tasks are loaded into stackSumSkyfile. +task stack.skycell.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/stack.skycell.log + + task.exec + if ($LABEL:n == 0) break + $run = stacktool -tosum + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$stackSkycell_DB + $run = $run -dbname $DB:$stackSkycell_DB + $stackSkycell_DB ++ + if ($stackSkycell_DB >= $DB:n) set stackSkycell_DB = 0 + end + add_poll_args run + add_poll_labels run + # change the limit (the last one on the command line takes precedence) + $run = $run -limit $POLL_LIMIT_STACK + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackSumSkyfile -key stack_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackSumSkyfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackSumSkyfile + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +### Run tasks for calculating the stack overlaps +### Tasks are taken from stackSumSkyfile. +task stack.skycell.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + book npages stackSumSkyfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new images in stackSumSkyfile (pantaskState == INIT) + book getpage stackSumSkyfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackSumSkyfile $pageName pantaskState RUN + book getword stackSumSkyfile $pageName stack_id -var STACK_ID + book getword stackSumSkyfile $pageName tess_id -var TESS_DIR + book getword stackSumSkyfile $pageName skycell_id -var SKYCELL_ID + book getword stackSumSkyfile $pageName workdir -var WORKDIR_TEMPLATE + book getword stackSumSkyfile $pageName path_base -var PATH_BASE + book getword stackSumSkyfile $pageName reduction -var REDUCTION + book getword stackSumSkyfile $pageName dbname -var DBNAME + book getword stackSumSkyfile $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + # XXX old code: + # host anyhost + # $WORKDIR = $WORKDIR_TEMPLATE + + basename $TESS_DIR -var TESS_ID + if ("$PATH_BASE" == "NULL") + sprintf outroot "%s/%s/%s/%s.%s.stk.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $STACK_ID + else + $outroot = $PATH_BASE + end + + stdout $LOGDIR/stack.skycell.log + stderr $LOGDIR/stack.skycell.log + + $run = stack_skycell.pl --threads @MAX_THREADS@ --stack_id $STACK_ID --outroot $outroot --redirect-output --run-state $RUN_STATE + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit stackSumSkyfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword stackSumSkyfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackSumSkyfile $options:0 pantaskState TIMEOUT + end +end + + +task stack.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it doesn't + # continue to occur). + $run = stacktool -revertsumskyfile -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$stack_revert_DB + $run = $run -dbname $DB:$stack_revert_DB + $stack_revert_DB ++ + if ($stack_revert_DB >= $DB:n) set stack_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Load tasks for doing the stack summary +### Tasks are loaded into stackPendingSummary +task stack.summary.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1200 + active false +# trange 07:00:00 08:00:00 -nmax 1 + npending 1 + + stdout NULL + stderr $LOGDIR/stack.summary.load + + task.exec + if ($LABEL:n == 0) break + $run = stacktool -tosummary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$stackSummary_DB + $run = $run -dbname $DB:$stackSummary_DB + $stackSummary_DB ++ + if ($stackSummary_DB >= $DB:n) set stackSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackPendingSummary -key sass_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackPendingSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackPendingSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for doing the stack summary +task stack.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages stackPendingSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in stackPendingSummary (pantaskState == INIT) + book getpage stackPendingSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackPendingSummary $pageName pantaskState RUN + book getword stackPendingSummary $pageName sass_id -var SASS_ID + book getword stackPendingSummary $pageName camera -var CAMERA + book getword stackPendingSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword stackPendingSummary $pageName dbname -var DBNAME + book getword stackPendingSummary $pageName tess_id -var TESS_DIR + book getword stackPendingSummary $pageName state -var RUN_STATE + book getword stackPendingSummary $pageName projection_cell -var projection_cell + + # set the host and workdir + host anyhost + strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR + + basename $TESS_DIR -var TESS_ID + + ## generate outroot specific to this association + sprintf outroot "%s/%s/%s.stk.%s.summary" $WORKDIR $TESS_ID $TESS_ID $SASS_ID + + stdout $LOGDIR/stack.summary.log + stderr $LOGDIR/stack.summary.log + + $run = skycell_jpeg.pl --stage stack --stage_id $SASS_ID --camera $CAMERA --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit stackPendingSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword stackPendingSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackPendingSummary $options:0 pantaskState TIMEOUT + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/staticsky.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/staticsky.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/staticsky.pro (revision 41106) @@ -0,0 +1,272 @@ +## staticsky.pro : tasks for staticsky photometry analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the +## staticsky photometry analysis. After a staticsky entry (with +## associated sky_id) is defined, psphotStack is performed (via script +## staticsky.pl) (tasks in staticskyResult). + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init staticskyResult + +### Database lists +$staticsky_DB = 0 +$staticsky_revert_DB = 0 + +### Check status of staticsky tasks +macro staticsky.status + book listbook staticskyResult +end + +### Reset staticsky tasks +macro staticsky.reset + book init staticskyResult +end + +### Turn staticsky tasks on +macro staticsky.on + task staticsky.load + active true + end + task staticsky.run + active true + end + task staticsky.revert + active false + end +end + +### Turn staticsky tasks off +macro staticsky.off + task staticsky.load + active false + end + task staticsky.run + active false + end + task staticsky.revert + active false + end +end + +macro staticsky.revert.on + task staticsky.revert + active true + end +end + +macro staticsky.revert.off + task staticsky.revert + active false + end +end + +$RA_POLL_MAX = 0 + +macro set.ra.max + if ($0 != 2) + echo "USAGE: set.ra.max (ra_max_deg)" + break + end + $RA_POLL_MAX = $1 +end +macro get.ra.max + echo $RA_POLL_MAX +end + +### Load tasks for staticsky +### Tasks are loaded into staticskyResult. +task staticsky.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/staticsky.log + + task.exec + if ($LABEL:n == 0) break + $run = staticskytool -todo + if ($RA_POLL_MAX > 0) + $run = $run -ra_max $RA_POLL_MAX + end + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$staticsky_DB + $run = $run -dbname $DB:$staticsky_DB -limit 40 + $staticsky_DB ++ + if ($staticsky_DB >= $DB:n) set staticsky_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout staticskyResult -key sky_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook staticskyResult + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup staticskyResult + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the staticsky analysis +### Tasks are taken from staticskyResult. +task staticsky.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + book npages staticskyResult -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new entries in staticskyResult (pantaskState == INIT) + book getpage staticskyResult 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword staticskyResult $pageName pantaskState RUN + book getword staticskyResult $pageName sky_id -var SKY_ID + book getword staticskyResult $pageName tess_id -var TESS_DIR + book getword staticskyResult $pageName skycell_id -var SKYCELL_ID + book getword staticskyResult $pageName workdir -var WORKDIR_TEMPLATE + book getword staticskyResult $pageName path_base -var PATH_BASE + book getword staticskyResult $pageName reduction -var REDUCTION + book getword staticskyResult $pageName dbname -var DBNAME + book getword staticskyResult $pageName state -var RUN_STATE + + # move this above when we get camera into the db correctly: + # book getword staticskyResult $pageName camera -var CAMERA + $CAMERA = GPC1 + # $CAMERA = MEGACAM + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + + if ("$PATH_BASE" == "NULL") + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.sky.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $SKY_ID + else + # run-state must be update use existing path_base + $outroot = $PATH_BASE + end + + stdout $LOGDIR/staticsky.log + stderr $LOGDIR/staticsky.log + + $run = staticsky.pl --sky_id $SKY_ID --run-state $RUN_STATE --outroot $outroot --redirect-output --camera $CAMERA --threads @MAX_THREADS@ + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit staticskyResult $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword staticskyResult $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword staticskyResult $options:0 pantaskState TIMEOUT + end +end + +task staticsky.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it doesn't + # continue to occur). + $run = staticskytool -revert -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$staticsky_revert_DB + $run = $run -dbname $DB:$staticsky_revert_DB + $staticsky_revert_DB ++ + if ($staticsky_revert_DB >= $DB:n) set staticsky_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /branches/ccl_branches/ipponly-20191108/ippTasks/summit.copy.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/summit.copy.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/summit.copy.pro (revision 41106) @@ -0,0 +1,720 @@ +## summit.copy.pro : tasks for the summit to IPP download : -*- sh -*- +## PanTasks scripts for Summit Copy + +## XXX note that this currently works with a single database as defined in .ipprc +## XXX tie the database for output to the database from which the datastore was determined + +# pztool -adddatastore -inst isp -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/skyprobe/index.txt +# pztool -adddatastore -inst ssp -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/ssp/index.txt +# pztool -adddatastore -inst gpc1 -telescope ps1 -uri http://conductor/ds/gpc1/index.txt +# pztool -adddatastore -inst allskycam -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/allskycam/index.txt + + +# NOTE: workdir / volume mangling and nebulous. these tasks copy the +# imfiles from the summit, placing the output files in a directory +# which is based on the chip/host relationship. If nebulous is being +# used, the volume name is set based on the host; if nebulous is not +# being used, the workdir is set to include the host name. The copy +# operation is targetted to the same host by pcontrol. The value of +# workdir is set to be a template into which the appropriate value of +# @HOST@ may be substituted in later scripts. + +# test for required global variables +check.globals + +# list of DataStores to pull data from +book init pzDataStore +# list of summit exps that need to be queried +book init pzPendingExp +# list of summit imfiles that need to be downloaded +book init pzPendingImfile +# list of pzDownloadExps that have completed downloading +book init pzPendingAdvance + +macro copy.status + book listbook pzPendingImfile +end + +macro copy.reset + book init pzPendingExp + book init pzPendingImfile + book init pzPendingAdvance +end + +# aliases because copy.* isn't intuitive +macro pztool.reset + copy.reset +end + +macro summitcopy.reset + summitcopy.reset +end + +macro copy.on + task pztool.datastore + active true + end + task pzgetexp + active true + end + task pztool.pendingexp + active true + end + task pzgetimfile + active true + end + task pztool.pendingimfile + active true + end + task summit_copy + active true + end + task pztool.clearfault + active true + end + task summit.toadvance + active true + end + task summit.advance + active true + end +end + +macro copy.off + task pztool.datastore + active false + end + task pzgetexp + active false + end + task pztool.pendingexp + active false + end + task pzgetimfile + active false + end + task pztool.pendingimfile + active false + end + task summit_copy + active false + end + task pztool.clearfault + active false + end + task summit.toadvance + active false + end + task summit.advance + active false + end +end + +# these variables will cycle through the known database names +$pztoolDatastore_DB = 0 +$pztoolPendingExp_DB = 0 +$pztoolPendingImfile_DB = 0 +$pztoolPendingAdvance_DB = 0 +$pztoolClearFault_DB = 0 +$pztoolAdvance_DB = 0; + +# build a book of datastores to poll for data +task pztool.datastore + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 1 + periods -poll 1 + periods -timeout 20 + npending 1 + + # only active in the day (06:00 to 19:00 HST, times are UT): + trange -reset + # trange 16:00 23:59 + # trange 00:00 05:00 + + task.exec +# echo "DB_DEBUG: PRE" $pztoolDatastore_DB $DB:$pztoolDatastore_DB + if ($DB:n == 0) + option DEFAULT + command pztool -datastore + else + # save the DB name for the exit tasks + option $DB:$pztoolDatastore_DB + command pztool -datastore -dbname $DB:$pztoolDatastore_DB + $pztoolDatastore_DB ++ + if ($pztoolDatastore_DB >= $DB:n) set pztoolDatastore_DB = 0 + end + + # More debug +# echo "DB_DEBUG: POST" $pztoolDatastore_DB $DB:$pztoolDatastore_DB + periods -exec 120 + end + + # success + task.exit 0 + # flush pzDataStore book +# book init pzDataStore + # convert 'stdout' to book format + ipptool2book stdout pzDataStore -key camera:telescope -uniq -setword dbname $options:0 + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +$datastore_index = 0 + +# run pzgetexp periodically to populate pzPendingExp in the database (no I/O) +# this task is querying the data store for a list of exposures ("filesets") +# and inserting these into a db table on the local cluster (pzDownloadExp) +task pzgetexp + host local + + periods -exec 30 + periods -poll 1 + periods -timeout 700 + npending 1 + + # only active in the day (06:00 to 19:00 HST, times are UT): + trange -reset + # trange 16:00 23:59 + # trange 00:00 05:00 + + task.exec + # find an exp that needs imfiles fetched + book getpage pzDataStore $datastore_index -var pageName + if ("$pageName" == "NULL") break + + # increment our pzDataStore index and loop back to 0 and the end of the + # book + $datastore_index ++ + book npages pzDataStore -var npages + if ($datastore_index == $npages ) + $datastore_index = 0 + end + + book getword pzDataStore $pageName camera -var CAMERA + book getword pzDataStore $pageName telescope -var TELESCOPE + book getword pzDataStore $pageName uri -var URI + book getword pzDataStore $pageName dbname -var DBNAME + + # store the current page + options $pageName + + $run = pzgetexp -uri $URI -inst $CAMERA -telescope $TELESCOPE -dbname $DBNAME -timeout 650 + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + + command $run + end + + task.exit 0 + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +# build a book of exps/filesetids that need to be queried +task pztool.pendingexp + host local + + periods -exec 10 + periods -poll 1 + periods -timeout 60 + npending 1 + + # only active in the day (06:00 to 19:00 HST, times are UT): + trange -reset + # trange 16:00 23:59 + # trange 00:00 05:00 + + task.exec + + # CZW 2016-04-08 This needs to be set to the date that pztool should start considering. + # I've chosen to use date's "relative items in date strings" calculation + # because it's easy to set a time frame. + $pztool_date_begin = `date -d "-30 days" +%Y-%m-%dT00:00:00` + + if ($DB:n == 0) + option DEFAULT + command pztool -pendingexp -limit 5 + else + # save the DB name for the exit tasks + option $DB:$pztoolPendingExp_DB + command pztool -pendingexp -dateobs_begin $pztool_date_begin -limit 10 -dbname $DB:$pztoolPendingExp_DB + $pztoolPendingExp_DB ++ + if ($pztoolPendingExp_DB >= $DB:n) set pztoolPendingExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingExp + end + + task.exit default + showcommand failure + if ($VERBOSE) + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +# run pzgetimfiles on pending exps. analogous to pzgetexp, this task +# is downloading a list of imfiles reported by the datastore for a +# given exposure ("fileset"), and placing the result in the local +# database table of imfiles +task pzgetimfile + host local + + periods -exec 0.05 + periods -poll 0.025 + periods -timeout 700 + npending 10 + + # only active in the day (06:00 to 19:00 HST, times are UT): + trange -reset + # trange 16:00 23:59 + # trange 00:00 05:00 + + task.exec + periods -exec 20 + + # if we are waiting on data, make the interval long + book npages pzPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # find an exp that needs imfiles fetched + book getpage pzPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingExp $pageName pantaskState RUN + + book getword pzPendingExp $pageName exp_name -var EXP_NAME + book getword pzPendingExp $pageName camera -var CAMERA + book getword pzPendingExp $pageName telescope -var TELESCOPE + book getword pzPendingExp $pageName dateobs -var DATEOBS + book getword pzPendingExp $pageName exp_type -var EXP_TYPE + book getword pzPendingExp $pageName uri -var URI + book getword pzPendingExp $pageName imfiles -var IMFILES + book getword pzPendingExp $pageName dbname -var DBNAME + + # store the current page + options $pageName + + $batman = $EXP_NAME + # Sidik says we should use a longer timeout + $run = pzgetimfiles -uri $URI -filesetid $batman -inst $CAMERA -telescope $TELESCOPE -dbname $DBNAME -timeout 650 + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # success + task.exit 0 + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword pzPendingExp $options:0 pantaskState CRASH + end + + task.exit timeout + showcommand timeout + book setword pzPendingExp $options:0 pantaskState TIMEOUT + end +end + + +# build a book of imfiles/files that need to be downloaded +task pztool.pendingimfile + host local + + periods -exec 10 + periods -poll 1 + periods -timeout 120 + npending 1 + + # only active in the day (06:00 to 19:00 HST, times are UT): + trange -reset + # trange 16:00 23:59 + # trange 00:00 05:00 + + # select entries from the current DB; cycle to the next DB, if it exists + # iff the DB list is not set, use the value defined in .ipprc + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -pendingimfile -limit 40 + else + # save the DB name for the exit tasks + option $DB:$pztoolPendingImfile_DB + command pztool -pendingimfile -limit 240 -dbname $DB:$pztoolPendingImfile_DB + $pztoolPendingImfile_DB ++ + if ($pztoolPendingImfile_DB >= $DB:n) set pztoolPendingImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope:class:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle pzPendingImfile + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingImfile + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +# retreive an imfile with dsget and then call pztool -copydone +task summit_copy + periods -exec 5 + periods -poll 0.05 + periods -timeout 1150 + + # only active in the day (06:00 to 19:00 HST, times are UT): + trange -reset + # trange 16:00 23:59 + # trange 00:00 05:00 + + task.exec + periods -exec 20 + + # if we are waiting on data, make the interval long + book npages pzPendingImfile -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # find an exp that needs imfiles fetched + book getpage pzPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingImfile $pageName pantaskState RUN + + book getword pzPendingImfile $pageName uri -var URI + book getword pzPendingImfile $pageName bytes -var BYTES + book getword pzPendingImfile $pageName md5sum -var MD5SUM + book getword pzPendingImfile $pageName dateobs -var DATEOBS + book getword pzPendingImfile $pageName summit_id -var SUMMIT_ID + book getword pzPendingImfile $pageName exp_name -var EXP_NAME + book getword pzPendingImfile $pageName camera -var CAMERA + book getword pzPendingImfile $pageName telescope -var TELESCOPE + book getword pzPendingImfile $pageName class -var CLASS + book getword pzPendingImfile $pageName class_id -var CLASS_ID + book getword pzPendingImfile $pageName dbname -var DBNAME + + set.host.for.camera $CAMERA $CLASS_ID + + # 2007-08-30T05:09:59Z + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 5 2 MONTH + substr $DATEOBS 8 2 DAY + + # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship + # this function uses workdir_template, default_host, volume_template, volume_default, + # it sets workdir and volume + + # Look to see if we have a stare type observation, and redirect it to an appropriate host + substr $EXP_NAME 10 1 EXPTYPE_KEY + if ("$EXPTYPE_KEY" == "a") + if ($NEBULOUS == 1) + $workdir_template = neb://@HOST@.1 + end + set.workdir.by.camera STARE $CLASS_ID $workdir_template $default_host workdir_base + else + if ($NEBULOUS == 1) + $workdir_template = neb://@HOST@.0 + end + set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base + end + + # figure out filename + # XXX may need to use sprintf here + $FILENAME = $workdir_base/$CAMERA/$YEAR\$MONTH\$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID.fits + $workdir = $workdir_template/$CAMERA/$YEAR\$MONTH\$DAY + + # workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb://@HOST@.0/gpc1/20080130 + + # filename examples: + # file://data/ipp005.0/gpc1/20080130/o4437g0025d/o4437g0025d.XY05.fits + # neb://@HOST@.0/gpc1/20080130/o4437g0025d/o4437g0025d.XY05.fits + + book setword pzPendingImfile $pageName filename $FILENAME + + stdout $LOGDIR/summit.copy.log + stderr $LOGDIR/summit.copy.log + + book getpage pzDataStore 0 -var PZDSPAGE -key dbname $DBNAME + book getword pzDataStore $PZDSPAGE use_compress -var USECOMPRESS + + # Debug line +# echo "DEBUG: " $DBNAME $EXP_NAME $CAMERA $PZDSPAGE $USECOMPRESS + + # unconditionally turn on requesting compression until we figuure + # out why the above doesn't work + if ($USECOMPRESS == "NULL") +# if ("$CAMERA" == "gpc1") + $USECOMPRESS = 1 + end + + $run = summit_copy.pl --uri $URI --filename $FILENAME --summit_id $SUMMIT_ID --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --dbname $DBNAME --timeout 600 --verbose --copies 2 + if ($USECOMPRESS == 1) + $run = $run --compress + end +# if (("$MD5SUM" != "NULL") && ("$MD5SUM" != "0") && (not($COMPRESS))) +# && (($YEAR > 2008) || (("$YEAR" = "2007") && ($MONTH > 8)))) +# $run = $run --md5 $MD5SUM +# end + if ($NEBULOUS) + $run = $run --nebulous + end + # add_standard_args run + + # store the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # More debug: +# echo "DEBUG2: " $run + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit pzPendingImfile $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword pzPendingImfile $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pzPendingImfile $options:0 pantaskState TIMEOUT + end +end + +task pztool.clearfault + host local + + # -exec is set much longer the first time this task runs + periods -exec 7 + periods -poll 1 + periods -timeout 30 + npending 1 + + task.exec + if ($DB:n == 0) + command pztool -clearcommonfaults + else + command pztool -clearcommonfaults -dbname $DB:$pztoolClearFault_DB + + # loop over the multiple databases quickly, then pause for 5 min + $pztoolClearFault_DB ++ + if ($pztoolClearFault_DB >= $DB:n) + set pztoolClearFault_DB = 0 + periods -exec 300 + else + periods -exec 30 + end + end + end + + # success + task.exit 0 + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +# build a book of exposures that have completed downloading +task summit.toadvance + host local + + periods -exec 30 + periods -poll 1 + periods -timeout 120 + npending 1 + + # select entries from the current DB; cycle to the next DB, if it exists + # iff the DB list is not set, use the value defined in .ipprc + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -toadvance -limit 40 + else + # save the DB name for the exit tasks + option $DB:$pztoolPendingAdvance_DB + command pztool -toadvance -limit 60 -dbname $DB:$pztoolPendingAdvance_DB + $pztoolPendingAdvance_DB ++ + if ($pztoolPendingAdvance_DB >= $DB:n) set pztoolPendingAdvance_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pzPendingAdvance -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle pzPendingAdvance + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingAdvance + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +task summit.advance + periods -exec 5 + periods -poll 0.05 + periods -timeout 650 + + task.exec + periods -exec 20 + + # if we are waiting on data, make the interval long + book npages pzPendingAdvance -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # find an exp that needs imfiles fetched + book getpage pzPendingAdvance 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingAdvance $pageName pantaskState RUN + + book getword pzPendingAdvance $pageName summit_id -var SUMMIT_ID + book getword pzPendingAdvance $pageName exp_name -var EXP_NAME + book getword pzPendingAdvance $pageName camera -var CAMERA + book getword pzPendingAdvance $pageName telescope -var TELESCOPE + book getword pzPendingAdvance $pageName dbname -var DBNAME + book getword pzPendingAdvance $pageName dateobs -var DATEOBS + + # 2007-08-30T05:09:59Z + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 5 2 MONTH + substr $DATEOBS 8 2 DAY + + # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship + # this function uses workdir_template, default_host, volume_template, volume_default, + # it sets workdir and volume + set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base + + $workdir = $workdir_template/$CAMERA/$YEAR\$MONTH\$DAY + + # workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb://@HOST@.0/gpc1/20080130 + + stdout $LOGDIR/summit.advance.log + stderr $LOGDIR/summit.advance.log + + $run = pztool -advance -summit_id $SUMMIT_ID -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -end_stage reg -workdir $workdir -dbname $DBNAME + + # store the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit pzPendingAdvance $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword pzPendingAdvance $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pzPendingAdvance $options:0 pantaskState TIMEOUT + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/survey.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/survey.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/survey.pro (revision 41106) @@ -0,0 +1,1900 @@ +## survey.pro : tasks related to automation of the PS1 survey : -*- sh -*- + +# test for required global variables +check.globals + +if (not($?haveSurveyBooks)) + book create SURVEY_DIFF + book create SURVEY_DIFF_WARPSTACK + book create SURVEY_DIFF_STACKSTACK + book create SURVEY_MAGIC + book create SURVEY_DESTREAK + book create SURVEY_DIST + book create SURVEY_ADDSTAR + book create SURVEY_MERGEDVODB + book create SURVEY_CHIP_BG + book create SURVEY_WARP_BG + book create SURVEY_PUBLISH + book create SURVEY_STATICSKYSINGLE + book create SURVEY_SKYCAL + book create SURVEY_LAPGROUP + book create SURVEY_RELEXP + book create SURVEY_RELSTACK + $haveSurveyBooks = TRUE +end + +$SURVEY_DIFF_DB = 0 +$SURVEY_DIFF_WARPSTACK_DB = 0 +$SURVEY_DIFF_STACKSTACK_DB = 0 +$SURVEY_MAGIC_DB = 0 +$SURVEY_DESTREAK_DB = 0 +$SURVEY_DIST_DB = 0 +$SURVEY_ADDSTAR_DB = 0 +$SURVEY_MERGEDVODB_DB = 0 +$SURVEY_CHIP_BG_DB = 0 +$SURVEY_WARP_BG_DB = 0 +$SURVEY_PUBLISH_DB = 0 +$SURVEY_SKYCAL_DB = 0 +$SURVEY_LAPGROUP_DB = 0 +$SURVEY_RELEXP_DB = 0 +$SURVEY_RELSTACK_DB = 0 +$SURVEY_STATICSKYSINGLE_DB = 0 +$SURVEY_EXEC = 120 +$SURVEY_POLL = 10 +$SURVEY_TIMEOUT = 600 +# the tasks for loading the release tabels for nightly science do not need to run very often +$SURVEY_RELEASE_EXEC = 300 + +macro survey.on + task survey.diff + active true + end + task survey.warpstack.diff + active true + end + task survey.stackstack.diff + active true + end + task survey.magic + active true + end + task survey.addstar + active true + end + task survey.mergedvodb + active true + end + task survey.destreak + active true + end + task survey.dist + active true + end + task survey.chip.bg + active true + end + task survey.warp.bg + active true + end + task survey.publish + active true + end + task survey.staticskysingle + active true + end + task survey.skycal + active true + end + task survey.lapgroup + active true + end + task survey.relexp + active true + end + task survey.relstack + active true + end +end + +macro survey.off + task survey.diff + active false + end + task survey.warpstack.diff + active false + end + task survey.stackstack.diff + active false + end + task survey.magic + active false + end + task survey.addstar + active false + end + task survey.mergedvodb + active false + end + task survey.destreak + active false + end + task survey.dist + active false + end + task survey.chip.bg + active false + end + task survey.warp.bg + active false + end + task survey.publish + active false + end + task survey.staticskysingle + active false + end + task survey.skycal + active false + end + task survey.lapgroup + active false + end + task survey.relexp + active false + end + task survey.relstack + active false + end +end + +# user functions to manipulate diff labels +macro survey.add.diff + if ($0 != 4) + echo "USAGE: survey.add.diff (label) (dist_group) (workdir base)" + break + end + book newpage SURVEY_DIFF $1 + book setword SURVEY_DIFF $1 DIST_GROUP $2 + book setword SURVEY_DIFF $1 WORKDIR $3 + book setword SURVEY_DIFF $1 STATE PENDING +end + +macro survey.del.diff + if ($0 != 2) + echo "USAGE: survey.del.diff (label)" + break + end + book delpage SURVEY_DIFF $1 +end + +macro survey.show.diff + if ($0 != 1) + echo "USAGE: survey.show.diff" + break + end + book listbook SURVEY_DIFF +end + +# user functions to manipulate warp/stack diff labels +macro survey.add.WSdiff + if ($0 != 7) + echo "USAGE: survey.add.WSdiff (key) (warp label) (stack label) (dist_group) (workdir base) (target label)" + break + end + book newpage SURVEY_DIFF_WARPSTACK $1 + book setword SURVEY_DIFF_WARPSTACK $1 WARP_LABEL $2 + book setword SURVEY_DIFF_WARPSTACK $1 STACK_LABEL $3 + book setword SURVEY_DIFF_WARPSTACK $1 DIST_GROUP $4 + book setword SURVEY_DIFF_WARPSTACK $1 WORKDIR $5 + book setword SURVEY_DIFF_WARPSTACK $1 TARGET_LABEL $6 + book setword SURVEY_DIFF_WARPSTACK $1 STATE PENDING +end + +macro survey.del.WSdiff + if ($0 != 2) + echo "USAGE: survey.del.WSdiff (warp label)" + break + end + book delpage SURVEY_DIFF_WARPSTACK $1 +end + +macro survey.show.WSdiff + if ($0 != 1) + echo "USAGE: survey.show.WSdiff" + break + end + book listbook SURVEY_DIFF_WARPSTACK +end + +# user functions to manipulate stack/stack diff labels +macro survey.add.SSdiff + if ($0 != 7) + echo "USAGE: survey.add.SSdiff (key) (input label) (stack label) (dist_group) (workdir base) (target label)" + break + end + book newpage SURVEY_DIFF_STACKSTACK $1 + book setword SURVEY_DIFF_STACKSTACK $1 INPUT_LABEL $2 + book setword SURVEY_DIFF_STACKSTACK $1 STACK_LABEL $3 + book setword SURVEY_DIFF_STACKSTACK $1 DIST_GROUP $4 + book setword SURVEY_DIFF_STACKSTACK $1 WORKDIR $5 + book setword SURVEY_DIFF_STACKSTACK $1 TARGET_LABEL $6 + book setword SURVEY_DIFF_STACKSTACK $1 STATE PENDING +end + +macro survey.del.SSdiff + if ($0 != 2) + echo "USAGE: survey.del.SSdiff (stack label)" + break + end + book delpage SURVEY_DIFF_STACKSTACK $1 +end + +macro survey.show.SSdiff + if ($0 != 1) + echo "USAGE: survey.show.SSdiff" + break + end + book listbook SURVEY_DIFF_STACKSTACK +end + +# user functions to manipulate magic labels +macro survey.add.magic + if (($0 != 3)&&($0 != 4)) + echo "USAGE: survey.add.magic (label) (workdir base) (multidiff)" + break + end + book newpage SURVEY_MAGIC $1 + book setword SURVEY_MAGIC $1 WORKDIR $2 + if ($0 == 3) + book setword SURVEY_MAGIC $1 MULTIDIFF FALSE + else + book setword SURVEY_MAGIC $1 MULTIDIFF $3 + end + book setword SURVEY_MAGIC $1 STATE PENDING +end + +macro survey.del.magic + if ($0 != 2) + echo "USAGE: survey.del.magic (label)" + break + end + book delpage SURVEY_MAGIC $1 +end + +macro survey.show.magic + if ($0 != 1) + echo "USAGE: survey.show.magic" + break + end + book listbook SURVEY_MAGIC +end + +macro survey.add.addstar + if ($0 != 6) + echo "USAGE: survey.add.addstar (tag) (label) (dvodb) (minidvodb_group) (stage)" + break + end + book newpage SURVEY_ADDSTAR $1 + book setword SURVEY_ADDSTAR $1 LABEL $2 + book setword SURVEY_ADDSTAR $1 DVODB $3 + book setword SURVEY_ADDSTAR $1 MINIDVODB_GROUP $4 + book setword SURVEY_ADDSTAR $1 STAGE $5 + book setword SURVEY_ADDSTAR $1 STATE PENDING +end + +macro survey.del.addstar + if ($0 != 2) + echo "USAGE: survey.del.addstar (tag)" + break + end + book delpage SURVEY_ADDSTAR $1 +end + +macro survey.show.addstar + if ($0 != 1) + echo "USAGE: survey.show.addstar" + break + end + book listbook SURVEY_ADDSTAR +end + +macro survey.add.mergedvodb + if ($0 != 4) + echo "USAGE: survey.add.mergedvodb (tag) (mergedvodb) (minidvodb_group)" + break + end + book newpage SURVEY_MERGEDVODB $1 + book setword SURVEY_MERGEDVODB $1 MERGEDVODB $2 + book setword SURVEY_MERGEDVODB $1 MINIDVODB_GROUP $3 +end + +macro survey.del.mergedvodb + if ($0 != 2) + echo "USAGE: survey.del.mergedvodb (tag)" + break + end + book delpage SURVEY_MERGEDVODB $1 +end + +macro survey.show.mergedvodb + if ($0 != 1) + echo "USAGE: survey.show.mergedvodb" + break + end + book listbook SURVEY_MERGEDVODB +end + + +# user functions to manipulate destreak labels +macro survey.add.destreak + if ($0 != 4) + echo "USAGE: survey.add.destreak (label) (workdir base) (recovery root)" + break + end + book newpage SURVEY_DESTREAK $1 + book setword SURVEY_DESTREAK $1 WORKDIR $2 + book setword SURVEY_DESTREAK $1 RECOVERYROOT $3 + book setword SURVEY_DESTREAK $1 STATE PENDING +end + +macro survey.del.destreak + if ($0 != 2) + echo "USAGE: survey.del.destreak (label)" + break + end + book delpage SURVEY_DESTREAK $1 +end + +macro survey.show.destreak + if ($0 != 1) + echo "USAGE: survey.show.destreak" + break + end + book listbook SURVEY_DESTREAK +end + +# user functions to manipulate dist labels +macro survey.add.dist + if ($0 != 4) + echo "USAGE: survey.add.dist (label) (workdir) (muggle)" + break + end + book newpage SURVEY_DIST $1 + book setword SURVEY_DIST $1 WORKDIR $2 + book setword SURVEY_DIST $1 MUGGLE $3 + book setword SURVEY_DIST $1 STATE PENDING +end + +macro survey.del.dist + if ($0 != 2) + echo "USAGE: survey.del.dist (label)" + break + end + book delpage SURVEY_DIST $1 +end + +macro survey.show.dist + if ($0 != 1) + echo "USAGE: survey.show.dist" + break + end + book listbook SURVEY_DIST +end + +# user functions to manipulate chip_bg labels +macro survey.add.chip.bg + if ($0 != 4) + echo "USAGE: survey.add.chip.bg (label) (cam_label) (dist_group)" + break + end + book newpage SURVEY_CHIP_BG $1 + book setword SURVEY_CHIP_BG $1 LABEL $1 + book setword SURVEY_CHIP_BG $1 CAM_LABEL $2 + book setword SURVEY_CHIP_BG $1 DIST_GROUP $3 + book setword SURVEY_CHIP_BG $1 STATE PENDING +end + +macro survey.del.chip.bg + if ($0 != 2) + echo "USAGE: survey.del.chip.bg (label)" + break + end + book delpage SURVEY_CHIP_BG $1 +end + +macro survey.show.chip.bg + if ($0 != 1) + echo "USAGE: survey.show.chip.bg" + break + end + book listbook SURVEY_CHIP_BG +end + +# user functions to manipulate warp_bg labels +macro survey.add.warp.bg + if ($0 != 4) + echo "USAGE: survey.add.warp.bg (label) (warp_label) (dist_group)" + break + end + book newpage SURVEY_WARP_BG $1 + book setword SURVEY_WARP_BG $1 LABEL $1 + book setword SURVEY_WARP_BG $1 WARP_LABEL $2 + book setword SURVEY_WARP_BG $1 DIST_GROUP $3 + book setword SURVEY_WARP_BG $1 STATE PENDING +end + +macro survey.del.warp.bg + if ($0 != 2) + echo "USAGE: survey.del.warp.bg (label)" + break + end + book delpage SURVEY_WARP_BG $1 +end + +macro survey.show.warp.bg + if ($0 != 1) + echo "USAGE: survey.show.warp.bg" + break + end + book listbook SURVEY_WARP_BG +end + +# user functions to manipulate publish labels +macro survey.add.publish + if ($0 != 5) + echo "USAGE: survey.add.publish (tag) (label) (client_id) (comment)" + break + end + book newpage SURVEY_PUBLISH $1 + book setword SURVEY_PUBLISH $1 LABEL $2 + book setword SURVEY_PUBLISH $1 CLIENT_ID $3 + book setword SURVEY_PUBLISH $1 COMMENT $4 + book setword SURVEY_PUBLISH $1 STATE PENDING +end + +macro survey.del.publish + if ($0 != 2) + echo "USAGE: survey.del.publish (label)" + break + end + book delpage SURVEY_PUBLISH $1 +end + +macro survey.show.publish + if ($0 != 1) + echo "USAGE: survey.show.publish" + break + end + book listbook SURVEY_PUBLISH +end + +macro survey.add.staticskysingle +# adds each of the filters for the label chosen +# for LAP - the final stacks chosen have %final% in them, and we +# want some ability to choose that +# for others can use % + + if ($0 != 7) + echo "USAGE: survey.add.staticskysingle (tag) (label) (workdir) (distgroup) (selectdatagroup) (filter)" + break + end + book newpage SURVEY_STATICSKYSINGLE $1 + book setword SURVEY_STATICSKYSINGLE $1 LABEL $2 + book setword SURVEY_STATICSKYSINGLE $1 WORKDIR $3 + book setword SURVEY_STATICSKYSINGLE $1 DIST_GROUP $4 + book setword SURVEY_STATICSKYSINGLE $1 SELECTDATAGROUP $5 + book setword SURVEY_STATICSKYSINGLE $1 FILTER $6 + book setword SURVEY_STATICSKYSINGLE $1 STATE PENDING +end + +macro survey.show.staticskysingle + if ($0 != 1) + echo "USAGE: survey.show.staticskysingle" + break + end + book listbook SURVEY_STATICSKYSINGLE +end + +macro survey.del.staticskysingle + if ($0 != 2) + echo "USAGE: survey.del.staticskysingle (tag)" + break + end + book delpage SURVEY_STATICSKYSINGLE $1 +end + +macro survey.add.skycal + if ($0 != 3) + echo "USAGE: survey.add.skycal (label) (dist_group)" + break + end + book newpage SURVEY_SKYCAL $1 + book setword SURVEY_SKYCAL $1 LABEL $1 + book setword SURVEY_SKYCAL $1 DIST_GROUP $2 + book setword SURVEY_SKYCAL $1 STATE PENDING +end + +macro survey.del.skycal + if ($0 != 2) + echo "USAGE: survey.del.skycal (label)" + break + end + book delpage SURVEY_SKYCAL $1 +end + +macro survey.show.skycal + if ($0 != 1) + echo "USAGE: survey.show.skycal" + break + end + book listbook SURVEY_SKYCAL +end + +macro survey.add.lapgroup + if ($0 != 3) + echo "USAGE: survey.add.lapgroup (label) (seq_id)" + break + end + book newpage SURVEY_LAPGROUP $1 + book setword SURVEY_LAPGROUP $1 LABEL $1 + book setword SURVEY_LAPGROUP $1 SEQ_ID $2 + book setword SURVEY_LAPGROUP $1 STATE PENDING +end + +macro survey.del.lapgroup + if ($0 != 2) + echo "USAGE: survey.del.lapgroup (label)" + break + end + book delpage SURVEY_LAPGROUP $1 +end + +macro survey.show.lapgroup + if ($0 != 1) + echo "USAGE: survey.show.lapgroup" + break + end + book listbook SURVEY_LAPGROUP +end + +macro survey.add.relexp + if ($0 != 3) + echo "USAGE: survey.add.relexp (label) (releasename)" + break + end + book newpage SURVEY_RELEXP $1 + book setword SURVEY_RELEXP $1 LABEL $1 + book setword SURVEY_RELEXP $1 RELEASE_NAME $2 + book setword SURVEY_RELEXP $1 STATE PENDING +end + +macro survey.del.relexp + if ($0 != 2) + echo "USAGE: survey.del.relexp (label)" + break + end + book delpage SURVEY_RELEXP $1 +end + +macro survey.show.relexp + if ($0 != 1) + echo "USAGE: survey.show.relexp" + break + end + book listbook SURVEY_RELEXP +end + +macro survey.add.relstack + if ($0 != 4) + echo "USAGE: survey.add.relstack (label) (releasename) (stacktype)" + break + end + book newpage SURVEY_RELSTACK $1 + book setword SURVEY_RELSTACK $1 LABEL $1 + book setword SURVEY_RELSTACK $1 RELEASE_NAME $2 + book setword SURVEY_RELSTACK $1 STACK_TYPE $3 + book setword SURVEY_RELSTACK $1 STATE PENDING +end + +macro survey.del.relstack + if ($0 != 2) + echo "USAGE: survey.del.relstack (label)" + break + end + book delpage SURVEY_RELSTACK $1 +end + +macro survey.show.relstack + if ($0 != 1) + echo "USAGE: survey.show.relstack" + break + end + book listbook SURVEY_RELSTACK +end + + +task survey.diff + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.diff.log + stderr $LOGDIR/survey.diff.log + + # generate diff warp-warp runs + task.exec + book npages SURVEY_DIFF -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_DIFF 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_DIFF $i -var label + book setword SURVEY_DIFF $label STATE NEW + end + book getpage SURVEY_DIFF 0 -var label -key STATE NEW + + # Select different database + $SURVEY_DIFF_DB ++ + if ($SURVEY_DIFF_DB >= $DB:n) set SURVEY_DIFF_DB = 0 + end + + book setword SURVEY_DIFF $label STATE DONE + book getword SURVEY_DIFF $label WORKDIR -var workdir + book getword SURVEY_DIFF $label DIST_GROUP -var dist_group + + $year = `date +%Y` + $month = `date +%m` + $day = `date +%d` + + $run = difftool -definewarpwarp -distance 0.1 -good_frac 0.1 -timediff 3600 + $run = $run -input_label $label + $run = $run -template_label $label + $run = $run -set_dist_group $dist_group + $run = $run -set_label $label + $run = $run -set_workdir $workdir/$label/$year/$month/$day + $run = $run -set_reduction WARPWARP + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_DIFF_DB + option $DB:$SURVEY_DIFF_DB + end + + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task survey.warpstack.diff + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + ## time in UT -- want to start ~0900 HST and run until next night starts. 24h boundary so two trange needed? + trange 19:00:00 23:59:00 + trange 00:00:00 05:00:00 + npending 1 + + stdout $LOGDIR/survey.WSdiff.log + stderr $LOGDIR/survey.WSdiff.log + + # generate diff warp-stack runs + task.exec + book npages SURVEY_DIFF_WARPSTACK -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_DIFF_WARPSTACK 0 -var wsKey -key STATE NEW + if ("$wsKey" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_DIFF_WARPSTACK $i -var wsKey + book setword SURVEY_DIFF_WARPSTACK $wsKey STATE NEW + end + book getpage SURVEY_DIFF_WARPSTACK 0 -var wsKey -key STATE NEW + + # Select different database + $SURVEY_DIFF_WARPSTACK_DB ++ + if ($SURVEY_DIFF_WARPSTACK_DB >= $DB:n) set SURVEY_DIFF_WARPSTACK_DB = 0 + end + + book setword SURVEY_DIFF_WARPSTACK $wsKey STATE DONE + book getword SURVEY_DIFF_WARPSTACK $wsKey WARP_LABEL -var warp_label + book getword SURVEY_DIFF_WARPSTACK $wsKey STACK_LABEL -var stack_label + book getword SURVEY_DIFF_WARPSTACK $wsKey WORKDIR -var workdir + book getword SURVEY_DIFF_WARPSTACK $wsKey DIST_GROUP -var dist_group + book getword SURVEY_DIFF_WARPSTACK $wsKey TARGET_LABEL -var target_label + + $year = `date +%Y` + $month = `date +%m` + $day = `date +%d` + +# Do we need anything else here? + $run = difftool -definewarpstack -good_frac 0.2 + $run = $run -warp_label $warp_label + $run = $run -stack_label $stack_label + $run = $run -set_dist_group $dist_group + $run = $run -set_label $target_label + $run = $run -set_workdir $workdir/$target_label/$year/$month/$day + $run = $run -set_reduction WARPSTACK + $run = $run -available + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_DIFF_WARPSTACK_DB + option $DB:$SURVEY_DIFF_WARPSTACK_DB + end + + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task survey.stackstack.diff + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT +# 13:00 was being done in early morning rather than afternoon HST (so UT value), want both times to have SSdiff ASAP? +# changing earlier for QUB to fetch @10pm for nightly processing there, should be enough time to finish MD stacks +# trange 13:00:00 14:00:00 + trange 20:00:00 21:00:00 + npending 1 + + stdout $LOGDIR/survey.SSdiff.log + stderr $LOGDIR/survey.SSdiff.log + + # generate diff stack-stack runs + task.exec + book npages SURVEY_DIFF_STACKSTACK -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_DIFF_STACKSTACK 0 -var ssKey -key STATE NEW + if ("$ssKey" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_DIFF_STACKSTACK $i -var ssKey + book setword SURVEY_DIFF_STACKSTACK $ssKey STATE NEW + end + book getpage SURVEY_DIFF_STACKSTACK 0 -var ssKey -key STATE NEW + + # Select different database + $SURVEY_DIFF_STACKSTACK_DB ++ + if ($SURVEY_DIFF_STACKSTACK_DB >= $DB:n) set SURVEY_DIFF_STACKSTACK_DB = 0 + end + + book setword SURVEY_DIFF_STACKSTACK $ssKey STATE DONE + book getword SURVEY_DIFF_STACKSTACK $ssKey INPUT_LABEL -var input_label + book getword SURVEY_DIFF_STACKSTACK $ssKey STACK_LABEL -var stack_label + book getword SURVEY_DIFF_STACKSTACK $ssKey WORKDIR -var workdir + book getword SURVEY_DIFF_STACKSTACK $ssKey DIST_GROUP -var dist_group + book getword SURVEY_DIFF_STACKSTACK $ssKey TARGET_LABEL -var target_label + + $year = `date +%Y` + $month = `date +%m` + $day = `date +%d` +# Do we need anything else here? + $run = difftool -definestackstack -good_frac 0.2 + $run = $run -input_label $input_label + $run = $run -template_label $stack_label + $run = $run -set_dist_group $dist_group + $run = $run -set_label $target_label + $run = $run -set_workdir $workdir/$target_label/$year/$month/$day + $run = $run -set_reduction STACKSTACK +# $run = $run -available + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_DIFF_STACKSTACK_DB + option $DB:$SURVEY_DIFF_STACKSTACK_DB + end + + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + +task survey.magic + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.magic.log + stderr $LOGDIR/survey.magic.log + + # generate magic warp-warp runs + task.exec + book npages SURVEY_MAGIC -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_MAGIC 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_MAGIC $i -var label + book setword SURVEY_MAGIC $label STATE NEW + end + book getpage SURVEY_MAGIC 0 -var label -key STATE NEW + + # Select different database + $SURVEY_MAGIC_DB ++ + if ($SURVEY_MAGIC_DB >= $DB:n) set SURVEY_MAGIC_DB = 0 + end + + book setword SURVEY_MAGIC $label STATE DONE + book getword SURVEY_MAGIC $label WORKDIR -var workdir + book getword SURVEY_MAGIC $label MULTIDIFF -var multidiff + + $run = magictool -rerun -definebyquery -label $label -diff_label $label -workdir $workdir/$label + + if (("$multidiff" == "TRUE")||("$multidiff" == "MULTIDIFF")) + $run = $run -multidiff + end + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_MAGIC_DB + option $DB:$SURVEY_MAGIC_DB + end + + echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +task survey.addstar + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.addstar.log + stderr $LOGDIR/survey.addstar.log + + # generate magic warp-warp runs + task.exec + book npages SURVEY_ADDSTAR -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_ADDSTAR 0 -var tag -key STATE NEW + if ("$tag" == "NULL") + # All labels have been done --- reset + # echo "Resetting labels" + for i 0 $N + book getpage SURVEY_ADDSTAR $i -var tag + book setword SURVEY_ADDSTAR $tag STATE NEW + end + book getpage SURVEY_ADDSTAR 0 -var tag -key STATE NEW + + # Select different database + $SURVEY_ADDSTAR_DB ++ + if ($SURVEY_ADDSTAR_DB >= $DB:n) set SURVEY_ADDSTAR_DB = 0 + end + + book setword SURVEY_ADDSTAR $tag STATE DONE + book getword SURVEY_ADDSTAR $tag LABEL -var label + book getword SURVEY_ADDSTAR $tag DVODB -var dvodb + book getword SURVEY_ADDSTAR $tag MINIDVODB_GROUP -var minidvodb_group + book getword SURVEY_ADDSTAR $tag STAGE -var stage + + + # $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb + $run = addtool + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_ADDSTAR_DB + option $DB:$SURVEY_ADDSTAR_DB + end + + $run = $run -definebyquery -label $label -set_dvodb $dvodb -set_minidvodb_group $minidvodb_group -set_minidvodb -set_label $minidvodb_group -stage $stage + # echo $run + if ("$stage" == "cam") + #only queue destreaked cams. stacks and staticsky don't need this + # if ("$DB:$SURVEY_ADDSTAR_DB" == "isp") + #this is the only way I can think of how to handle this (but it is messy). If it is a database that we KNOW doesn't use magicked (ie, isp), then do not queue destreaked + # now with no magic, all are uncensored + $run = $run -uncensored +# else + #if not isp, then run destreaked version only +# $run = $run -destreaked +# end + end + if ("$stage" == "staticsky") + #only queue uncensored staticsky + $run = $run -uncensored + end + if ("$stage" == "stack") + #only queue uncensored stacks + $run = $run -uncensored + end + if ("$stage" == "skycal") + #skycal doesn't have magic, however, we still need to tell addtool + $run = $run -uncensored + end + if ("$stage" == "fullforce") + $run = $run -uncensored + end + if ("$stage" == "diff") + $run = $run -uncensored + end + # we need to handle isp/gpc1: gpc1 wants destreaked, isp wants uncensored. Perhaps the best way is by default do destreaked (the most paranoid), and if db = isp (is this the best way?) then do uncensored + + + echo $run + command $run + end + + # success + + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +task survey.mergedvodb + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.mergedvodb.log + stderr $LOGDIR/survey.mergedvodb.log + + task.exec + book npages SURVEY_MERGEDVODB -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW + if ("$tag" == "NULL") + # All labels have been done --- reset + # echo "Resetting labels" + for i 0 $N + book getpage SURVEY_MERGEDVODB $i -var tag + book setword SURVEY_MERGEDVODB $tag STATE NEW + end + book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW + + # Select different database + $SURVEY_MERGEDVODB_DB ++ + if ($SURVEY_MERGEDVODB_DB >= $DB:n) set SURVEY_MERGEDVODB_DB = 0 + end + book setword SURVEY_MERGEDVODB $tag STATE DONE + book getword SURVEY_MERGEDVODB $tag MERGEDVODB -var mergedvodb + book getword SURVEY_MERGEDVODB $tag MINIDVODB_GROUP -var minidvodb_group + + # $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb + #$run = mergetool + $run = mergedvodb_queue.pl --outroot $mergedvodb --mergedvodb $mergedvodb --minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + $run = $run --dbname $DB:$SURVEY_MERGEDVODB_DB + option $DB:$SURVEY_MERGEDVODB_DB + end + #this run doesn't work because there's no workdir + #$run = $run -definebyquery -mergedvodb $mergedvodb -minidvodb_group $minidvodb_group + + + # echo $run + + echo $run + command $run + end + + # success + + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + +task survey.destreak + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.destreak.log + stderr $LOGDIR/survey.destreak.log + + # generate destreak runs + task.exec + book npages SURVEY_DESTREAK -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_DESTREAK 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_DESTREAK $i -var label + book setword SURVEY_DESTREAK $label STATE NEW + end + book getpage SURVEY_DESTREAK 0 -var label -key STATE NEW + + # Select different database + $SURVEY_DESTREAK_DB ++ + if ($SURVEY_DESTREAK_DB >= $DB:n) set SURVEY_DESTREAK_DB = 0 + end + + book setword SURVEY_DESTREAK $label STATE DONE + book getword SURVEY_DESTREAK $label WORKDIR -var workdir + book getword SURVEY_DESTREAK $label RECOVERYROOT -var recoveryroot + + $run = magic_destreak_defineruns.pl --label $label --workdir $workdir/$label + if ("$recoveryroot" != "NULL") + $run = $run --recoveryroot $recoveryroot/$label + end + + if ($DB:n == 0) + option DEFAULT + else + $run = $run --dbname $DB:$SURVEY_DESTREAK_DB + option $DB:$SURVEY_DESTREAK_DB + end + +# echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task survey.dist + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.dist.log + stderr $LOGDIR/survey.dist.log + + # generate distribution runs + task.exec + book npages SURVEY_DIST -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_DIST 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_DIST $i -var label + book setword SURVEY_DIST $label STATE NEW + end + book getpage SURVEY_DIST 0 -var label -key STATE NEW + + # Select different database + $SURVEY_DIST_DB ++ + if ($SURVEY_DIST_DB >= $DB:n) set SURVEY_DIST_DB = 0 + end + + book getword SURVEY_DIST $label WORKDIR -var workdir_base + $year = `date +%Y` + $month = `date +%m` + $day = `date +%d` + $workdir = $workdir_base/$label/$year/$month/$day + + book getword SURVEY_DIST $label MUGGLE -var muggle + + book setword SURVEY_DIST $label STATE DONE + + # note workdir is set by the script based on site.config + $run = dist_defineruns.pl --label $label --workdir $workdir + if ($muggle != 0) + $run = $run --no_magic + end + + if ($DB:n == 0) + option DEFAULT + else + $run = $run --dbname $DB:$SURVEY_DIST_DB + option $DB:$SURVEY_DIST_DB + end + + if ($VERBOSE > 1 ) + echo $run + end + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task survey.chip.bg + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.chip.bg.log + stderr $LOGDIR/survey.chip.bg.log + + # generate chip_bg runs + task.exec + book npages SURVEY_CHIP_BG -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_CHIP_BG 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_CHIP_BG $i -var label + book setword SURVEY_CHIP_BG $label STATE NEW + end + book getpage SURVEY_CHIP_BG 0 -var label -key STATE NEW + + # Select different database + $SURVEY_CHIP_BG ++ + if ($SURVEY_CHIP_BG >= $DB:n) set SURVEY_CHIP_BG = 0 + end + + book setword SURVEY_CHIP_BG $label STATE DONE + book getword SURVEY_CHIP_BG $label CAM_LABEL -var cam_label + book getword SURVEY_CHIP_BG $label DIST_GROUP -var dist_group +# book getword SURVEY_CHIP_BG $label MUGGLE -var muggle + + $run = bgtool -definechip -label $label -cam_label $cam_label -set_dist_group $dist_group +# if ($muggle == 0) +# $run = $run -destreaked +# end + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_CHIP_BG + option $DB:$SURVEY_CHIP_BG + end + + if ($VERBOSE > 1) + echo $run + end + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task survey.warp.bg + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.warp.bg.log + stderr $LOGDIR/survey.warp.bg.log + + # generate warp_bg runs + task.exec + book npages SURVEY_WARP_BG -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_WARP_BG 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_WARP_BG $i -var label + book setword SURVEY_WARP_BG $label STATE NEW + end + book getpage SURVEY_WARP_BG 0 -var label -key STATE NEW + + # Select different database + $SURVEY_WARP_BG ++ + if ($SURVEY_WARP_BG >= $DB:n) set SURVEY_WARP_BG = 0 + end + + book setword SURVEY_WARP_BG $label STATE DONE + book getword SURVEY_WARP_BG $label WARP_LABEL -var warp_label + book getword SURVEY_WARP_BG $label DIST_GROUP -var dist_group + + $run = bgtool -definewarp -set_label $label -chip_bg_label $label -warp_label $warp_label -set_dist_group $dist_group +# if ($muggle == 0) +# $run = $run -destreaked +# end + + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_WARP_BG + option $DB:$SURVEY_WARP_BG + end + + if ($VERBOSE > 1) + echo $run + end + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task survey.publish + host local + + # we are going to use a long timeout (900 sec) between passes of the list + # and a short timeout (10 sec) while we are doing the full list + periods -exec 10 + periods -poll 2 + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.publish.log + stderr $LOGDIR/survey.publish.log + + # generate publish runs + task.exec + book npages SURVEY_PUBLISH -var N + if ($N == 0) + # echo "No labels for processing" + break + endif + + # short timeout while we have something to do + periods -exec 10 + + # survey.publish allows multiple entries per label. + # The key is called tag and must be unique + book getpage SURVEY_PUBLISH 0 -var tag -key STATE NEW + if ("$tag" == "NULL") + # All tags have been done --- reset + # echo "Resetting tags" + for i 0 $N + book getpage SURVEY_PUBLISH $i -var tag + book setword SURVEY_PUBLISH $tag STATE NEW + end + book getpage SURVEY_PUBLISH 0 -var tag -key STATE NEW + + # Select different database, set a long timeout + $SURVEY_PUBLISH_DB ++ + if ($SURVEY_PUBLISH_DB >= $DB:n) set SURVEY_PUBLISH_DB = 0 + periods -exec 300 + date -var mytime + echo "done with publish list @ $mytime" + end + + book setword SURVEY_PUBLISH $tag STATE DONE + book getword SURVEY_PUBLISH $tag LABEL -var label + book getword SURVEY_PUBLISH $tag CLIENT_ID -var client_id + book getword SURVEY_PUBLISH $tag COMMENT -var comment + + $run = pubtool -definerun -label $label -client_id $client_id + + if ("$comment" != "NULL") + $run = $run -comment $comment + end + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_PUBLISH_DB + option $DB:$SURVEY_PUBLISH_DB + end + + # echo $run + command $run + + end + + # success + task.exit 0 + #echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + +task survey.staticskysingle + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.staticskysingle.log + stderr $LOGDIR/survey.staticskysingle.log + + task.exec + book npages SURVEY_STATICSKYSINGLE -var N + if ($N == 0) + # echo "No STATICSKYSINGLE labels for processing" + break + endif + book getpage SURVEY_STATICSKYSINGLE 0 -var tag -key STATE NEW + if ("$tag" == "NULL") + # All tags have been done --- reset + # echo "Resetting tags" + for i 0 $N + book getpage SURVEY_STATICSKYSINGLE $i -var tag + book setword SURVEY_STATICSKYSINGLE $tag STATE NEW + end + book getpage SURVEY_STATICSKYSINGLE 0 -var tag -key STATE NEW + + # Select different database + $SURVEY_STATICSKYSINGLE_DB ++ + if ($SURVEY_STATICSKYSINGLE_DB >= $DB:n) set SURVEY_STATICSKYSINGLE_DB = 0 + end + + book setword SURVEY_STATICSKYSINGLE $tag STATE DONE + book getword SURVEY_STATICSKYSINGLE $tag LABEL -var label + book getword SURVEY_STATICSKYSINGLE $tag WORKDIR -var workdir + book getword SURVEY_STATICSKYSINGLE $tag DIST_GROUP -var dist_group + book getword SURVEY_STATICSKYSINGLE $tag SELECTDATAGROUP -var selectdatagroup + book getword SURVEY_STATICSKYSINGLE $tag FILTER -var filter + + $year = `date +%Y` + $month = `date +%m` + $day = `date +%d` + + $run = staticskytool -definebyquery + $run = $run -set_workdir $workdir/$label/$year/$month/$day + $run = $run -set_dist_group $dist_group + $run = $run -set_label $label + $run = $run -select_label $label + $run = $run -select_data_group $selectdatagroup + $run = $run -select_filter $filter +# $run = $run -pretend -simple + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_STATICSKYSINGLE_DB + option $DB:$SURVEY_STATICSKYSINGLE_DB + end + +# echo $run + + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# Survey task for Skycal stage + +task survey.skycal + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.skycal.log + stderr $LOGDIR/survey.skycal.log + + task.exec + book npages SURVEY_SKYCAL -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_SKYCAL 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_SKYCAL $i -var label + book setword SURVEY_SKYCAL $label STATE NEW + end + book getpage SURVEY_SKYCAL 0 -var label -key STATE NEW + + # Select different database + $SURVEY_SKYCAL_DB ++ + if ($SURVEY_SKYCAL_DB >= $DB:n) set SURVEY_SKYCAL_DB = 0 + end + + book setword SURVEY_SKYCAL $label STATE DONE + book getword SURVEY_SKYCAL $label DIST_GROUP -var dist_group + + # note: currently skycal uses the staticskyRun workdir + + $run = staticskytool -defineskycalrun -set_label $label -select_label $label -set_dist_group $dist_group + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_SKYCAL_DB + option $DB:$SURVEY_SKYCAL_DB + end + + # echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task survey.lapgroup + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.lapgroup.log + stderr $LOGDIR/survey.lapgroup.log + + task.exec + book npages SURVEY_LAPGROUP -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_LAPGROUP 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_LAPGROUP $i -var label + book setword SURVEY_LAPGROUP $label STATE NEW + end + book getpage SURVEY_LAPGROUP 0 -var label -key STATE NEW + + # Select different database + $SURVEY_LAPGROUP_DB ++ + if ($SURVEY_LAPGROUP_DB >= $DB:n) set SURVEY_LAPGROUP_DB = 0 + end + + book setword SURVEY_LAPGROUP $label STATE DONE + book getword SURVEY_LAPGROUP $label SEQ_ID -var SEQ_ID + + # For now the list of filters is hardcoded here + $run = laptool -definegroup -seq_id $SEQ_ID -set_label $label -filter g.00000 -filter r.00000 -filter i.00000 -filter z.00000 -filter y.00000 + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_LAPGROUP_DB + option $DB:$SURVEY_LAPGROUP_DB + end + + # echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task survey.relexp + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_RELEASE_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.relexp.log + stderr $LOGDIR/survey.relexp.log + + task.exec + book npages SURVEY_RELEXP -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_RELEXP 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_RELEXP $i -var label + book setword SURVEY_RELEXP $label STATE NEW + end + book getpage SURVEY_RELEXP 0 -var label -key STATE NEW + + # Select different database + $SURVEY_RELEXP_DB ++ + if ($SURVEY_RELEXP_DB >= $DB:n) set SURVEY_RELEXP_DB = 0 + end + + book setword SURVEY_RELEXP $label STATE DONE + book getword SURVEY_RELEXP $label RELEASE_NAME -var RELEASE_NAME + + $run = releasetool -definerelexp -label $label -release_name $RELEASE_NAME -set_state processed + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_RELEXP_DB + option $DB:$SURVEY_RELEXP_DB + end + + # echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task survey.relstack + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_RELEASE_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.relstack.log + stderr $LOGDIR/survey.relstack.log + + task.exec + book npages SURVEY_RELSTACK -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_RELSTACK 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_RELSTACK $i -var label + book setword SURVEY_RELSTACK $label STATE NEW + end + book getpage SURVEY_RELSTACK 0 -var label -key STATE NEW + + # Select different database + $SURVEY_RELSTACK_DB ++ + if ($SURVEY_RELSTACK_DB >= $DB:n) set SURVEY_RELSTACK_DB = 0 + end + + book setword SURVEY_RELSTACK $label STATE DONE + book getword SURVEY_RELSTACK $label RELEASE_NAME -var RELEASE_NAME + book getword SURVEY_RELSTACK $label STACK_TYPE -var STACK_TYPE + + # XXX: Can we use the survey task for stack types other than nightly? + $run = releasetool -definerelstack -label $label -release_name $RELEASE_NAME -set_state processed -set_stack_type $STACK_TYPE + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_RELSTACK_DB + option $DB:$SURVEY_RELSTACK_DB + end + + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/surveys.mhpcc.config =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/surveys.mhpcc.config (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/surveys.mhpcc.config (revision 41106) @@ -0,0 +1,70 @@ +# Note from Ken Chambers on the OBS_MODE keyword: +# OBS_MODE - Official Keyword - string use for Survey ID, 3PI, MD, SS, STS, M31, CAL, AF1, AF2, AF3, ENG, NA - from scheduler + +surveys MULTI +surveys METADATA + camera STR isp + survey STR NULL + label STR @CAMERA@.@DATE@ + dvodb STR /data/ipp004.0/isp/catdir.isp + worksubdir STR neb://@HOST@.0/isp/@DATE@ + end_stage STR camera + tess_id STR RINGS.V0 +END + + + +#all of these are really old - never use them, except maybe as an example? +#surveys METADATA +# survey STR 3PI +# label STR ThreePi.@CAMERA@.@FILTER@.@DATE@ +# dvodb STR /data/ipp004.0/gpc1/catdir.ThreePi +# worksubdir STR ThreePi +# end_stage STR warp +# tess_id STR ALLSKY +#END + +#surveys METADATA +# survey STR MD +# label STR MediumDeep.@CAMERA@.@FILTER@.@DATE@ +# dvodb STR /data/ipp004.0/gpc1/MediumDeep +# worksubdir STR MediumDeep +# end_stage STR warp +# tess_id STR ALLSKY +#END + +#surveys METADATA +# survey STR SS +# label STR SolarSystem.@CAMERA@.@FILTER@.@DATE@ +# dvodb STR /data/ipp004.0/gpc1/SolarSystem +# worksubdir STR SolarSystem +# end_stage STR warp +# tess_id STR ALLSKY +#END + +#surveys METADATA +# survey STR STS +# label STR STS.@CAMERA@.@FILTER@.@DATE@ +# dvodb STR /data/ipp004.0/gpc1/STS +# worksubdir STR STS +# end_stage STR warp +# tess_id STR ALLSKY +#END + +#surveys METADATA +# survey STR M31 +# label STR M31.@CAMERA@.@FILTER@.@DATE@ +# dvodb STR /data/ipp004.0/gpc1/M31 +# worksubdir STR M31 +# end_stage STR warp +# tess_id STR ALLSKY +#END + +#surveys METADATA +# survey STR CAL +# label STR CalFields.@CAMERA@.@FILTER@.@DATE@ +# dvodb STR /data/ipp004.0/gpc1/CalFields +# worksubdir STR CalFields +# end_stage STR warp +# tess_id STR ALLSKY +#END Index: /branches/ccl_branches/ipponly-20191108/ippTasks/vp.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/vp.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/vp.pro (revision 41106) @@ -0,0 +1,249 @@ +## vp.pro : tasks for video photometry : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/vp +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init vpPendingRun + +### Database lists +$vp_DB = 0 +$vp_revert_DB = 0 + +$VP_POLL_LIMIT = 20 + +### Check status of vping tasks +macro vp.status + book listbook vpPendingRun +end + +### Reset vping tasks +macro vp.reset + book init vpPendingRun +end + +### Turn vp tasks on +macro vp.on + task vp.load + active true + end + task vp.run + active true + end +end + +### Turn vp tasks off +macro vp.off + task vp.load + active false + end + task vp.run + active false + end +end + +macro vp.revert.on + task vp.revert + active true + end +end + +macro vp.revert.off + task vp.revert + active false + end +end + +macro set.vp.poll + $VP_POLL_LIMIT = $1 +end +macro get.vp.poll + echo $VP_POLL_LIMIT +end + + +### Load tasks for doing video photometry +### Tasks are loaded into vpPendingRun. +task vp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/vp.load.log + + task.exec + if ($LABEL:n == 0) break + $run = vptool -pendingrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$vp_DB + $run = $run -dbname $DB:$vp_DB + $vp_DB ++ + if ($vp_DB >= $DB:n) set vp_DB = 0 + end + add_poll_args run + add_poll_labels run + $run = $run -limit $VP_POLL_LIMIT + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout vpPendingRun -key vp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook vpPendingRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup vpPendingRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +### Run video photometery tasks +### Tasks are taken from vpPendingRun. +task vp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + + task.exec + book npages vpPendingRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new runs in vpPendingRun (pantaskState == INIT) + book getpage vpPendingRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword vpPendingRun $pageName pantaskState RUN + book getword vpPendingRun $pageName vp_id -var VP_ID + book getword vpPendingRun $pageName camera -var CAMERA + book getword vpPendingRun $pageName exp_tag -var EXP_TAG + book getword vpPendingRun $pageName workdir -var WORKDIR + book getword vpPendingRun $pageName dest_id -var DEST_ID + book getword vpPendingRun $pageName dest_name -var DEST_NAME + book getword vpPendingRun $pageName ds_dbname -var DS_DBNAME + book getword vpPendingRun $pageName ds_dbhost -var DS_DBHOST + book getword vpPendingRun $pageName dbname -var DBNAME + + # set the host and workdir based on the skycell hash +# set.host.for.skycell $SKYCELL_ID +# set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf outroot "%s/%s.vp.%d" $WORKDIR $EXP_TAG $VP_ID + + stdout $LOGSUBDIR/vp.log + stderr $LOGSUBDIR/vp.log + + host anyhost + + $run = videophot_process.pl --vp_id $VP_ID --outroot $outroot --redirect-output --camera $CAMERA + if ($DEST_ID) + $run = $run --dest_id $DEST_ID --product $DEST_NAME --ds_dbname $DS_DBNAME --ds_dbhost $DS_DBHOST + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit vpPendingRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword vpPendingRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword vpPendingRun $options:0 pantaskState TIMEOUT + end +end + + +task vp.revert + host local + + periods -poll 5 + periods -exec 600 + periods -exec 30 + periods -timeout 120 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/vp.revert.log + + task.exec + if ($LABEL:n == 0) break + $run = vptool -revertrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$vp_revert_DB + $run = $run -dbname $DB:$vp_revert_DB + $vp_revert_DB ++ + if ($vp_revert_DB >= $DB:n) set vp_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/ccl_branches/ipponly-20191108/ippTasks/warp.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/ippTasks/warp.pro (revision 41106) +++ /branches/ccl_branches/ipponly-20191108/ippTasks/warp.pro (revision 41106) @@ -0,0 +1,687 @@ +## warp.pro : image warping tasks : -*- sh -*- +## This file contains panTasks definitions for performing the image warping. + +### This is done in two (main) steps. After a warp (with associated warp_id) is defined, +### overlaps between the exposure being warped and skycells are calculated (tasks in warpInputExp). +### Then for each skycell, the warp is made (tasks in warpPendingSkycell). + +### Setups +check.globals + +### Initialise the books containing the tasks to do +book init warpInputExp +book init warpPendingSkyCell +book init warpPendingCleanup +book init warpPendingSummary + +### Database lists +$warpExp_DB = 0 +$warpSkycell_DB = 0 +$warp_revert_overlap_DB = 0 +$warp_revert_warped_DB = 0 +$warpSummary_DB = 0 + +### Check status of warping tasks +macro warp.status + book listbook warpInputExp + book listbook warpPendingSkyCell +end + +### Reset warping tasks +macro warp.reset + book init warpInputExp + book init warpPendingSkyCell + book init warpPendingSummary +end + +### Turn warping tasks on +macro warp.on + task warp.exp.load + active true + end + task warp.exp.run + active true + end + task warp.skycell.load + active true + end + task warp.skycell.run + active true + end + task warp.advancerun + active true + end + task warp.revert.overlap + active true + end + task warp.revert.warped + active true + end +end + +### Turn warping tasks off +macro warp.off + task warp.exp.load + active false + end + task warp.exp.run + active false + end + task warp.skycell.load + active false + end + task warp.skycell.run + active false + end + task warp.advancerun + active false + end + task warp.revert.overlap + active false + end + task warp.revert.warped + active false + end +end + +macro warp.revert.on + task warp.revert.warped + active true + end + task warp.revert.overlap + active true + end +end + +macro warp.revert.off + task warp.revert.warped + active false + end + task warp.revert.overlap + active false + end +end + +macro warp.summary.on + task warp.summary.load + active true + end + task warp.summary.run + active true + end +end + +macro warp.summary.off + task warp.summary.load + active false + end + task warp.summary.run + active false + end +end + +### Load tasks for calculating the warp overlaps +### Tasks are loaded into warpInputExp. +task warp.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1200 + npending 1 + + stdout NULL + stderr $LOGDIR/warp.exp.log + + task.exec + if ($LABEL:n == 0) break + $run = warptool -tooverlap + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warpExp_DB + $run = $run -dbname $DB:$warpExp_DB + $warpExp_DB ++ + if ($warpExp_DB >= $DB:n) set warpExp_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpInputExp -key warp_id:fake_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpInputExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpInputExp + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpInputExp. +task warp.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages warpInputExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new images in warpInputExp (pantaskState == INIT) + book getpage warpInputExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpInputExp $pageName pantaskState RUN + book getword warpInputExp $pageName warp_id -var WARP_ID + book getword warpInputExp $pageName camera -var CAMERA + book getword warpInputExp $pageName workdir -var WORKDIR_TEMPLATE + book getword warpInputExp $pageName dbname -var DBNAME + # XXX change tess_id to tess_dir when schema is changed + book getword warpInputExp $pageName tess_id -var TESS_DIR + book getword warpInputExp $pageName exp_tag -var EXP_TAG + + # set the host and workdir (default) + set.host.for.camera $CAMERA $WARP_ID + set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate outroot specific to this exposure + sprintf logfile "%s/%s/%s.wrp.%s.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID + + stdout $LOGDIR/warp.exp.log + stderr $LOGDIR/warp.exp.log + + # XXX warp_overlap.pl differs from the standard script : it does not have an 'outroot' argument, and it does not take '--redirect' + $run = warp_overlap.pl --warp_id $WARP_ID --camera $CAMERA --tess_dir $TESS_DIR --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit warpInputExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword warpInputExp $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpInputExp $options:0 pantaskState TIMEOUT + end +end + + +### Load tasks for doing the warps +### Tasks are loaded into warpPendingSkyCell. +task warp.skycell.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1200 + npending 1 + + stdout NULL + stderr $LOGDIR/warp.skycell.log + + task.exec + if ($LABEL:n == 0) break + $run = warptool -towarped + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warpSkycell_DB + $run = $run -dbname $DB:$warpSkycell_DB + $warpSkycell_DB ++ + if ($warpSkycell_DB >= $DB:n) set warpSkycell_DB = 0 + end + add_poll_args run + add_poll_labels run + # increase the poll limit for warp over the default to + # help it keep up with chip processing + # NOTE : it is not a problem for warptool to have multiple + # -limit entries: the last one is used + $run = $run -limit {$POLL_LIMIT * 2} + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + # XXX change tess_id to tess_dir when db is updated + ipptool2book stdout warpPendingSkyCell -key warp_id:skycell_id:tess_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingSkyCell + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingSkyCell + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpPendingSkyCell. +task warp.skycell.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages warpPendingSkyCell -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in warpPendingSkyCell (pantaskState == INIT) + book getpage warpPendingSkyCell 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingSkyCell $pageName pantaskState RUN + book getword warpPendingSkyCell $pageName warp_id -var WARP_ID + book getword warpPendingSkyCell $pageName warp_skyfile_id -var WARP_SKYFILE_ID + book getword warpPendingSkyCell $pageName skycell_id -var SKYCELL_ID + book getword warpPendingSkyCell $pageName camera -var CAMERA + book getword warpPendingSkyCell $pageName workdir -var WORKDIR_TEMPLATE + book getword warpPendingSkyCell $pageName path-base -var PATH_BASE + book getword warpPendingSkyCell $pageName dbname -var DBNAME + # XXX change tess_id to tess_dir when schema is changed + book getword warpPendingSkyCell $pageName tess_id -var TESS_DIR + book getword warpPendingSkyCell $pageName reduction -var REDUCTION + book getword warpPendingSkyCell $pageName exp_tag -var EXP_TAG + book getword warpPendingSkyCell $pageName state -var RUN_STATE + book getword warpPendingSkyCell $pageName magicked -var CHIP_MAGICKED + if ($CHIP_MAGICKED > 0) + $MAGICKED_ARG = "--magicked $CHIP_MAGICKED" + else + $MAGICKED_ARG = "" + end + + # set the host and workdir based on the skycell hash + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + if ("$PATH_BASE" == "NULL") + ## generate outroot specific to this exposure + sprintf outroot "%s/%s/%s.wrp.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID $SKYCELL_ID + else + $outroot = $PATH_BASE + end + + stdout $LOGDIR/warp.skycell.log + stderr $LOGDIR/warp.skycell.log + + $run = warp_skycell.pl --threads @MAX_THREADS@ --warp_id $WARP_ID --warp_skyfile_id $WARP_SKYFILE_ID --skycell_id $SKYCELL_ID --tess_dir $TESS_DIR --camera $CAMERA --outroot $outroot --redirect-output --run-state $RUN_STATE $MAGICKED_ARG + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit warpPendingSkyCell $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword warpPendingSkyCell $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingSkyCell $options:0 pantaskState TIMEOUT + end +end + +# this variable will cycle through the known database names +$warp_advance_DB = 0 + +# advance exposures for which all imfiles have completed processing +# sets the exposure state to full and queues warp processing if requested +task warp.advancerun + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/warp.advancerun.log + + task.exec + if ($LABEL:n == 0) break + $run = warptool -advancerun -limit 10 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warp_advance_DB + $run = $run -dbname $DB:$warp_advance_DB + $warp_advance_DB ++ + if ($warp_advance_DB >= $DB:n) set warp_advance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task warp.revert.overlap + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = warptool -revertoverlap + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warp_revert_overlap_DB + $run = $run -dbname $DB:$warp_revert_overlap_DB + $warp_revert_overlap_DB ++ + if ($warp_revert_overlap_DB >= $DB:n) set warp_revert_overlap_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task warp.revert.warped + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it doesn't + # continue to occur). + $run = warptool -revertwarped -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warp_revert_warped_DB + $run = $run -dbname $DB:$warp_revert_warped_DB + $warp_revert_warped_DB ++ + if ($warp_revert_warped_DB >= $DB:n) set warp_revert_warped_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Load tasks for doing the warps +### Tasks are loaded into warpPendingSkyCell. +task warp.summary.load + host local + active false + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 1200 + npending 1 + + stdout NULL + stderr $LOGDIR/warp.summary.log + + task.exec + if ($LABEL:n == 0) break + $run = warptool -tosummary + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warpSummary_DB + $run = $run -dbname $DB:$warpSummary_DB + $warpSummary_DB ++ + if ($warpSummary_DB >= $DB:n) set warpSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + # XXX change tess_id to tess_dir when db is updated + ipptool2book stdout warpPendingSummary -key warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpPendingSkyCell. +task warp.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages warpPendingSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in warpPendingSkyCell (pantaskState == INIT) + book getpage warpPendingSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingSummary $pageName pantaskState RUN + book getword warpPendingSummary $pageName warp_id -var WARP_ID + book getword warpPendingSummary $pageName camera -var CAMERA + book getword warpPendingSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword warpPendingSummary $pageName dbname -var DBNAME + book getword warpPendingSummary $pageName tess_id -var TESS_DIR + book getword warpPendingSummary $pageName exp_tag -var EXP_TAG + book getword warpPendingSummary $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + host anyhost + strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR +# set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + +# if ("$PATH_BASE" == "NULL") + ## generate outroot specific to this exposure + basename $TESS_DIR -var TESS_ID + + sprintf outroot "%s/%s/%s.wrp.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID $TESS_ID + + + stdout $LOGDIR/warp.summary.log + stderr $LOGDIR/warp.summary.log + + $run = skycell_jpeg.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --outroot $outroot + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit warpPendingSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword warpPendingSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingSummary $options:0 pantaskState TIMEOUT + end +end