Index: branches/eam_branches/ipp-20100621/ippTasks/Makefile.am =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/Makefile.am (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/Makefile.am (revision 28794) @@ -34,5 +34,7 @@ minidvodb.pro \ nightly_stacks.pro \ + burntool.pro \ lossy_compress.pro \ + background.pro \ diffphot.pro Index: branches/eam_branches/ipp-20100621/ippTasks/background.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/background.pro (revision 28794) +++ branches/eam_branches/ipp-20100621/ippTasks/background.pro (revision 28794) @@ -0,0 +1,536 @@ +## background.pro : globals and support macros : -*- sh -*- + +check.globals + +book init chipBackgroundRuns +book init warpBackgroundRuns + +macro bg.status + book listbook chipBackgroundRuns + book listbook warpBackgroundRuns +end + +macro bg.reset + book init chipBackgroundRuns + book init warpBackgroundRuns +end + +macro bg.on + task bg.chip.load + active true + end + task bg.chip.run + active true + end + task bg.chip.advance + active true + end + task bg.chip.revert + active true + end + task bg.warp.load + active true + end + task bg.warp.run + active true + end + task bg.warp.advance + active true + end + task bg.warp.revert + active true + end +end + +macro bg.off + task bg.chip.load + active false + end + task bg.chip.run + active false + end + task bg.chip.advance + active false + end + task bg.chip.revert + active false + end + task bg.warp.load + active false + end + task bg.warp.run + active false + end + task bg.warp.advance + active false + end + task bg.warp.revert + active false + end +end + +macro bg.revert.on + task bg.chip.revert + active true + end + task bg.warp.revert + active true + end +end + +macro bg.revert.off + task bg.chip.revert + active false + end + task bg.warp.revert + active false + end +end + + +# These variables cycle through the known database names +$bg_chip_load_DB = 0 +$bg_chip_advance_DB = 0 +$bg_chip_revert_DB = 0 +$bg_warp_load_DB = 0 +$bg_warp_advance_DB = 0 +$bg_warp_revert_DB = 0 + +task bg.chip.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.chip.load.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -tochip + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_chip_load_DB + $run = $run -dbname $DB:$bg_chip_load_DB + $bg_chip_load_DB ++ + if ($bg_chip_load_DB >= $DB:n) set bg_chip_load_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipBackgroundRuns -key chip_bg_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipBackgroundRuns + end + process_cleanup chipBackgroundRuns + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.chip.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + periods -exec $RUNEXEC + + book npages chipBackgroundRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + book getpage chipBackgroundRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipBackgroundRuns $pageName pantaskState RUN + book getword chipBackgroundRuns $pageName camera -var CAMERA + book getword chipBackgroundRuns $pageName chip_bg_id -var CHIP_BG_ID + book getword chipBackgroundRuns $pageName class_id -var CLASS_ID + book getword chipBackgroundRuns $pageName exp_tag -var EXP_TAG + book getword chipBackgroundRuns $pageName workdir -var WORKDIR_TEMPLATE + book getword chipBackgroundRuns $pageName dbname -var DBNAME + book getword chipBackgroundRuns $pageName reduction -var REDUCTION + + set.host.for.camera $CAMERA $CLASS_ID + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf outroot "%s/%s/%s.bgc.%s" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_BG_ID + stdout $LOGDIR/bg.chip.run.log + stderr $LOGDIR/bg.chip.run.log + + $run = background_chip.pl --threads @MAX_THREADS@ --chip_bg_id $CHIP_BG_ID --class_id $CLASS_ID --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit chipBackgroundRuns $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + exec bgtool -addchip -dbname $DBNAME -chip_bg_id $CHIP_BG_ID -class_id $CLASS_ID -fault $EXIT_CRASH_ERR + process_exit chipBackgroundRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipBackgroundRuns $options:0 pantaskState TIMEOUT + end +end + +# advance exposures for which all imfiles have completed processing +# sets the exposure state to full and queues warp processing if requested +task bg.chip.advance + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.chip.advance.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -advancechip + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$chip_advance_DB + $run = $run -dbname $DB:$bg_chip_advance_DB + $bg_chip_advance_DB ++ + if ($bg_chip_advance_DB >= $DB:n) set bg_chip_advance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.chip.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.chip.revert.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -revertchip + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_chip_revert_DB + $run = $run -dbname $DB:$bg_chip_revert_DB + $bg_chip_revert_DB ++ + if ($bg_chip_revert_DB >= $DB:n) set bg_chip_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.warp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.warp.load.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -towarp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_warp_load_DB + $run = $run -dbname $DB:$bg_warp_load_DB + $bg_warp_load_DB ++ + if ($bg_warp_load_DB >= $DB:n) set bg_warp_load_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpBackgroundRuns -key warp_bg_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpBackgroundRuns + end + process_cleanup warpBackgroundRuns + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.warp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + periods -exec $RUNEXEC + + book npages warpBackgroundRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + book getpage warpBackgroundRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpBackgroundRuns $pageName pantaskState RUN + book getword warpBackgroundRuns $pageName camera -var CAMERA + book getword warpBackgroundRuns $pageName warp_bg_id -var WARP_BG_ID + book getword warpBackgroundRuns $pageName skycell_id -var SKYCELL_ID + book getword warpBackgroundRuns $pageName tess_id -var TESS_DIR + book getword warpBackgroundRuns $pageName exp_tag -var EXP_TAG + book getword warpBackgroundRuns $pageName workdir -var WORKDIR_TEMPLATE + book getword warpBackgroundRuns $pageName dbname -var DBNAME + book getword warpBackgroundRuns $pageName reduction -var REDUCTION + + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf outroot "%s/%s/%s.bgw.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_BG_ID $SKYCELL_ID + stdout $LOGDIR/bg.warp.run.log + stderr $LOGDIR/bg.warp.run.log + + $run = background_warp.pl --threads @MAX_THREADS@ --warp_bg_id $WARP_BG_ID --skycell_id $SKYCELL_ID --tess_dir $TESS_DIR --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit warpBackgroundRuns $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + exec bgtool -addwarp -dbname $DBNAME -warp_bg_id $WARP_BG_ID -skycell_id $SKYCELL_ID -fault $EXIT_CRASH_ERR + process_exit warpBackgroundRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpBackgroundRuns $options:0 pantaskState TIMEOUT + end +end + +# advance exposures for which all imfiles have completed processing +# sets the exposure state to full and queues warp processing if requested +task bg.warp.advance + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.warp.advance.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -advancewarp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warp_advance_DB + $run = $run -dbname $DB:$bg_warp_advance_DB + $bg_warp_advance_DB ++ + if ($bg_warp_advance_DB >= $DB:n) set bg_warp_advance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.warp.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.warp.revert.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -revertwarp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_warp_revert_DB + $run = $run -dbname $DB:$bg_warp_revert_DB + $bg_warp_revert_DB ++ + if ($bg_warp_revert_DB >= $DB:n) set bg_warp_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: branches/eam_branches/ipp-20100621/ippTasks/burntool.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/burntool.pro (revision 28794) +++ branches/eam_branches/ipp-20100621/ippTasks/burntool.pro (revision 28794) @@ -0,0 +1,568 @@ +## burntool.pro : -*- sh -*- + +check.globals + +macro burntool.it.on +# bt.initday.on + bt.registration.on + bt.burntool.on + bt.chips.on +end + +macro burntool.it.off +# bt.initday.off + bt.registration.off + bt.burntool.off + bt.chips.off +end + +macro bt.on + burntool.it.on +end + +macro bt.off + burntool.it.off +end + +macro bt.initday.on + task bt.initday.load + active true + end +end + +macro bt.registration.on + task bt.registration.load + active true + end +end + +macro bt.burntool.on + task bt.burntool.load + active true + end + task bt.burntool.run + active true + end +end + +macro bt.chips.on + task bt.chips.load + active true + end +end + +macro bt.initday.off + task bt.initday.load + active false + end +end + +macro bt.registration.off + task bt.registration.load + active false + end +end + +macro bt.burntool.off + task bt.burntool.load + active false + end + task bt.burntool.run + active false + end +end + +macro bt.chips.off + task bt.chips.load + active false + end +end + +$bt_regPAGE = 0 +$bt_burnPAGE = 0 +$bt_RburnPAGE = 0 +$bt_RburnCELL = 0 +$bt_chipPAGE = 0 +$bt_RchipPAGE = 0 + +book init btData +book init btBurntool +# +# Macros to control the book. +# +macro bt.add.date + book newpage btData $1 + book setword btData $1 nsState NEW +end + +macro bt.show.dates + book npages btData -var Npages + for i 0 $Npages + book getpage btData $i -var date + book getword btData $date nsState -var state + echo $date $state + end + book npages btBurntool -var Npages +end + +macro bt.del.date + book delpage btData $1 +end + +macro bt.set.date + book setword btData $1 nsState $2 +end + + + +# +# Start a new date to work on +# +task bt.initday.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + active false + npending 1 + + task.exec + stdout $LOGDIR/bt.initday.log + stderr $LOGDIR/bt.initday.log + $today = `date +%Y-%m-%d` + book newpage btData $today + book setword btData $today nsState NEW + + command echo $today + end + + task.exit 0 + # nothing to do here? + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if registration is complete +# +task bt.registration.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/bt.registration.log + + book getpage btData $bt_regPAGE -var date + book getword btData $date nsState -var bt_STATE + book npages btData -var Npages + + if ($VERBOSE > 5) + echo "bt.registration.load: " $bt_regPAGE $date $bt_STATE $Npages + end + + $bt_regPAGE ++ + if ($bt_regPAGE >= $Npages) set bt_regPAGE = 0 + option $date + + if ("$bt_STATE" != "NEW") break + $run = automate_stacks.pl --check_registration --date $date + command $run + end + + # success + task.exit 0 +# convert 'stdout' to book format + book delpage btData $options:0 + ipptool2book stdout btData -uniq -key date + + if ($VERBOSE > 2) + book listbook btData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the chips have been properly burntooled and are ready for queueing +# +task bt.chips.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/bt.chips.log + + book getpage btData $bt_chipPAGE -var date + book getword btData $date nsState -var bt_STATE + book npages btData -var Npages + + if ($VERBOSE > 5) + echo "bt.chips.load: " $bt_chipPAGE $date $bt_STATE $Npages + end + + $bt_chipPAGE ++ + if ($bt_chipPAGE >= $Npages) set bt_chipPAGE = 0 + option $date + + if (("$bt_STATE" != "REGISTERED")&&("$bt_STATE" != "BURNING")) break + $run = automate_stacks.pl --check_chips --date $date + + if ("$bt_STATE" == "BURNING") + $run = $run --isburning + end + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage btData $options:0 + ipptool2book stdout btData -uniq -key date + + # remove the burntool page if we're done with it. + book getword btData $options:0 nsState -var bt_STATE + if ("$bt_STATE" == "QUEUECHIPS") + book delpage btBurntool $options:0 + book delpage btData $options:0 + end + + if ($VERBOSE > 2) + book listbook btData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Generate a list of date ranges and chips that need to be processed with burntool +# +task bt.burntool.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/bt.burntool.log + + book getpage btData $bt_burnPAGE -var date + book getword btData $date nsState -var bt_STATE + book npages btData -var Npages + + if ($VERBOSE > 5) + echo "bt.burntool.load: " $bt_burnPAGE $date $bt_STATE $Npages + end + + $bt_burnPAGE ++ + if ($bt_burnPAGE >= $Npages) set bt_burnPAGE = 0 + option $date + + if ("$bt_STATE" != "NEEDSBURNING") break + + $run = automate_stacks.pl --define_burntool --date $date + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format +# book delpage btBurntool $options:0 + ipptool2book stdout btBurntool -uniq -key date + book setword btData $options:0 nsState "QUEUEBURNING" + + if ($VERBOSE > 2) + book listbook btData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Magically run burntool on the data, based on the information stored in our book. +# +task bt.burntool.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + npending 30 + task.exec + periods -exec $RUNEXEC + stdout NULL + stderr $LOGDIR/bt.burntool.log + + book getpage btData $bt_RburnPAGE -var date + book getword btData $date nsState -var bt_STATE + book npages btData -var Npages + + if ($VERBOSE > 5) + echo "bt.burntool.run: " $bt_RburnPAGE $date $bt_STATE $Npages + end + + $bt_RburnPAGE ++ + if ($bt_RburnPAGE >= $Npages) set bt_RburnPAGE = 0 + if (("$bt_STATE" != "QUEUEBURNING")&&("$bt_STATE" != "BURNING")) break + $BURNTOOLING = 1 + # Find out where in the list of jobs we are + book getword btBurntool $date btN -var btN + book getword btBurntool $date btNCounter -var btNcounter + + if ($VERBOSE > 5) + echo "bt.burntool.run: Status: " $btNcounter $date + end + + if ("$bt_STATE" == "QUEUEBURNING") + $new_state = "QUEUEBURNING" + end + if ("$bt_STATE" == "BURNING") + $new_state = "BURNING" + end + if ($btNcounter + 1 > $btN) + $new_state = "BURNING" + end + if ($btNcounter > $btN) + $btNcounter = 0 + end + if ($VERBOSE > 5) + echo "bt.burntool.run: Status: " $btNcounter $new_state + end + + # Increment the counter in the book for the next job. + $counter_update = $btNcounter + 1 + book setword btBurntool $date btNCounter $counter_update +# book setword btData $options:0 nsState $options:1 + book setword btData $date nsState $new_state + + # Get the current status of this job, and skip if it doesn't need to process. + sprintf status_label "bt%dStatus" $btNcounter + book getword btBurntool $date $status_label -var status + + if ($VERBOSE > 5) + echo "bt.burntool.run: Status: " $btNcounter $status $date $status_label + end + if (("$status" == "FINISHED")||("$status" == "RUN")) break + + # Continue loading information to process this job + sprintf start_date_label "bt%dBegin" $btNcounter + sprintf end_date_label "bt%dEnd" $btNcounter + sprintf class_count_label "bt%dClass" $btNcounter + + book getword btBurntool $date $start_date_label -var start_date + book getword btBurntool $date $end_date_label -var end_date + book getword btBurntool $date $class_count_label -var chip_counter + + # Lookup class_id/host pairs + list word -split $hostmatch:$chip_counter + $class_id = $word:0 + $host = $word:1 + host $host +# set.host.for.camera GPC1 $class_id + $logfile = "/data/$host.0/burntool_logs/$class_id.$start_date.log" + + $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile + + echo "bt.burntool.run: " $date $btN $btNcounter $start_date $end_date $chip_counter $class_id $host $logfile $run + echo "bt.burntool.run: " $date $btN $btNcounter $chip_counter $new_state + + book setword btBurntool $date $status_label RUN + option $date $new_state $status_label + + command $run + periods -exec 0.05 +# command /bin/true + end + # success + task.exit 0 +# convert 'stdout' to book format + # Set data state based on if we're queueing or waiting + # Set the job state for success. + book setword btBurntool $options:0 $options:2 FINISHED + if ($VERBOSE > 2) + book listbook btData + end + end + + # locked list + task.exit default + book setword btBurntool $options:0 $options:2 FAIL + showcommand failure + end + task.exit crash + book setword btBurntool $options:0 $options:2 FAIL + showcommand crash + end + #operation times out? + task.exit timeout + book setword btBurntool $options:0 $option:2 FAIL + showcommand timeout + end +end + + +# +# This is all burntool related stuff. +# + +macro burntool + if ($0 != 3) + echo "USAGE: burntool (dateobs_begin) (dateobs_end)" + break + end + + for i 0 $hostmatch:n + job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile + end +end +macro loadhosts + for i 0 $allhosts:n + controller host add $allhosts:$i + end +end + +# sorry this list is messy, it's supposed to be that way to try and keep to burntools from running on the same host at the same time. + +list hostmatch + XY01 ipp014 + XY03 ipp038 + XY05 ipp023 + XY10 ipp039 + XY12 ipp024 + XY15 ipp040 + XY17 ipp020 + XY21 ipp041 + XY23 ipp042 + XY25 ipp043 + XY27 ipp028 + XY31 ipp044 + XY33 ipp029 + XY35 ipp045 + XY37 ipp030 + XY41 ipp046 + XY43 ipp031 + XY45 ipp047 + XY47 ipp032 + XY51 ipp048 + XY53 ipp033 + XY55 ipp049 + XY57 ipp034 + XY61 ipp050 + XY63 ipp035 + XY65 ipp051 + XY67 ipp036 + XY72 ipp052 + XY74 ipp015 + XY76 ipp053 + XY02 ipp014 + XY04 ipp038 + XY06 ipp023 + XY11 ipp039 + XY13 ipp024 + XY14 ipp040 + XY16 ipp020 + XY20 ipp041 + XY22 ipp042 + XY24 ipp043 + XY26 ipp028 + XY30 ipp044 + XY32 ipp029 + XY34 ipp045 + XY36 ipp030 + XY40 ipp046 + XY42 ipp031 + XY44 ipp047 + XY46 ipp032 + XY50 ipp048 + XY52 ipp033 + XY54 ipp049 + XY56 ipp034 + XY60 ipp050 + XY62 ipp035 + XY64 ipp051 + XY66 ipp036 + XY71 ipp052 + XY73 ipp015 + XY75 ipp053 +end + +list allhosts + ipp043 + ipp014 + ipp015 + ipp023 + ipp024 + ipp020 + ipp028 + ipp029 + ipp030 + ipp031 + ipp032 + ipp033 + ipp034 + ipp035 + ipp036 + ipp038 + ipp039 + ipp040 + ipp041 + ipp042 + ipp044 + ipp045 + ipp046 + ipp047 + ipp048 + ipp049 + ipp050 + ipp051 + ipp052 + ipp053 +end Index: branches/eam_branches/ipp-20100621/ippTasks/detrend.norm.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/detrend.norm.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/detrend.norm.pro (revision 28794) @@ -64,8 +64,36 @@ end +macro detnorm.revert.off + task detrend.norm.revert + active false + end + task detrend.normexp.revert + active false + end + task detrend.normstat.revert + active false + end +end + +macro detnorm.revert.on + task detrend.norm.revert + active true + end + task detrend.normexp.revert + active true + end + task detrend.normstat.revert + active true + end +end + + # these variables will cycle through the known database names $detPendingNormStatImfile_DB = 0 $detPendingNormImfile_DB = 0 $detPendingNormExp_DB = 0 +$detPendingNormStatImfile_DB_revert = 0 +$detPendingNormImfile_DB_revert = 0 +$detPendingNormExp_DB_revert = 0 # select images ready for copy @@ -440,2 +468,140 @@ end end + +task detrend.norm.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertnormalizedimfile -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormImfile_DB_revert + $run = $run -dbname $DB:$detPendingNormImfile_DB_revert + $detPendingNormImfile_DB_revert ++ + if ($detPendingNormImfile_DB_revert >= $DB:n) set detPendingNormImfile_DB_revert = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task detrend.normexp.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertnormalizedexp -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormExp_DB_revert + $run = $run -dbname $DB:$detPendingNormExp_DB_revert + $detPendingNormExp_DB_revert ++ + if ($detPendingNormExp_DB_revert >= $DB:n) set detPendingNormExp_DB_revert = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task detrend.normstat.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertnormalizedstat -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingNormStatImfile_DB_revert + $run = $run -dbname $DB:$detPendingNormStatImfile_DB_revert + $detPendingNormStatImfile_DB_revert ++ + if ($detPendingNormStatImfile_DB_revert >= $DB:n) set detPendingNormStatImfile_DB_revert = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: branches/eam_branches/ipp-20100621/ippTasks/detrend.process.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/detrend.process.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/detrend.process.pro (revision 28794) @@ -51,8 +51,27 @@ end +macro detproc.revert.off + task detrend.process.revert + active false + end + task detrend.processexp.revert + active false + end +end + +macro detproc.revert.on + task detrend.process.revert + active true + end + task detrend.processexp.revert + active true + end +end # these variables will cycle through the known database names $detPendingProcessedImfile_DB = 0 $detPendingProcessedExp_DB = 0 +$detPendingProcessedImfile_revert_DB = 0 +$detPendingProcessedExp_revert_DB = 0 # select images ready for copy @@ -313,2 +332,94 @@ end end + +task detrend.process.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertprocessedimfile -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedImfile_revert_DB + $run = $run -dbname $DB:$detPendingProcessedImfile_revert_DB + $detPendingProcessedImfile_revert_DB ++ + if ($detPendingProcessedImfile_revert_DB >= $DB:n) set detPendingProcessedImfile_revert_DB = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task detrend.processexp.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertprocessedexp -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedExp_revert_DB + $run = $run -dbname $DB:$detPendingProcessedExp_revert_DB + $detPendingProcessedExp_revert_DB ++ + if ($detPendingProcessedExp_revert_DB >= $DB:n) set detPendingProcessedExp_revert_DB = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: branches/eam_branches/ipp-20100621/ippTasks/detrend.resid.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/detrend.resid.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/detrend.resid.pro (revision 28794) @@ -49,7 +49,27 @@ end +macro detresid.revert.on + task detrend.resid.revert + active true + end + task detrend.residexp.revert + active true + end +end + +macro detresid.revert.off + task detrend.resid.revert + active false + end + task detrend.residexp.revert + active false + end +end + # these variables will cycle through the known database names $detPendingResidImfile_DB = 0 $detPendingResidExp_DB = 0 +$detPendingResidImfile_revert_DB = 0 +$detPendingResidExp_revert_DB = 0 # select images ready for copy @@ -316,2 +336,95 @@ end end + +task detrend.resid.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertresidimfile -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingResidImfile_revert_DB + $run = $run -dbname $DB:$detPendingResidImfile_revert_DB + $detPendingResidImfile_revert_DB ++ + if ($detPendingResidImfile_revert_DB >= $DB:n) set detPendingResidImfile_revert_DB = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task detrend.residexp.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + + $run = dettool -revertresidexp -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingResidExp_revert_DB + $run = $run -dbname $DB:$detPendingResidExp_revert_DB + $detPendingResidExp_revert_DB ++ + if ($detPendingResidExp_revert_DB >= $DB:n) set detPendingResidExp_revert_DB = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: branches/eam_branches/ipp-20100621/ippTasks/detrend.stack.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/detrend.stack.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/detrend.stack.pro (revision 28794) @@ -35,6 +35,19 @@ end +macro detstack.revert.off + task detrend.stack.revert + active false + end +end + +macro detstack.revert.on + task detrend.stack.revert + active true + end +end + # this variable will cycle through the known database names $detPendingStackedImfile_DB = 0 +$detPendingStackedImfile_revert_DB = 0 # select images ready for detrend_stack.pl @@ -165,2 +178,49 @@ end end + + +task detrend.stack.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + + $run = dettool -revertstacked -all-run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$detPendingStackedImfile_revert_DB + $run = $run -dbname $DB:$detPendingStackedImfile_revert_DB + $detPendingStackedImfile_revert_DB ++ + if ($detPendingStackedImfile_revert_DB >= $DB:n) set detPendingStackedImfile_revert_DB = 0 + end + echo $run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: branches/eam_branches/ipp-20100621/ippTasks/dist.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/dist.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/dist.pro (revision 28794) @@ -28,4 +28,6 @@ list DIST_STAGE -add "stack" list DIST_STAGE -add "SSdiff" +list DIST_STAGE -add "chip_bg" +list DIST_STAGE -add "warp_bg" $currentStage = 0 @@ -145,6 +147,11 @@ # to compute an index into the host table strlen $stage_id length - $start = $length - 2 - substr $stage_id $start 2 index + if ($length >= 2) + $start = $length - 2 + substr $stage_id $start 2 index + else + # stage_id < 10 caused a very annoying couple of hours for bills to debug + substr $stage_id 0 1 index + end $component_id = $index % $count book getword ipphosts distribution $component_id -var myhost @@ -409,6 +416,7 @@ # using $DIST_ID as the "component" works fine here since we only look - # at the last two digits - set.dist.workdir.by.component $STAGE_ID "exposure" $OUTDIR_TEMPLATE OUTDIR + # at the last two digits. But make sure that there 2 digits + $fake_component = $STAGE_ID + 10 + set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR if ("$OUTDIR" == "NULL") echo ERROR failed to set workdir for $DIST_ID Index: branches/eam_branches/ipp-20100621/ippTasks/dqstats.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/dqstats.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/dqstats.pro (revision 28794) @@ -37,4 +37,16 @@ task dqstats.revert active false + end +end + +macro dqstats.revert.on + task dqstats.revert + active true + end +end + +macro dqstats.revert.off + task dqstats.revert + active true end end @@ -109,5 +121,4 @@ ## we want only a single outstanding dqstats job. - host local npending 1 @@ -187,4 +198,5 @@ periods -timeout 120.0 npending 1 + active false stdout NULL Index: branches/eam_branches/ipp-20100621/ippTasks/ipphosts.mhpcc.config =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/ipphosts.mhpcc.config (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/ipphosts.mhpcc.config (revision 28794) @@ -253,36 +253,34 @@ END -# this list is no longer used -# XXX : delete if this does not cause trouble -## ipphosts METADATA -## camera STR distribution -## count S32 26 -## 0 STR ipp021 -## 1 STR ipp023 -## 2 STR ipp024 -## 3 STR ipp026 -## 4 STR ipp028 -## 5 STR ipp029 -## 6 STR ipp030 -## 7 STR ipp031 -## 8 STR ipp032 -## 9 STR ipp033 -## 10 STR ipp034 -## 11 STR ipp035 -## 12 STR ipp036 -## 13 STR ipp038 -## 14 STR ipp039 -## 15 STR ipp040 -## 16 STR ipp041 -## 17 STR ipp043 -## 18 STR ipp044 -## 19 STR ipp045 -## 20 STR ipp046 -## 21 STR ipp047 -## 22 STR ipp048 -## 23 STR ipp050 -## 24 STR ipp051 -## 25 STR ipp052 -## END +ipphosts METADATA + camera STR distribution + count S32 26 + 0 STR ipp021 + 1 STR ipp023 + 2 STR ipp024 + 3 STR ipp026 + 4 STR ipp028 + 5 STR ipp029 + 6 STR ipp030 + 7 STR ipp031 + 8 STR ipp032 + 9 STR ipp033 + 10 STR ipp034 + 11 STR ipp035 + 12 STR ipp036 + 13 STR ipp038 + 14 STR ipp039 + 15 STR ipp040 + 16 STR ipp041 + 17 STR ipp043 + 18 STR ipp044 + 19 STR ipp045 + 20 STR ipp046 + 21 STR ipp047 + 22 STR ipp048 + 23 STR ipp050 + 24 STR ipp051 + 25 STR ipp052 +END ipphosts METADATA Index: branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro (revision 28794) @@ -16,4 +16,5 @@ ns.initday.off ns.detrends.off + ns.dqstats.off ns.registration.off ns.burntool.off @@ -23,19 +24,9 @@ macro ns.on - ns.initday.on - ns.detrends.off - ns.registration.on - ns.burntool.on - ns.chips.on - ns.stacks.on + nightly.stacks.on end macro ns.off - ns.initday.off - ns.detrends.off - ns.registration.off - ns.burntool.off - ns.chips.off - ns.stacks.off + nightly.stacks.off end Index: branches/eam_branches/ipp-20100621/ippTasks/pantasks.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/pantasks.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/pantasks.pro (revision 28794) @@ -242,4 +242,18 @@ end +macro detrend.revert.off + detproc.revert.off + detstack.revert.off + detnorm.revert.off + detresid.revert.off +end + +macro detrend.revert.on + detproc.revert.on + detstack.revert.on + detnorm.revert.on + detresid.revert.on +end + macro detrend.reset detproc.reset @@ -251,5 +265,4 @@ macro all.on - register.on detrend.on flatcorr.on @@ -264,5 +277,4 @@ macro all.off - register.off detrend.off flatcorr.off @@ -721,2 +733,19 @@ end +macro tasks.revert.off + chip.revert.off + camera.revert.off + fake.revert.off + warp.revert.off + diff.revert.off + stack.revert.off +end +macro tasks.revert.on + chip.revert.on + camera.revert.on + fake.revert.on + warp.revert.on + diff.revert.on + stack.revert.on +end + Index: branches/eam_branches/ipp-20100621/ippTasks/pstamp.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/pstamp.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/pstamp.pro (revision 28794) @@ -188,4 +188,7 @@ add_poll_args run add_poll_labels run + # limit number of requests in the queue to avoid blocking everything + # when jobs take an infinite amount of time to parse + $run = $run -limit 10 command $run end Index: branches/eam_branches/ipp-20100621/ippTasks/rcserver.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/rcserver.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/rcserver.pro (revision 28794) @@ -12,4 +12,5 @@ ### Database lists $rcPendingFS_DB = 0 +$rcRevertFS_DB = 0 ### Check status of tasks @@ -38,4 +39,13 @@ task rcserver.makefileset.run active false + end + task rcserver.revert + active false + end +end + +macro rcserver.revert.on + task rcserver.revert + active true end end @@ -156,2 +166,49 @@ end end + + +task rcserver.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = disttool -revertfileset + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$rcRevertFS_DB + $run = $run -dbname $DB:$rcRevertFS_DB + $rcRevertFS_DB ++ + if ($rcRevertFS_DB >= $DB:n) set rcRevertFS_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: branches/eam_branches/ipp-20100621/ippTasks/simtest.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/simtest.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/simtest.pro (revision 28794) @@ -159,19 +159,2 @@ $SIMTEST_AUTO = simtest.flatcorr.auto end - -macro tasks.revert.off - chip.revert.off - camera.revert.off - fake.revert.off - warp.revert.off - diff.revert.off - stack.revert.off -end -macro tasks.revert.on - chip.revert.on - camera.revert.on - fake.revert.on - warp.revert.on - diff.revert.on - stack.revert.on -end Index: branches/eam_branches/ipp-20100621/ippTasks/stack.pro =================================================================== --- branches/eam_branches/ipp-20100621/ippTasks/stack.pro (revision 28439) +++ branches/eam_branches/ipp-20100621/ippTasks/stack.pro (revision 28794) @@ -126,5 +126,5 @@ # save the DB name for the exit tasks option $DB:$stackSkycell_DB - $run = $run -dbname $DB:$stackSkycell_DB -limit 40 + $run = $run -dbname $DB:$stackSkycell_DB $stackSkycell_DB ++ if ($stackSkycell_DB >= $DB:n) set stackSkycell_DB = 0 @@ -132,4 +132,6 @@ add_poll_args run add_poll_labels run + # reduce the limit (the last one takes precedence) + $run = $run -limit 40 command $run end