Index: trunk/ippTasks/Makefile.am =================================================================== --- trunk/ippTasks/Makefile.am (revision 27125) +++ trunk/ippTasks/Makefile.am (revision 27149) @@ -31,5 +31,6 @@ dqstats.pro \ science.cleanup.pro \ - nightly_stacks.pro + nightly_stacks.pro \ + lossy_compress.pro other_files = \ Index: trunk/ippTasks/lossy_compress.pro =================================================================== --- trunk/ippTasks/lossy_compress.pro (revision 27149) +++ trunk/ippTasks/lossy_compress.pro (revision 27149) @@ -0,0 +1,215 @@ +## lossy_compress.pro : tasks to lossy compress old raw images that have negligible science value : -*- sh -*- +## use the books compPendingImfile and compPendingExp + +# test for required global variables +check.globals + +book init compPendingImfile +book init compPendingExp + +macro lossycomp.reset + book init compPendingImfile + book init compPendingExp +end + +macro lossycomp.status + book list + book listbook compPendingImfile + book listbook compPendingExp +end + +macro lossycomp.on + task lossycomp.imfile.load + active true + end + task lossycomp.imfile.run + active true + end + task lossycomp.exp.finish + active true + end +end + +macro lossycomp.off + task lossycomp.imfile.load + active false + end + task lossycomp.imfile.run + active false + end + task lossycomp.exp.finish + active false + end +end + +# these variables will cycle through the known database names +$compPendingImfile_DB = 0 +$comp_finish_DB = 0 + +# select images ready to be compressed +task lossycomp.imfile.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/lossycomp.imfile.load.log + + #select entries from the current DB, cycle to the next DB if possible + task.exec + $run = regtool -pendingcompressimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$compPendingImfile_DB + $run = $run -dbname $DB:$compPendingImfile_DB + $compPendingImfile_DB ++ + if ($compPendingImfile_DB >= $DB:n) set compPendingImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout compPendingImfile -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle compPendingImfile + if ($VERBOSE > 2) + book listbook compPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup compPendingImfile + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +# run the lossy_compress_imfile.pl script on the pending images +task lossycomp.imfile.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + if ($NETWORK == 0) break + book npages compPendingImfile -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + periods -exec 0.05 + + # look for new images + book getpage compPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword compPendingImfile $pageName pantaskState RUN + book getword compPendingImfile $pageName exp_id -var EXP_ID + book getword compPendingImfile $pageName exp_name -var EXP_NAME + book getword compPendingImfile $pageName exp_tag -var EXP_TAG + book getword compPendingImfile $pageName tmp_class_id -var TMP_CLASS_ID + book getword compPendingImfile $pageName class_id -var CLASS_ID + book getword compPendingImfile $pageName camera -var CAMERA + book getword compPendingImfile $pageName uri -var URI + book getword compPendingImfile $pageName bytes -var BYTES + book getword compPendingImfile $pageName md5sum -var MD5SUM + book getword compPendingImfile $pageName workdir -var WORKDIR + book getword compPendingImfile $pageName data_state -var STATE + book getword compPendingImfile $pageName dbname -var DBNAME + + # specify choice of remote host + set.host.for.camera $CAMERA $TMP_CLASS_ID + + # set logfile name + if ("$WORKDIR" == "NULL") + sprintf logfile "compress_log/%s.%d.lossycomp.%s.log" $EXP_NAME $EXP_ID $CLASS_ID + else + sprintf logfile "%s/%s/%s.lossycomp.%s.log" $WORKDIR $EXP_TAG $EXP_TAG $TMP_CLASS_ID + end + + stdout $LOGDIR/lossycomp.imfile.run.log + stdout $LOGDIR/lossycomp.imfile.run.log + + $run = lossy_compress_imfile.pl --exp_id $EXP_ID --class_id $CLASS_ID --exp_name $EXP_NAME --uri $URI --camera $CAMERA --state $STATE --logfile $logfile --bytes $BYTES --md5sum $MD5SUM + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit compPendingImfile $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword compPendingImfile $options:0 pantaskState CRASH + end + task.exit timeout + showcommand timeout + book setword compPendingImfile $options:0 pantaskState TIMEOUT + end +end + +# select exposures ready for lossy_compress_exp.pl +task lossycomp.exp.finish + host local + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/lossycomp.exp.load.log + + task.exec + $run = regtool -finishcompressexp -limit 10 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$comp_finish_DB + $run = $run -dbname $DB:$comp_finish_DB + $comp_finish_DB ++ + if ($comp_finish_DB >= $DB:n) set comp_finish_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end