Index: /trunk/ippTasks/detrend.mkruns.pro =================================================================== --- /trunk/ippTasks/detrend.mkruns.pro (revision 11209) +++ /trunk/ippTasks/detrend.mkruns.pro (revision 11210) @@ -1,2 +1,4 @@ +## XXX this script needs to be updated to use books, not queues +## XXX this script needs to be reconsidered... ## This script defines tasks which regularly initiate the detrend Index: /trunk/ippTasks/detrend.norm.pro =================================================================== --- /trunk/ippTasks/detrend.norm.pro (revision 11209) +++ /trunk/ippTasks/detrend.norm.pro (revision 11210) @@ -1,12 +1,5 @@ - -## this file contains the tasks for running the detrend normalization stage - -## these tasks use the queue DetrendNorm -## the DetrendNorm queue contains: -## STATE DET_ID ITER - -queueinit DetrendNorm -queueinit DetrendNormStat -queueinit DetrendNormExp +## this file contains the tasks for running the detrend normalization stages +## these tasks use the books detPendingNormStatImfile detPendingNormImfile detPendingNormExp + if ($?network == 0) $network = 1 @@ -15,9 +8,4 @@ $parallel = 0 end - -$DetrendNormFail = 0 -$DetrendNormStatFail = 0 -$DetrendNormExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -28,18 +16,20 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init detPendingNormStatImfile +book init detPendingNormImfile +book init detPendingNormExp + macro detnorm.status - queueprint DetrendNorm - queueprint DetrendNormStat - queueprint DetrendNormExp - echo "DetrendNormFail: $DetrendNormFail" - echo "DetrendNormStatFail: $DetrendNormStatFail" - echo "DetrendNormExpFail: $DetrendNormExpFail" + book listbook detPendingNormStatImfile + book listbook detPendingNormImfile + book listbook detPendingNormExp end # select images ready for copy -# new entries are added to queue DetrendNormStat +# new entries are added to detPendingNormStatImfile # compare the new list with the ones already selected task dettool.normstat.load @@ -50,38 +40,23 @@ periods -timeout 30 - stdout $LOGSUBDIR/dettool.normstat.log + stdout NULL stderr $LOGSUBDIR/dettool.normstat.log task.exec - command dettool -tonormalizedstat -simple -limit {$DetrendNormStatFail + 20} + command dettool -tonormalizedstat -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $DET_TYPE = $word:1 - $ITER = $word:2 - $CAMERA = $word:3 - queuepush DetrendNormStat -uniq -key 1:3 "NEW $DET_ID $DET_TYPE $ITER $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout detPendingNormStatImfile -key det_id:iteration -uniq + if ($VERBOSE > 2) + book listbook detPendingNormStatImfile end # delete existing entries which are DONE - queuedrop DetrendNormStat -key 0 DONE - end - - # locked list + book delpage detPendingNormStatImfile -key state DONE + end + # default exit status task.exit default @@ -102,28 +77,17 @@ task.exec - queuesize DetrendNormStat -var N + book npages detPendingNormStatImfile -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendNormStat queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendNormStat -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendNormStat -replace -key 1:3 "RUN $line" - - # the DetrendNormStat queue contains: - # STATE DET_ID ITER - list word -split $line - $DET_ID = $word:0 - $DET_TYPE = $word:1 - $ITER = $word:2 - $CAMERA = $word:3 - - # XXX add $WORKDIR/$LOG_DIR - stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log - stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + # look for new images in detPendingNormStatImfile + book getpage detPendingNormStatImfile 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword detPendingNormStatImfile $pageName state RUN + book getword detPendingNormStatImfile $pageName det_id -var DET_ID + book getword detPendingNormStatImfile $pageName det_type -var DET_TYPE + book getword detPendingNormStatImfile $pageName iteration -var ITERATION + book getword detPendingNormStatImfile $pageName camera -var CAMERA # specify choice of remote host: @@ -134,34 +98,33 @@ end + # XXX add $WORKDIR/$LOG_DIR + # XXX use ipp_filename.pl to lookup output file names + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATIION.log + + # save the pageName for future reference below + options $pageName + # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) - echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE - end - command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE - end - - # success - task.exit 0 - queuepush DetrendNormStat -replace -key 1:3 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendNormStat -replace -key 1:3 "FAIL $options:0" - $DetrendNormStatFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendNormStat -replace -key 1:3 "TIMEOUT $options:0" - $DetrendNormStatFail ++ + echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE + end + command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE + end + + # default exit status + task.exit default + process_exit detPendingNormStatImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormStatImfile $options:0 state TIMEOUT end end # select images ready for copy -# new entries are added to queue DetrendNorm +# new entries are added to detPendingNormImfile # compare the new list with the ones already selected task dettool.norm.load @@ -172,41 +135,23 @@ periods -timeout 30 - stdout $LOGSUBDIR/dettool.norm.log + stdout NULL stderr $LOGSUBDIR/dettool.norm.log task.exec - command dettool -tonormalize -simple -limit {$DetrendNormFail + 20} + command dettool -tonormalize -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_TYPE = $word:0 - $CAMERA = $word:1 - $URI = $word:2 - $DET_ID = $word:3 - $ITER = $word:4 - $CLASS_ID = $word:5 - $NORM = $word:6 - queuepush DetrendNorm -uniq -key 4:5 "NEW $DET_TYPE $CAMERA $URI $DET_ID $ITER $CLASS_ID $NORM" + # convert 'stdout' to book format + ipptool2book stdout detPendingNormImfile -key det_id:iteration -uniq + if ($VERBOSE > 2) + book listbook detPendingNormImfile end # delete existing entries which are DONE - queuedrop DetrendNorm -key 0 DONE - end - - # locked list + book delpage detPendingNormImfile -key state DONE + end + # default exit status task.exit default @@ -227,30 +172,20 @@ task.exec - queuesize DetrendNorm -var N + book npages detPendingNormImfile -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendNorm queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendNorm -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendNorm -replace -key 4:5 "RUN $line" - - # the DetrendNorm queue contains: - # STATE DET_ID ITER - list word -split $line - $DET_TYPE = $word:0 - $CAMERA = $word:1 - $URI = $word:2 - $DET_ID = $word:3 - $ITER = $word:4 - $CLASS_ID = $word:5 - $NORM = $word:6 - - stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log - stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + # look for new images in detPendingNormImfile + book getpage detPendingNormImfile 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword detPendingNormImfile $pageName state RUN + book getword detPendingNormImfile $pageName det_type -var DET_TYPE + book getword detPendingNormImfile $pageName camera -var CAMERA + book getword detPendingNormImfile $pageName uri -var URI + book getword detPendingNormImfile $pageName det_id -var DET_ID + book getword detPendingNormImfile $pageName iteration -var ITERATION + book getword detPendingNormImfile $pageName class_id -var CLASS_ID + book getword detPendingNormImfile $pageName norm -var NORM # specify choice of remote host: @@ -261,34 +196,32 @@ end + # XXX use ipp_filename.pl to lookup output file names + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log + + # save the pageName for future reference below + options $pageName + # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) - echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE - end - command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE - end - - # success - task.exit 0 - queuepush DetrendNorm -replace -key 4:5 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendNorm -replace -key 4:5 "FAIL $options:0" - $DetrendNormFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendNorm -replace -key 4:5 "TIMEOUT $options:0" - $DetrendNormFail ++ + echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE + end + command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE + end + + # default exit status + task.exit default + process_exit detPendingNormImfile $options:0 state FAIL + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormImfile $options:0 state TIMEOUT end end # select images ready for copy -# new entries are added to queue DetrendNormExp +# new entries are added to detPendingNormExp # compare the new list with the ones already selected task dettool.normexp.load @@ -299,41 +232,23 @@ periods -timeout 30 - stdout $LOGSUBDIR/dettool.normexp.log + stdout NULL stderr $LOGSUBDIR/dettool.normexp.log task.exec - command dettool -tonormalizedexp -simple -limit {$DetrendNormExpFail + 20} + command dettool -tonormalizedexp -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $CAMERA = $word:3 - #$TELESCOPE= $word:4 - #$EXP_TYPE = $word:5 - #$IMFILES = $word:6 - queuepush DetrendNormExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout detPendingNormExp -key det_id:iteration -uniq + if ($VERBOSE > 2) + book listbook detPendingNormExp end # delete existing entries which are DONE - queuedrop DetrendNormExp -key 0 DONE - end - - # locked list + book delpage detPendingNormExp -key state DONE + end + # default exit status task.exit default @@ -354,32 +269,17 @@ task.exec - queuesize DetrendNormExp -var N - + book npages detPendingNormExp -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendNormExp queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendNormExp -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendNormExp -replace -key 1:2 "RUN $line" - - # the DetrendNormExp queue contains: - # STATE DET_ID ITER - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $CAMERA = $word:3 - #$TELESCOPE= $word:4 - #$EXP_TYPE = $word:5 - #$IMFILES = $word:6 - - # XXX add $WORKDIR/$LOG_DIR - stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log - stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + # look for new images in detPendingNormExp + book getpage detPendingNormExp 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword detPendingNormExp $pageName state RUN + book getword detPendingNormExp $pageName det_id -var DET_ID + book getword detPendingNormExp $pageName iteration -var ITERATION + book getword detPendingNormExp $pageName det_type -var DET_TYPE + book getword detPendingNormExp $pageName camera -var CAMERA # specify choice of remote host: @@ -390,29 +290,27 @@ end + # XXX add $WORKDIR/$LOG_DIR + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log + + # save the pageName for future reference below + options $pageName + # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) - echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE - end - command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE - end - - # success - task.exit 0 - queuepush DetrendNormExp -replace -key 1:2 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendNormExp -replace -key 1:2 "FAIL $options:0" - $DetrendNormExpFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendNormExp -replace -key 1:2 "TIMEOUT $options:0" - $DetrendNormExpFail ++ - end -end + echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE + end + command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE + end + + # default exit status + task.exit default + process_exit detPendingNormExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormExp $options:0 state TIMEOUT + end +end Index: /trunk/ippTasks/detrend.process.pro =================================================================== --- /trunk/ippTasks/detrend.process.pro (revision 11209) +++ /trunk/ippTasks/detrend.process.pro (revision 11210) @@ -1,18 +1,5 @@ ## this file contains the tasks for running the detrend processing stage - -# query for pending imfiles to process: -# dettool -raw -simple -# 1 bias 850131b fpa mc 850131b.fits BIAS r.MP9601 1.656000 5.496118 0.349071 0.000000 1155.934692 107.037283 46.214947 37.160000 105.480000 -133.067001 0.000000 -# this returns many columns for manual operations. we strip all but the first few: - -# process a single imfile -# detrend_process.pl 1 850131b mc bias 850131b.fits 850131b - -## these tasks use the queue DetrendProcessImfiles -## the DetrendProcessImfiles queue contains: -## STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI - -queueinit DetrendProcessImfiles -queueinit DetrendProcessExposures +## these tasks use the books detPendingProcessedImfile and detPendingProcessedExp + if ($?network == 0) $network = 1 @@ -21,8 +8,4 @@ $parallel = 0 end - -$DetrendProcessImfileFail = 0 -$DetrendProcessExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -33,16 +16,18 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init detPendingProcessedImfile +book init detPendingProcessedExp + macro detproc.status - queueprint DetrendProcessImfiles - queueprint DetrendProcessExposures - echo "DetrendProcessImfileFail: $DetrendProcessImfileFail" - echo "DetrendProcessExpFail: $DetrendProcessExpFail" + book listbook detPendingProcessedImfile + book listbook detPendingProcessedExp end # select images ready for copy -# new entries are added to queue DetrendProcessImfiles +# new entries are added to detPendingProcessedImfile # compare the new list with the ones already selected task dettool.process.load @@ -53,38 +38,21 @@ periods -timeout 30 - stdout $LOGSUBDIR/detproc.imfile.log + stdout NULL stderr $LOGSUBDIR/detproc.imfile.log task.exec - command dettool -toprocessedimfile -simple -limit {$DetrendProcessImfileFail + 20} + command dettool -toprocessedimfile -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $DET_TYPE = $word:1 - $EXP_TAG = $word:2 - $CLASS = $word:3 - $CLASS_ID = $word:4 - $URI = $word:5 - $CAMERA = $word:21 - queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $DET_TYPE $EXP_TAG $CLASS $CLASS_ID $URI $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedImfile -key det_id:exp_tag:class_id -uniq + if ($VERBOSE > 2) + book listbook detPendingProcessedImfile end # delete existing entries which are DONE - queuedrop DetrendProcessImfiles -key 0 DONE + book delpage detPendingProcessedImfile -key state DONE end @@ -100,5 +68,5 @@ end -# copy new images, sending job to desired host +# run detrend_process_imfile.ps on pending images task dettool.process.run periods -poll $runpoll @@ -107,29 +75,31 @@ task.exec - queuesize DetrendProcessImfiles -var N + book npages detPendingProcessedImfile -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendProcessImfiles queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendProcessImfiles -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendProcessImfiles -replace -key 1:3:5 "RUN $line" - - ## the DetrendProcessImfiles queue contains: - # STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $DET_TYPE = $word:1 - $EXP_TAG = $word:2 - $CLASS = $word:3 - $CLASS_ID = $word:4 - $URI = $word:5 - $CAMERA = $word:6 - + # look for new images in detPendingProcessedImfile + book getpage detPendingProcessedImfile 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword detPendingProcessedImfile $pageName state RUN + book getword detPendingProcessedImfile $pageName det_id -var DET_ID + book getword detPendingProcessedImfile $pageName det_type -var DET_TYPE + book getword detPendingProcessedImfile $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedImfile $pageName class -var CLASS + book getword detPendingProcessedImfile $pageName class_id -var CLASS_ID + book getword detPendingProcessedImfile $pageName uri -var URI + book getword detPendingProcessedImfile $pageName camera -var CAMERA + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + ## XXX use OUTPATH from db to set output path + ## XXX use ipp_filename.pl DETPROC.LOG $outroot $CLASS_ID` $word = `basename $EXP_TAG | tr '.' ' '` list word -split $word @@ -139,4 +109,88 @@ exec mkdir -p $LOGSUBDIR/$base + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command detrend_process_imfile.pl --det_id $DET_ID --det_type $DET_TYPE --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --input_uri $URI --camera $CAMERA + end + command detrend_process_imfile.pl --det_id $DET_ID --det_type $DET_TYPE --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --input_uri $URI --camera $CAMERA + end + + # default exit status + task.exit default + process_exit detPendingProcessedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedImfile $options:0 state TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingProcessedExp +# compare the new list with the ones already selected +task dettool.processexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout NULL + stderr $LOGSUBDIR/detproc.exp.log + + task.exec + command dettool -toprocessedexp -limit 20 + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedExp -key det_id:iteration:exp_tag -uniq + if ($VERBOSE > 2) + book listbook detPendingProcessedExp + end + + # delete existing entries which are DONE + book delpage detPendingProcessedExp -key state DONE + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.processexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingProcessedExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new exposures in detPendingProcessedExp + book getpage detPendingProcessedExp 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword detPendingProcessedExp $pageName state RUN + book getword detPendingProcessedExp $pageName det_id -var DET_ID + book getword detPendingProcessedExp $pageName iteration -var ITERATION + book getword detPendingProcessedExp $pageName det_type -var DET_TYPE + book getword detPendingProcessedExp $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedExp $pageName camera -var CAMERA + # specify choice of remote host: if ($parallel) @@ -146,133 +200,16 @@ end - # create example job options as a demonstration - options "$line" - if ($VERBOSE > 1) - echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA - end - command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA - end - - # success - task.exit 0 - queuepush DetrendProcessImfiles -replace -key 1:3:5 "DONE $options:0" - # detrend_process_imfile.pl --force_exist --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0" - $DetrendProcessImfileFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0" - $DetrendProcessImfileFail ++ - end -end - -# select images ready for copy -# new entries are added to queue DetrendProcessExposures -# compare the new list with the ones already selected -task dettool.processexp.load - host local - - periods -poll $loadpoll - periods -exec $loadexec - periods -timeout 30 - - stdout $LOGSUBDIR/detproc.exp.log - stderr $LOGSUBDIR/detproc.exp.log - - task.exec - command dettool -toprocessedexp -simple -limit {$DetrendProcessExpFail + 20} - end - - # success - task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $EXP_TAG = $word:3 - $CAMERA = $word:4 - queuepush DetrendProcessExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_TAG $CAMERA" - end - - # delete existing entries which are DONE - queuedrop DetrendProcessExposures -key 0 DONE - end - - # error - task.exit default - showcommand failure - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# copy new images, sending job to desired host -task dettool.processexp.run - periods -poll $runpoll - periods -exec $runexec - periods -timeout 30 - - task.exec - queuesize DetrendProcessExposures -var N - if ($N == 0) break - if ($network == 0) break - - # look for new images on the internal DetrendProcessExposures queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendProcessExposures -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendProcessExposures -replace -key 1:2:4 "RUN $line" - - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $EXP_TAG = $word:3 - $CAMERA = $word:4 - # XXX this is still a lame rule $word = `basename $EXP_TAG | tr '.' ' '` list word -split $word $base = $word:0 - stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log exec mkdir -p $LOGSUBDIR/$base - # specify choice of remote host: - if ($parallel) - host anyhost - else - host local - end + # save the pageName for future reference below + options $pageName # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA @@ -282,13 +219,6 @@ # success - task.exit 0 - queuepush DetrendProcessExposures -replace -key 1:2:4 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendProcessExposures -replace -key 1:2:4 "FAIL $options:0" - $DetrendProcessExpFail ++ + task.exit default + process_exit detPendingProcessedExp $options:0 $JOB_STATUS end @@ -296,6 +226,5 @@ task.exit timeout showcommand timeout - queuepush DetrendProcessExposures -replace -key 1:2:4 "TIMEOUT $options:0" - $DetrendProcessExpFail ++ - end -end + book setword detPendingProcessedExp $options:0 state TIMEOUT + end +end Index: /trunk/ippTasks/detrend.reject.pro =================================================================== --- /trunk/ippTasks/detrend.reject.pro (revision 11209) +++ /trunk/ippTasks/detrend.reject.pro (revision 11210) @@ -1,17 +1,5 @@ +## this file contains the tasks for running the detrend processing stage +## these tasks use the book detRejectExp -## this file contains the tasks for running the detrend processing stage - -# dettool -residdetrun -simple -# 1 0 bias -# DET_ID ITER TYPE - -# detrend_reject_exp.pl 1 0 bias -# detrend_reject_exp.pl DET_ID ITER TYPE - -## these tasks use the queue DetrendResidExp -## the DetrendRejectExp queue contains: -## STATE DET_ID ITER TYPE - -queueinit DetrendRejectExp if ($?network == 0) $network = 1 @@ -20,7 +8,4 @@ $parallel = 0 end - -$DetrendRejectExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -31,14 +16,16 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init detRejectExp + macro detreject.status - queueprint DetrendRejectExp - echo "DetrendRejectExpFail: $DetrendRejectExpFail" + book listbook detRejectExp end # select images ready for copy -# new entries are added to queue DetrendRejectExp +# new entries are added to detRejectExp # compare the new list with the ones already selected task dettool.reject.load @@ -49,34 +36,21 @@ periods -timeout 30 - stdout $LOGSUBDIR/dettool.reject.log + stdout NULL stderr $LOGSUBDIR/dettool.reject.log task.exec - command dettool -residdetrun -simple -limit {$DetrendRejectExpFail + 20} + command dettool -residdetrun -limit 20 end # success task.exit 0 - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $CAMERA = $word:4 - queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $MODE $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout detRejectExp -key det_id:iteration -uniq + if ($VERBOSE > 2) + book listbook detRejectExp end # delete existing entries which are DONE - queuedrop DetrendRejectExp -key 0 DONE + book delpage detRejectExp -key state DONE end @@ -99,30 +73,18 @@ task.exec - queuesize DetrendRejectExp -var N + book npages detRejectExp -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendRejectExp queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendRejectExp -var line -key 0 NEW - if ("$line" == "NULL") break + # look for new images in detRejectExp + book getpage detRejectExp 0 -var pageName -key state NULL + if ($pageName == NULL) break - strpop line state - queuepush DetrendRejectExp -replace -key 1:2 "RUN $line" - - ## the DetrendRejectExp queue contains: - ## STATE DET_ID ITER TYPE - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $CAMERA = $word:4 - - # I'd like to add CAMERA to the log file... - stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log - stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + book setword detRejectExp $pageName state RUN + book getword detRejectExp $pageName det_id -var DET_ID + book getword detRejectExp $pageName iteration -var ITERATION + book getword detRejectExp $pageName det_type -var DET_TYPE + book getword detRejectExp $pageName mode -var MODE + book getword detRejectExp $pageName camera -var CAMERA # specify choice of remote host: @@ -133,22 +95,21 @@ end + # I'd like to add CAMERA to the log file... + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log + + # save the pageName for future reference below + options $pageName + # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) - echo command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE --camera $CAMERA + echo command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA end - command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE --camera $CAMERA + command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA end - - # success - task.exit 0 - queuepush DetrendRejectExp -replace -key 1:2 "DONE $options:0" - end # default exit status task.exit default - showcommand failure - queuepush DetrendRejectExp -replace -key 1:2 "FAIL $options:0" - $DetrendRejectExpFail ++ + process_exit detRejectExp $options:0 state FAIL end @@ -156,6 +117,5 @@ task.exit timeout showcommand failure - queuepush DetrendRejectExp -replace -key 1:2 "TIMEOUT $options:0" - $DetrendRejectExpFail ++ + book setword detRejectExp $options:0 state TIMEOUT end end Index: /trunk/ippTasks/detrend.resid.pro =================================================================== --- /trunk/ippTasks/detrend.resid.pro (revision 11209) +++ /trunk/ippTasks/detrend.resid.pro (revision 11210) @@ -1,19 +1,5 @@ - ## this file contains the tasks for running the detrend processing stage - -# dettool -toresidimfile -simple -# 0 bias 1 850131b mc o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.565582 -# ITER TYPE DET_ID EXP_TAG CLASS_ID URI-IN - -# detrend_create_resid.pl 1 0 850131b mc bias bias_mc_1_0.fit o_850131b.mc.fit 850131b -# detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG) -# URL-DET: $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit - -## these tasks use the queue DetrendResidImfiles -## the DetrendResidImfiles queue contains: -## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - -queueinit DetrendResidImfiles -queueinit DetrendResidExposures +## these tasks use the books detPendingResidImfile and detPendingResidExp + if ($?network == 0) $network = 1 @@ -22,8 +8,4 @@ $parallel = 0 end - -$DetrendResidImfileFail = 0 -$DetrendResidExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -34,16 +16,18 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init detPendingResidImfile +book init detPendingResidExp + macro detresid.status - queueprint DetrendResidImfiles - queueprint DetrendResidExposures - echo "DetrendResidImfileFail: $DetrendResidImfileFail" - echo "DetrendResidExpFail: $DetrendResidExpFail" + book listbook detPendingResidImfile + book listbook detPendingResidExp end # select images ready for copy -# new entries are added to queue DetrendResidImfiles +# new entries are added to detPendingResidImfile # compare the new list with the ones already selected task dettool.resid.load @@ -54,45 +38,23 @@ periods -timeout 30 - stdout $LOGSUBDIR/detresid.log + stdout NULL stderr $LOGSUBDIR/detresid.log task.exec - command dettool -toresidimfile -simple -limit {$DetrendResidImfileFail + 20} + command dettool -toresidimfile -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $CLASS_ID = $word:5 - $URI = $word:6 - $DET_URI = $word:7 - $CAMERA = $word:8 - ## must include the detID and iter in the key (if we have two running at the same time...) - ## Also needed to be in the output file (or path) - queuepush DetrendResidImfiles -uniq -key 1:2:5:6 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $CLASS_ID $URI $DET_URI $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout detPendingResidImfile -key det_id:iteration:exp_tag:class_id -uniq + if ($VERBOSE > 2) + book listbook detPendingResidImfile end # delete existing entries which are DONE - queuedrop DetrendResidImfiles -key 0 DONE - end - - # locked list + book delpage detPendingResidImfile -key state DONE + end + # default exit status task.exit default @@ -113,37 +75,22 @@ task.exec - queuesize DetrendResidImfiles -var N + book npages detPendingResidImfile -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendResidImfiles queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendResidImfiles -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "RUN $line" - - ## the DetrendResidImfiles queue contains: - ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $CLASS_ID = $word:5 - $URI = $word:6 - $DET_URI = $word:7 - $CAMERA = $word:8 - - $word = `basename $EXP_TAG | tr '.' ' '` - list word -split $word - $base = $word:0 - stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log - stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log - exec mkdir -p $LOGSUBDIR/$base + # look for new images in detPendingResidImfile + book getpage detPendingResidImfile 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword detPendingResidImfile $pageName state RUN + book getword detPendingResidImfile $pageName det_id -var DET_ID + book getword detPendingResidImfile $pageName iteration -var ITERATION + book getword detPendingResidImfile $pageName det_type -var DET_TYPE + book getword detPendingResidImfile $pageName mode -var MODE + book getword detPendingResidImfile $pageName exp_tag -var EXP_TAG + book getword detPendingResidImfile $pageName class_id -var CLASS_ID + book getword detPendingResidImfile $pageName uri -var URI + book getword detPendingResidImfile $pageName det_uri -var DET_URI + book getword detPendingResidImfile $pageName camera -var CAMERA # specify choice of remote host: @@ -154,35 +101,36 @@ end - # create example job options as a demonstration - options "$line" - # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???) - # need to parse the subdir path for the detrend URL - # $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits - echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE - command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE - end - - # success - task.exit 0 - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "FAIL $options:0" - $DetrendResidImfileFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendResidImfiles -replace -key 1:2:6:6 "TIMEOUT $options:0" - $DetrendResidImfileFail ++ + # XXX use ipp_filename.pl to lookup output file names + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.$CLASS_ID.log + stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.$CLASS_ID.log + exec mkdir -p $LOGSUBDIR/$base + + # save the pageName for future reference below + options $pageName + + # create command + if ($VERBOSE > 1) + echo command detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + end + command detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + end + + # default exit status + task.exit default + process_exit detPendingResidImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidImfile $options:0 state TIMEOUT end end # select images ready for copy -# new entries are added to queue DetrendResidExposures +# new entries are added to detPendingResidExp # compare the new list with the ones already selected task dettool.residexp.load @@ -193,41 +141,23 @@ periods -timeout 30 - stdout $LOGSUBDIR/detresid.exp.log + stdout NULL stderr $LOGSUBDIR/detresid.exp.log task.exec - command dettool -toresidexp -simple -limit {$DetrendResidExpFail + 20} + command dettool -toresidexp -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $INCLUDE = $word:5 - $CAMERA = $word:6 - queuepush DetrendResidExposures -uniq -key 1:2:5 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $INCLUDE $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout detPendingResidExp -key det_id:iteration:exp_tag -uniq + if ($VERBOSE > 2) + book listbook detPendingResidExp end # delete existing entries which are DONE - queuedrop DetrendResidExposures -key 0 DONE - end - - # locked list + book delpage detPendingResidExp -key state DONE + end + # default exit status task.exit default @@ -248,37 +178,20 @@ task.exec - queuesize DetrendResidExposures -var N + book npages detPendingResidExp -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendResidExposures queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendResidExposures -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendResidExposures -replace -key 1:2:5 "RUN $line" - - ## the DetrendResidExposures queue contains: - ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $INCLUDE = $word:5 - $CAMERA = $word:6 - - # XXX this is still a lame rule - $word = `basename $EXP_TAG | tr '.' ' '` - list word -split $word - $base = $word:0 - - stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log - stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log - exec mkdir -p $LOGSUBDIR/$base + # look for new images in detPendingResidExp + book getpage detPendingResidExp 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword detPendingResidExp $pageName state RUN + book getword detPendingResidExp $pageName det_id -var DET_ID + book getword detPendingResidExp $pageName iteration -var ITERATION + book getword detPendingResidExp $pageName det_type -var DET_TYPE + book getword detPendingResidExp $pageName mode -var MODE + book getword detPendingResidExp $pageName exp_tag -var EXP_TAG + book getword detPendingResidExp $pageName include -var INCLUDE + book getword detPendingResidExp $pageName camera -var CAMERA # specify choice of remote host: @@ -289,29 +202,31 @@ end + # XXX use ipp_filename.pl to lookup output file names + $word = `basename $EXP_TAG | tr '.' ' '` + list word -split $word + $base = $word:0 + stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.log + stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.log + exec mkdir -p $LOGSUBDIR/$base + + # save the pageName for future reference below + options $pageName + # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) - echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA - end - command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA - end - - # success - task.exit 0 - queuepush DetrendResidExposures -replace -key 1:2:5 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendResidExposures -replace -key 1:2:5 "FAIL $options:0" - $DetrendResidExpFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendResidExposures -replace -key 1:2:5 "TIMEOUT $options:0" - $DetrendResidExpFail ++ - end -end + echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA + end + command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA + end + + # default exit status + task.exit default + process_exit detPendingResidExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidExp $options:0 state TIMEOUT + end +end Index: /trunk/ippTasks/detrend.stack.pro =================================================================== --- /trunk/ippTasks/detrend.stack.pro (revision 11209) +++ /trunk/ippTasks/detrend.stack.pro (revision 11210) @@ -1,17 +1,5 @@ ## this file contains the tasks for running the detrend stacking stage +## these tasks use the book detPendingStackedImfile -# get a list of imfiles to stack -# dettool -tostack -simple -# 1 0 bias mc -# DET_ID ITER TYPE CLASS_ID - -# function to stack the imfiles: -# detrend_stack.pl 1 0 mc bias - -## these tasks use the queue DetrendStackClass -## the DetrendStackClass queue contains: -## STATE DET_ID ITER TYPE CLASS_ID CAMERA - -queueinit DetrendStackClass if ($?network == 0) $network = 1 @@ -20,7 +8,4 @@ $parallel = 0 end - -$DetrendStackClassFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -31,14 +16,16 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init detPendingStackedImfile + macro detstack.status - queueprint DetrendStackClass - echo "DetrendStackClassFail: $DetrendStackClassFail" + book listbook detPendingStackedImfile end -# select images ready for copy -# new entries are added to queue DetrendStackClass +# select images ready for detrend_stack.pl +# new entries are added to detPendingStackedImfile # compare the new list with the ones already selected task dettool.stack.load @@ -49,34 +36,21 @@ periods -timeout 30 - stdout $LOGSUBDIR/detstack.log + stdout NULL stderr $LOGSUBDIR/detstack.log task.exec - command dettool -tostacked -simple -limit {$DetrendStackClassFail + 20} + command dettool -tostacked -limit 20 end # success task.exit 0 - local i Nqueue - - # compare output with newImage queue - # only add entries which do not exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $CLASS_ID = $word:3 - $CAMERA = $word:4 - queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $CLASS_ID $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout detPendingStackedImfile -key det_id:iteration:class_id -uniq + if ($VERBOSE > 2) + book listbook detPendingStackedImfile end # delete existing entries which are DONE - queuedrop DetrendStackClass -key 0 DONE + book delpage detPendingStackedImfile -key state DONE end @@ -99,31 +73,18 @@ task.exec - queuesize DetrendStackClass -var N + book npages detPendingStackedImfile -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendStackClass queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendStackClass -var line -key 0 NEW - if ("$line" == "NULL") break + # look for new images in detPendingStackedImfile + book getpage detPendingStackedImfile 0 -var pageName -key state NULL + if ($pageName == NULL) break - strpop line state - queuepush DetrendStackClass -replace -key 1:2:4 "RUN $line" - - # the DetrendStackClass queue contains: - # STATE DET_ID ITER TYPE CLASS_ID - ## NOTE: the queue has the STATE prepended - # use local variables - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $CLASS_ID = $word:3 - $CAMERA = $word:4 - - # XXX add $WORKDIR/$LOG_DIR - stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log - stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log + book setword detPendingStackedImfile $pageName state RUN + book getword detPendingStackedImfile $pageName det_id -var DET_ID + book getword detPendingStackedImfile $pageName iteration -var ITERATION + book getword detPendingStackedImfile $pageName det_type -var DET_TYPE + book getword detPendingStackedImfile $pageName class_id -var CLASS_ID + book getword detPendingStackedImfile $pageName camera -var CAMERA # specify choice of remote host: @@ -134,22 +95,22 @@ end - # create example job options as a demonstration - options "$line" + # XXX add $WORKDIR/$LOG_DIR + # XXX use ipp_filename.pl to lookup output file names + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.$CLASS_ID.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.$CLASS_ID.log + + # save the pageName for future reference below + options $pageName + + # create the command line if ($VERBOSE > 1) - echo command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA + echo command detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA end - command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA - end - - # success - task.exit 0 - queuepush DetrendStackClass -replace -key 1:2:4 "DONE $options:0" + command detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA end # default exit status task.exit default - showcommand failure - queuepush DetrendStackClass -replace -key 1:2:4 "FAIL $options:0" - $DetrendStackClassFail ++ + process_exit detPendingStackedImfile $options:0 $JOB_STATUS end @@ -157,6 +118,5 @@ task.exit timeout showcommand timeout - queuepush DetrendStackClass -replace -key 1:2:4 "TIMEOUT $options:0" - $DetrendStackClassFail ++ + book setword detPendingStackedImfile $options:0 state TIMEOUT end end Index: /trunk/ippTasks/notes.txt =================================================================== --- /trunk/ippTasks/notes.txt (revision 11209) +++ /trunk/ippTasks/notes.txt (revision 11210) @@ -1,2 +1,24 @@ + +2007.01.08 + + I have updated pantasks to used named tags in named groups to make + life much easier when changing the schema of the db tables. The + function ipptool2book convernts the output from an ippTool query in + the form of a set of metadata structures (saved in stdout) into a + book format. + + dettool -tonormalizedstat DetrendNormStat + dettool -tonormalize DetrendNorm + dettool -tonormalizedexp DetrendNormExp + dettool -toprocessedimfile DetrendProcessImfiles + dettool -toprocessedexp DetrendProcessExposures + dettool -residdetrun DetrendRejectExp + dettool -toresidimfile DetrendResidImfiles + dettool -toresidexp DetrendResidExposures + dettool -tostacked DetrendStackClass + p0tool -pendingimfile Phase0Imfiles + p0tool -pendingexp Phase0Exposures + p2tool -pendingimfile Phase2Imfiles + p3tool -pendingexp Phase3Exposures 2006.08.28 Index: /trunk/ippTasks/pantasks.pro =================================================================== --- /trunk/ippTasks/pantasks.pro (revision 11209) +++ /trunk/ippTasks/pantasks.pro (revision 11210) @@ -18,8 +18,32 @@ $runexec = 1 +$EXIT_SUCCESS = 0 +$EXIT_UNKNOWN_ERR = 1 +$EXIT_SYS_ERR = 2 +$EXIT_CONFIG_ERR = 3 +$EXIT_PROG_ERR = 4 +$EXIT_DATA_ERR = 5 + +$DB:n = 0 + +macro init.simtest + $LOGDIR = `ipp_datapath.pl path://SIMTEST` + $LOGDIR = $LOGDIR/logs + exec mkdir -p $LOGDIR + + list DB -add simtest + + if ($parallel) + controller exit true + controller host add alala + end +end + macro init.essence $LOGDIR = `ipp_datapath.pl path://ESSENCE` $LOGDIR = $LOGDIR/logs exec mkdir -p $LOGDIR + + list DB -add essence_v2 if ($parallel) @@ -36,4 +60,6 @@ $LOGDIR = $LOGDIR/logs exec mkdir -p $LOGDIR + + list DB -add isptest if ($parallel) @@ -55,5 +81,5 @@ macro load.tasks input $scripts/phase0.pro - input $scripts/phase2.pro +# input $scripts/phase2.pro # input $scripts/phase3.pro @@ -75,4 +101,6 @@ if ($0 == 2) echo "$1 for: $command" + echo "job exit status: $JOB_STATUS" + echo "job dtime: $JOB_DTIME" else echo "command: $command" @@ -80,2 +108,56 @@ end end + +macro process_exit + if ($0 != 4) + echo "USAGE: process_exit (bookName) (pageName) (exitCode)" + break + end + + $bookName = $1 + $pageName = $2 + $exitCode = $3 + + # success + if ($exitCode == $EXIT_SUCCESS) + # the handler scripts update DB the tables; here we just update the page + book setword $bookName $pageName state DONE + return + end + + # failure related to the data files + # jobs which result in DATAERR must have db state updated + if ($exitCode == $EXIT_DATA_ERR) + showcommand failure + book setword $bookName $pageName state DATA_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_SYS_ERR) + halt + showcommand "system failure" + book setword $bookName $pageName state SYS_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_CONFIG_ERR) + halt + showcommand "config error" + book setword $bookName $pageName state CONFIG_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_PROG_ERR) + halt + showcommand "programming error" + book setword $bookName $pageName state PROG_ERR + return + end + + # any other exit status + showcommand failure + book setword $bookName $pageName state UNKNOWN_ERR +end Index: /trunk/ippTasks/phase0.pro =================================================================== --- /trunk/ippTasks/phase0.pro (revision 11209) +++ /trunk/ippTasks/phase0.pro (revision 11210) @@ -1,11 +1,5 @@ ## this file contains the tasks for running the phase 0 stage - -## these tasks use the queue Phase0Imfiles -## the Phase0Imfiles queue contains: -## STATE EXP_TAG CLASS CLASS_ID URI-FITS -## queue keys are counted from 0 - -queueinit Phase0Imfiles -queueinit Phase0Exposures +## these tasks use the books p0PendingImfile and p0PendingExp + if ($?network == 0) $network = 1 @@ -14,8 +8,4 @@ $parallel = 0 end - -$Phase0ImfileFail = 0 -$Phase0ExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -26,16 +16,20 @@ break end + $LOGSUBDIR = $LOGDIR/phase0 exec mkdir -p $LOGSUBDIR +book init p0PendingImfile +book init p0PendingExp + macro phase0.status - queueprint Phase0Imfiles - queueprint Phase0Exposures - echo "Phase0ImfileFail: $Phase0ImfileFail" - echo "Phase0ExpFail: $Phase0ExpFail" -end + book listbook p0PendingImfile + book listbook p0PendingExp +end + +$phase0_DB = 0 # select images ready for phase0 analysis -# new entries are added to queue Phase0Imfiles +# new entries are added to p0PendingImfile # compare the new list with the ones already selected task phase0.imfile.load @@ -46,33 +40,34 @@ periods -timeout 30 - stdout $LOGSUBDIR/phase0.imfile.log + # XXX test that this option works + npending 1 + + # silently drop stdout + stdout NULL stderr $LOGSUBDIR/phase0.imfile.log - task.exec - command p0tool -pendingimfile -simple -limit {$Phase0ImfileFail + 20} + # select entries from the current DB; cycle to the next DB, if it exists + # if the DB list is not set, use the value defined in .ipprc + task.exec + if ($DB:n == 0) + command p0tool -pendingimfile -limit 20 + else + command p0tool -pendingimfile -limit 20 -dbname $DB:$phase0_DB + $phase0_DB ++ + if ($phase0_DB >= $DB:n) set phase0_DB = 0 + end end # success - task.exit 0 - local i Nqueue - - # compare output with Phase0Imfiles queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $EXP_TAG = $word:0 - $CLASS = $word:1 - $CLASS_ID = $word:2 - $URI = $word:3 - queuepush Phase0Imfiles -uniq -key 1 "NEW $EXP_TAG $CLASS $CLASS_ID $URI" - end - - # delete existing entries which are DONE - queuedrop Phase0Imfiles -key 0 DONE + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout p0PendingImfile -key exp_tag -uniq + if ($VERBOSE > 2) + book listbook p0PendingImfile + end + + # delete existing entries which are DONE or DATAERR + book delpage p0PendingImfile -key state DONE + book delpage p0PendingImfile -key state DATA_ERR end @@ -88,7 +83,5 @@ end - - -# run the phase0imfile script on pending images +# run the phase0_imfile.pl script on pending images task phase0.imfile.run periods -poll $runpoll @@ -97,24 +90,114 @@ task.exec - queuesize Phase0Imfiles -var N + book npages p0PendingImfile -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendImfilesToProcess queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop Phase0Imfiles -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush Phase0Imfiles -replace -key 1 "RUN $line" - - ## the Phase0Imfiles queue contains (line has popped STATE) - ## STATE EXP_TAG CLASS CLASS_ID URI-FITS - list word -split $line - $EXP_TAG = $word:0 - $CLASS = $word:1 - $CLASS_ID = $word:2 - $URI = $word:3 + # look for new images in p0PendingImfile + book getpage p0PendingImfile 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword p0PendingImfile $pageName state RUN + + book getword p0PendingImfile $pageName exp_tag -var EXP_TAG + book getword p0PendingImfile $pageName class -var CLASS + book getword p0PendingImfile $pageName class_id -var CLASS_ID + book getword p0PendingImfile $pageName uri -var URI + book getword p0PendingImfile $pageName outpath -var OUTPATH + book getword p0PendingImfile $pageName dbname -var DBNAME + + # specify choice of remote host + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + $outroot = $OUTPATH/$EXP_TAG + $logfile = `ipp_filename.pl PHASE0.LOG $outroot $CLASS_ID` + stdout $logfile + stderr $logfile + exec mkdir -p $OUTPATH + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --dbname $DBNAME + end + command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --dbname $DBNAME + end + + # default exit status + task.exit default + process_exit p0PendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword p0PendingImfile $options:0 state TIMEOUT + end +end + +# select exposures ready for phase0_exp.pl +task phase0.exp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout NULL + stderr $LOGSUBDIR/phase0.exp.log + + task.exec + command p0tool -pendingexp -limit 20 + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout p0PendingExp -key exp_tag -uniq + if ($VERBOSE > 2) + book listbook p0PendingExp + end + + # delete existing entries which are DONE + book delpage p0PendingExp -key state DONE + book delpage p0PendingExp -key state DATA_ERR + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the phase0_exp.pl script on pending exposures +task phase0.exp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages p0PendingExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in p0PendingExp (state == NULL) + book getpage p0PendingExp 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword p0PendingExp $pageName state RUN + book getword p0PendingExp $pageName exp_tag -var EXP_TAG + book getword p0PendingExp $pageName outpath -var OUTPATH # specify choice of remote host:(need to choose based on chips) @@ -125,146 +208,15 @@ end - # XXX this is still a lame rule - $word = `basename $EXP_TAG | tr '.' ' '` - list word -split $word - $base = $word:0 - stdout $LOGSUBDIR/$base/$EXP_TAG.p0.log - stderr $LOGSUBDIR/$base/$EXP_TAG.p0.log - exec mkdir -p $LOGSUBDIR/$base + ## generate output log based on filerule + $outroot = $OUTPATH/$EXP_TAG + $logfile = `ipp_filename.pl PHASE0.LOG $outroot FPA` + stdout $logfile + stderr $logfile + exec mkdir -p $OUTPATH + + # save the pageName for future reference below + options $pageName # create the command line - options "$line" - if ($VERBOSE > 1) - echo command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI - end - command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI - end - - # success - task.exit 0 - # phase0imfile.pl updates DB tables, here we just update the queue - queuepush Phase0Imfiles -replace -key 1 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush Phase0Imfiles -replace -key 1 "FAIL $options:0" - $Phase0ImfileFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush Phase0Imfiles -replace -key 1 "TIMEOUT $options:0" - $Phase0ImfileFail ++ - end -end - - - -## these tasks use the queue Phase0Exposures -## the Phase0Exposures queue contains: -## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS - -# select images ready for phase0 analysis -# new entries are added to queue Phase0Exposures -# compare the new list with the ones already selected -task phase0.exp.load - host local - - periods -poll $loadpoll - periods -exec $loadexec - periods -timeout 30 - - stdout $LOGSUBDIR/phase0.exp.log - stderr $LOGSUBDIR/phase0.exp.log - - task.exec - command p0tool -pendingexp -simple -limit {$Phase0ExpFail + 20} - end - - # success - task.exit 0 - local i Nqueue - - # compare output with Phase0Exposures queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $EXP_TAG = $word:0 - # $EXP_ID = $word:1 - # $CAMERA = $word:2 - # $TELESCOPE = $word:3 - # $DATEOBS = $word:4 - # $TYPE = $word:5 - # $IMFILES = $word:6 - queuepush Phase0Exposures -uniq -key 1 "NEW $EXP_TAG" - end - - # delete existing entries which are DONE - queuedrop Phase0Exposures -key 0 DONE - end - - # default exit status - task.exit default - showcommand - end - - # operation times out? - task.exit timeout - showcommand - end -end - - - -# run the phase0imfile script on pending images -task phase0.exp.run - periods -poll $runpoll - periods -exec $runexec - periods -timeout 30 - - task.exec - queuesize Phase0Exposures -var N - if ($N == 0) break - if ($network == 0) break - - # look for new images on the internal DetrendImfilesToProcess queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop Phase0Exposures -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush Phase0Exposures -replace -key 1 "RUN $line" - - ## the Phase0Exposures queue contains: - ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS - list word -split $line - $EXP_TAG = $word:0 - - # XXX this is still a lame rule - $word = `basename $EXP_TAG | tr '.' ' '` - list word -split $word - $base = $word:0 - stdout $LOGSUBDIR/$base/$EXP_TAG.p0.log - stderr $LOGSUBDIR/$base/$EXP_TAG.p0.log - exec mkdir -p $LOGSUBDIR/$base - - # specify choice of remote host:(need to choose based on chips) - if ($parallel) - host anyhost - else - host local - end - - # create the command line - options "$line" if ($VERBOSE > 1) echo command phase0_exp.pl --exp_tag $EXP_TAG @@ -274,21 +226,12 @@ # success - task.exit 0 - # phase0exp.pl updates DB tables, here we just update the queue - queuepush Phase0Exposures -replace -key 1 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand - queuepush Phase0Exposures -replace -key 1 "FAIL $options:0" - $Phase0ExpFail + task.exit default + process_exit p0PendingExp $options:0 $JOB_STATUS end # operation times out? task.exit timeout - showcommand - queuepush Phase0Exposures -replace -key 1 "TIMEOUT $options:0" - $Phase0ExpFail + showcommand timeout + book setword p0PendingExp $options:0 state TIMEOUT end end Index: /trunk/ippTasks/phase2.pro =================================================================== --- /trunk/ippTasks/phase2.pro (revision 11209) +++ /trunk/ippTasks/phase2.pro (revision 11210) @@ -1,10 +1,5 @@ ## this file contains the tasks for running the phase 0 stage +## these tasks use the book p2PendingImfile -## these tasks use the queue Phase2Imfiles -## the Phase2Imfiles queue contains: -## STATE EXP_TAG CLASS CLASS_ID URI-FITS -## queue keys are counted from 0 - -queueinit Phase2Imfiles if ($?network == 0) $network = 1 @@ -13,7 +8,4 @@ $parallel = 0 end - -$Phase2ImfileFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -24,15 +16,17 @@ break end + $LOGSUBDIR = $LOGDIR/phase2 exec mkdir -p $LOGSUBDIR +book init p2PendingImfile + macro phase2.status - queueprint Phase2Imfiles - echo "Phase2ImfileFail: $Phase2ImfileFail" + book listbook p2PendingImfile end # select images ready for phase2 analysis -# new entries are added to queue Phase2Imfiles -# compare the new list with the ones already selected +# new entries are added to p2PendingImfile +# skip already-present entries task phase2.imfile.load host local @@ -42,35 +36,21 @@ periods -timeout 30 - stdout $LOGSUBDIR/phase2.imfile.log + stdout NULL stderr $LOGSUBDIR/phase2.imfile.log task.exec - # command p2tool -pendingimfile -simple -limit {$Phase2ImfileFail + 20} - command p2tool -pendingimfile -simple + command p2tool -pendingimfile -limit 20 end # success task.exit 0 - local i Nqueue - - # save stdout and stderr - - # compare output with Phase2Imfiles queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $EXP_TAG = $word:0 - $CLASS_ID = $word:1 - $URI = $word:2 - queuepush Phase2Imfiles -uniq -key 1:2 "NEW $EXP_TAG $CLASS_ID $URI" + # convert 'stdout' to book format + ipptool2book stdout p2PendingImfile -key exp_tag:class_id -uniq + if ($VERBOSE > 2) + book listbook p2PendingImfile end # delete existing entries which are DONE - queuedrop Phase2Imfiles -key 0 DONE + book delpage p2PendingImfile -key state DONE end @@ -86,5 +66,5 @@ end -# run the phase2imfile script on pending images +# run the phase2.pl script on pending images task phase2.imfile.run periods -poll $runpoll @@ -93,25 +73,21 @@ task.exec - queuesize Phase2Imfiles -var N + book npages p2PendingImfile -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendImfilesToProcess queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop Phase2Imfiles -var line -key 0 NEW - if ("$line" == "NULL") break + # look for new images in p2PendingImfile (state == NULL) + book getpage p2PendingImfile 0 -var pageName -key state NULL + if ($pageName == NULL) break - strpop line state - queuepush Phase2Imfiles -replace -key 1:2 "RUN $line" + book setword p2PendingImfile $pageName state RUN + book getword p2PendingImfile $pageName camera -var CAMERA + book getword p2PendingImfile $pageName exp_tag -var EXP_TAG + book getword p2PendingImfile $pageName outpath -var OUTPATH + book getword p2PendingImfile $pageName class_id -var CLASS_ID + book getword p2PendingImfile $pageName uri -var URI - ## the Phase2Imfiles queue contains (line has popped STATE) - ## STATE EXP_TAG CLASS CLASS_ID URI-FITS - list word -split $line - $EXP_TAG = $word:0 - $CLASS_ID = $word:1 - $URI = $word:2 - - # specify choice of remote host:(need to choose based on chips) + # specify choice of remote host + # XXX need to choose based on chips if ($parallel) host anyhost @@ -120,40 +96,30 @@ end - ## add 'p2' to the log files? - # XXX this is still a lame rule - $word = `basename $EXP_TAG | tr '.' ' '` - list word -split $word - $base = $word:0 - stdout $LOGSUBDIR/$base/$EXP_TAG.$CLASS_ID.log - stderr $LOGSUBDIR/$base/$EXP_TAG.$CLASS_ID.log - exec mkdir -p $LOGSUBDIR/$base + ## generate output log based on filerule + $outroot = $OUTPATH/$EXP_TAG + $logfile = `ipp_filename.pl PHASE2.LOG $outroot $CLASS_ID` + stdout $logfile + stderr $logfile + exec mkdir -p $OUTPATH + + # save the pageName for future reference below + options $pageName # create the command line - options "$line" if ($VERBOSE > 1) - echo command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI + echo command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --camera $CAMERA end - ## XXX this is annoying: phase2.pl now requires camera :: it should not need this, it has the file. - command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --camera CTIO_MOSAIC2 - end - - # success - task.exit 0 - # phase2imfile.pl updates DB tables, here we just update the queue - queuepush Phase2Imfiles -replace -key 1:2 "DONE $options:0" + command phase2.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --camera $CAMERA --outpath $OUTPATH end # default exit status task.exit default - showcommand failure - queuepush Phase2Imfiles -replace -key 1:2 "FAIL $options:0" - $Phase2ImfileFail ++ + process_exit p2PendingImfile $options:0 $JOB_STATUS end - # operation times out? + # operation timed out? task.exit timeout showcommand timeout - queuepush Phase2Imfiles -replace -key 1:2 "TIMEOUT $options:0" - $Phase2ImfileFail ++ + book setword p2PendingImfile $options:0 state TIMEOUT end end Index: /trunk/ippTasks/phase3.pro =================================================================== --- /trunk/ippTasks/phase3.pro (revision 11209) +++ /trunk/ippTasks/phase3.pro (revision 11210) @@ -1,10 +1,5 @@ -## this file contains the tasks for running the phase 0 stage +## this file contains the tasks for running the phase 3 stage +## these tasks use the book p3PendingExp -## these tasks use the queue Phase3Exposures -## the Phase3Exposures queue contains: -## STATE EXP_TAG CLASS CLASS_ID URI-FITS -## queue keys are counted from 0 - -queueinit Phase3Exposures if ($?network == 0) $network = 1 @@ -13,7 +8,4 @@ $parallel = 0 end - -$Phase3ExposureFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -24,15 +16,17 @@ break end + $LOGSUBDIR = $LOGDIR/phase3 exec mkdir -p $LOGSUBDIR +book init p3PendingExp + macro phase3.status - queueprint Phase3Exposures - echo "Phase3ExposureFail: $Phase3ExposureFail" + book listbook p3PendingExp end # select images ready for phase3 analysis -# new entries are added to queue Phase3Exposures -# compare the new list with the ones already selected +# new entries are added to p3PendingExp +# skip already-present entries task phase3.exp.load host local @@ -46,26 +40,17 @@ task.exec - command p3tool -pendingexp -simple + command p3tool -pendingexp -limit 20 end # success task.exit 0 - local i Nqueue - - # compare output with Phase3Exposures queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $EXP_TAG = $word:0 - queuepush Phase3Exposures -uniq -key 1 "NEW $EXP_TAG" + # convert 'stdout' to book format + ipptool2book stdout p3PendingExp -key exp_tag -uniq + if ($VERBOSE > 2) + book listbook p3PendingExp end # delete existing entries which are DONE - queuedrop Phase3Exposures -key 0 DONE + book delpage p3PendingExp -key state DONE end @@ -88,21 +73,22 @@ task.exec - queuesize Phase3Exposures -var N + book npages p3PendingExp -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendExposuresToProcess queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop Phase3Exposures -var line -key 0 NEW - if ("$line" == "NULL") break + # look for new images in p3PendingExp (state == NULL) + book getpage p3PendingExp 0 -var pageName -key state NULL + if ($pageName == NULL) break - strpop line state - queuepush Phase3Exposures -replace -key 1 "RUN $line" + book setword p3PendingExp $pageName state RUN + book getword p3PendingExp $pageName camera -var CAMERA + book getword p3PendingExp $pageName exp_tag -var EXP_TAG - ## the Phase3Exposures queue contains: - ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS - list word -split $line - $EXP_TAG = $word:0 + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end # XXX this is still a lame rule @@ -114,23 +100,18 @@ exec mkdir -p $LOGSUBDIR/$base - # specify choice of remote host:(need to choose based on chips) - if ($parallel) - host anyhost - else - host local - end + # save the pageName for future reference below + options $pageName # create the command line - options "$line" if ($VERBOSE > 1) echo command phase3.pl --exp_tag $EXP_TAG end - command phase3.pl --exp_tag $EXP_TAG --camera CTIO_MOSAIC2 + command phase3.pl --exp_tag $EXP_TAG --camera $CAMERA end # success task.exit 0 - # phase3exp.pl updates DB tables, here we just update the queue - queuepush Phase3Exposures -replace -key 1 "DONE $options:0" + # phase3exp.pl updates DB tables, here we just update the page + book setword p3PendingExp $options:0 state DONE end @@ -138,6 +119,5 @@ task.exit default showcommand failure - queuepush Phase3Exposures -replace -key 1 "FAIL $options:0" - $Phase3ExposureFail ++ + book setword p3PendingExp $options:0 state FAIL end @@ -145,6 +125,5 @@ task.exit timeout showcommand timeout - queuepush Phase3Exposures -replace -key 1 "TIMEOUT $options:0" - $Phase3ExposureFail ++ + book setword p3PendingExp $options:0 state TIMEOUT end end