Index: /branches/eam_branch_00/ippTasks/detrend.norm.pro =================================================================== --- /branches/eam_branch_00/ippTasks/detrend.norm.pro (revision 11010) +++ /branches/eam_branch_00/ippTasks/detrend.norm.pro (revision 11011) @@ -1,12 +1,4 @@ - -## this file contains the tasks for running the detrend normalization stage - -## these tasks use the queue DetrendNorm -## the DetrendNorm queue contains: -## STATE DET_ID ITER - -queueinit DetrendNorm -queueinit DetrendNormStat -queueinit DetrendNormExp +## this file contains the tasks for running the detrend normalization stages + if ($?network == 0) $network = 1 @@ -15,9 +7,4 @@ $parallel = 0 end - -$DetrendNormFail = 0 -$DetrendNormStatFail = 0 -$DetrendNormExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -28,14 +15,16 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init DetrendNormStat +book init DetrendNorm +book init DetrendNormExp + macro detnorm.status - queueprint DetrendNorm - queueprint DetrendNormStat - queueprint DetrendNormExp - echo "DetrendNormFail: $DetrendNormFail" - echo "DetrendNormStatFail: $DetrendNormStatFail" - echo "DetrendNormExpFail: $DetrendNormExpFail" + book listbook DetrendNormStat + book listbook DetrendNorm + book listbook DetrendNormExp end @@ -54,34 +43,19 @@ task.exec - command dettool -tonormalizedstat -simple -limit {$DetrendNormStatFail + 20} + command dettool -tonormalizedstat -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $DET_TYPE = $word:1 - $ITER = $word:2 - $CAMERA = $word:3 - queuepush DetrendNormStat -uniq -key 1:3 "NEW $DET_ID $DET_TYPE $ITER $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout DetrendNormStat -key det_id:iter -uniq + if ($VERBOSE > 2) + book listbook DetrendNormStat end # delete existing entries which are DONE - queuedrop DetrendNormStat -key 0 DONE - end - - # locked list + book delpage DetrendNormStat -key state DONE + end + # default exit status task.exit default @@ -102,28 +76,17 @@ task.exec - queuesize DetrendNormStat -var N + book npages DetrendNormStat -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendNormStat queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendNormStat -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendNormStat -replace -key 1:3 "RUN $line" - - # the DetrendNormStat queue contains: - # STATE DET_ID ITER - list word -split $line - $DET_ID = $word:0 - $DET_TYPE = $word:1 - $ITER = $word:2 - $CAMERA = $word:3 - - # XXX add $WORKDIR/$LOG_DIR - stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log - stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + # look for new images in DetrendNormStat + book getpage DetrendNormStat 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword DetrendNormStat $pageName state RUN + book getword DetrendNormStat $pageName DET_ID -var DET_ID + book getword DetrendNormStat $pageName DET_TYPE -var DET_TYPE + book getword DetrendNormStat $pageName ITER -var ITER + book getword DetrendNormStat $pageName CAMERA -var CAMERA # specify choice of remote host: @@ -134,6 +97,12 @@ end + # XXX add $WORKDIR/$LOG_DIR + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + + # save the pageName for future reference below + options $pageName + # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE @@ -144,24 +113,22 @@ # success task.exit 0 - queuepush DetrendNormStat -replace -key 1:3 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendNormStat -replace -key 1:3 "FAIL $options:0" - $DetrendNormStatFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendNormStat -replace -key 1:3 "TIMEOUT $options:0" - $DetrendNormStatFail ++ + book setword DetrendNormStat $options:0 state DONE + end + + # default exit status + task.exit default + showcommand failure + book setword DetrendNormStat $options:0 state FAIL + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword DetrendNormStat $options:0 state TIMEOUT end end # select images ready for copy -# new entries are added to queue DetrendNorm +# new entries are added to DetrendNorm # compare the new list with the ones already selected task dettool.norm.load @@ -176,37 +143,19 @@ task.exec - command dettool -tonormalize -simple -limit {$DetrendNormFail + 20} + command dettool -tonormalize -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_TYPE = $word:0 - $CAMERA = $word:1 - $URI = $word:2 - $DET_ID = $word:3 - $ITER = $word:4 - $CLASS_ID = $word:5 - $NORM = $word:6 - queuepush DetrendNorm -uniq -key 4:5 "NEW $DET_TYPE $CAMERA $URI $DET_ID $ITER $CLASS_ID $NORM" + # convert 'stdout' to book format + ipptool2book stdout DetrendNorm -key det_id:iter -uniq + if ($VERBOSE > 2) + book listbook DetrendNorm end # delete existing entries which are DONE - queuedrop DetrendNorm -key 0 DONE - end - - # locked list + book delpage DetrendNorm -key state DONE + end + # default exit status task.exit default @@ -227,30 +176,20 @@ task.exec - queuesize DetrendNorm -var N + book npages DetrendNorm -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendNorm queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendNorm -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendNorm -replace -key 4:5 "RUN $line" - - # the DetrendNorm queue contains: - # STATE DET_ID ITER - list word -split $line - $DET_TYPE = $word:0 - $CAMERA = $word:1 - $URI = $word:2 - $DET_ID = $word:3 - $ITER = $word:4 - $CLASS_ID = $word:5 - $NORM = $word:6 - - stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log - stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + # look for new images in DetrendNorm + book getpage DetrendNorm 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword DetrendNorm $pageName state RUN + book getword DetrendNorm $pageName det_type -var DET_TYPE + book getword DetrendNorm $pageName camera -var CAMERA + book getword DetrendNorm $pageName uri -var URI + book getword DetrendNorm $pageName det_id -var DET_ID + book getword DetrendNorm $pageName iter -var ITER + book getword DetrendNorm $pageName class_id -var CLASS_ID + book getword DetrendNorm $pageName norm -var NORM # specify choice of remote host: @@ -261,6 +200,11 @@ end + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + + # save the pageName for future reference below + options $pageName + # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE @@ -271,19 +215,17 @@ # success task.exit 0 - queuepush DetrendNorm -replace -key 4:5 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendNorm -replace -key 4:5 "FAIL $options:0" - $DetrendNormFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendNorm -replace -key 4:5 "TIMEOUT $options:0" - $DetrendNormFail ++ + book setword DetrendNorm $options:0 state DONE + end + + # default exit status + task.exit default + showcommand failure + book setword DetrendNorm $options:0 state FAIL + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword DetrendNorm $options:0 state TIMEOUT end end @@ -303,37 +245,19 @@ task.exec - command dettool -tonormalizedexp -simple -limit {$DetrendNormExpFail + 20} + command dettool -tonormalizedexp -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $CAMERA = $word:3 - #$TELESCOPE= $word:4 - #$EXP_TYPE = $word:5 - #$IMFILES = $word:6 - queuepush DetrendNormExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout DetrendNormExp -key det_id:iter -uniq + if ($VERBOSE > 2) + book listbook DetrendNormExp end # delete existing entries which are DONE - queuedrop DetrendNormExp -key 0 DONE - end - - # locked list + book delpage DetrendNormExp -key state DONE + end + # default exit status task.exit default @@ -354,32 +278,17 @@ task.exec - queuesize DetrendNormExp -var N - + book npages DetrendNormExp -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendNormExp queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendNormExp -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendNormExp -replace -key 1:2 "RUN $line" - - # the DetrendNormExp queue contains: - # STATE DET_ID ITER - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $CAMERA = $word:3 - #$TELESCOPE= $word:4 - #$EXP_TYPE = $word:5 - #$IMFILES = $word:6 - - # XXX add $WORKDIR/$LOG_DIR - stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log - stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + # look for new images in DetrendNormExp + book getpage DetrendNormExp 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword DetrendNormExp $pageName state RUN + book getword DetrendNormExp $pageName det_id -var DET_ID + book getword DetrendNormExp $pageName iter -var ITER + book getword DetrendNormExp $pageName det_type -var DET_TYPE + book getword DetrendNormExp $pageName camera -var CAMERA # specify choice of remote host: @@ -390,6 +299,12 @@ end + # XXX add $WORKDIR/$LOG_DIR + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + + # save the pageName for future reference below + options $pageName + # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE @@ -400,19 +315,17 @@ # success task.exit 0 - queuepush DetrendNormExp -replace -key 1:2 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendNormExp -replace -key 1:2 "FAIL $options:0" - $DetrendNormExpFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendNormExp -replace -key 1:2 "TIMEOUT $options:0" - $DetrendNormExpFail ++ - end -end + book setword DetrendNormExp $options:0 state DONE + end + + # default exit status + task.exit default + showcommand failure + book setword DetrendNormExp $options:0 state FAIL + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword DetrendNormExp $options:0 state TIMEOUT + end +end Index: /branches/eam_branch_00/ippTasks/detrend.process.pro =================================================================== --- /branches/eam_branch_00/ippTasks/detrend.process.pro (revision 11010) +++ /branches/eam_branch_00/ippTasks/detrend.process.pro (revision 11011) @@ -1,18 +1,5 @@ ## this file contains the tasks for running the detrend processing stage - -# query for pending imfiles to process: -# dettool -raw -simple -# 1 bias 850131b fpa mc 850131b.fits BIAS r.MP9601 1.656000 5.496118 0.349071 0.000000 1155.934692 107.037283 46.214947 37.160000 105.480000 -133.067001 0.000000 -# this returns many columns for manual operations. we strip all but the first few: - -# process a single imfile -# detrend_process.pl 1 850131b mc bias 850131b.fits 850131b - -## these tasks use the queue DetrendProcessImfiles -## the DetrendProcessImfiles queue contains: -## STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI - -queueinit DetrendProcessImfiles -queueinit DetrendProcessExposures +## these tasks use the books DetrendProcessImfiles and DetrendProcessExposures + if ($?network == 0) $network = 1 @@ -21,8 +8,4 @@ $parallel = 0 end - -$DetrendProcessImfileFail = 0 -$DetrendProcessExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -33,12 +16,14 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init DetrendProcessImfiles +book init DetrendProcessExposures + macro detproc.status - queueprint DetrendProcessImfiles - queueprint DetrendProcessExposures - echo "DetrendProcessImfileFail: $DetrendProcessImfileFail" - echo "DetrendProcessExpFail: $DetrendProcessExpFail" + book listbook DetrendProcessImfiles + book listbook DetrendProcessExposures end @@ -57,34 +42,17 @@ task.exec - command dettool -toprocessedimfile -simple -limit {$DetrendProcessImfileFail + 20} + command dettool -toprocessedimfile -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $DET_TYPE = $word:1 - $EXP_TAG = $word:2 - $CLASS = $word:3 - $CLASS_ID = $word:4 - $URI = $word:5 - $CAMERA = $word:21 - queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $DET_TYPE $EXP_TAG $CLASS $CLASS_ID $URI $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout DetrendProcessImfiles -key det_id:exp_tag:class_id -uniq + if ($VERBOSE > 2) + book listbook DetrendProcessImfiles end # delete existing entries which are DONE - queuedrop DetrendProcessImfiles -key 0 DONE + book delpage DetrendProcessImfiles -key state DONE end @@ -100,5 +68,5 @@ end -# copy new images, sending job to desired host +# run detrend_process_imfile.ps on pending images task dettool.process.run periods -poll $runpoll @@ -107,28 +75,27 @@ task.exec - queuesize DetrendProcessImfiles -var N + book npages DetrendProcessImfiles -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendProcessImfiles queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendProcessImfiles -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendProcessImfiles -replace -key 1:3:5 "RUN $line" - - ## the DetrendProcessImfiles queue contains: - # STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $DET_TYPE = $word:1 - $EXP_TAG = $word:2 - $CLASS = $word:3 - $CLASS_ID = $word:4 - $URI = $word:5 - $CAMERA = $word:6 + # look for new images in DetrendProcessImfiles queue + book getpage DetrendProcessImfiles 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword DetrendProcessImfiles $pageName state RUN + book getword DetrendProcessImfiles $pageName det_id -var DET_ID + book getword DetrendProcessImfiles $pageName det_type -var DET_TYPE + book getword DetrendProcessImfiles $pageName exp_tag -var EXP_TAG + book getword DetrendProcessImfiles $pageName class -var CLASS + book getword DetrendProcessImfiles $pageName class_id -var CLASS_ID + book getword DetrendProcessImfiles $pageName uri -var URI + book getword DetrendProcessImfiles $pageName camera -var CAMERA + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end $word = `basename $EXP_TAG | tr '.' ' '` @@ -139,4 +106,91 @@ exec mkdir -p $LOGSUBDIR/$base + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + setcommand detrend_process_imfile.pl --det_id $DET_ID --det_type $DET_TYPE --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --input_uri $URI --camera $CAMERA + end + + # success + task.exit 0 + book setword DetrendProcessImfiles $options:0 state DONE + end + + # default exit status + task.exit default + showcommand failure + book setword DetrendProcessImfiles $options:0 state FAIL + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword DetrendProcessImfiles $options:0 state TIMEOUT + end +end + +# select images ready for copy +# new entries are added to queue DetrendProcessExposures +# compare the new list with the ones already selected +task dettool.processexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/detproc.exp.log + stderr $LOGSUBDIR/detproc.exp.log + + task.exec + command dettool -toprocessedexp -limit 20 + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout DetrendProcessExposures -key det_id:iter:exp_tag -uniq + if ($VERBOSE > 2) + book listbook DetrendProcessExposures + end + + # delete existing entries which are DONE + book delpage DetrendProcessExposures -key state DONE + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.processexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages DetrendProcessExposures -var N + if ($N == 0) break + if ($network == 0) break + + # look for new exposures in DetrendProcessExposures + book getpage DetrendProcessExposures 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword DetrendProcessExposures $pageName state RUN + book getword DetrendProcessExposures $pageName det_id -var DET_ID + book getword DetrendProcessExposures $pageName iter -var ITER + book getword DetrendProcessExposures $pageName det_type -var DET_TYPE + book getword DetrendProcessExposures $pageName exp_tag -var EXP_TAG + book getword DetrendProcessExposures $pageName camera -var CAMERA + # specify choice of remote host: if ($parallel) @@ -146,133 +200,16 @@ end - # create example job options as a demonstration - options "$line" - if ($VERBOSE > 1) - echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA - end - command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA - end - - # success - task.exit 0 - queuepush DetrendProcessImfiles -replace -key 1:3:5 "DONE $options:0" - # detrend_process_imfile.pl --force_exist --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0" - $DetrendProcessImfileFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0" - $DetrendProcessImfileFail ++ - end -end - -# select images ready for copy -# new entries are added to queue DetrendProcessExposures -# compare the new list with the ones already selected -task dettool.processexp.load - host local - - periods -poll $loadpoll - periods -exec $loadexec - periods -timeout 30 - - stdout $LOGSUBDIR/detproc.exp.log - stderr $LOGSUBDIR/detproc.exp.log - - task.exec - command dettool -toprocessedexp -simple -limit {$DetrendProcessExpFail + 20} - end - - # success - task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $EXP_TAG = $word:3 - $CAMERA = $word:4 - queuepush DetrendProcessExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_TAG $CAMERA" - end - - # delete existing entries which are DONE - queuedrop DetrendProcessExposures -key 0 DONE - end - - # error - task.exit default - showcommand failure - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# copy new images, sending job to desired host -task dettool.processexp.run - periods -poll $runpoll - periods -exec $runexec - periods -timeout 30 - - task.exec - queuesize DetrendProcessExposures -var N - if ($N == 0) break - if ($network == 0) break - - # look for new images on the internal DetrendProcessExposures queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendProcessExposures -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendProcessExposures -replace -key 1:2:4 "RUN $line" - - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $EXP_TAG = $word:3 - $CAMERA = $word:4 - # XXX this is still a lame rule $word = `basename $EXP_TAG | tr '.' ' '` list word -split $word $base = $word:0 - stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log exec mkdir -p $LOGSUBDIR/$base - # specify choice of remote host: - if ($parallel) - host anyhost - else - host local - end + # save the pageName for future reference below + options $pageName # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA @@ -283,5 +220,5 @@ # success task.exit 0 - queuepush DetrendProcessExposures -replace -key 1:2:4 "DONE $options:0" + book setword DetrendProcessExposures $options:0 state DONE end @@ -289,6 +226,5 @@ task.exit default showcommand failure - queuepush DetrendProcessExposures -replace -key 1:2:4 "FAIL $options:0" - $DetrendProcessExpFail ++ + book setword DetrendProcessExposures $options:0 state FAIL end @@ -296,6 +232,5 @@ task.exit timeout showcommand timeout - queuepush DetrendProcessExposures -replace -key 1:2:4 "TIMEOUT $options:0" - $DetrendProcessExpFail ++ - end -end + book setword DetrendProcessExposures $options:0 state TIMEOUT + end +end Index: /branches/eam_branch_00/ippTasks/detrend.reject.pro =================================================================== --- /branches/eam_branch_00/ippTasks/detrend.reject.pro (revision 11010) +++ /branches/eam_branch_00/ippTasks/detrend.reject.pro (revision 11011) @@ -1,17 +1,4 @@ - ## this file contains the tasks for running the detrend processing stage -# dettool -residdetrun -simple -# 1 0 bias -# DET_ID ITER TYPE - -# detrend_reject_exp.pl 1 0 bias -# detrend_reject_exp.pl DET_ID ITER TYPE - -## these tasks use the queue DetrendResidExp -## the DetrendRejectExp queue contains: -## STATE DET_ID ITER TYPE - -queueinit DetrendRejectExp if ($?network == 0) $network = 1 @@ -20,7 +7,4 @@ $parallel = 0 end - -$DetrendRejectExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -31,10 +15,12 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init DetrendRejectExp + macro detreject.status - queueprint DetrendRejectExp - echo "DetrendRejectExpFail: $DetrendRejectExpFail" + book listbook DetrendRejectExp end @@ -53,30 +39,17 @@ task.exec - command dettool -residdetrun -simple -limit {$DetrendRejectExpFail + 20} + command dettool -residdetrun -limit 20 end # success task.exit 0 - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $CAMERA = $word:4 - queuepush DetrendRejectExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $MODE $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout DetrendRejectExp -key det_id:iter -uniq + if ($VERBOSE > 2) + book listbook DetrendRejectExp end # delete existing entries which are DONE - queuedrop DetrendRejectExp -key 0 DONE + book delpage DetrendRejectExp -key state DONE end @@ -99,30 +72,18 @@ task.exec - queuesize DetrendRejectExp -var N + book npages DetrendRejectExp -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendRejectExp queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendRejectExp -var line -key 0 NEW - if ("$line" == "NULL") break + # look for new images in DetrendRejectExp + book getpage DetrendRejectExp 0 -var pageName -key state NULL + if ($pageName == NULL) break - strpop line state - queuepush DetrendRejectExp -replace -key 1:2 "RUN $line" - - ## the DetrendRejectExp queue contains: - ## STATE DET_ID ITER TYPE - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $CAMERA = $word:4 - - # I'd like to add CAMERA to the log file... - stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log - stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + book setword DetrendRejectExp $pageName state RUN + book getword DetrendRejectExp $pageName DET_ID -var DET_ID + book getword DetrendRejectExp $pageName ITER -var ITER + book getword DetrendRejectExp $pageName DET_TYPE -var DET_TYPE + book getword DetrendRejectExp $pageName MODE -var MODE + book getword DetrendRejectExp $pageName CAMERA -var CAMERA # specify choice of remote host: @@ -133,6 +94,12 @@ end + # I'd like to add CAMERA to the log file... + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log + + # save the pageName for future reference below + options $pageName + # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) echo command detrend_reject_exp.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE --camera $CAMERA @@ -143,5 +110,5 @@ # success task.exit 0 - queuepush DetrendRejectExp -replace -key 1:2 "DONE $options:0" + book setword DetrendRejectExp $options:0 state DONE end @@ -149,6 +116,5 @@ task.exit default showcommand failure - queuepush DetrendRejectExp -replace -key 1:2 "FAIL $options:0" - $DetrendRejectExpFail ++ + book setword DetrendRejectExp $options:0 state FAIL end @@ -156,6 +122,5 @@ task.exit timeout showcommand failure - queuepush DetrendRejectExp -replace -key 1:2 "TIMEOUT $options:0" - $DetrendRejectExpFail ++ + book setword DetrendRejectExp $options:0 state TIMEOUT end end Index: /branches/eam_branch_00/ippTasks/detrend.resid.pro =================================================================== --- /branches/eam_branch_00/ippTasks/detrend.resid.pro (revision 11010) +++ /branches/eam_branch_00/ippTasks/detrend.resid.pro (revision 11011) @@ -1,19 +1,4 @@ - ## this file contains the tasks for running the detrend processing stage -# dettool -toresidimfile -simple -# 0 bias 1 850131b mc o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.565582 -# ITER TYPE DET_ID EXP_TAG CLASS_ID URI-IN - -# detrend_create_resid.pl 1 0 850131b mc bias bias_mc_1_0.fit o_850131b.mc.fit 850131b -# detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG) -# URL-DET: $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit - -## these tasks use the queue DetrendResidImfiles -## the DetrendResidImfiles queue contains: -## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - -queueinit DetrendResidImfiles -queueinit DetrendResidExposures if ($?network == 0) $network = 1 @@ -22,8 +7,4 @@ $parallel = 0 end - -$DetrendResidImfileFail = 0 -$DetrendResidExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -34,12 +15,14 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init DetrendResidImfiles +book init DetrendResidExposures + macro detresid.status - queueprint DetrendResidImfiles - queueprint DetrendResidExposures - echo "DetrendResidImfileFail: $DetrendResidImfileFail" - echo "DetrendResidExpFail: $DetrendResidExpFail" + book listbook DetrendResidImfiles + book listbook DetrendResidExposures end @@ -58,41 +41,19 @@ task.exec - command dettool -toresidimfile -simple -limit {$DetrendResidImfileFail + 20} + command dettool -toresidimfile -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $CLASS_ID = $word:5 - $URI = $word:6 - $DET_URI = $word:7 - $CAMERA = $word:8 - ## must include the detID and iter in the key (if we have two running at the same time...) - ## Also needed to be in the output file (or path) - queuepush DetrendResidImfiles -uniq -key 1:2:5:6 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $CLASS_ID $URI $DET_URI $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout DetrendResidImfiles -key det_id:iter:exp_tag:class_id -uniq + if ($VERBOSE > 2) + book listbook DetrendResidImfiles end # delete existing entries which are DONE - queuedrop DetrendResidImfiles -key 0 DONE - end - - # locked list + book delpage DetrendResidImfiles -key state DONE + end + # default exit status task.exit default @@ -113,30 +74,29 @@ task.exec - queuesize DetrendResidImfiles -var N + book npages DetrendResidImfiles -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendResidImfiles queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendResidImfiles -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "RUN $line" - - ## the DetrendResidImfiles queue contains: - ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $CLASS_ID = $word:5 - $URI = $word:6 - $DET_URI = $word:7 - $CAMERA = $word:8 + # look for new images in DetrendResidImfiles + book getpage DetrendResidImfiles 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword DetrendResidImfiles $pageName state RUN + book getword DetrendResidImfiles $pageName DET_ID -var DET_ID + book getword DetrendResidImfiles $pageName ITER -var ITER + book getword DetrendResidImfiles $pageName DET_TYPE -var DET_TYPE + book getword DetrendResidImfiles $pageName MODE -var MODE + book getword DetrendResidImfiles $pageName EXP_TAG -var EXP_TAG + book getword DetrendResidImfiles $pageName CLASS_ID -var CLASS_ID + book getword DetrendResidImfiles $pageName URI -var URI + book getword DetrendResidImfiles $pageName DET_URI -var DET_URI + book getword DetrendResidImfiles $pageName CAMERA -var CAMERA + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end $word = `basename $EXP_TAG | tr '.' ' '` @@ -147,4 +107,96 @@ exec mkdir -p $LOGSUBDIR/$base + # save the pageName for future reference below + options $pageName + + # create command + if ($VERBOSE > 1) + echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + end + command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + end + + # success + task.exit 0 + book setword DetrendResidImfiles $options:0 state DONE + end + + # default exit status + task.exit default + showcommand failure + book setword DetrendResidImfiles $options:0 state FAIL + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword DetrendResidImfiles $options:0 state TIMEOUT + end +end + +# select images ready for copy +# new entries are added to DetrendResidExposures +# compare the new list with the ones already selected +task dettool.residexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout $LOGSUBDIR/detresid.exp.log + stderr $LOGSUBDIR/detresid.exp.log + + task.exec + command dettool -toresidexp -limit 20 + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout DetrendResidExposures -key det_id:iter:exp_tag -uniq + if ($VERBOSE > 2) + book listbook DetrendResidExposures + end + + # delete existing entries which are DONE + book delpage DetrendResidExposures -key state DONE + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.residexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages DetrendResidExposures -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in DetrendResidExposures + book getpage DetrendResidExposures 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword DetrendResidExposures $pageName state RUN + book getword DetrendResidExposures $pageName DET_ID -var DET_ID + book getword DetrendResidExposures $pageName ITER -var ITER + book getword DetrendResidExposures $pageName DET_TYPE -var DET_TYPE + book getword DetrendResidExposures $pageName MODE -var MODE + book getword DetrendResidExposures $pageName EXP_TAG -var EXP_TAG + book getword DetrendResidExposures $pageName INCLUDE -var INCLUDE + book getword DetrendResidExposures $pageName CAMERA -var CAMERA + # specify choice of remote host: if ($parallel) @@ -154,141 +206,16 @@ end - # create example job options as a demonstration - options "$line" - # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???) - # need to parse the subdir path for the detrend URL - # $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits - echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE - command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE - end - - # success - task.exit 0 - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendResidImfiles -replace -key 1:2:5:6 "FAIL $options:0" - $DetrendResidImfileFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendResidImfiles -replace -key 1:2:6:6 "TIMEOUT $options:0" - $DetrendResidImfileFail ++ - end -end - -# select images ready for copy -# new entries are added to queue DetrendResidExposures -# compare the new list with the ones already selected -task dettool.residexp.load - host local - - periods -poll $loadpoll - periods -exec $loadexec - periods -timeout 30 - - stdout $LOGSUBDIR/detresid.exp.log - stderr $LOGSUBDIR/detresid.exp.log - - task.exec - command dettool -toresidexp -simple -limit {$DetrendResidExpFail + 20} - end - - # success - task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $INCLUDE = $word:5 - $CAMERA = $word:6 - queuepush DetrendResidExposures -uniq -key 1:2:5 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $INCLUDE $CAMERA" - end - - # delete existing entries which are DONE - queuedrop DetrendResidExposures -key 0 DONE - end - - # locked list - # default exit status - task.exit default - showcommand failure - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# copy new images, sending job to desired host -task dettool.residexp.run - periods -poll $runpoll - periods -exec $runexec - periods -timeout 30 - - task.exec - queuesize DetrendResidExposures -var N - if ($N == 0) break - if ($network == 0) break - - # look for new images on the internal DetrendResidExposures queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendResidExposures -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendResidExposures -replace -key 1:2:5 "RUN $line" - - ## the DetrendResidExposures queue contains: - ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $MODE = $word:3 - $EXP_TAG = $word:4 - $INCLUDE = $word:5 - $CAMERA = $word:6 - # XXX this is still a lame rule $word = `basename $EXP_TAG | tr '.' ' '` list word -split $word $base = $word:0 - stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log exec mkdir -p $LOGSUBDIR/$base - # specify choice of remote host: - if ($parallel) - host anyhost - else - host local - end + # save the pageName for future reference below + options $pageName # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA @@ -299,19 +226,17 @@ # success task.exit 0 - queuepush DetrendResidExposures -replace -key 1:2:5 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendResidExposures -replace -key 1:2:5 "FAIL $options:0" - $DetrendResidExpFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendResidExposures -replace -key 1:2:5 "TIMEOUT $options:0" - $DetrendResidExpFail ++ - end -end + book setword DetrendResidExposures $options:0 state DONE + end + + # default exit status + task.exit default + showcommand failure + book setword DetrendResidExposures $options:0 state FAIL + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword DetrendResidExposures $options:0 state TIMEOUT + end +end Index: /branches/eam_branch_00/ippTasks/detrend.stack.pro =================================================================== --- /branches/eam_branch_00/ippTasks/detrend.stack.pro (revision 11010) +++ /branches/eam_branch_00/ippTasks/detrend.stack.pro (revision 11011) @@ -1,17 +1,4 @@ ## this file contains the tasks for running the detrend stacking stage -# get a list of imfiles to stack -# dettool -tostack -simple -# 1 0 bias mc -# DET_ID ITER TYPE CLASS_ID - -# function to stack the imfiles: -# detrend_stack.pl 1 0 mc bias - -## these tasks use the queue DetrendStackClass -## the DetrendStackClass queue contains: -## STATE DET_ID ITER TYPE CLASS_ID CAMERA - -queueinit DetrendStackClass if ($?network == 0) $network = 1 @@ -20,7 +7,4 @@ $parallel = 0 end - -$DetrendStackClassFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -31,6 +15,9 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR + +book init DetrendStackClass macro detstack.status @@ -39,6 +26,6 @@ end -# select images ready for copy -# new entries are added to queue DetrendStackClass +# select images ready for detrend_stack.pl +# new entries are added to DetrendStackClass # compare the new list with the ones already selected task dettool.stack.load @@ -53,30 +40,17 @@ task.exec - command dettool -tostacked -simple -limit {$DetrendStackClassFail + 20} + command dettool -tostacked -limit 20 end # success task.exit 0 - local i Nqueue - - # compare output with newImage queue - # only add entries which do not exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $CLASS_ID = $word:3 - $CAMERA = $word:4 - queuepush DetrendStackClass -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $CLASS_ID $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout DetrendStackClass -key det_id:iter:class_id -uniq + if ($VERBOSE > 2) + book listbook DetrendStackClass end # delete existing entries which are DONE - queuedrop DetrendStackClass -key 0 DONE + book delpage DetrendStackClass -key state DONE end @@ -99,31 +73,18 @@ task.exec - queuesize DetrendStackClass -var N + book npages DetrendStackClass -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendStackClass queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendStackClass -var line -key 0 NEW - if ("$line" == "NULL") break + # look for new images in DetrendStackClass + book getpage DetrendStackClass 0 -var pageName -key state NULL + if ($pageName == NULL) break - strpop line state - queuepush DetrendStackClass -replace -key 1:2:4 "RUN $line" - - # the DetrendStackClass queue contains: - # STATE DET_ID ITER TYPE CLASS_ID - ## NOTE: the queue has the STATE prepended - # use local variables - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $CLASS_ID = $word:3 - $CAMERA = $word:4 - - # XXX add $WORKDIR/$LOG_DIR - stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log - stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log + book setword DetrendStackClass $pageName state RUN + book getword DetrendStackClass $pageName det_id -var DET_ID + book getword DetrendStackClass $pageName iter -var ITER + book getword DetrendStackClass $pageName det_type -var DET_TYPE + book getword DetrendStackClass $pageName class_id -var CLASS_ID + book getword DetrendStackClass $pageName camera -var CAMERA # specify choice of remote host: @@ -134,6 +95,12 @@ end - # create example job options as a demonstration - options "$line" + # XXX add $WORKDIR/$LOG_DIR + stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log + stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.log + + # save the pageName for future reference below + options $pageName + + # create the command line if ($VERBOSE > 1) echo command detrend_stack.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA @@ -144,5 +111,5 @@ # success task.exit 0 - queuepush DetrendStackClass -replace -key 1:2:4 "DONE $options:0" + book setword DetrendStackClass $options:0 state DONE end @@ -150,6 +117,5 @@ task.exit default showcommand failure - queuepush DetrendStackClass -replace -key 1:2:4 "FAIL $options:0" - $DetrendStackClassFail ++ + book setword DetrendStackClass $options:0 state FAIL end @@ -157,6 +123,5 @@ task.exit timeout showcommand timeout - queuepush DetrendStackClass -replace -key 1:2:4 "TIMEOUT $options:0" - $DetrendStackClassFail ++ + book setword DetrendStackClass $options:0 state TIMEOUT end end Index: /branches/eam_branch_00/ippTasks/pantasks.pro =================================================================== --- /branches/eam_branch_00/ippTasks/pantasks.pro (revision 11010) +++ /branches/eam_branch_00/ippTasks/pantasks.pro (revision 11011) @@ -80,2 +80,14 @@ end end + +macro setcommand + local n + $command = "" + for n 1 $0 + $command = $command $$n + end + if ($VERBOSE > 1) + echo "command $command" + end + command $command +end Index: /branches/eam_branch_00/ippTasks/phase0.pro =================================================================== --- /branches/eam_branch_00/ippTasks/phase0.pro (revision 11010) +++ /branches/eam_branch_00/ippTasks/phase0.pro (revision 11011) @@ -1,11 +1,5 @@ ## this file contains the tasks for running the phase 0 stage - -## these tasks use the queue Phase0Imfiles -## the Phase0Imfiles queue contains: -## STATE EXP_TAG CLASS CLASS_ID URI-FITS -## queue keys are counted from 0 - -queueinit Phase0Imfiles -queueinit Phase0Exposures +## these tasks use the books Phase0Imfiles and Phase0Exposures + if ($?network == 0) $network = 1 @@ -14,8 +8,4 @@ $parallel = 0 end - -$Phase0ImfileFail = 0 -$Phase0ExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -26,12 +16,14 @@ break end + $LOGSUBDIR = $LOGDIR/phase0 exec mkdir -p $LOGSUBDIR +book init Phase0Imfiles +book init Phase0Exposures + macro phase0.status - queueprint Phase0Imfiles - queueprint Phase0Exposures - echo "Phase0ImfileFail: $Phase0ImfileFail" - echo "Phase0ExpFail: $Phase0ExpFail" + book listbook Phase0Imfiles + book listbook Phase0Exposures end @@ -50,29 +42,17 @@ task.exec - command p0tool -pendingimfile -simple -limit {$Phase0ImfileFail + 20} + command p0tool -pendingimfile -limit 20 end # success task.exit 0 - local i Nqueue - - # compare output with Phase0Imfiles queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $EXP_TAG = $word:0 - $CLASS = $word:1 - $CLASS_ID = $word:2 - $URI = $word:3 - queuepush Phase0Imfiles -uniq -key 1 "NEW $EXP_TAG $CLASS $CLASS_ID $URI" + # convert 'stdout' to book format + ipptool2book stdout Phase0Imfiles -key exp_tag -uniq + if ($VERBOSE > 2) + book listbook Phase0Imfiles end # delete existing entries which are DONE - queuedrop Phase0Imfiles -key 0 DONE + book delpage Phase0Imfiles -key state DONE end @@ -88,7 +68,5 @@ end - - -# run the phase0imfile script on pending images +# run the phase0_imfile.pl script on pending images task phase0.imfile.run periods -poll $runpoll @@ -97,26 +75,19 @@ task.exec - queuesize Phase0Imfiles -var N + book npages Phase0Imfiles -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendImfilesToProcess queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop Phase0Imfiles -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush Phase0Imfiles -replace -key 1 "RUN $line" - - ## the Phase0Imfiles queue contains (line has popped STATE) - ## STATE EXP_TAG CLASS CLASS_ID URI-FITS - list word -split $line - $EXP_TAG = $word:0 - $CLASS = $word:1 - $CLASS_ID = $word:2 - $URI = $word:3 - - # specify choice of remote host:(need to choose based on chips) + # look for new images in Phase0Imfiles + book getpage Phase0Imfiles 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword Phase0Imfiles $pageName state RUN + book getword Phase0Imfiles $pageName exp_tag -var EXP_TAG + book getword Phase0Imfiles $pageName class -var CLASS + book getword Phase0Imfiles $pageName class_id -var CLASS_ID + book getword Phase0Imfiles $pageName uri -var URI + + # specify choice of remote host if ($parallel) host anyhost @@ -133,6 +104,8 @@ exec mkdir -p $LOGSUBDIR/$base + # save the pageName for future reference below + options $pageName + # create the command line - options "$line" if ($VERBOSE > 1) echo command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI @@ -143,6 +116,6 @@ # success task.exit 0 - # phase0imfile.pl updates DB tables, here we just update the queue - queuepush Phase0Imfiles -replace -key 1 "DONE $options:0" + # phase0_imfile.pl updates DB tables, here we just update the queue + book setword Phase0Imfiles $options:0 state DONE end @@ -150,6 +123,5 @@ task.exit default showcommand failure - queuepush Phase0Imfiles -replace -key 1 "FAIL $options:0" - $Phase0ImfileFail ++ + book setword Phase0Imfiles $options:0 state FAIL end @@ -157,18 +129,9 @@ task.exit timeout showcommand timeout - queuepush Phase0Imfiles -replace -key 1 "TIMEOUT $options:0" - $Phase0ImfileFail ++ - end -end - - - -## these tasks use the queue Phase0Exposures -## the Phase0Exposures queue contains: -## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS - -# select images ready for phase0 analysis -# new entries are added to queue Phase0Exposures -# compare the new list with the ones already selected + book setword Phase0Imfiles $options:0 state TIMEOUT + end +end + +# select exposures ready for phase0_exp.pl task phase0.exp.load host local @@ -182,48 +145,31 @@ task.exec - command p0tool -pendingexp -simple -limit {$Phase0ExpFail + 20} + command p0tool -pendingexp -limit 20 end # success task.exit 0 - local i Nqueue - - # compare output with Phase0Exposures queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $EXP_TAG = $word:0 - # $EXP_ID = $word:1 - # $CAMERA = $word:2 - # $TELESCOPE = $word:3 - # $DATEOBS = $word:4 - # $TYPE = $word:5 - # $IMFILES = $word:6 - queuepush Phase0Exposures -uniq -key 1 "NEW $EXP_TAG" + # convert 'stdout' to book format + ipptool2book stdout Phase0Exposures -key exp_tag -uniq + if ($VERBOSE > 2) + book listbook Phase0Exposures end # delete existing entries which are DONE - queuedrop Phase0Exposures -key 0 DONE + book delpage Phase0Exposures -key state DONE end # default exit status task.exit default - showcommand - end - - # operation times out? - task.exit timeout - showcommand - end -end - - - -# run the phase0imfile script on pending images + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the phase0_exp.pl script on pending exposures task phase0.exp.run periods -poll $runpoll @@ -232,21 +178,21 @@ task.exec - queuesize Phase0Exposures -var N + book npages Phase0Exposures -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendImfilesToProcess queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop Phase0Exposures -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush Phase0Exposures -replace -key 1 "RUN $line" - - ## the Phase0Exposures queue contains: - ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS - list word -split $line - $EXP_TAG = $word:0 + # look for new images in Phase0Exposures (state == NULL) + book getpage Phase0Exposures 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword Phase0Exposures $pageName state RUN + book getword Phase0Exposures $pageName exp_tag -var EXP_TAG + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end # XXX this is still a lame rule @@ -258,13 +204,8 @@ exec mkdir -p $LOGSUBDIR/$base - # specify choice of remote host:(need to choose based on chips) - if ($parallel) - host anyhost - else - host local - end + # save the pageName for future reference below + options $pageName # create the command line - options "$line" if ($VERBOSE > 1) echo command phase0_exp.pl --exp_tag $EXP_TAG @@ -276,5 +217,5 @@ task.exit 0 # phase0exp.pl updates DB tables, here we just update the queue - queuepush Phase0Exposures -replace -key 1 "DONE $options:0" + book setword Phase0Exposures $options:0 state DONE end @@ -282,6 +223,5 @@ task.exit default showcommand - queuepush Phase0Exposures -replace -key 1 "FAIL $options:0" - $Phase0ExpFail + book setword Phase0Exposures $options:0 state FAIL end @@ -289,6 +229,5 @@ task.exit timeout showcommand - queuepush Phase0Exposures -replace -key 1 "TIMEOUT $options:0" - $Phase0ExpFail + book setword Phase0Exposures $options:0 state TIMEOUT end end Index: /branches/eam_branch_00/ippTasks/phase2.pro =================================================================== --- /branches/eam_branch_00/ippTasks/phase2.pro (revision 11010) +++ /branches/eam_branch_00/ippTasks/phase2.pro (revision 11011) @@ -1,12 +1,5 @@ ## this file contains the tasks for running the phase 0 stage +## these tasks use the book p2PendingImfile -## these tasks use the queue Phase2Imfiles -## the Phase2Imfiles queue contains: -## STATE EXP_TAG CLASS CLASS_ID URI-FITS -## queue keys are counted from 0 - -# XXX change book names to match ippTool values? -# XXX replace with book init p2pending -queueinit Phase2Imfiles if ($?network == 0) $network = 1 @@ -15,7 +8,4 @@ $parallel = 0 end - -$Phase2ImfileFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -26,15 +16,17 @@ break end + $LOGSUBDIR = $LOGDIR/phase2 exec mkdir -p $LOGSUBDIR +book init p2PendingImfile + macro phase2.status - book listbook p2pending - echo "Phase2ImfileFail: $Phase2ImfileFail" + book listbook p2PendingImfile end # select images ready for phase2 analysis -# new entries are added to queue Phase2Imfiles -# compare the new list with the ones already selected +# new entries are added to p2PendingImfile +# skip already-present entries task phase2.imfile.load host local @@ -48,6 +40,4 @@ task.exec - # command p2tool -pendingimfile -simple -limit {$Phase2ImfileFail + 20} - # XXX make sure we remove FAIL and TIMEOUT entries command p2tool -pendingimfile -limit 20 end @@ -55,15 +45,12 @@ # success task.exit 0 - # stdout from the task is stored on queue 'stdout' # convert 'stdout' to book format - # only add entries which don't exist in the book - # 'state' has value of NULL - ipptool2book stdout p2pending -key exp_tag:class_id -uniq + ipptool2book stdout p2PendingImfile -key exp_tag:class_id -uniq if ($VERBOSE > 2) - book listbook p2pending + book listbook p2PendingImfile end # delete existing entries which are DONE - book delpage p2pending -key state DONE + book delpage p2PendingImfile -key state DONE end @@ -79,5 +66,5 @@ end -# run the phase2imfile script on pending images +# run the phase2.pl script on pending images task phase2.imfile.run periods -poll $runpoll @@ -86,20 +73,20 @@ task.exec - book npages p2pending -var N + book npages p2PendingImfile -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal p2pending book - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - book getpage p2pending 0 -var pageName -key state NULL - book setword p2pending $pageName state RUN + # look for new images in p2PendingImfile (state == NULL) + book getpage p2PendingImfile 0 -var pageName -key state NULL + if ($pageName == NULL) break - book getword p2pending $pageName camera -var CAMERA - book getword p2pending $pageName exp_tag -var EXP_TAG - book getword p2pending $pageName class_id -var CLASS_ID - book getword p2pending $pageName uri -var URI + book setword p2PendingImfile $pageName state RUN + book getword p2PendingImfile $pageName camera -var CAMERA + book getword p2PendingImfile $pageName exp_tag -var EXP_TAG + book getword p2PendingImfile $pageName class_id -var CLASS_ID + book getword p2PendingImfile $pageName uri -var URI - # specify choice of remote host:(need to choose based on chips) + # specify choice of remote host + # XXX need to choose based on chips if ($parallel) host anyhost @@ -118,5 +105,5 @@ exec mkdir -p $LOGSUBDIR/$base - # save the p2pending book pageName for future reference below + # save the pageName for future reference below options $pageName @@ -130,7 +117,6 @@ # success task.exit 0 - # phase2imfile.pl updates DB tables, here we just update the queue - book setword p2pending $options:0 state DONE - # queuepush Phase2Imfiles -replace -key 1:2 "DONE $options:0" + # phase2.pl updates DB tables, here we just update the queue + book setword p2PendingImfile $options:0 state DONE end @@ -138,16 +124,11 @@ task.exit default showcommand failure - book setword p2pending $options:0 state FAIL - # queuepush Phase2Imfiles -replace -key 1:2 "FAIL $options:0" - # drop failed entries from the book lists? - $Phase2ImfileFail ++ + book setword p2PendingImfile $options:0 state FAIL end - # operation times out? + # operation timed out? task.exit timeout showcommand timeout - book setword p2pending $options:0 state TIMEOUT - # queuepush Phase2Imfiles -replace -key 1:2 "TIMEOUT $options:0" - $Phase2ImfileFail ++ + book setword p2PendingImfile $options:0 state TIMEOUT end end Index: /branches/eam_branch_00/ippTasks/phase3.pro =================================================================== --- /branches/eam_branch_00/ippTasks/phase3.pro (revision 11010) +++ /branches/eam_branch_00/ippTasks/phase3.pro (revision 11011) @@ -1,10 +1,5 @@ -## this file contains the tasks for running the phase 0 stage +## this file contains the tasks for running the phase 3 stage +## these tasks use the book Phase3Exposures -## these tasks use the queue Phase3Exposures -## the Phase3Exposures queue contains: -## STATE EXP_TAG CLASS CLASS_ID URI-FITS -## queue keys are counted from 0 - -queueinit Phase3Exposures if ($?network == 0) $network = 1 @@ -13,7 +8,4 @@ $parallel = 0 end - -$Phase3ExposureFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -24,15 +16,17 @@ break end + $LOGSUBDIR = $LOGDIR/phase3 exec mkdir -p $LOGSUBDIR +book init Phase3Exposures + macro phase3.status - queueprint Phase3Exposures - echo "Phase3ExposureFail: $Phase3ExposureFail" + book listbook Phase3Exposures end # select images ready for phase3 analysis -# new entries are added to queue Phase3Exposures -# compare the new list with the ones already selected +# new entries are added to Phase3Exposures +# skip already-present entries task phase3.exp.load host local @@ -46,26 +40,17 @@ task.exec - command p3tool -pendingexp -simple + command p3tool -pendingexp -limit 20 end # success task.exit 0 - local i Nqueue - - # compare output with Phase3Exposures queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $EXP_TAG = $word:0 - queuepush Phase3Exposures -uniq -key 1 "NEW $EXP_TAG" + # convert 'stdout' to book format + ipptool2book stdout Phase3Exposures -key exp_tag -uniq + if ($VERBOSE > 2) + book listbook Phase3Exposures end # delete existing entries which are DONE - queuedrop Phase3Exposures -key 0 DONE + book delpage Phase3Exposures -key state DONE end @@ -88,21 +73,22 @@ task.exec - queuesize Phase3Exposures -var N + book npages Phase3Exposures -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendExposuresToProcess queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop Phase3Exposures -var line -key 0 NEW - if ("$line" == "NULL") break + # look for new images in Phase3Exposures (state == NULL) + book getpage Phase3Exposures 0 -var pageName -key state NULL + if ($pageName == NULL) break - strpop line state - queuepush Phase3Exposures -replace -key 1 "RUN $line" + book setword Phase3Exposures $pageName state RUN + book getword Phase3Exposures $pageName camera -var CAMERA + book getword Phase3Exposures $pageName exp_tag -var EXP_TAG - ## the Phase3Exposures queue contains: - ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS - list word -split $line - $EXP_TAG = $word:0 + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end # XXX this is still a lame rule @@ -114,17 +100,12 @@ exec mkdir -p $LOGSUBDIR/$base - # specify choice of remote host:(need to choose based on chips) - if ($parallel) - host anyhost - else - host local - end + # save the pageName for future reference below + options $pageName # create the command line - options "$line" if ($VERBOSE > 1) echo command phase3.pl --exp_tag $EXP_TAG end - command phase3.pl --exp_tag $EXP_TAG --camera CTIO_MOSAIC2 + command phase3.pl --exp_tag $EXP_TAG --camera $CAMERA end @@ -132,5 +113,5 @@ task.exit 0 # phase3exp.pl updates DB tables, here we just update the queue - queuepush Phase3Exposures -replace -key 1 "DONE $options:0" + book setword Phase3Exposures $options:0 state DONE end @@ -138,6 +119,5 @@ task.exit default showcommand failure - queuepush Phase3Exposures -replace -key 1 "FAIL $options:0" - $Phase3ExposureFail ++ + book setword Phase3Exposures $options:0 state FAIL end @@ -145,6 +125,5 @@ task.exit timeout showcommand timeout - queuepush Phase3Exposures -replace -key 1 "TIMEOUT $options:0" - $Phase3ExposureFail ++ + book setword Phase3Exposures $options:0 state TIMEOUT end end