Index: trunk/ippTasks/detrend.process.pro =================================================================== --- trunk/ippTasks/detrend.process.pro (revision 10745) +++ trunk/ippTasks/detrend.process.pro (revision 11210) @@ -1,18 +1,5 @@ ## this file contains the tasks for running the detrend processing stage - -# query for pending imfiles to process: -# dettool -raw -simple -# 1 bias 850131b fpa mc 850131b.fits BIAS r.MP9601 1.656000 5.496118 0.349071 0.000000 1155.934692 107.037283 46.214947 37.160000 105.480000 -133.067001 0.000000 -# this returns many columns for manual operations. we strip all but the first few: - -# process a single imfile -# detrend_process.pl 1 850131b mc bias 850131b.fits 850131b - -## these tasks use the queue DetrendProcessImfiles -## the DetrendProcessImfiles queue contains: -## STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI - -queueinit DetrendProcessImfiles -queueinit DetrendProcessExposures +## these tasks use the books detPendingProcessedImfile and detPendingProcessedExp + if ($?network == 0) $network = 1 @@ -21,8 +8,4 @@ $parallel = 0 end - -$DetrendProcessImfileFail = 0 -$DetrendProcessExpFail = 0 - if ($?VERBOSE == 0) echo "VERBOSE not defined: load pantasks.pro first" @@ -33,16 +16,18 @@ break end + $LOGSUBDIR = $LOGDIR/detrend exec mkdir -p $LOGSUBDIR +book init detPendingProcessedImfile +book init detPendingProcessedExp + macro detproc.status - queueprint DetrendProcessImfiles - queueprint DetrendProcessExposures - echo "DetrendProcessImfileFail: $DetrendProcessImfileFail" - echo "DetrendProcessExpFail: $DetrendProcessExpFail" + book listbook detPendingProcessedImfile + book listbook detPendingProcessedExp end # select images ready for copy -# new entries are added to queue DetrendProcessImfiles +# new entries are added to detPendingProcessedImfile # compare the new list with the ones already selected task dettool.process.load @@ -53,38 +38,21 @@ periods -timeout 30 - stdout $LOGSUBDIR/detproc.imfile.log + stdout NULL stderr $LOGSUBDIR/detproc.imfile.log task.exec - command dettool -toprocessedimfile -simple -limit {$DetrendProcessImfileFail + 20} + command dettool -toprocessedimfile -limit 20 end # success task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $DET_TYPE = $word:1 - $EXP_TAG = $word:2 - $CLASS = $word:3 - $CLASS_ID = $word:4 - $URI = $word:5 - $CAMERA = $word:21 - queuepush DetrendProcessImfiles -uniq -key 1:3:5 "NEW $DET_ID $DET_TYPE $EXP_TAG $CLASS $CLASS_ID $URI $CAMERA" + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedImfile -key det_id:exp_tag:class_id -uniq + if ($VERBOSE > 2) + book listbook detPendingProcessedImfile end # delete existing entries which are DONE - queuedrop DetrendProcessImfiles -key 0 DONE + book delpage detPendingProcessedImfile -key state DONE end @@ -100,5 +68,5 @@ end -# copy new images, sending job to desired host +# run detrend_process_imfile.ps on pending images task dettool.process.run periods -poll $runpoll @@ -107,29 +75,31 @@ task.exec - queuesize DetrendProcessImfiles -var N + book npages detPendingProcessedImfile -var N if ($N == 0) break if ($network == 0) break - # look for new images on the internal DetrendProcessImfiles queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendProcessImfiles -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendProcessImfiles -replace -key 1:3:5 "RUN $line" - - ## the DetrendProcessImfiles queue contains: - # STATE DET_ID TYPE EXP_TAG CLASS CLASS_ID URI - ## NOTE: the queue has the STATE prepended - list word -split $line - $DET_ID = $word:0 - $DET_TYPE = $word:1 - $EXP_TAG = $word:2 - $CLASS = $word:3 - $CLASS_ID = $word:4 - $URI = $word:5 - $CAMERA = $word:6 - + # look for new images in detPendingProcessedImfile + book getpage detPendingProcessedImfile 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword detPendingProcessedImfile $pageName state RUN + book getword detPendingProcessedImfile $pageName det_id -var DET_ID + book getword detPendingProcessedImfile $pageName det_type -var DET_TYPE + book getword detPendingProcessedImfile $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedImfile $pageName class -var CLASS + book getword detPendingProcessedImfile $pageName class_id -var CLASS_ID + book getword detPendingProcessedImfile $pageName uri -var URI + book getword detPendingProcessedImfile $pageName camera -var CAMERA + + # specify choice of remote host: + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + ## XXX use OUTPATH from db to set output path + ## XXX use ipp_filename.pl DETPROC.LOG $outroot $CLASS_ID` $word = `basename $EXP_TAG | tr '.' ' '` list word -split $word @@ -139,4 +109,88 @@ exec mkdir -p $LOGSUBDIR/$base + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command detrend_process_imfile.pl --det_id $DET_ID --det_type $DET_TYPE --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --input_uri $URI --camera $CAMERA + end + command detrend_process_imfile.pl --det_id $DET_ID --det_type $DET_TYPE --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --input_uri $URI --camera $CAMERA + end + + # default exit status + task.exit default + process_exit detPendingProcessedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedImfile $options:0 state TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingProcessedExp +# compare the new list with the ones already selected +task dettool.processexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + + stdout NULL + stderr $LOGSUBDIR/detproc.exp.log + + task.exec + command dettool -toprocessedexp -limit 20 + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedExp -key det_id:iteration:exp_tag -uniq + if ($VERBOSE > 2) + book listbook detPendingProcessedExp + end + + # delete existing entries which are DONE + book delpage detPendingProcessedExp -key state DONE + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task dettool.processexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingProcessedExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new exposures in detPendingProcessedExp + book getpage detPendingProcessedExp 0 -var pageName -key state NULL + if ($pageName == NULL) break + + book setword detPendingProcessedExp $pageName state RUN + book getword detPendingProcessedExp $pageName det_id -var DET_ID + book getword detPendingProcessedExp $pageName iteration -var ITERATION + book getword detPendingProcessedExp $pageName det_type -var DET_TYPE + book getword detPendingProcessedExp $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedExp $pageName camera -var CAMERA + # specify choice of remote host: if ($parallel) @@ -146,133 +200,16 @@ end - # create example job options as a demonstration - options "$line" - if ($VERBOSE > 1) - echo command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA - end - command detrend_process_imfile.pl --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA - end - - # success - task.exit 0 - queuepush DetrendProcessImfiles -replace -key 1:3:5 "DONE $options:0" - # detrend_process_imfile.pl --force_exist --det_id $DET_ID --exp_tag $EXP_TAG --class $CLASS --class_id $CLASS_ID --det_type $DET_TYPE --input_uri $URI --camera $CAMERA - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendProcessImfiles -replace -key 1:3:5 "FAIL $options:0" - $DetrendProcessImfileFail ++ - end - - # operation times out? - task.exit timeout - showcommand timeout - queuepush DetrendProcessImfiles -replace -key 1:3:5 "TIMEOUT $options:0" - $DetrendProcessImfileFail ++ - end -end - -# select images ready for copy -# new entries are added to queue DetrendProcessExposures -# compare the new list with the ones already selected -task dettool.processexp.load - host local - - periods -poll $loadpoll - periods -exec $loadexec - periods -timeout 30 - - stdout $LOGSUBDIR/detproc.exp.log - stderr $LOGSUBDIR/detproc.exp.log - - task.exec - command dettool -toprocessedexp -simple -limit {$DetrendProcessExpFail + 20} - end - - # success - task.exit 0 - # XXX is it necessary for these to be local? - # drop after this task macro is done? - local i Nqueue - - # compare output with newImage queue - # only add entries which don't exist in queue - queuesize stdout -var Nqueue - for i 0 $Nqueue - queuepop stdout -var line - if ($VERBOSE > 2) - echo $line - end - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $EXP_TAG = $word:3 - $CAMERA = $word:4 - queuepush DetrendProcessExposures -uniq -key 1:2:4 "NEW $DET_ID $ITER $DET_TYPE $EXP_TAG $CAMERA" - end - - # delete existing entries which are DONE - queuedrop DetrendProcessExposures -key 0 DONE - end - - # error - task.exit default - showcommand failure - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# copy new images, sending job to desired host -task dettool.processexp.run - periods -poll $runpoll - periods -exec $runexec - periods -timeout 30 - - task.exec - queuesize DetrendProcessExposures -var N - if ($N == 0) break - if ($network == 0) break - - # look for new images on the internal DetrendProcessExposures queue - # caution with these 'if' statements: syntax errors - # will make the task fail without given a good status - queuepop DetrendProcessExposures -var line -key 0 NEW - if ("$line" == "NULL") break - - strpop line state - queuepush DetrendProcessExposures -replace -key 1:2:4 "RUN $line" - - list word -split $line - $DET_ID = $word:0 - $ITER = $word:1 - $DET_TYPE = $word:2 - $EXP_TAG = $word:3 - $CAMERA = $word:4 - # XXX this is still a lame rule $word = `basename $EXP_TAG | tr '.' ' '` list word -split $word $base = $word:0 - stdout $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log stderr $LOGSUBDIR/$base/$EXP_TAG.detproc.$DET_ID.log exec mkdir -p $LOGSUBDIR/$base - # specify choice of remote host: - if ($parallel) - host anyhost - else - host local - end + # save the pageName for future reference below + options $pageName # create example job options as a demonstration - options "$line" if ($VERBOSE > 1) echo command detrend_process_exp.pl --det_id $DET_ID --exp_tag $EXP_TAG --camera $CAMERA @@ -282,13 +219,6 @@ # success - task.exit 0 - queuepush DetrendProcessExposures -replace -key 1:2:4 "DONE $options:0" - end - - # default exit status - task.exit default - showcommand failure - queuepush DetrendProcessExposures -replace -key 1:2:4 "FAIL $options:0" - $DetrendProcessExpFail ++ + task.exit default + process_exit detPendingProcessedExp $options:0 $JOB_STATUS end @@ -296,6 +226,5 @@ task.exit timeout showcommand timeout - queuepush DetrendProcessExposures -replace -key 1:2:4 "TIMEOUT $options:0" - $DetrendProcessExpFail ++ - end -end + book setword detPendingProcessedExp $options:0 state TIMEOUT + end +end