Changeset 11011 for branches/eam_branch_00/ippTasks/detrend.resid.pro
- Timestamp:
- Jan 10, 2007, 12:12:35 PM (20 years ago)
- File:
-
- 1 edited
-
branches/eam_branch_00/ippTasks/detrend.resid.pro (modified) (8 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branch_00/ippTasks/detrend.resid.pro
r10745 r11011 1 2 1 ## this file contains the tasks for running the detrend processing stage 3 2 4 # dettool -toresidimfile -simple5 # 0 bias 1 850131b mc o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.5655826 # ITER TYPE DET_ID EXP_TAG CLASS_ID URI-IN7 8 # detrend_create_resid.pl 1 0 850131b mc bias bias_mc_1_0.fit o_850131b.mc.fit 850131b9 # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG)10 # URL-DET: $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit11 12 ## these tasks use the queue DetrendResidImfiles13 ## the DetrendResidImfiles queue contains:14 ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN15 16 queueinit DetrendResidImfiles17 queueinit DetrendResidExposures18 3 if ($?network == 0) 19 4 $network = 1 … … 22 7 $parallel = 0 23 8 end 24 25 $DetrendResidImfileFail = 026 $DetrendResidExpFail = 027 28 9 if ($?VERBOSE == 0) 29 10 echo "VERBOSE not defined: load pantasks.pro first" … … 34 15 break 35 16 end 17 36 18 $LOGSUBDIR = $LOGDIR/detrend 37 19 exec mkdir -p $LOGSUBDIR 38 20 21 book init DetrendResidImfiles 22 book init DetrendResidExposures 23 39 24 macro detresid.status 40 queueprint DetrendResidImfiles 41 queueprint DetrendResidExposures 42 echo "DetrendResidImfileFail: $DetrendResidImfileFail" 43 echo "DetrendResidExpFail: $DetrendResidExpFail" 25 book listbook DetrendResidImfiles 26 book listbook DetrendResidExposures 44 27 end 45 28 … … 58 41 59 42 task.exec 60 command dettool -toresidimfile - simple -limit {$DetrendResidImfileFail + 20}43 command dettool -toresidimfile -limit 20 61 44 end 62 45 63 46 # success 64 47 task.exit 0 65 # XXX is it necessary for these to be local? 66 # drop after this task macro is done? 67 local i Nqueue 68 69 # compare output with newImage queue 70 # only add entries which don't exist in queue 71 queuesize stdout -var Nqueue 72 for i 0 $Nqueue 73 queuepop stdout -var line 74 if ($VERBOSE > 2) 75 echo $line 76 end 77 list word -split $line 78 $DET_ID = $word:0 79 $ITER = $word:1 80 $DET_TYPE = $word:2 81 $MODE = $word:3 82 $EXP_TAG = $word:4 83 $CLASS_ID = $word:5 84 $URI = $word:6 85 $DET_URI = $word:7 86 $CAMERA = $word:8 87 ## must include the detID and iter in the key (if we have two running at the same time...) 88 ## Also needed to be in the output file (or path) 89 queuepush DetrendResidImfiles -uniq -key 1:2:5:6 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $CLASS_ID $URI $DET_URI $CAMERA" 48 # convert 'stdout' to book format 49 ipptool2book stdout DetrendResidImfiles -key det_id:iter:exp_tag:class_id -uniq 50 if ($VERBOSE > 2) 51 book listbook DetrendResidImfiles 90 52 end 91 53 92 54 # delete existing entries which are DONE 93 queuedrop DetrendResidImfiles -key 0 DONE 94 end 95 96 # locked list 55 book delpage DetrendResidImfiles -key state DONE 56 end 57 97 58 # default exit status 98 59 task.exit default … … 113 74 114 75 task.exec 115 queuesizeDetrendResidImfiles -var N76 book npages DetrendResidImfiles -var N 116 77 if ($N == 0) break 117 78 if ($network == 0) break 118 79 119 # look for new images on the internal DetrendResidImfiles queue 120 # caution with these 'if' statements: syntax errors 121 # will make the task fail without given a good status 122 queuepop DetrendResidImfiles -var line -key 0 NEW 123 if ("$line" == "NULL") break 124 125 strpop line state 126 queuepush DetrendResidImfiles -replace -key 1:2:5:6 "RUN $line" 127 128 ## the DetrendResidImfiles queue contains: 129 ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN 130 ## NOTE: the queue has the STATE prepended 131 list word -split $line 132 $DET_ID = $word:0 133 $ITER = $word:1 134 $DET_TYPE = $word:2 135 $MODE = $word:3 136 $EXP_TAG = $word:4 137 $CLASS_ID = $word:5 138 $URI = $word:6 139 $DET_URI = $word:7 140 $CAMERA = $word:8 80 # look for new images in DetrendResidImfiles 81 book getpage DetrendResidImfiles 0 -var pageName -key state NULL 82 if ($pageName == NULL) break 83 84 book setword DetrendResidImfiles $pageName state RUN 85 book getword DetrendResidImfiles $pageName DET_ID -var DET_ID 86 book getword DetrendResidImfiles $pageName ITER -var ITER 87 book getword DetrendResidImfiles $pageName DET_TYPE -var DET_TYPE 88 book getword DetrendResidImfiles $pageName MODE -var MODE 89 book getword DetrendResidImfiles $pageName EXP_TAG -var EXP_TAG 90 book getword DetrendResidImfiles $pageName CLASS_ID -var CLASS_ID 91 book getword DetrendResidImfiles $pageName URI -var URI 92 book getword DetrendResidImfiles $pageName DET_URI -var DET_URI 93 book getword DetrendResidImfiles $pageName CAMERA -var CAMERA 94 95 # specify choice of remote host: 96 if ($parallel) 97 host anyhost 98 else 99 host local 100 end 141 101 142 102 $word = `basename $EXP_TAG | tr '.' ' '` … … 147 107 exec mkdir -p $LOGSUBDIR/$base 148 108 109 # save the pageName for future reference below 110 options $pageName 111 112 # create command 113 if ($VERBOSE > 1) 114 echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE 115 end 116 command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE 117 end 118 119 # success 120 task.exit 0 121 book setword DetrendResidImfiles $options:0 state DONE 122 end 123 124 # default exit status 125 task.exit default 126 showcommand failure 127 book setword DetrendResidImfiles $options:0 state FAIL 128 end 129 130 # operation times out? 131 task.exit timeout 132 showcommand timeout 133 book setword DetrendResidImfiles $options:0 state TIMEOUT 134 end 135 end 136 137 # select images ready for copy 138 # new entries are added to DetrendResidExposures 139 # compare the new list with the ones already selected 140 task dettool.residexp.load 141 host local 142 143 periods -poll $loadpoll 144 periods -exec $loadexec 145 periods -timeout 30 146 147 stdout $LOGSUBDIR/detresid.exp.log 148 stderr $LOGSUBDIR/detresid.exp.log 149 150 task.exec 151 command dettool -toresidexp -limit 20 152 end 153 154 # success 155 task.exit 0 156 # convert 'stdout' to book format 157 ipptool2book stdout DetrendResidExposures -key det_id:iter:exp_tag -uniq 158 if ($VERBOSE > 2) 159 book listbook DetrendResidExposures 160 end 161 162 # delete existing entries which are DONE 163 book delpage DetrendResidExposures -key state DONE 164 end 165 166 # default exit status 167 task.exit default 168 showcommand failure 169 end 170 171 # operation times out? 172 task.exit timeout 173 showcommand timeout 174 end 175 end 176 177 # copy new images, sending job to desired host 178 task dettool.residexp.run 179 periods -poll $runpoll 180 periods -exec $runexec 181 periods -timeout 30 182 183 task.exec 184 book npages DetrendResidExposures -var N 185 if ($N == 0) break 186 if ($network == 0) break 187 188 # look for new images in DetrendResidExposures 189 book getpage DetrendResidExposures 0 -var pageName -key state NULL 190 if ($pageName == NULL) break 191 192 book setword DetrendResidExposures $pageName state RUN 193 book getword DetrendResidExposures $pageName DET_ID -var DET_ID 194 book getword DetrendResidExposures $pageName ITER -var ITER 195 book getword DetrendResidExposures $pageName DET_TYPE -var DET_TYPE 196 book getword DetrendResidExposures $pageName MODE -var MODE 197 book getword DetrendResidExposures $pageName EXP_TAG -var EXP_TAG 198 book getword DetrendResidExposures $pageName INCLUDE -var INCLUDE 199 book getword DetrendResidExposures $pageName CAMERA -var CAMERA 200 149 201 # specify choice of remote host: 150 202 if ($parallel) … … 154 206 end 155 207 156 # create example job options as a demonstration157 options "$line"158 # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???)159 # need to parse the subdir path for the detrend URL160 # $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits161 echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE162 command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE163 end164 165 # success166 task.exit 0167 queuepush DetrendResidImfiles -replace -key 1:2:5:6 "DONE $options:0"168 end169 170 # default exit status171 task.exit default172 showcommand failure173 queuepush DetrendResidImfiles -replace -key 1:2:5:6 "FAIL $options:0"174 $DetrendResidImfileFail ++175 end176 177 # operation times out?178 task.exit timeout179 showcommand timeout180 queuepush DetrendResidImfiles -replace -key 1:2:6:6 "TIMEOUT $options:0"181 $DetrendResidImfileFail ++182 end183 end184 185 # select images ready for copy186 # new entries are added to queue DetrendResidExposures187 # compare the new list with the ones already selected188 task dettool.residexp.load189 host local190 191 periods -poll $loadpoll192 periods -exec $loadexec193 periods -timeout 30194 195 stdout $LOGSUBDIR/detresid.exp.log196 stderr $LOGSUBDIR/detresid.exp.log197 198 task.exec199 command dettool -toresidexp -simple -limit {$DetrendResidExpFail + 20}200 end201 202 # success203 task.exit 0204 # XXX is it necessary for these to be local?205 # drop after this task macro is done?206 local i Nqueue207 208 # compare output with newImage queue209 # only add entries which don't exist in queue210 queuesize stdout -var Nqueue211 for i 0 $Nqueue212 queuepop stdout -var line213 if ($VERBOSE > 2)214 echo $line215 end216 list word -split $line217 $DET_ID = $word:0218 $ITER = $word:1219 $DET_TYPE = $word:2220 $MODE = $word:3221 $EXP_TAG = $word:4222 $INCLUDE = $word:5223 $CAMERA = $word:6224 queuepush DetrendResidExposures -uniq -key 1:2:5 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $INCLUDE $CAMERA"225 end226 227 # delete existing entries which are DONE228 queuedrop DetrendResidExposures -key 0 DONE229 end230 231 # locked list232 # default exit status233 task.exit default234 showcommand failure235 end236 237 # operation times out?238 task.exit timeout239 showcommand timeout240 end241 end242 243 # copy new images, sending job to desired host244 task dettool.residexp.run245 periods -poll $runpoll246 periods -exec $runexec247 periods -timeout 30248 249 task.exec250 queuesize DetrendResidExposures -var N251 if ($N == 0) break252 if ($network == 0) break253 254 # look for new images on the internal DetrendResidExposures queue255 # caution with these 'if' statements: syntax errors256 # will make the task fail without given a good status257 queuepop DetrendResidExposures -var line -key 0 NEW258 if ("$line" == "NULL") break259 260 strpop line state261 queuepush DetrendResidExposures -replace -key 1:2:5 "RUN $line"262 263 ## the DetrendResidExposures queue contains:264 ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN265 ## NOTE: the queue has the STATE prepended266 list word -split $line267 $DET_ID = $word:0268 $ITER = $word:1269 $DET_TYPE = $word:2270 $MODE = $word:3271 $EXP_TAG = $word:4272 $INCLUDE = $word:5273 $CAMERA = $word:6274 275 208 # XXX this is still a lame rule 276 209 $word = `basename $EXP_TAG | tr '.' ' '` 277 210 list word -split $word 278 211 $base = $word:0 279 280 212 stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log 281 213 stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log 282 214 exec mkdir -p $LOGSUBDIR/$base 283 215 284 # specify choice of remote host: 285 if ($parallel) 286 host anyhost 287 else 288 host local 289 end 216 # save the pageName for future reference below 217 options $pageName 290 218 291 219 # create example job options as a demonstration 292 options "$line"293 220 if ($VERBOSE > 1) 294 221 echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA … … 299 226 # success 300 227 task.exit 0 301 queuepush DetrendResidExposures -replace -key 1:2:5 "DONE $options:0" 302 end 303 304 # default exit status 305 task.exit default 306 showcommand failure 307 queuepush DetrendResidExposures -replace -key 1:2:5 "FAIL $options:0" 308 $DetrendResidExpFail ++ 309 end 310 311 # operation times out? 312 task.exit timeout 313 showcommand timeout 314 queuepush DetrendResidExposures -replace -key 1:2:5 "TIMEOUT $options:0" 315 $DetrendResidExpFail ++ 316 end 317 end 228 book setword DetrendResidExposures $options:0 state DONE 229 end 230 231 # default exit status 232 task.exit default 233 showcommand failure 234 book setword DetrendResidExposures $options:0 state FAIL 235 end 236 237 # operation times out? 238 task.exit timeout 239 showcommand timeout 240 book setword DetrendResidExposures $options:0 state TIMEOUT 241 end 242 end
Note:
See TracChangeset
for help on using the changeset viewer.
