Changeset 11210 for trunk/ippTasks/detrend.norm.pro
- Timestamp:
- Jan 19, 2007, 5:24:59 PM (20 years ago)
- File:
-
- 1 edited
-
trunk/ippTasks/detrend.norm.pro (modified) (12 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippTasks/detrend.norm.pro
r10745 r11210 1 2 ## this file contains the tasks for running the detrend normalization stage 3 4 ## these tasks use the queue DetrendNorm 5 ## the DetrendNorm queue contains: 6 ## STATE DET_ID ITER 7 8 queueinit DetrendNorm 9 queueinit DetrendNormStat 10 queueinit DetrendNormExp 1 ## this file contains the tasks for running the detrend normalization stages 2 ## these tasks use the books detPendingNormStatImfile detPendingNormImfile detPendingNormExp 3 11 4 if ($?network == 0) 12 5 $network = 1 … … 15 8 $parallel = 0 16 9 end 17 18 $DetrendNormFail = 019 $DetrendNormStatFail = 020 $DetrendNormExpFail = 021 22 10 if ($?VERBOSE == 0) 23 11 echo "VERBOSE not defined: load pantasks.pro first" … … 28 16 break 29 17 end 18 30 19 $LOGSUBDIR = $LOGDIR/detrend 31 20 exec mkdir -p $LOGSUBDIR 32 21 22 book init detPendingNormStatImfile 23 book init detPendingNormImfile 24 book init detPendingNormExp 25 33 26 macro detnorm.status 34 queueprint DetrendNorm 35 queueprint DetrendNormStat 36 queueprint DetrendNormExp 37 echo "DetrendNormFail: $DetrendNormFail" 38 echo "DetrendNormStatFail: $DetrendNormStatFail" 39 echo "DetrendNormExpFail: $DetrendNormExpFail" 27 book listbook detPendingNormStatImfile 28 book listbook detPendingNormImfile 29 book listbook detPendingNormExp 40 30 end 41 31 42 32 # select images ready for copy 43 # new entries are added to queue DetrendNormStat33 # new entries are added to detPendingNormStatImfile 44 34 # compare the new list with the ones already selected 45 35 task dettool.normstat.load … … 50 40 periods -timeout 30 51 41 52 stdout $LOGSUBDIR/dettool.normstat.log42 stdout NULL 53 43 stderr $LOGSUBDIR/dettool.normstat.log 54 44 55 45 task.exec 56 command dettool -tonormalizedstat -simple -limit {$DetrendNormStatFail + 20}46 command dettool -tonormalizedstat -limit 20 57 47 end 58 48 59 49 # success 60 50 task.exit 0 61 # XXX is it necessary for these to be local? 62 # drop after this task macro is done? 63 local i Nqueue 64 65 # compare output with newImage queue 66 # only add entries which don't exist in queue 67 queuesize stdout -var Nqueue 68 for i 0 $Nqueue 69 queuepop stdout -var line 70 if ($VERBOSE > 2) 71 echo $line 72 end 73 list word -split $line 74 $DET_ID = $word:0 75 $DET_TYPE = $word:1 76 $ITER = $word:2 77 $CAMERA = $word:3 78 queuepush DetrendNormStat -uniq -key 1:3 "NEW $DET_ID $DET_TYPE $ITER $CAMERA" 51 # convert 'stdout' to book format 52 ipptool2book stdout detPendingNormStatImfile -key det_id:iteration -uniq 53 if ($VERBOSE > 2) 54 book listbook detPendingNormStatImfile 79 55 end 80 56 81 57 # delete existing entries which are DONE 82 queuedrop DetrendNormStat -key 0 DONE 83 end 84 85 # locked list 58 book delpage detPendingNormStatImfile -key state DONE 59 end 60 86 61 # default exit status 87 62 task.exit default … … 102 77 103 78 task.exec 104 queuesize DetrendNormStat-var N79 book npages detPendingNormStatImfile -var N 105 80 if ($N == 0) break 106 81 if ($network == 0) break 107 82 108 # look for new images on the internal DetrendNormStat queue 109 # caution with these 'if' statements: syntax errors 110 # will make the task fail without given a good status 111 queuepop DetrendNormStat -var line -key 0 NEW 112 if ("$line" == "NULL") break 113 114 strpop line state 115 queuepush DetrendNormStat -replace -key 1:3 "RUN $line" 116 117 # the DetrendNormStat queue contains: 118 # STATE DET_ID ITER 119 list word -split $line 120 $DET_ID = $word:0 121 $DET_TYPE = $word:1 122 $ITER = $word:2 123 $CAMERA = $word:3 124 125 # XXX add $WORKDIR/$LOG_DIR 126 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 127 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 83 # look for new images in detPendingNormStatImfile 84 book getpage detPendingNormStatImfile 0 -var pageName -key state NULL 85 if ($pageName == NULL) break 86 87 book setword detPendingNormStatImfile $pageName state RUN 88 book getword detPendingNormStatImfile $pageName det_id -var DET_ID 89 book getword detPendingNormStatImfile $pageName det_type -var DET_TYPE 90 book getword detPendingNormStatImfile $pageName iteration -var ITERATION 91 book getword detPendingNormStatImfile $pageName camera -var CAMERA 128 92 129 93 # specify choice of remote host: … … 134 98 end 135 99 100 # XXX add $WORKDIR/$LOG_DIR 101 # XXX use ipp_filename.pl to lookup output file names 102 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 103 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATIION.log 104 105 # save the pageName for future reference below 106 options $pageName 107 136 108 # create example job options as a demonstration 137 options "$line"138 109 if ($VERBOSE > 1) 139 echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE 140 end 141 command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE 142 end 143 144 # success 145 task.exit 0 146 queuepush DetrendNormStat -replace -key 1:3 "DONE $options:0" 147 end 148 149 # default exit status 150 task.exit default 151 showcommand failure 152 queuepush DetrendNormStat -replace -key 1:3 "FAIL $options:0" 153 $DetrendNormStatFail ++ 154 end 155 156 # operation times out? 157 task.exit timeout 158 showcommand timeout 159 queuepush DetrendNormStat -replace -key 1:3 "TIMEOUT $options:0" 160 $DetrendNormStatFail ++ 110 echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE 111 end 112 command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE 113 end 114 115 # default exit status 116 task.exit default 117 process_exit detPendingNormStatImfile $options:0 $JOB_STATUS 118 end 119 120 # operation times out? 121 task.exit timeout 122 showcommand timeout 123 book setword detPendingNormStatImfile $options:0 state TIMEOUT 161 124 end 162 125 end 163 126 164 127 # select images ready for copy 165 # new entries are added to queue DetrendNorm128 # new entries are added to detPendingNormImfile 166 129 # compare the new list with the ones already selected 167 130 task dettool.norm.load … … 172 135 periods -timeout 30 173 136 174 stdout $LOGSUBDIR/dettool.norm.log137 stdout NULL 175 138 stderr $LOGSUBDIR/dettool.norm.log 176 139 177 140 task.exec 178 command dettool -tonormalize -simple -limit {$DetrendNormFail + 20}141 command dettool -tonormalize -limit 20 179 142 end 180 143 181 144 # success 182 145 task.exit 0 183 # XXX is it necessary for these to be local? 184 # drop after this task macro is done? 185 local i Nqueue 186 187 # compare output with newImage queue 188 # only add entries which don't exist in queue 189 queuesize stdout -var Nqueue 190 for i 0 $Nqueue 191 queuepop stdout -var line 192 if ($VERBOSE > 2) 193 echo $line 194 end 195 list word -split $line 196 $DET_TYPE = $word:0 197 $CAMERA = $word:1 198 $URI = $word:2 199 $DET_ID = $word:3 200 $ITER = $word:4 201 $CLASS_ID = $word:5 202 $NORM = $word:6 203 queuepush DetrendNorm -uniq -key 4:5 "NEW $DET_TYPE $CAMERA $URI $DET_ID $ITER $CLASS_ID $NORM" 146 # convert 'stdout' to book format 147 ipptool2book stdout detPendingNormImfile -key det_id:iteration -uniq 148 if ($VERBOSE > 2) 149 book listbook detPendingNormImfile 204 150 end 205 151 206 152 # delete existing entries which are DONE 207 queuedrop DetrendNorm -key 0 DONE 208 end 209 210 # locked list 153 book delpage detPendingNormImfile -key state DONE 154 end 155 211 156 # default exit status 212 157 task.exit default … … 227 172 228 173 task.exec 229 queuesize DetrendNorm-var N174 book npages detPendingNormImfile -var N 230 175 if ($N == 0) break 231 176 if ($network == 0) break 232 177 233 # look for new images on the internal DetrendNorm queue 234 # caution with these 'if' statements: syntax errors 235 # will make the task fail without given a good status 236 queuepop DetrendNorm -var line -key 0 NEW 237 if ("$line" == "NULL") break 238 239 strpop line state 240 queuepush DetrendNorm -replace -key 4:5 "RUN $line" 241 242 # the DetrendNorm queue contains: 243 # STATE DET_ID ITER 244 list word -split $line 245 $DET_TYPE = $word:0 246 $CAMERA = $word:1 247 $URI = $word:2 248 $DET_ID = $word:3 249 $ITER = $word:4 250 $CLASS_ID = $word:5 251 $NORM = $word:6 252 253 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 254 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 178 # look for new images in detPendingNormImfile 179 book getpage detPendingNormImfile 0 -var pageName -key state NULL 180 if ($pageName == NULL) break 181 182 book setword detPendingNormImfile $pageName state RUN 183 book getword detPendingNormImfile $pageName det_type -var DET_TYPE 184 book getword detPendingNormImfile $pageName camera -var CAMERA 185 book getword detPendingNormImfile $pageName uri -var URI 186 book getword detPendingNormImfile $pageName det_id -var DET_ID 187 book getword detPendingNormImfile $pageName iteration -var ITERATION 188 book getword detPendingNormImfile $pageName class_id -var CLASS_ID 189 book getword detPendingNormImfile $pageName norm -var NORM 255 190 256 191 # specify choice of remote host: … … 261 196 end 262 197 198 # XXX use ipp_filename.pl to lookup output file names 199 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 200 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 201 202 # save the pageName for future reference below 203 options $pageName 204 263 205 # create example job options as a demonstration 264 options "$line"265 206 if ($VERBOSE > 1) 266 echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE 267 end 268 command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE 269 end 270 271 # success 272 task.exit 0 273 queuepush DetrendNorm -replace -key 4:5 "DONE $options:0" 274 end 275 276 # default exit status 277 task.exit default 278 showcommand failure 279 queuepush DetrendNorm -replace -key 4:5 "FAIL $options:0" 280 $DetrendNormFail ++ 281 end 282 283 # operation times out? 284 task.exit timeout 285 showcommand timeout 286 queuepush DetrendNorm -replace -key 4:5 "TIMEOUT $options:0" 287 $DetrendNormFail ++ 207 echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE 208 end 209 command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE 210 end 211 212 # default exit status 213 task.exit default 214 process_exit detPendingNormImfile $options:0 state FAIL 215 end 216 217 # operation times out? 218 task.exit timeout 219 showcommand timeout 220 book setword detPendingNormImfile $options:0 state TIMEOUT 288 221 end 289 222 end 290 223 291 224 # select images ready for copy 292 # new entries are added to queue DetrendNormExp225 # new entries are added to detPendingNormExp 293 226 # compare the new list with the ones already selected 294 227 task dettool.normexp.load … … 299 232 periods -timeout 30 300 233 301 stdout $LOGSUBDIR/dettool.normexp.log234 stdout NULL 302 235 stderr $LOGSUBDIR/dettool.normexp.log 303 236 304 237 task.exec 305 command dettool -tonormalizedexp -simple -limit {$DetrendNormExpFail + 20}238 command dettool -tonormalizedexp -limit 20 306 239 end 307 240 308 241 # success 309 242 task.exit 0 310 # XXX is it necessary for these to be local? 311 # drop after this task macro is done? 312 local i Nqueue 313 314 # compare output with newImage queue 315 # only add entries which don't exist in queue 316 queuesize stdout -var Nqueue 317 for i 0 $Nqueue 318 queuepop stdout -var line 319 if ($VERBOSE > 2) 320 echo $line 321 end 322 list word -split $line 323 $DET_ID = $word:0 324 $ITER = $word:1 325 $DET_TYPE = $word:2 326 $CAMERA = $word:3 327 #$TELESCOPE= $word:4 328 #$EXP_TYPE = $word:5 329 #$IMFILES = $word:6 330 queuepush DetrendNormExp -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA" 243 # convert 'stdout' to book format 244 ipptool2book stdout detPendingNormExp -key det_id:iteration -uniq 245 if ($VERBOSE > 2) 246 book listbook detPendingNormExp 331 247 end 332 248 333 249 # delete existing entries which are DONE 334 queuedrop DetrendNormExp -key 0 DONE 335 end 336 337 # locked list 250 book delpage detPendingNormExp -key state DONE 251 end 252 338 253 # default exit status 339 254 task.exit default … … 354 269 355 270 task.exec 356 queuesize DetrendNormExp -var N 357 271 book npages detPendingNormExp -var N 358 272 if ($N == 0) break 359 273 if ($network == 0) break 360 274 361 # look for new images on the internal DetrendNormExp queue 362 # caution with these 'if' statements: syntax errors 363 # will make the task fail without given a good status 364 queuepop DetrendNormExp -var line -key 0 NEW 365 if ("$line" == "NULL") break 366 367 strpop line state 368 queuepush DetrendNormExp -replace -key 1:2 "RUN $line" 369 370 # the DetrendNormExp queue contains: 371 # STATE DET_ID ITER 372 list word -split $line 373 $DET_ID = $word:0 374 $ITER = $word:1 375 $DET_TYPE = $word:2 376 $CAMERA = $word:3 377 #$TELESCOPE= $word:4 378 #$EXP_TYPE = $word:5 379 #$IMFILES = $word:6 380 381 # XXX add $WORKDIR/$LOG_DIR 382 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 383 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log 275 # look for new images in detPendingNormExp 276 book getpage detPendingNormExp 0 -var pageName -key state NULL 277 if ($pageName == NULL) break 278 279 book setword detPendingNormExp $pageName state RUN 280 book getword detPendingNormExp $pageName det_id -var DET_ID 281 book getword detPendingNormExp $pageName iteration -var ITERATION 282 book getword detPendingNormExp $pageName det_type -var DET_TYPE 283 book getword detPendingNormExp $pageName camera -var CAMERA 384 284 385 285 # specify choice of remote host: … … 390 290 end 391 291 292 # XXX add $WORKDIR/$LOG_DIR 293 stdout $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 294 stderr $LOGSUBDIR/$CAMERA.$DET_TYPE.$DET_ID.$ITERATION.log 295 296 # save the pageName for future reference below 297 options $pageName 298 392 299 # create example job options as a demonstration 393 options "$line"394 300 if ($VERBOSE > 1) 395 echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE 396 end 397 command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE 398 end 399 400 # success 401 task.exit 0 402 queuepush DetrendNormExp -replace -key 1:2 "DONE $options:0" 403 end 404 405 # default exit status 406 task.exit default 407 showcommand failure 408 queuepush DetrendNormExp -replace -key 1:2 "FAIL $options:0" 409 $DetrendNormExpFail ++ 410 end 411 412 # operation times out? 413 task.exit timeout 414 showcommand timeout 415 queuepush DetrendNormExp -replace -key 1:2 "TIMEOUT $options:0" 416 $DetrendNormExpFail ++ 417 end 418 end 301 echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE 302 end 303 command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE 304 end 305 306 # default exit status 307 task.exit default 308 process_exit detPendingNormExp $options:0 $JOB_STATUS 309 end 310 311 # operation times out? 312 task.exit timeout 313 showcommand timeout 314 book setword detPendingNormExp $options:0 state TIMEOUT 315 end 316 end
Note:
See TracChangeset
for help on using the changeset viewer.
