Changeset 42405 for trunk/tools/eam/rawfix.20230221/src/rawfix.pt
- Timestamp:
- Mar 9, 2023, 2:53:11 PM (3 years ago)
- File:
-
- 1 edited
-
trunk/tools/eam/rawfix.20230221/src/rawfix.pt (modified) (1 diff)
Legend:
- Unmodified
- Added
- Removed
-
trunk/tools/eam/rawfix.20230221/src/rawfix.pt
r42404 r42405 20 20 end 21 21 22 if (not($?RAWFIX_INIT)) 23 book init md5sum.book 24 book init md5host.book 25 $RAWFIX_INIT = 1 26 end 27 28 # select nights which are ready for md5sum calcs 29 # output: md5sum.book 30 task nights.md5sum.load 31 host local 32 periods -poll $TPOLL 33 periods -exec $TEXEC 34 periods -timeout 30 35 36 stdout $LOGDIR/md5sum.load.stdout.log 37 stderr $LOGDIR/md5sum.load.stderr.log 38 39 # only run one of these tasks at a time 40 npending 1 41 42 task.exec 43 command check.rawfix.sh state md5sum.new 44 end 45 46 task.exit 0 47 # convert the stdout 'queue' to entries in a book 48 # fields in the table will become words in the page 49 queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum.book 50 end 51 52 # exit values other than 0: 53 task.exit default 54 showcommand failure 55 end 56 57 task.exit crash 58 showcommand crash 59 end 60 61 # operation times out? 62 task.exit timeout 63 showcommand timeout 64 end 65 end 66 67 # output: md5host.book 68 task nights.md5sum.loadhosts 69 70 periods -poll $RPOLL 71 periods -exec $REXEC 72 periods -timeout 60 73 npending 1 74 75 stdout $LOGDIR/md5sum.loadhosts.stdout.log 76 stderr $LOGDIR/md5sum.loadhosts.stderr.log 77 78 task.exec 79 # if we are unable to run the 'exec', use a long retry time 80 periods -exec $REXEC 81 82 book npages md5sum.book -var N 83 if ($N == 0) break 84 if ($NETWORK == 0) break 85 86 # look for new nights in md5sum.book (pantaskState == INIT) 87 book getpage md5sum.book 0 -var pageName -key pantaskState INIT 88 if ("$pageName" == "NULL") break 89 90 book setword md5sum.book $pageName pantaskState RUN 91 book getword md5sum.book $pageName dateobs -var DATEOBS 92 option $pageName 93 # save the page so we can use it below 94 95 # write an entry point in the log for this night 96 sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $DATEOBS 97 date -var mydate 98 exec echo "# start runhosts $mydate" >> $tmpline 99 100 command get_hosts_md5s.sh $DATEOBS 101 end 102 103 task.exit 0 104 # convert the stdout 'queue' to entries in a book 105 # fields in the table will become words in the page 106 queuesize stdout -var Nhosts 107 queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book 108 book setword md5sum.book $options:0 NHOST {$Nhosts - 1} 109 book setword md5sum.book $options:0 NDONE 0 110 # use the queuesize (Nhosts) to track how many jobs are done / to be done 111 end 112 113 # exit values other than 0: 114 task.exit default 115 showcommand failure 116 book setword md5sum.book $options:0 pantasksQuality FAIL 117 end 118 119 task.exit crash 120 showcommand crash 121 book setword md5sum.book $options:0 pantasksQuality CRASH 122 end 123 124 # operation times out? 125 task.exit timeout 126 showcommand timeout 127 book setword md5sum.book $options:0 pantasksQuality TIMEOUT 128 end 129 end 130 131 # output: md5host.book 132 task nights.md5sum.runhosts 133 134 periods -poll $RPOLL 135 periods -exec $REXEC 136 periods -timeout 60 137 138 stdout $LOGDIR/md5sum.runhosts.stdout.log 139 stderr $LOGDIR/md5sum.runhosts.stderr.log 140 141 task.exec 142 # if we are unable to run the 'exec', use a long retry time 143 periods -exec $REXEC 144 145 book npages md5host.book -var N 146 if ($N == 0) break 147 if ($NETWORK == 0) break 148 149 # look for new entries in md5host.book (pantaskState == INIT) 150 book getpage md5host.book 0 -var pageName -key pantaskState INIT 151 if ("$pageName" == "NULL") break 152 153 book setword md5host.book $pageName pantaskState RUN 154 book getword md5host.book $pageName dateobs -var DATEOBS 155 book getword md5host.book $pageName md5host -var MD5HOST 156 157 option $pageName $DATEOBS 158 # save DATEOBS so we can find the md5sum.book entry 159 160 # XXX TEST: $USE_HOST = ipp060 161 162 $USE_HOST = $MD5HOST 163 host -required $USE_HOST 164 ## check if this host needs to be launched 165 if (not($?host:$USE_HOST)) 166 echo "host not defined, creating : $USE_HOST" 167 $host:$USE_HOST = 0 168 end 169 if (not($host:$USE_HOST)) 170 control host add $USE_HOST 171 $host:$USE_HOST = 1 172 end 173 174 ## even when testing, use the original hostname here: 175 $input = $mypath/$DATEOBS/$MD5HOST.inst.txt 176 177 # need to make sure this will work on the ipp b-nodes: 178 # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs 179 periods -exec 0.05 180 command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input 181 end 182 183 task.exit 0 184 nights.mdsum.exit OK 185 end 186 187 # exit values other than 0: 188 task.exit default 189 nights.mdsum.exit FAIL 190 showcommand failure 191 end 192 193 task.exit crash 194 nights.mdsum.exit CRASH 195 showcommand crash 196 end 197 198 # operation times out? 199 task.exit timeout 200 nights.mdsum.exit TIMEOUT 201 showcommand timeout 202 end 203 end 204 205 # output: md5host.book 206 task nights.md5sum.clean 207 208 periods -poll $RPOLL 209 periods -exec $REXEC 210 periods -timeout 60 211 npending 1 212 213 stdout $LOGDIR/md5sum.clean.stdout.log 214 stderr $LOGDIR/md5sum.clean.stderr.log 215 216 task.exec 217 # if we are unable to run the 'exec', use a long retry time 218 periods -exec $REXEC 219 220 book npages md5sum.book -var N 221 if ($N == 0) break 222 if ($NETWORK == 0) break 223 224 # look for completed nights in md5sum.book (pantaskState == DONE) 225 book getpage md5sum.book 0 -var pageName -key pantaskState DONE 226 if ("$pageName" == "NULL") break 227 228 # book listpage md5sum.book $pageName 229 230 # book setword md5sum.book $pageName pantaskState RUN 231 book getword md5sum.book $pageName dateobs -var DATEOBS 232 book getword md5sum.book $pageName pantasksQuality -var QUALITY 233 234 option $pageName 235 # save the page so we can use it below 236 237 if ("$QUALITY" == "OK") 238 command update.rawfix.sh $DATEOBS md5sum.done 239 else 240 command update.rawfix.sh $DATEOBS md5sum.fail 241 end 242 end 243 244 task.exit 0 245 # now that the entry is set to done in the db, remove the pantasks book entry 246 echo "cleaning this page: $options:0" 247 book delpage md5sum.book $options:0 248 end 249 250 # exit values other than 0: 251 task.exit default 252 showcommand failure 253 book setword md5sum.book $options:0 pantasksQuality FAIL 254 end 255 256 task.exit crash 257 showcommand crash 258 book setword md5sum.book $options:0 pantasksQuality CRASH 259 end 260 261 # operation times out? 262 task.exit timeout 263 showcommand timeout 264 book setword md5sum.book $options:0 pantasksQuality TIMEOUT 265 end 266 end 22 input rawfix.setup.pt 23 input rawfix.md5sum.pt 267 24 268 25 ######################### UTILITIES ################################# 269 270 macro nights.mdsum.exit271 if ($0 != 2)272 echo "USAGE: nights.mdsum.exit (quality)"273 break274 end275 276 book setword md5host.book $options:0 pantaskState DONE277 book getword md5sum.book $options:1 NHOST -var Nhosts278 book getword md5sum.book $options:1 NDONE -var Ndone279 $Ndone ++280 book setword md5sum.book $options:1 NDONE $Ndone281 if ("$1" != "OK")282 book setword md5sum.book $options:1 pantasksQuality $1283 sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $options:1284 exec echo "failure $1 $options:0" >> $tmpline285 else286 exec echo "success $1 $options:0" >> $tmpline287 end288 if ($Ndone == $Nhosts)289 book setword md5sum.book $options:1 pantaskState DONE290 end291 book delpage md5host.book $options:0292 end293 26 294 27 macro showcommand -c "print out the current command (used within tasks)"
Note:
See TracChangeset
for help on using the changeset viewer.
