Index: /trunk/ippTasks/summit.copy.pro =================================================================== --- /trunk/ippTasks/summit.copy.pro (revision 10066) +++ /trunk/ippTasks/summit.copy.pro (revision 10067) @@ -1,104 +1,166 @@ - ## PanTasks scripts for Summit Copy -# queues: expPending (expID) (URL) (type) (source) +if ($?parallel == 0) + $parallel = 0 +end -$LastExpID = "" +# queue for summit exps that need to be queried +queueinit pzPendingExp +# queue for summit imfiles that need to be downloaded +queueinit pzPendingImfile -# newImagesPending : search for images to be processed -task exp.pending - command getNewExposures $LastExpID +# run pzgetexp periodically to populate pzPendingExp +task pzgetexp + command pzgetexp -uri http://otis/ds/skyprobe/ -inst skyprobe -telescope ps1 host local # timeout shorter than exec so jobs do not build up - periods -exec 5 - periods -poll 1 - periods -timeout 4 + periods -exec 30 + periods -poll 10 + periods -timeout 30 - # success - task.exit 0 - local i Nstdout - # keep only new, unique entries (URL is key) - queuesize stdout -var Nstdout - for i 0 $Nstdout - queuepop stdout -var line - queuepush expPending -uniq -key 0 "$line new" - end + task.exit 0 + end - # extract last exposure in list - list tmp -split $line - $LastExpID = $tmp:0 + task.exit default + echo "phaseZ pzgetexp: failure" + end + task.exit timeout + echo "phaseZ pzgetexp: timeout" end end -# imfile.pending : get the list imfiles for each exposure -task imfile.pending - periods -exec 1 - periods -timeout 10 +# build a queue of exps/filesetids that need to be queried +task pztool.pendingexp + command pztool -pendingexp -simple + host local - task.exec - local Npending - queuesize expPending -var Npending - if ($Npending == 0) break + # timeout shorter than exec so jobs do not build up + periods -exec 30 + periods -poll 10 + periods -timeout 30 - queuepop expPending -var line -key 2 new - if ("$line" == "NULL") break + # success + task.exit 0 + local i Nstdout + queuesize stdout -var Nstdout + for i 0 $Nstdout + queuepop stdout -var line + list word -split $line + $exp_id = $word:0 + $camera = $word:1 + $telescope = $word:2 + $dateobs = $word:3 + $exp_type = $word:4 + $uri = $word:5 + $imfiles = $word:6 + queuepush pzPendingExp -uniq -key 1:2:3 "NEW $uri $exp_id $camera $telescope $dateobs exp_type $uri $imfiles" + end - list tmp -split $line - $URL = $tmp:0 - $expID = $tmp:1 - $type = $tmp:2 - $camera = $tmp:3 - queuepush expPending "$expID $camera run" + queuedrop pzPendingExp -key 0 DONE + end - host local - command getNewImfiles $URL - end - - # success - task.exit 0 - local i Nstdout - # keep only new, unique entries (URL is key) - queuesize stdout -var Nstdout - for i 0 $Nstdout - queuepop stdout -var line - queuepush imfilePending -uniq -key 0 "$line new" + task.exit default + echo "phaseZ pending exp: failure" end - end + task.exit timeout + echo "phaseZ pending exp: timeout" + end end -# imfile.download : submit download job to the appropriate node -task imfile.download - periods -exec 1 - periods -timeout 10 +# run pzgetimfiles on pending exps +task pzgetimfile + periods -exec 30 + periods -poll 10 + periods -timeout 30 - task.exec - local Npending - queuesize imfilePending -var Npending - if ($Npending == 0) break + task.exec + queuesize pzPendingExp -var N + if ($N == 0) break - queuepop imfilePending -var line -key 2 new - if ("$line" == "NULL") break + # get a line out of the queue + queuepop pzPendingExp -var line -key 0 NEW - list tmp -split $line - $URL = $tmp:0 - $expID = $tmp:1 - $type = $tmp:2 - $camera = $tmp:3 - queuepush imfilePending "$expID $camera run" + # take the state out of the line + strpop line state - host local - command copyImfile $URL - end + # put the line back into the queue with a stea of RUN + queuepush pzPendingExp -replace -key 1:2:3 "RUN $line" - # success - task.exit 0 - local i Nstdout - # keep only new, unique entries (URL is key) - queuesize stdout -var Nstdout - for i 0 $Nstdout - queuepop stdout -var line - queuepush imfilePending -uniq -key 0 "$line new" + # the STATE has already been removed from the line + list word -split $line + $exp_id = $word:0 + $camera = $word:1 + $telescope = $word:2 + $dateobs = $word:3 + $exp_type = $word:4 + $uri = $word:5 + $imfiles = $word:6 + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end + + # store the current line + options "$line" + + echo command ./pzgetimfiles -uri $uri -filesetid $exp_id -inst $camera -telescope $telescope + command ./pzgetimfiles -uri $uri -filesetid $exp_id -inst $camera -telescope $telescope end - end + + # success + task.exit 0 + queuepush pzPendingExp -replace -key 1:2:3 "DONE $options:0" + end + + task.exit default + queuepush pzPendingExp -replace -key 1:2:3 "FAIL $options:0" + end + + task.exit timeout + queuepush pzPendingExp -replace -key 1:2:3 "TIMEOUT $options:0" + end end + +# build a queue of imfiles/files that need to be downloaded +task pztool.pendingimfile + command pztool -pendingimfile -simple + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 30 + periods -poll 10 + periods -timeout 30 + + # success + task.exit 0 + local i Nstdout + queuesize stdout -var Nstdout + for i 0 $Nstdout + queuepop stdout -var line + list word -split $line + $exp_id = $word:0 + $camera = $word:1 + $telescope = $word:2 + $bytes = $word:3 + $md5sum = $word:4 + $class = $word:5 + $class_id = $word:6 + $uri = $word:7 + queuepush pzPendingImfile -uniq -key 1:2:3:6:7 "NEW $exp_id $camera $telescope $bytes $md5sum $class $class_id $uri" + end + + # delete existing entries which are DONE + queuedrop pzPendingImfile -key 0 DONE + end + + task.exit default + echo "phaseZ pending exp: failure" + end + task.exit timeout + echo "phaseZ pending exp: timeout" + end +end