Index: trunk/ippTasks/summit.copy.pro =================================================================== --- trunk/ippTasks/summit.copy.pro (revision 14601) +++ trunk/ippTasks/summit.copy.pro (revision 14607) @@ -1,7 +1,8 @@ ## PanTasks scripts for Summit Copy -$uri = http://otis/ds/skyprobe/ -$camera = skyprobe -$telescope = ps1 +#$uri = http://otis/ds/skyprobe/ +$uri = http://otis4.ifa.hawaii.edu/ds/allskycam/ +$camera = allskycam +$telescope = zl $workdir = /home/moanui/jhoblitt/workdir @@ -10,10 +11,12 @@ end -# queue for summit exps that need to be queried -queueinit pzPendingExp -# queue for summit imfiles that need to be downloaded -queueinit pzPendingImfile -# queue for downloaded files that need to registered with pztool -copydone -queueinit pzCopyDone +if ($?network == 0) + $network = 1 +end + +# book for summit exps that need to be queried +book init pzPendingExp +# book for summit imfiles that need to be downloaded +book init pzPendingImfile # run pzgetexp periodically to populate pzPendingExp @@ -31,12 +34,12 @@ task.exit default - echo "phaseZ pzgetexp: failure" + showcommand failure end task.exit timeout - echo "phaseZ pzgetexp: timeout" + showcommand timeout end end -# build a queue of exps/filesetids that need to be queried +# build a book of exps/filesetids that need to be queried task pztool.pendingexp command pztool -pendingexp -simple @@ -49,29 +52,18 @@ # success - task.exit 0 - local i Nstdout - queuesize stdout -var Nstdout - for i 0 $Nstdout - queuepop stdout -var line -# list word -split $line -# $exp_id = $word:0 -# $camera = $word:1 -# $telescope = $word:2 -# $dateobs = $word:3 -# $exp_type = $word:4 -# $uri = $word:5 -# $imfiles = $word:6 -# queuepush pzPendingExp -uniq -key 1:2:3 "NEW $uri $exp_id $camera $telescope $dateobs $exp_type $uri $imfiles" - queuepush pzPendingExp -uniq -key 1:2:3 "NEW $line" - end - - queuedrop pzPendingExp -key 0 DONE - end - - task.exit default - echo "phaseZ pending exp: failure" - end - task.exit timeout - echo "phaseZ pending exp: timeout" + task.exit 0 + # convert 'stdout' to book format +# ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword pantaskState INIT + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingExp + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout end end @@ -84,60 +76,59 @@ task.exec - queuesize pzPendingExp -var N + if ($network == 0) break + + book npages pzPendingExp -var N if ($N == 0) periods -exec 20 break end - periods -exec 0.2 - # get a line out of the queue - queuepop pzPendingExp -var line -key 0 NEW - - # take the state out of the line - strpop line state - - # put the line back into the queue with a stea of RUN - queuepush pzPendingExp -replace -key 1:2:3 "RUN $line" - - # the STATE has already been removed from the line - list word -split $line - $exp_id = $word:0 - $camera = $word:1 - $telescope = $word:2 - $dateobs = $word:3 - $exp_type = $word:4 - $uri = $word:5 - $imfiles = $word:6 - - # specify choice of remote host:(need to choose based on chips) -# if ($parallel) -# host anyhost -# else -# host local -# end - - # store the current line - options "$line" - - echo command pzgetimfiles -uri $uri -filesetid $exp_id -inst $camera -telescope $telescope - command pzgetimfiles -uri $uri -filesetid $exp_id -inst $camera -telescope $telescope + # find an exp that needs imfiles fetched + book getpage pzPendingExp 0 -var pageName -key pantaskState INIT + if ($pageName == NULL) break + + + # set that exp to run + book setword pzPendingExp $pageName pantaskState RUN + + book getword pzPendingExp $pageName exp_id -var EXP_ID + book getword pzPendingExp $pageName camera -var CAMERA + book getword pzPendingExp $pageName telescope -var TELESCOPE + book getword pzPendingExp $pageName dateobs -var DATEOBS + book getword pzPendingExp $pageName exp_type -var EXP_TYPE + book getword pzPendingExp $pageName uri -var URI + book getword pzPendingExp $pageName imfiles -var IMFILES + + # store the current page + options $pageName + + $run = pzgetimfiles -uri $URI -filesetid $EXP_ID -inst $CAMERA -telescope $TELESCOPE + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run end # success task.exit 0 - queuepush pzPendingExp -replace -key 1:2:3 "DONE $options:0" + process_exit pzPendingExp $options:0 $JOB_STATUS end task.exit default - queuepush pzPendingExp -replace -key 1:2:3 "FAIL $options:0" + showcommand failure + book setword pzPendingExp $options:0 pantaskState $JOB_STATUS end task.exit timeout - queuepush pzPendingExp -replace -key 1:2:3 "TIMEOUT $options:0" - end -end - -# build a queue of imfiles/files that need to be downloaded + showcommand timeout + book setword pzPendingExp $options:0 pantaskState TIMEOUT + end +end + + +# build a book of imfiles/files that need to be downloaded task pztool.pendingimfile command pztool -pendingimfile -simple @@ -151,34 +142,20 @@ # success task.exit 0 - local i Nstdout - queuesize stdout -var Nstdout - for i 0 $Nstdout - queuepop stdout -var line -# list word -split $line -# $exp_id = $word:0 -# $camera = $word:1 -# $telescope = $word:2 -# $file_id = $word:3 -# $bytes = $word:4 -# $md5sum = $word:5 -# $class = $word:6 -# $class_id = $word:7 -# $uri = $word:8 -# $exp_tag = $word:9 - - queuepush pzPendingImfile -uniq -key 1:2:3:7:8 "NEW $line" - end - - # delete existing entries which are DONE - queuedrop pzPendingImfile -key 0 DONE + # convert 'stdout' to book format +# ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope -uniq -setword pantaskState INIT + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingImfile end task.exit default - echo "phaseZ pending exp: failure" + showcommand failure end task.exit timeout - echo "phaseZ pending exp: timeout" - end -end + showcommand timeout + end +end + # get dsget to retreive an imfile and then call pztool -copydone @@ -189,125 +166,68 @@ task.exec - queuesize pzPendingImfile -var N - if ($N == 0) break - - # get a line out of the queue - queuepop pzPendingImfile -var line -key 0 NEW - - # take the state out of the line - strpop line state - - # put the line back into the queue with a stea of RUN - queuepush pzPendingImfile -replace -key 1:2:3:7:8 "RUN $line" - - # the STATE has already been removed from the line - list word -split $line - $exp_id = $word:0 - $camera = $word:1 - $telescope = $word:2 - $file_id = $word:3 - $bytes = $word:4 - $md5sum = $word:5 - $class = $word:6 - $class_id = $word:7 - $uri = $word:8 - $exp_tag = $word:9 - -# # specify choice of remote host:(need to choose based on chips) -# if ($parallel) -# host anyhost -# else -# host local -# end - - # store the current line without the state - options "$line" - - echo "parsing line $line" - echo command dsget --uri $uri --filename $workdir/$exp_id.$class_id.fits --bytes $bytes --md5 $md5sum - command dsget --uri $uri --filename $workdir/$exp_id.$class_id.fits --bytes $bytes --md5 $md5sum + if ($network == 0) break + + book npages pzPendingImfile -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.2 + + # find an exp that needs imfiles fetched + book getpage pzPendingImfile 0 -var pageName -key pantaskState INIT + if ($pageName == NULL) break + + + # set that exp to run + book setword pzPendingImfile $pageName pantaskState RUN + + book getword pzPendingImfile $pageName uri -var URI + book getword pzPendingImfile $pageName exp_name -var EXP_NAME + book getword pzPendingImfile $pageName class_id -var CLASS_ID + book getword pzPendingImfile $pageName bytes -var BYTES + book getword pzPendingImfile $pageName md5sum -var MD5SUM + + # store the current page + options $pageName + + $run = dsget --uri $URI --filename $workdir/$EXP_NAME.$CLASS_ID.fits --bytes $BYTES --md5 $MD5SUM + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run end # success task.exit 0 - # could run copydone from here but how would the exit status be - # handled? - - list word -split $options:0 - $exp_id = $word:0 - $camera = $word:1 - $telescope = $word:2 - $file_id = $word:3 - $bytes = $word:4 - $md5sum = $word:5 - $class = $word:6 - $class_id = $word:7 - $uri = $word:8 - $exp_tag = $word:9 - queuepush pzCopyDone -key 1:2:3:7:8 "NEW $options:0 $workdir/$exp_id.$class_id.fits" - queuepush pzPendingExp -replace -key 1:2:3:7:8 "DONE $options:0" + book getword options $pageName uri -var URI + book getword options $pageName exp_name -var EXP_NAME + book getword options $pageName class_id -var CLASS_ID + book getword options $pageName bytes -var BYTES + book getword options $pageName md5sum -var MD5SUM + + book delpage pzPendingImfiles optoins + + local $run = pztool -copydone -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -class $CLASS -class_id $CLASS_ID -uri $workdir/$EXP_NAME.$CLASS_ID.fits + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + + process_exit pzPendingExp $options:0 $JOB_STATUS end task.exit default - queuepush pzPendingExp -replace -key 1:2:3:7:8 "FAIL $options:0" + showcommand failure + book setword pzPendingImfile $options:0 pantaskState $JOB_STATUS end task.exit timeout - queuepush pzPendingExp -replace -key 1:2:3:7:8 "TIMEOUT $options:0" - end -end - -task pztool.copydone - periods -exec 5 - periods -poll 5 - periods -timeout 15 - - host local - - task.exec - queuesize pzCopyDone -var N - if ($N == 0) break - - # get a line out of the queue - queuepop pzCopyDone -var line -key 0 NEW - - # take the state out of the line - strpop line state - - # put the line back into the queue with a stea of RUN - queuepush pzCopyDone -replace -key 1:2:3:7:8 "RUN $line" - - # the STATE has already been removed from the line - list word -split $line - $exp_id = $word:0 - $camera = $word:1 - $telescope = $word:2 - $file_id = $word:3 - $bytes = $word:4 - $md5sum = $word:5 - $class = $word:6 - $class_id = $word:7 - $uri = $word:8 - $exp_tag = $word:9 - $new_uri = $word:10 - - # store the current line without the state - options "$line" - - echo command pztool -copydone -exp_id $exp_id -inst $camera -telescope $telescope -class $class -class_id $class_id -uri $uri - command pztool -copydone -exp_id $exp_id -inst $camera -telescope $telescope -class $class -class_id $class_id -uri $new_uri - end - - # success - task.exit 0 - queuepush pzCopyDone -replace -key 1:2:3:6:7 "DONE $options:0" - end - - task.exit default - queuepush pzCopyDone -replace -key 1:2:3:6:7 "FAIL $options:0" - end - - task.exit timeout - queuepush pzCopyDone -replace -key 1:2:3:6:7 "TIMEOUT $options:0" - end -end + showcommand timeout + book setword pzPendingImfile $options:0 pantaskState TIMEOUT + end +end