Index: /branches/haf_branches/ipp.20100512/ippTasks/addstar.pro =================================================================== --- /branches/haf_branches/ipp.20100512/ippTasks/addstar.pro (revision 28168) +++ /branches/haf_branches/ipp.20100512/ippTasks/addstar.pro (revision 28169) @@ -254,50 +254,2 @@ end end - -task minidvodb.create - host local - - periods -poll 5.0 - periods -exec 60.0 - periods -timeout 1200 - npending 1 - active false - - stdout NULL - stderr $LOGDIR/revert.log - - task.exec - if ($LABEL:n == 0) break - $run = addtool -revertprocessedexp - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$addstar_revert_DB - $run = $run -dbname $DB:$addstar_revert_DB - $addstar_revert_DB ++ - if ($addstar_revert_DB >= $DB:n) set addstar_revert_DB = 0 - end - add_poll_labels run - command $run - end - - # success - task.exit 0 - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - Index: /branches/haf_branches/ipp.20100512/ippTasks/minidvodb.pro =================================================================== --- /branches/haf_branches/ipp.20100512/ippTasks/minidvodb.pro (revision 28168) +++ /branches/haf_branches/ipp.20100512/ippTasks/minidvodb.pro (revision 28169) @@ -6,17 +6,43 @@ check.globals + +#There is a book for each task, because I don't use labels. + if (not($?haveBooks)) book create MINIDVODB + book create MINIDVODB_MERGE + book create MINIDVODB_CREATE + book create MINIDVODB_ACTIVE $haveBooks = TRUE end -if (not($?haveSurveyBooks)) - book create SURVEY_MINIDVODB - $haveSurveyBooks = TRUE -end + $MINIDVODB_DB = 0 $MINIDVODB_WAIT_DB = 0 +$MINIDVODB_MERGE_DB = 0 +$MINIDVODB_CREATE_DB = 0 +$MINIDVODB_ACTIVE_DB = 0 book init minidvodbWaitlist +book init minidvodbMergelist +book init minidvodbCreatelist +book init minidvodbActivelist + +macro minidvodb.create.status + book listbook minidvodbCreatelist +end + +macro minidvodb.create.reset + book init minidvodbCreatelist +end + +macro minidvodb.active.status + book listbook minidvodbActivelist +end + +macro minidvodb.active.reset + book init minidvodbActivelist +end + macro minidvodb.wait.status @@ -28,17 +54,40 @@ end - - +macro minidvodb.merge.status + book listbook minidvodbMergelist +end + +macro minidvodb.merge.reset + book init minidvodbMergelist +end + +#this is the create task, there is no load, it just runs macro minidvodb.create.on - task minidvodbcreate + task minidvodb.create active true end end macro minidvodb.create.off - task minidvodbcreate + task minidvodb.create active false end end - +##these are the tasks that flip from new -> active +macro minidvodb.active.on + task minidvodb.active.load + active true + end + task minidvodb.active.run + active true + end +end +macro minidvodb.active.off + task minidvodb.active.load + active false + end + task minidvodb.active.run + active false + end +end ##these are the tasks that check to see if addRun processing is finished @@ -61,15 +110,53 @@ end - -## at some point, I need to add/separate the ones for the survey task +##these merge the dbs + +macro minidvodb.merge.on + task minidvodb.merge.load + active true + end + task minidvodb.merge.run + active true + end +end +macro minidvodb.merge.off + task minidvodb.merge.load + active false + end + task minidvodb.merge.run + active false + end +end + +## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. macro add.minidvodb - if ($0 != 3) - echo "USAGE: add.minidvodb (minidvodb_group) (dvodb)" + if ($0 != 5) + echo "USAGE: add.minidvodb (minidvodb_group) (dvodb) (interval days) (camera)" break end + + #wait - shoudl be renamed MINIDVODB_WAIT book newpage MINIDVODB $1 book setword MINIDVODB $1 MINIDVODB_GROUP $1 book setword MINIDVODB $1 DVODB $2 book setword MINIDVODB $1 STATE PENDING + #merge + book newpage MINIDVODB_MERGE $1 + book setword MINIDVODB_MERGE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_MERGE $1 DVODB $2 + book setword MINIDVODB_MERGE $1 STATE PENDING + #active + book newpage MINIDVODB_ACTIVE $1 + book setword MINIDVODB_ACTIVE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_ACTIVE $1 DVODB $2 + book setword MINIDVODB_ACTIVE $1 STATE PENDING + + #create note that camera should be GPC1 for it to work. I couldn't figure out how to easily get this out. + book newpage MINIDVODB_CREATE $1 + book setword MINIDVODB_CREATE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_CREATE $1 DVODB $2 + book setword MINIDVODB_CREATE $1 DVODB_DAYS $3 + book setword MINIDVODB_CREATE $1 CAMERA $4 + book setword MINIDVODB_CREATE $1 STATE PENDING end @@ -80,4 +167,7 @@ end book delpage MINIDVODB $1 + book delpage MINIDVODB_MERGE $1 + book delpage MINIDVODB_CREATE $1 + book delpage MINIDVODB_ACTIVE $1 end @@ -87,6 +177,19 @@ break end + echo "minidvodb wait" book listbook MINIDVODB -end + echo "minidvodb merge" + book listbook MINIDVODB_MERGE + echo "minidvodb create" + book listbook MINIDVODB_CREATE + echo "minidvodb_active" + book listbook MINIDVODB_ACTIVE +end + + + + + + @@ -100,5 +203,5 @@ stdout NULL - stderr $LOGDIR/minidvodb.wait.run.log + stderr $LOGDIR/minidvodb.wait.load.log task.exec @@ -123,9 +226,5 @@ if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 end - - - - - +#using check as opposed to list because it sees if it is done with the addRun state yet. $run = addtool -checkminidvodbrunaddrun -state waiting $run = $run -minidvodb_group $minidvodb_group @@ -172,5 +271,5 @@ -# run the chip_imfile.pl script on pending images + task minidvodb.wait.run periods -poll $RUNPOLL @@ -184,7 +283,5 @@ book npages minidvodbWaitlist -var N if ($N == 0) break - if ($NETWORK == 0) break - if ($BURNTOOLING == 1) break - + # look for new images in minidvodbWaitlist (pantaskState == INIT) book getpage minidvodbWaitlist 0 -var pageName -key pantaskState INIT @@ -196,5 +293,5 @@ stdout $LOGDIR/minidvodb.wait.run.log stderr $LOGDIR/minidvodb.wait.run.log - + $run = addtool -updateminidvodbrun -minidvodb_id $MINIDVODB_ID -set_state to_be_merged @@ -209,8 +306,6 @@ if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 end -# save the pageName for future reference below + # save the pageName for future reference below options $pageName - - # create the command line @@ -222,5 +317,5 @@ command $run end -# default exit status + # default exit status task.exit default process_exit minidvodbWaitlist $options:0 $JOB_STATUS @@ -229,10 +324,5 @@ # locked list task.exit crash - ### Getting a lot of chip crashes (no idea why), so remove verbosity for now - #showcommand crash echo "hostname: $JOB_HOSTNAME" - - # Set a fault code in the database - process_exit minidvodbWaitlist $options:0 $EXIT_CRASH_ERR end @@ -247,54 +337,31 @@ - - - - - - - - - - - - - - - - - - - -task minidvodbcreate - host local - - periods -poll 10 - periods -exec 12000000000000 +task minidvodb.merge.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC periods -timeout 30 - - npending 1 -task.exec -active false - - stdout $LOGDIR/minidvodb.create.log - stderr $LOGDIR/minidvodb.create.log - - # generate diff warp-warp runs - #task.exec - book npages MINIDVODB -var N + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.merge.load.log + + task.exec + book npages MINIDVODB_MERGE -var N if ($N == 0) -# echo "No minidvodbs for processing" +# echo "No labels for processing" break endif - book getpage MINIDVODB 0 -var minidvodb -key STATE NEW - if ("$minidvodb" == "NULL") + book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") # All labels have been done --- reset # echo "Resetting labels" for i 0 $N - book getpage MINIDVODB $i -var label - book setword MINIDVODB $minidvodb STATE NEW + book getpage MINIDVODB_MERGE $i -var minidvodb_group + book setword MINIDVODB_MERGE $minidvodb_group STATE NEW end - book getpage MINIDVODB 0 -var minidvodb -key STATE NEW + book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW # Select different database @@ -302,22 +369,18 @@ if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 end - - book setword MINIDVODB $minidvodb STATE DONE - book getword MINIDVODB $minidvodb DVODB -var dvodb - - $run = minidvodb_createdb.pl - $run = $run --camera GPC1 - $run = $run --dvodb $dvodb - $run = $run --minidvodb_group $minidvodb_group - $run = $run --outroot $dvodb - $run = $run --redirect-output + #finds the minidvodbs in a state of 'to_be_merged' + $run = addtool -listminidvodbrun -state to_be_merged + $run = $run -minidvodb_group $minidvodb_group if ($DB:n == 0) option DEFAULT else - $run = $run --dbname $DB:$MINIDVODB_DB + # save the DB name for the exit tasks option $DB:$MINIDVODB_DB - end - -# echo $run + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + add_poll_args run + command $run end @@ -325,6 +388,12 @@ # success task.exit 0 -# echo "Success" - end + # convert 'stdout' to book format + ipptool2book stdout minidvodbMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbMergelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbMergelist + end # locked list @@ -344,2 +413,310 @@ + + +task minidvodb.merge.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbMergelist -var N + if ($N == 0) break + + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage minidvodbMergelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword minidvodbMergelist $pageName pantaskState RUN + book getword minidvodbMergelist $pageName minidvodb_id -var MINIDVODB_ID + book getword minidvodbMergelist $pageName state -var STATE + stdout $LOGDIR/minidvodb.merge.run.log + stderr $LOGDIR/minidvodb.merge.run.log + + $run = addtool -updateminidvodbrun -minidvodb_id $MINIDVODB_ID -set_state merging + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit minidvodbMergelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + + echo "hostname: $JOB_HOSTNAME" + + # Set a fault code in the database + + process_exit minidvodbMergelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbMergelist $options:0 pantaskState TIMEOUT + end +end + + +#this is the complicated script - it creates the minidvodbRuns +#since none may exist by default, I decided not to have a load.task +#it calls a perl script, which creates a new one if: +#there is none in an active state (true if there is none at all) +#the current one is older than the dvodb_age +#there are none in new (we have a task that moves it to 'active') +#the current active one has > 30000 add_ids in it + +#also confusing: it succeeds if it doesn't create it (if it doesn't want to create one) +#and it succeeds if it creates one. (otherwise there would be a lot of false failures, +# like in replication) +#it fails if it has problems with one of the addtool commands in the script. + +task minidvodb.create + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 240 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.create.log + + task.exec + + periods -poll 60 +#wait a bit before trying again + + book npages MINIDVODB_CREATE -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_CREATE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_CREATE $i -var minidvodb_group + book setword MINIDVODB_CREATE $minidvodb_group STATE NEW + end + book getpage MINIDVODB_CREATE 0 -var minidvodb_group -key STATE NEW + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + + book setword MINIDVODB_CREATE $minidvodb_group STATE DONE + book getword MINIDVODB_CREATE $minidvodb_group DVODB -var dvodb + book getword MINIDVODB_CREATE $minidvodb_group DVODB_DAYS -var dvodb_days + book getword MINIDVODB_CREATE $minidvodb_group CAMERA -var camera + $run = minidvodb_createdb.pl --camera $camera --outroot $dvodb --dvodb $dvodb --minidvodb_group $minidvodb_group --interval $dvodb_days + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run --dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + command $run + end + + # success + task.exit 0 + if ($VERBOSE > 2) + showcommand + end + end + + # locked list + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + +task minidvodb.active.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.active.load.log + + task.exec + book npages MINIDVODB_ACTIVE -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_ACTIVE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_ACTIVE $i -var minidvodb_group + book setword MINIDVODB_ACTIVE $minidvodb_group STATE NEW + end + book getpage MINIDVODB_ACTIVE 0 -var minidvodb_group -key STATE NEW + + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + + + $run = addtool -listminidvodbrun -state new + $run = $run -minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + add_poll_args run + + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout minidvodbActivelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbActivelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbActivelist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +task minidvodb.active.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbActivelist -var N + if ($N == 0) break + + # look for new images in minidvodbActivelist (pantaskState == INIT) + book getpage minidvodbActivelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword minidvodbActivelist $pageName pantaskState RUN + book getword minidvodbActivelist $pageName minidvodb_group -var MINIDVODB_GROUP + stdout $LOGDIR/minidvodb.active.run.log + stderr $LOGDIR/minidvodb.active.run.log + + $run = addtool -flipminidvodbrun -minidvodb_group $MINIDVODB_GROUP + + +if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end +# save the pageName for future reference below + options $pageName + + + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end +# default exit status + task.exit default + process_exit minidvodbActivelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + #showcommand crash + echo "hostname: $JOB_HOSTNAME" + # Set a fault code in the database + + process_exit minidvodbActivelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbActivelist $options:0 pantaskState TIMEOUT + end +end +