IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 33030 for trunk/ippTasks


Ignore:
Timestamp:
Dec 27, 2011, 3:09:17 PM (15 years ago)
Author:
heather
Message:

crap. I don't know why it thinks some of these are modified, but diffing them
against a fresh checkout of the trunk shows no differences.

These are the merge of addstar changes into the trunk. This adds tables,
breaking apart minidvodbRun into 2 parts (building/relphot/addstar) and
(verify/merging), which will make it easier to rebuild dvodbs in the
future. There's some debugging of various tasks here (addstar.revert, for
example), and a lot of cleanup of code.

Location:
trunk
Files:
5 edited
1 copied

Legend:

Unmodified
Added
Removed

  • trunk

  • trunk/ippTasks/Makefile.am

    r32983 r33030  
    3838        minidvodb.pro \
    3939        minidvodbcopy.pro \
     40        mergedvodb.pro \
    4041        nightly_science.pro \
    4142        burntool.pro \

  • trunk/ippTasks/addstar.pro

    r32832 r33030  
    5353
    5454macro addstar.revert.off
    55   task addstar.revert
     55  task addstar.revert.cam
     56    active false
     57  end
     58 task addstar.revert.stack
     59    active false
     60  end
     61 task addstar.revert.staticsky
     62    active false
     63  end
     64 task addstar.revert.staticsky_multi
    5665    active false
    5766  end
    5867end
    5968macro addstar.revert.on
    60   task addstar.revert
     69  task addstar.revert.cam
     70    active true
     71  end
     72  task addstar.revert.stack
     73    active true
     74  end
     75  task addstar.revert.staticsky_multi
     76    active true
     77  end
     78  task addstar.revert.staticsky
    6179    active true
    6280  end
    … …  
    6583
    6684# this variable will cycle through the known database names
    67 $addstar_DB = 0
    68 $addstar_revert_DB = 0
     85$addstar_DB = 0
     86#this may not work for more databases (addstar) will do that later)
     87$addstar_revert_DB_C = 0
     88$addstar_revert_DB_S = 0
     89$addstar_revert_DB_SS = 0
     90$addstar_revert_DB_SSM = 0
    6991
    7092
    … …  
    318340    book getword addPendingExp $pageName stage_extra1 -var STAGE_EXTRA1
    319341    book getword addPendingExp $pageName stageroot -var STAGEROOT
    320     book getword addPendingExp $pageName stage -var STAGE 
     342    book getword addPendingExp $pageName stage -var STAGE
     343    book getword addPendingExp $pageName stage_id -var STAGE_ID
    321344    book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE
    322345    book getword addPendingExp $pageName reduction -var REDUCTION
    … …  
    367390    if ("$STAGE" == "staticsky_multi")
    368391      $run = $run --multi_num $STAGE_EXTRA1
     392    end
     393    if ("$STAGE" == "cam")
     394      $run = $run --stage_id $STAGE_ID
    369395    end
    370396    if ("$IMAGE_ONLY" == "T")
    … …  
    411437end
    412438
    413 task addstar.revert
     439task addstar.revert.cam
    414440  host         local
    415441
    … …  
    425451  task.exec
    426452    if ($LABEL:n == 0) break
    427     $run = addtool -revertprocessedexp
    428     if ($DB:n == 0)
    429       option DEFAULT
    430     else
    431       # save the DB name for the exit tasks
    432       option $DB:$addstar_revert_DB
    433       $run = $run -dbname $DB:$addstar_revert_DB
    434       $addstar_revert_DB ++
    435       if ($addstar_revert_DB >= $DB:n) set addstar_revert_DB = 0
     453    $run = addtool -revertprocessedexp -stage cam
     454    if ($DB:n == 0)
     455      option DEFAULT
     456    else
     457      # save the DB name for the exit tasks
     458      option $DB:$addstar_revert_DB_C
     459      $run = $run -dbname $DB:$addstar_revert_DB_C
     460      $addstar_revert_DB_C ++
     461      if ($addstar_revert_DB_C >= $DB:n) set addstar_revert_DB_C = 0
    436462    end
    437463    add_poll_labels run
    … …  
    457483  end
    458484end
     485task addstar.revert.stack
     486  host         local
     487
     488  periods      -poll 5.0
     489  periods      -exec 60.0
     490  periods      -timeout 1200
     491  npending     1
     492  active        false
     493
     494  stdout NULL
     495  stderr $LOGDIR/revert.log
     496
     497  task.exec
     498    if ($LABEL:n == 0) break
     499    $run = addtool -revertprocessedexp -stage stack
     500    if ($DB:n == 0)
     501      option DEFAULT
     502    else
     503      # save the DB name for the exit tasks
     504      option $DB:$addstar_revert_DB_S
     505      $run = $run -dbname $DB:$addstar_revert_DB_S
     506      $addstar_revert_DB_S ++
     507      if ($addstar_revert_DB_S >= $DB:n) set addstar_revert_DB_S = 0
     508    end
     509    add_poll_labels run
     510    command $run
     511  end
     512
     513  # success
     514  task.exit    0
     515  end
     516
     517  # locked list
     518  task.exit    default
     519    showcommand failure
     520  end
     521
     522  task.exit    crash
     523    showcommand crash
     524  end
     525
     526  # operation times out?
     527  task.exit    timeout
     528    showcommand timeout
     529  end
     530end
     531task addstar.revert.staticsky
     532  host         local
     533
     534  periods      -poll 5.0
     535  periods      -exec 60.0
     536  periods      -timeout 1200
     537  npending     1
     538  active        false
     539
     540  stdout NULL
     541  stderr $LOGDIR/revert.log
     542
     543  task.exec
     544    if ($LABEL:n == 0) break
     545    $run = addtool -revertprocessedexp -stage staticsky
     546    if ($DB:n == 0)
     547      option DEFAULT
     548    else
     549      # save the DB name for the exit tasks
     550      option $DB:$addstar_revert_DB_SS
     551      $run = $run -dbname $DB:$addstar_revert_DB_SS
     552      $addstar_revert_DB_SS ++
     553      if ($addstar_revert_DB_SS >= $DB:n) set addstar_revert_DB_SS = 0
     554    end
     555    add_poll_labels run
     556    command $run
     557  end
     558
     559  # success
     560  task.exit    0
     561  end
     562
     563  # locked list
     564  task.exit    default
     565    showcommand failure
     566  end
     567
     568  task.exit    crash
     569    showcommand crash
     570  end
     571
     572  # operation times out?
     573  task.exit    timeout
     574    showcommand timeout
     575  end
     576end
     577task addstar.revert.staticsky_multi
     578  host         local
     579
     580  periods      -poll 5.0
     581  periods      -exec 60.0
     582  periods      -timeout 1200
     583  npending     1
     584  active        false
     585
     586  stdout NULL
     587  stderr $LOGDIR/revert.log
     588
     589  task.exec
     590    if ($LABEL:n == 0) break
     591    $run = addtool -revertprocessedexp -stage staticsky_multi
     592    if ($DB:n == 0)
     593      option DEFAULT
     594    else
     595      # save the DB name for the exit tasks
     596      option $DB:$addstar_revert_DB_SSM
     597      $run = $run -dbname $DB:$addstar_revert_DB_SSM
     598      $addstar_revert_DB_SSM ++
     599      if ($addstar_revert_DB_SSM >= $DB:n) set addstar_revert_DB_SSM = 0
     600    end
     601    add_poll_labels run
     602    command $run
     603  end
     604
     605  # success
     606  task.exit    0
     607  end
     608
     609  # locked list
     610  task.exit    default
     611    showcommand failure
     612  end
     613
     614  task.exit    crash
     615    showcommand crash
     616  end
     617
     618  # operation times out?
     619  task.exit    timeout
     620    showcommand timeout
     621  end
     622end

  • trunk/ippTasks/minidvodb.pro

    r29634 r33030  
    1111if (not($?haveminidvodbBooks))
    1212 book create MINIDVODB
    13  book create MINIDVODB_MERGE
     13 book create MINIDVODB_PREMERGE
    1414 book create MINIDVODB_CREATE
    1515 book create MINIDVODB_ACTIVE   
    … …  
    2020
    2121$MINIDVODB_WAIT_DB = 0
    22 $MINIDVODB_MERGE_DB = 0
     22$MINIDVODB_PREMERGE_DB = 0
    2323$MINIDVODB_CREATE_DB = 0
    2424$MINIDVODB_ACTIVE_DB = 0
    2525
    2626book init minidvodbWaitlist
    27 book init minidvodbMergelist
     27book init minidvodbPreMergelist
    2828book init minidvodbCreatelist
    2929book init minidvodbActivelist
    … …  
    5454end
    5555
    56 macro minidvodb.merge.status
    57   book listbook minidvodbMergelist
    58 end
    59 
    60 macro minidvodb.merge.reset
    61   book init minidvodbMergelist
     56macro minidvodb.premerge.status
     57  book listbook minidvodbPreMergelist
     58end
     59
     60macro minidvodb.premerge.reset
     61  book init minidvodbPreMergelist
    6262end
    6363
    … …  
    112112##these merge the dbs
    113113
    114 macro minidvodb.merge.on
    115   task minidvodb.merge.load
     114macro minidvodb.premerge.on
     115  task minidvodb.premerge.load
    116116    active true
    117117  end
    118   task minidvodb.merge.run
     118  task minidvodb.premerge.run
    119119    active true
    120120  end 
    121121end
    122 macro minidvodb.merge.off
    123   task minidvodb.merge.load
     122macro minidvodb.premerge.off
     123  task minidvodb.premerge.load
    124124    active false
    125125  end
    126   task minidvodb.merge.run
     126  task minidvodb.premerge.run
    127127    active false
    128128  end 
    … …  
    142142  book setword MINIDVODB $1 STATE PENDING
    143143  #merge 
    144   book newpage MINIDVODB_MERGE $1
    145   book setword MINIDVODB_MERGE $1 MINIDVODB_GROUP $1
    146   book setword MINIDVODB_MERGE $1 DVODB $3
    147   book setword MINIDVODB_MERGE $1 STATE PENDING
     144  book newpage MINIDVODB_PREMERGE $1
     145  book setword MINIDVODB_PREMERGE $1 MINIDVODB_GROUP $1
     146  book setword MINIDVODB_PREMERGE $1 DVODB $3
     147  book setword MINIDVODB_PREMERGE $1 STATE PENDING
    148148  #active 
    149149  book newpage MINIDVODB_ACTIVE $1
    … …  
    168168  end
    169169  book delpage MINIDVODB $1
    170   book delpage MINIDVODB_MERGE $1
     170  book delpage MINIDVODB_PREMERGE $1
    171171  book delpage MINIDVODB_CREATE $1 
    172172  book delpage MINIDVODB_ACTIVE $1
    … …  
    180180  echo "minidvodb wait"
    181181  book listbook MINIDVODB
    182   echo "minidvodb merge"
    183   book listbook MINIDVODB_MERGE
     182  echo "minidvodb premerge"
     183  book listbook MINIDVODB_PREMERGE
    184184  echo "minidvodb create"
    185185  book listbook MINIDVODB_CREATE
    … …  
    339339
    340340
    341 task           minidvodb.merge.load
     341task           minidvodb.premerge.load
    342342  host         local
    343343
    … …  
    348348
    349349  stdout NULL
    350   stderr $LOGDIR/minidvodb.merge.load.log
     350  stderr $LOGDIR/minidvodb.premerge.load.log
    351351
    352352  task.exec
    353     book npages MINIDVODB_MERGE -var N
     353    book npages MINIDVODB_PREMERGE -var N
    354354    if ($N == 0)
    355355#      echo "No labels for processing"
    … …  
    357357    endif
    358358
    359     book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW
     359    book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW
    360360    if ("$minidvodb_group" == "NULL")
    361361      # All labels have been done --- reset
    362362#      echo "Resetting labels"
    363363      for i 0 $N
    364         book getpage MINIDVODB_MERGE $i -var minidvodb_group
    365         book setword MINIDVODB_MERGE $minidvodb_group STATE NEW
     364        book getpage MINIDVODB_PREMERGE $i -var minidvodb_group
     365        book setword MINIDVODB_PREMERGE $minidvodb_group STATE NEW
    366366      end
    367       book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW
     367      book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW
    368368
    369369      # Select different database
    … …  
    372372    end
    373373    #finds the minidvodbs in a state of 'to_be_merged'
    374     book setword MINIDVODB_MERGE $minidvodb_group STATE DONE
     374    book setword MINIDVODB_PREMERGE $minidvodb_group STATE DONE
    375375    $run = addtool -listminidvodbrun -state to_be_merged -limit 1
    376376    $run = $run -minidvodb_group $minidvodb_group
    … …  
    392392  task.exit    0
    393393    # convert 'stdout' to book format
    394     ipptool2book stdout minidvodbMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     394    ipptool2book stdout minidvodbPreMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT
    395395    if ($VERBOSE > 2)
    396       book listbook minidvodbMergelist
     396      book listbook minidvodbPreMergelist
    397397    end
    398398    # delete existing entries in the appropriate pantaskStates
    399     process_cleanup minidvodbMergelist
     399    process_cleanup minidvodbPreMergelist
    400400  end
    401401
    … …  
    418418
    419419
    420 task           minidvodb.merge.run
     420task           minidvodb.premerge.run
    421421  periods      -poll $RUNPOLL
    422422  periods      -exec $RUNEXEC
    … …  
    432432    periods -exec $RUNEXEC
    433433
    434     book npages minidvodbMergelist -var N
     434    book npages minidvodbPreMergelist -var N
    435435    if ($N == 0) break
    436436   
    437437    # look for new images in minidvodbWaitlist (pantaskState == INIT)
    438     book getpage minidvodbMergelist 0 -var pageName -key pantaskState INIT
     438    book getpage minidvodbPreMergelist 0 -var pageName -key pantaskState INIT
    439439    if ("$pageName" == "NULL") break
    440440
    441     book setword minidvodbMergelist $pageName pantaskState RUN
    442     book getword minidvodbMergelist $pageName minidvodb_id -var MINIDVODB_ID
    443     book getword minidvodbMergelist $pageName minidvodb_group -var MINIDVODB_GROUP
    444     book getword minidvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH
    445     book getword minidvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH
    446     book getword minidvodbMergelist $pageName state -var STATE
    447     stdout $LOGDIR/minidvodb.merge.run.log
    448     stderr $LOGDIR/minidvodb.merge.run.log
    449 
    450     $run = minidvodb_merge.pl --camera GPC1 --mergedvodb $MERGEDVODB_PATH --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID
     441    book setword minidvodbPreMergelist $pageName pantaskState RUN
     442    book getword minidvodbPreMergelist $pageName minidvodb_id -var MINIDVODB_ID
     443    book getword minidvodbPreMergelist $pageName minidvodb_group -var MINIDVODB_GROUP
     444    book getword minidvodbPreMergelist $pageName minidvodb_path -var MINIDVODB_PATH
     445    book getword minidvodbPreMergelist $pageName state -var STATE
     446    stdout $LOGDIR/minidvodb.premerge.run.log
     447    stderr $LOGDIR/minidvodb.premerge.run.log
     448
     449    $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID
    451450   
    452451  if ($DB:n == 0)
    … …  
    472471  # default exit status
    473472    task.exit    default
    474     process_exit minidvodbMergelist $options:0 $JOB_STATUS
     473    process_exit minidvodbPreMergelist $options:0 $JOB_STATUS
    475474  end
    476475
    … …  
    482481    # Set a fault code in the database
    483482   
    484     process_exit minidvodbMergelist $options:0 $EXIT_CRASH_ERR
     483    process_exit minidvodbPreMergelist $options:0 $EXIT_CRASH_ERR
    485484  end
    486485
    … …  
    488487    task.exit    timeout
    489488        showcommand timeout
    490     book setword minidvodbMergelist $options:0 pantaskState TIMEOUT
     489    book setword minidvodbPreMergelist $options:0 pantaskState TIMEOUT
    491490  end
    492491end

  • trunk/ippTasks/survey.pro

    r32832 r33030  
    33# test for required global variables
    44check.globals
    5 
    6 # there is a bug in 'getbook', use a different method below
    7 # book getbook SURVEY_DIFF -var isBook
    8 # if ("$isBook" != "SURVEY_DIFF")
    9 #   book create SURVEY_DIFF
    10 # end
    11 # book getbook SURVEY_MAGIC -var isBook
    12 # if ("$isBook" == "NULL")
    13 #   book create SURVEY_MAGIC
    14 # end
    15 # book getbook SURVEY_DESTREAK -var isBook
    16 # if ("$isBook" == "NULL")
    17 #   book create SURVEY_DESTREAK
    18 # end
    19 # book getbook SURVEY_DIST -var isBook
    20 # if ("$isBook" == "NULL")
    21 #   book create SURVEY_DIST
    22 # end
    235
    246if (not($?haveSurveyBooks))
    … …  
    3012 book create SURVEY_DIST
    3113 book create SURVEY_ADDSTAR   
     14 book create SURVEY_MERGEDVODB
    3215 book create SURVEY_CHIP_BG
    3316 book create SURVEY_WARP_BG
    … …  
    4427$SURVEY_DIST_DB = 0
    4528$SURVEY_ADDSTAR_DB = 0
     29$SURVEY_MERGEDVODB_DB = 0
    4630$SURVEY_CHIP_BG_DB = 0
    4731$SURVEY_WARP_BG_DB = 0
    … …  
    6852    active true
    6953  end
     54  task survey.mergedvodb
     55    active true
     56  end
    7057  task survey.destreak
    7158    active true
    … …  
    10491    active false
    10592  end
     93  task survey.mergedvodb
     94    active false
     95  end 
    10696  task survey.destreak
    10797    active false
    … …  
    270260  book listbook SURVEY_ADDSTAR
    271261end
     262
     263macro survey.add.mergedvodb
     264  if ($0 != 4)
     265    echo "USAGE: survey.add.mergedvodb (tag) (mergedvodb) (minidvodb_group)"
     266    break
     267  end
     268  book newpage SURVEY_MERGEDVODB $1
     269  book setword SURVEY_MERGEDVODB $1 MERGEDVODB $2
     270  book setword SURVEY_MERGEDVODB $1 MINIDVODB_GROUP $3
     271end
     272
     273macro survey.del.mergedvodb
     274  if ($0 != 2)
     275    echo "USAGE: survey.del.mergedvodb (tag)"
     276    break
     277  end
     278  book delpage SURVEY_MERGEDVODB $1
     279end
     280
     281macro survey.show.mergedvodb
     282  if ($0 != 1)
     283    echo "USAGE: survey.show.mergedvodb"
     284    break
     285  end
     286  book listbook SURVEY_MERGEDVODB
     287end
     288
    272289
    273290# user functions to manipulate destreak labels
    … …  
    826843    if ("$stage" == "cam")
    827844        #only queue destreaked cams. stacks and staticsky don't need this
    828         if ("$DB:$SURVEY_ADDSTAR_DB" == "isp")
     845    #    if ("$DB:$SURVEY_ADDSTAR_DB" == "isp")
    829846            #this is the only way I can think of how to handle this (but it is messy). If it is a database that we KNOW doesn't use magicked (ie, isp), then do not queue destreaked
     847           # now with no magic, all are uncensored
    830848           $run = $run -uncensored
    831         else
     849#       else
    832850           #if not isp, then run destreaked version only
    833            $run = $run -destreaked
    834         end
     851#          $run = $run -destreaked
     852#       end
    835853    end
    836854    if ("$stage" == "staticsky")
    … …  
    875893  end
    876894end
     895
     896
     897task survey.mergedvodb
     898  host local
     899 
     900  periods      -poll $SURVEY_POLL
     901  periods      -exec $SURVEY_EXEC
     902  periods      -timeout $SURVEY_TIMEOUT
     903  npending     1
     904
     905  stdout $LOGDIR/survey.mergedvodb.log
     906  stderr $LOGDIR/survey.mergedvodb.log
     907
     908  task.exec
     909    book npages SURVEY_MERGEDVODB -var N
     910    if ($N == 0)
     911#      echo "No labels for processing"
     912      break
     913    endif
     914
     915    book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW
     916    if ("$tag" == "NULL")
     917      # All labels have been done --- reset
     918    # echo "Resetting labels"
     919      for i 0 $N
     920        book getpage SURVEY_MERGEDVODB $i -var tag
     921        book setword SURVEY_MERGEDVODB $tag STATE NEW
     922      end
     923      book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW
     924
     925      # Select different database
     926      $SURVEY_MERGEDVODB_DB ++
     927      if ($SURVEY_MERGEDVODB_DB >= $DB:n) set SURVEY_MERGEDVODB_DB = 0
     928    end
     929   book setword SURVEY_MERGEDVODB $tag STATE DONE
     930   book getword SURVEY_MERGEDVODB $tag MERGEDVODB -var mergedvodb
     931   book getword SURVEY_MERGEDVODB $tag MINIDVODB_GROUP -var minidvodb_group
     932     
     933 #   $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb
     934    #$run = mergetool
     935    $run = mergedvodb_queue.pl --outroot $mergedvodb --mergedvodb $mergedvodb --minidvodb_group $minidvodb_group
     936    if ($DB:n == 0)
     937      option DEFAULT
     938    else
     939      $run = $run --dbname $DB:$SURVEY_MERGEDVODB_DB
     940      option $DB:$SURVEY_MERGEDVODB_DB
     941    end
     942    #this run doesn't work because there's no workdir
     943    #$run = $run -definebyquery -mergedvodb $mergedvodb -minidvodb_group $minidvodb_group
     944   
     945   
     946    # echo $run
     947
     948  echo $run
     949    command $run
     950  end
     951
     952  # success
     953   
     954  task.exit    0
     955#    echo "Success"
     956  end
     957
     958  # locked list
     959  task.exit    default
     960    showcommand failure
     961  end
     962
     963  task.exit    crash
     964    showcommand crash
     965  end
     966
     967  # operation times out?
     968  task.exit    timeout
     969    showcommand timeout
     970  end
     971end
     972
    877973
    878974
Note: See TracChangeset for help on using the changeset viewer.