IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 3, 2010, 8:50:52 AM (16 years ago)
Author:
eugene
Message:

updates from trunk

Location:
branches/simtest_nebulous_branches
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/simtest_nebulous_branches

  • branches/simtest_nebulous_branches/ippTasks/dist.pro

    r25030 r27840  
    1515$distToAdvance_DB = 0
    1616$distQueue_DB = 0
     17$dist_revert_DB = 0
     18
     19### list of stages
     20#list of stages
     21$DIST_STAGE:n = 0
     22list DIST_STAGE -add "raw"
     23list DIST_STAGE -add "chip"
     24list DIST_STAGE -add "camera"
     25list DIST_STAGE -add "fake"
     26list DIST_STAGE -add "warp"
     27list DIST_STAGE -add "diff"
     28list DIST_STAGE -add "stack"
     29list DIST_STAGE -add "SSdiff"
     30
     31$currentStage = 0
    1732
    1833### Check status of tasks
    … …  
    4257    active true
    4358  end
    44   task dist.queueruns
    45 #   We aren't ready to run this task yet. It will queue much too much
    46 #    active true
    47     active false
    48   end
    49 end
     59  task dist.revert
     60    active false
     61  end
     62end
     63
    5064macro dist.off
    5165  task dist.process.load
    … …  
    6175    active false
    6276  end
    63   task dist.queueruns
    64     active false
    65   end
    66 end
     77  task dist.revert
     78    active false
     79  end
     80end
     81
     82macro dist.revert.on
     83  task dist.revert
     84    active true
     85  end
     86end
     87
     88macro dist.revert.off
     89  task dist.revert
     90    active false
     91  end
     92end
     93
     94macro get.host.for.component
     95    if ($0 != 3)
     96        echo "USAGE: get.host.for.component (componentID) (varname)"
     97        break
     98    end
     99    local component host varname
     100    $component = $1
     101    $varname = $2
     102    substr $component 0 3 COMP_HEAD
     103    if ("$COMP_HEAD" == "sky")
     104        book getword ipphosts dist_skycell count -var count
     105        local j myValue skyhash
     106        list word -splitbychar . $component
     107        $skyhash = 0
     108        for j 0 $word:n
     109            inthash $word:$j $count -var myValue
     110            $skyhash = $skyhash + $myValue
     111        end
     112        inthash $skyhash $count -var skyhash
     113        sprintf skyname "sky%02d" $skyhash
     114        book getword ipphosts dist_skycell $skyname -var host
     115    else
     116        book getword ipphosts dist_chip $component -var host
     117    end
     118    $$varname = $host
     119end
     120
     121macro set.dist.workdir.by.component
     122    if ($0 != 5)
     123        echo "USAGE: set.workdir.by.component (stage_id) (componentID) (template) (varname)"
     124        break
     125    end
     126    local host stage_id component default template varname length start count selector component_id random_number
     127
     128    $stage_id = $1
     129    $component = $2
     130    $template = $3
     131    $varname = $4
     132    if ("$template" == "NULL")
     133        echo "ERROR: WORKDIR template not set."
     134        break
     135    end
     136    book getword ipphosts distribution count -var count
     137    if ("$count" == "NULL")
     138        echo "ERROR: distributin hosts list is empty"
     139        break
     140    end
     141    $host = "foo"
     142    if ("$component" == "exposure")
     143        # take last two letters of selector
     144        # treat it as an integer and use modulous of length of distribution hosts
     145        # to compute an index into the host table
     146        strlen $stage_id length
     147        $start = $length - 2
     148        substr $stage_id $start 2 index
     149        $component_id = $index % $count
     150        book getword ipphosts distribution $component_id -var myhost
     151    else
     152        get.host.for.component $component myhost
     153    end
     154
     155    if ("$myhost" == "NULL")
     156        echo "ERROR: failed to find host for $component"
     157        break
     158    end
     159
     160    strsub $template @HOST@ $myhost -var $varname
     161
     162    #echo template is $template
     163    #echo outdir is $$varname
     164end
     165
    67166
    68167task           dist.process.load
    … …  
    74173  npending     1
    75174
    76   stdout NULL
    77   stderr $LOGDIR/dist.process.log
    78175
    79176  task.exec
    80     $run = disttool -pendingcomponent
     177     # stdout NULL
     178     # stderr $LOGSUBDIR/dist.process.load.log
     179
     180    $run = disttool -pendingcomponent -stage $DIST_STAGE:$currentStage
     181    $currentStage ++
     182    if ($currentStage >= $DIST_STAGE:n) set currentStage = 0
     183
    81184    if ($DB:n == 0)
    82185      option DEFAULT
    … …  
    85188      option $DB:$distToProcess_DB
    86189      $run = $run -dbname $DB:$distToProcess_DB
    87       $distToProcess_DB ++
    88       if ($distToProcess_DB >= $DB:n) set distToProcess_DB = 0
     190
     191      # only increment the database number after we have gone through all of
     192      # the stages
     193      if ($currentStage == 0)
     194          $distToProcess_DB ++
     195          if ($distToProcess_DB >= $DB:n) set distToProcess_DB = 0
     196      end
    89197    end
    90198    add_poll_args run
    … …  
    122230
    123231  task.exec
     232    if ($NETWORK == 0) break
    124233    book npages distToProcess -var N
    125234    if ($N == 0)
    … …  
    128237    end
    129238    periods -exec 0.05
    130 
    131     if ($NETWORK == 0) break
    132239   
    133240    # look for new components to process (pantaskState == INIT)
    … …  
    135242    if ("$pageName" == "NULL") break
    136243
     244#    echo running $pageName
     245
    137246    book setword distToProcess $pageName pantaskState RUN
    138247    book getword distToProcess $pageName dist_id -var DIST_ID
    139248    book getword distToProcess $pageName camera -var CAMERA
    140     book getword distToProcess $pageName stage -var STAGE
    141     book getword distToProcess $pageName stage_id -var STAGE_ID
     249    book getword distToProcess $pageName stage -var DIST_STAGE
     250    book getword distToProcess $pageName stage_id -var DIST_STAGE_ID
    142251    book getword distToProcess $pageName clean -var CLEAN
    143252    book getword distToProcess $pageName component -var COMPONENT
    … …  
    149258    book getword distToProcess $pageName no_magic -var NO_MAGIC
    150259    book getword distToProcess $pageName magicked -var MAGICKED
    151     book getword distToProcess $pageName outdir -var OUTDIR
     260    book getword distToProcess $pageName alt_path_base -var ALT_PATH_BASE
     261    book getword distToProcess $pageName outdir -var OUTDIR_TEMPLATE
    152262    book getword distToProcess $pageName dbname -var DBNAME
    153263
    … …  
    160270        $EXTRA_ARGS = $EXTRA_ARGS --magicked
    161271    end
    162     # no_magic is output as integer due to the union in the sql
    163     if ($NO_MAGIC)
     272    # is this right for stack and fake?
     273    if ("$NO_MAGIC" == "T")
    164274        $EXTRA_ARGS = $EXTRA_ARGS --no_magic
    165275    end
    … …  
    167277        $EXTRA_ARGS = $EXTRA_ARGS --poor_quality
    168278    end
    169 
    170 #    set.host.for.camera $CAMERA $MAGIC_ID
    171 #    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
    172 #    host anyhost
    173 
     279    if ("$ALT_PATH_BASE" != "")
     280        $EXTRA_ARGS = $EXTRA_ARGS --alt_path_base $ALT_PATH_BASE
     281    end
     282
     283
     284    set.dist.workdir.by.component $DIST_STAGE_ID $COMPONENT $OUTDIR_TEMPLATE OUTDIR
     285    if ("$OUTDIR" == "NULL")
     286        echo ERROR failed to set workdir for $COMPONENT
     287        break
     288    end
    174289
    175290    substr $COMPONENT 0 3 COMP_HEAD
    … …  
    181296    end
    182297
    183 
    184298    sprintf logfile "%s/dist.%s.%s.log" $OUTDIR $DIST_ID $COMPONENT
    185     stdout $logfile
    186     stderr $logfile
    187 
    188     $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile
     299#    stdout $logfile
     300#    stderr $logfile
     301
     302    $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $DIST_STAGE --stage_id $DIST_STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile
    189303
    190304    add_standard_args run
    … …  
    212326end
    213327
    214 
    215328task           dist.advance.load
    216329  host         local
    … …  
    221334  npending     1
    222335
    223   stdout NULL
    224   stderr $LOGDIR/dist.advance.load.log
    225 
    226336  task.exec
     337  #  stderr $LOGSUBDIR/dist.advance.load.log
     338
    227339    $run = disttool -toadvance
    228340    if ($DB:n == 0)
    … …  
    263375end
    264376
     377
    265378task           dist.advance.run
    266379  periods      -poll $RUNPOLL
    … …  
    270383  task.exec
    271384    book npages distToAdvance -var N
    272     if ($N == 0) break
    273385    if ($NETWORK == 0) break
     386    if ($N == 0)
     387        periods -exec 10
     388        break
     389    end
     390    periods -exec .05
    274391   
    275392    # look for new components to process (pantaskState == INIT)
    … …  
    281398    book getword distToAdvance $pageName stage   -var STAGE
    282399    book getword distToAdvance $pageName stage_id -var STAGE_ID
    283     book getword distToAdvance $pageName outdir -var OUTDIR
     400    book getword distToAdvance $pageName outdir -var OUTDIR_TEMPLATE
    284401    book getword distToAdvance $pageName clean -var CLEAN
    285402    book getword distToAdvance $pageName dbname -var DBNAME
    … …  
    291408    host anyhost
    292409
     410    # using $DIST_ID as the "component" works fine here since we only look
     411    # at the last two digits
     412    set.dist.workdir.by.component $STAGE_ID "exposure" $OUTDIR_TEMPLATE OUTDIR
     413    if ("$OUTDIR" == "NULL")
     414        echo ERROR failed to set workdir for $DIST_ID
     415        break
     416    end
     417
    293418    sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID
    294     stdout $logfile
    295     stderr $logfile
     419#    stdout $logfile
     420#    stderr $logfile
    296421
    297422    $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile
    … …  
    320445end
    321446
    322 task           dist.queueruns
    323 #  host         local
    324 
    325   periods      -poll $RUNPOLL
    326   periods      -exec 30
    327   periods      -timeout 45
     447task dist.revert
     448  host         local
     449
     450  periods      -poll 5.0
     451  periods      -exec 60.0
     452  periods      -timeout 120.0
    328453  npending     1
    329 
    330454  active false
    331 
    332 #  stdout $LOGDIR/dist.queuruns
    333 #  stderr $LOGDIR/dist.queueruns
     455 
     456  stdout NULL
     457  stderr $LOGDIR/revert.log
    334458
    335459  task.exec
    336     $MYARGS = ""
    337     # assume that we need magic unless we are running simtest
    338     if ($?SIMTEST_CAMERA != 0)
    339         $MYARGS = --no_magic
    340     end
    341 
    342     $run = dist_queue_runs.pl $MYARGS --stage_limit 16 --logfile $LOGDIR/dist.queueruns
    343 
     460    if ($LABEL:n == 0) break
     461    $run = disttool -revertcomponent
    344462    if ($DB:n == 0)
    345        $DBNAME = DEFAULT
     463      option DEFAULT
    346464    else
    347       # save the DB name for add_standard_args
    348       $DBNAME =  $DB:$distQueue_DB
    349       $distQueue_DB ++
    350       if ($distQueue_DB >= $DB:n) set distQueue_DB = 0
    351     end
    352 
    353     host anyhost
    354 
    355     add_standard_args run
    356 
    357     # create the command line
    358     if ($VERBOSE > 1)
    359       echo command $run
    360     end
     465      # save the DB name for the exit tasks
     466      option $DB:$dist_revert_DB
     467      $run = $run -dbname $DB:$dist_revert_DB
     468      $dist_revert_DB ++
     469      if ($dist_revert_DB >= $DB:n) set dist_revert_DB = 0
     470    end
     471    add_poll_labels run
     472    echo $run
    361473    command $run
    362474  end
    363475
    364   task.exit     $EXIT_SUCCESS
    365     # nothing to do
    366   end
    367 
    368   # default exit status
     476  # success
     477  task.exit    0
     478  end
     479
     480  # locked list
    369481  task.exit    default
    370482    showcommand failure
    371483  end
    372484
    373   # operation timed out?
     485  task.exit    crash
     486    showcommand crash
     487  end
     488
     489  # operation times out?
    374490  task.exit    timeout
    375491    showcommand timeout
    376492  end
    377 
    378   # operation timed out?
    379   task.exit    crash
    380     showcommand crash
    381   end
    382 end
     493end
Note: See TracChangeset for help on using the changeset viewer.