IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Aug 7, 2009, 4:08:25 PM (17 years ago)
Author:
Paul Price
Message:

Merging trunk (r25026) to get up-to-date on old branch.

Location:
branches/pap
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/pap

  • branches/pap/ippTasks/magic.pro

    r23913 r25027  
    55
    66$LOGSUBDIR = $LOGDIR/magic
    7 exec mkdir -p $LOGSUBDIR
     7mkdir $LOGSUBDIR
    88
    99### Initialise the books containing the tasks to do
    … …  
    1111book init magicToProcess
    1212book init magicToDS
     13book init magicDSToRevert
    1314
    1415### Database lists
    … …  
    1617$magicToProcess_DB = 0
    1718$magicToDS_DB = 0
     19$magicDSToRevert_DB = 0
     20
     21#list of stages
     22$STAGE:n = 0
     23list STAGE -add "raw"
     24list STAGE -add "chip"
     25list STAGE -add "camera"
     26list STAGE -add "warp"
     27list STAGE -add "diff"
     28$magicDSRevertStage = 0
    1829
    1930### Check status of tasks
    … …  
    2233  book listbook magicToProcess
    2334  book listbook magicToDS
     35  book listbook magicDSToRevert
    2436end
    2537
    … …  
    2941  book init magicToProcess
    3042  book init magicToDS
     43  book init magicDSToRevert
    3144end
    3245
    … …  
    5366end
    5467
     68macro magic.ds.revert.on
     69    task magic.ds.revert.load
     70        active true
     71    end
     72    task magic.ds.revert.run
     73        active true
     74    end
     75end
    5576### Turn tasks off
    5677macro magic.off
    … …  
    7495  end
    7596end
     97
     98macro magic.ds.revert.off
     99    task magic.ds.revert.load
     100        active false
     101    end
     102    task magic.ds.revert.run
     103        active false
     104    end
     105end
     106macro magic.ds.off
     107    task magic.destreak.load
     108        active false
     109    end
     110    task magic.destreak.run
     111        active false
     112    end
     113end
     114macro magic.ds.on
     115    task magic.destreak.load
     116        active true
     117    end
     118    task magic.destreak.run
     119        active true
     120    end
     121end
     122
     123macro magic.ds.status
     124    echo magicToDS
     125    book listbook magicToDS
     126    echo ""
     127    echo magicDSToRevert
     128    book listbook magicDSToRevert
     129end
     130
     131
    76132
    77133task           magic.tree.load
    … …  
    98154    end
    99155    add_poll_args run
     156    add_poll_labels run
    100157    command $run
    101158  end
    … …  
    157214    $WORKDIR = $WORKDIR_TEMPLATE
    158215
    159     basename $TESS_DIR -var TESS_ID
    160 
    161     $TESS_ID = "FIXNS"
    162 
    163216    sprintf outroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID
    164217
    … …  
    228281    end
    229282    add_poll_args run
     283    add_poll_labels run
    230284    command $run
    231285  end
    … …  
    265319  task.exec
    266320    book npages magicToProcess -var N
    267     if ($N == 0) break
     321    if ($N == 0)
     322      periods -exec 10
     323      break
     324    end
     325    periods -exec 0.05
     326
    268327    if ($NETWORK == 0) break
    269328   
    … …  
    280339    book getword magicToProcess $pageName dbname -var DBNAME
    281340
    282 #    set.host.for.camera $CAMERA $MAGIC_ID
    283 #    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
    284     host anyhost
     341#    XXX: we need new macros that select the host and workdir based on the NODE
     342#    we also need to make the magic scripts able to deal with the workdir being spread
     343#    around. That should not be too hard
     344#    For now select nodes by skycell
     345
     346    substr $NODE 0 3 NODE_HEAD
     347    if ("$NODE_HEAD" == "sky")
     348        set.host.for.skycell $NODE
     349    else
     350        host anyhost
     351    end
     352
     353#    currently DetectStreaks expects that the workdir is not in nebulous
     354#    set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
    285355    $WORKDIR = $WORKDIR_TEMPLATE
    286356
    … …  
    288358
    289359    ## generate output log based on filerule (convert the URI to a PATH)
    290     $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot`
     360    ## caution with neb:// names here:
     361    # $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot`
     362    $logfile = $outroot.log
    291363    if ("$logfile" == "")
    292364      echo "WARNING: logfile not defined in magic.process.run"
    … …  
    333405
    334406  periods      -poll $LOADPOLL
    335   periods      -exec $LOADEXEC
    336   periods      -timeout 20
     407  # this query can take a long time
     408  periods      -exec 10
     409  periods      -timeout 120
    337410  npending     1
    338411
    … …  
    341414
    342415  task.exec
    343     $run = magicdstool -todestreak -limit 32
     416    $run = magicdstool -todestreak -limit 120
    344417    if ($DB:n == 0)
    345418      option DEFAULT
    … …  
    351424      if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0
    352425    end
    353 #    XXX magicDSRun doesn't have label so can't use add_poll_args
    354 #    add_poll_args run
     426    add_poll_args run
     427    add_poll_labels run
    355428    command $run
    356429  end
    … …  
    402475    book getword magicToDS $pageName camera -var CAMERA
    403476    book getword magicToDS $pageName streaks_uri -var STREAKS
     477    book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS
    404478    book getword magicToDS $pageName stage -var STAGE
    405479    book getword magicToDS $pageName stage_id -var STAGE_ID
    … …  
    411485    book getword magicToDS $pageName recoveryroot -var RECROOT
    412486    book getword magicToDS $pageName re_place -var REPLACE
    413     book getword magicToDS $pageName remove -var REMOVE
    414487    book getword magicToDS $pageName dbname -var DBNAME
    415488
    416489    sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT
    417490
    418     host anyhost
    419 
    420     # TODO: do not add recoveryroot, replace, or remove if they are null or zero
    421 
    422     $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --remove $REMOVE
     491    substr $COMPONENT 0 3 COMP_HEAD
     492    if ("$COMP_HEAD" == "sky")
     493        set.host.for.skycell $COMPONENT
     494    else
     495        # assume component is a class_id, if not we will default to anyhost
     496        set.host.for.camera $CAMERA $COMPONENT
     497    end
     498
     499    # TODO: do not add recoveryroot or replace if they are null or zero
     500
     501    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE
    423502
    424503    add_standard_args run
    … …  
    457536end
    458537
     538task           magic.ds.revert.load
     539  host         local
     540
     541  periods      -poll $LOADPOLL
     542  periods      -exec $LOADEXEC
     543  periods      -timeout 20
     544  npending     1
     545  active       false
     546
     547  stdout NULL
     548  stderr $LOGDIR/magic.ds.revert.log
     549
     550  task.exec
     551    $run = magicdstool -torevert -stage $STAGE:$magicDSRevertStage
     552    $magicDSRevertStage ++
     553    if ($magicDSRevertStage >= $STAGE:n) set magicDSRevertStage = 0
     554
     555    if ($DB:n == 0)
     556      option DEFAULT
     557    else
     558
     559      # save the DB name for the exit tasks
     560      option $DB:$magicDSToRevert_DB
     561      $run = $run -dbname $DB:$magicDSToRevert_DB
     562
     563      # only bump database number after we have gone through all of the stages
     564      if ($magicDSRevertStage == 0)
     565             $magicDSToRevert_DB ++
     566      end
     567      if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0
     568    end
     569    add_poll_args run
     570    add_poll_labels run
     571    command $run
     572  end
     573
     574  # success
     575  task.exit    0
     576    # convert 'stdout' to book format
     577    ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT
     578    if ($VERBOSE > 2)
     579      book listbook magicDSToRevert
     580    end
     581
     582    # delete existing entries in the appropriate pantaskStates
     583    process_cleanup magicDSToRevert
     584  end
     585
     586  # locked list
     587  task.exit    default
     588    showcommand failure
     589  end
     590
     591  task.exit    crash
     592    showcommand crash
     593  end
     594
     595  # operation times out?
     596  task.exit    timeout
     597    showcommand timeout
     598  end
     599end
     600
     601task           magic.ds.revert.run
     602  periods      -poll $RUNPOLL
     603  periods      -exec $RUNEXEC
     604  periods      -timeout 60
     605  active       false
     606
     607  task.exec
     608    book npages magicDSToRevert -var N
     609    if ($N == 0) break
     610    if ($NETWORK == 0) break
     611   
     612    # look for new images (pantaskState == INIT)
     613    book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT
     614    if ("$pageName" == "NULL") break
     615
     616    book setword magicDSToRevert $pageName pantaskState RUN
     617    book getword magicDSToRevert $pageName exp_id -var EXP_ID
     618    book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID
     619    book getword magicDSToRevert $pageName camera -var CAMERA
     620    book getword magicDSToRevert $pageName stage -var STAGE
     621    book getword magicDSToRevert $pageName stage_id -var STAGE_ID
     622    book getword magicDSToRevert $pageName component -var COMPONENT
     623    book getword magicDSToRevert $pageName path_base -var PATH_BASE
     624    book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE
     625    book getword magicDSToRevert $pageName outroot -var OUTROOT
     626    book getword magciDSToRevert $pageName bytes -var BYTES
     627    book getword magciDSToRevert $pageName md5sum -var md5sum
     628#    book getword magicDSToRevert $pageName recoveryroot -var RECROOT
     629    book getword magicDSToRevert $pageName re_place -var REPLACE
     630    book getword magicDSToRevert $pageName dbname -var DBNAME
     631
     632    sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT
     633
     634    substr $COMPONENT 0 3 COMP_HEAD
     635    if ("$COMP_HEAD" == "sky")
     636        set.host.for.skycell $COMPONENT
     637    else
     638        # assume component is a class_id, if not we will default to anyhost
     639        set.host.for.camera $CAMERA $COMPONENT
     640    end
     641
     642    $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE
     643
     644    add_standard_args run
     645
     646    # save the pageName for future reference below
     647    options $pageName
     648
     649    # create the command line
     650    if ($VERBOSE > 1)
     651      echo command $run
     652    end
     653    command $run
     654  end
     655
     656  # default exit status
     657  task.exit    0
     658    process_exit magicDSToRevert $options:0 $JOB_STATUS
     659   end
     660
     661  # locked list
     662  task.exit    default
     663    showcommand failure
     664    process_exit magicDSToRevert $options:0 $JOB_STATUS
     665  end
     666
     667  task.exit    crash
     668    showcommand crash
     669    book setword magicDSToRevert $options:0 pantaskState CRASH
     670  end
     671
     672  # operation timed out?
     673  task.exit    timeout
     674    showcommand timeout
     675    book setword magicDSToRevert $options:0 pantaskState TIMEOUT
     676  end
     677end
     678
Note: See TracChangeset for help on using the changeset viewer.