IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Aug 16, 2008, 1:06:59 PM (18 years ago)
Author:
eugene
Message:

adding cleanup stages; tidy logfile names

File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/ippTasks/detrend.norm.pro

    r18607 r19089  
    99book init detPendingNormImfile
    1010book init detPendingNormExp
     11
     12book init detCleanupNormStatImfile
     13book init detCleanupNormImfile
     14book init detCleanupNormExp
    1115
    1216macro detnorm.reset
     
    1418  book init detPendingNormImfile
    1519  book init detPendingNormExp
     20
     21  book init detCleanupNormStatImfile
     22  book init detCleanupNormImfile
     23  book init detCleanupNormExp
    1624end
    1725
     
    2028  book listbook detPendingNormImfile
    2129  book listbook detPendingNormExp
     30
     31  book listbook detCleanupNormStatImfile
     32  book listbook detCleanupNormImfile
     33  book listbook detCleanupNormExp
    2234end
    2335
     
    4153    active true
    4254  end
     55
     56  task detrend.cleanup.norm.load
     57    active true
     58  end
     59  task detrend.cleanup.norm.run
     60    active true
     61  end
     62  task detrend.cleanup.normexp.load
     63    active true
     64  end
     65  task detrend.cleanup.normexp.run
     66    active true
     67  end
     68  task detrend.cleanup.normstat.load
     69    active true
     70  end
     71  task detrend.cleanup.normstat.run
     72    active true
     73  end
    4374end
    4475
     
    6091  end
    6192  task detrend.normstat.run
     93    active false
     94  end
     95  task detrend.cleanup.norm.load
     96    active false
     97  end
     98  task detrend.cleanup.norm.run
     99    active false
     100  end
     101  task detrend.cleanup.normexp.load
     102    active false
     103  end
     104  task detrend.cleanup.normexp.run
     105    active false
     106  end
     107  task detrend.cleanup.normstat.load
     108    active false
     109  end
     110  task detrend.cleanup.normstat.run
    62111    active false
    63112  end
     
    68117$detPendingNormImfile_DB = 0
    69118$detPendingNormExp_DB = 0
     119
     120$detCleanupNormStatImfile_DB = 0
     121$detCleanupNormImfile_DB = 0
     122$detCleanupNormExp_DB = 0
    70123
    71124# select images ready for copy
     
    81134
    82135  stdout NULL
    83   stderr $LOGDIR/detrend.normstat.load.log
     136  stderr $LOGDIR/detrend.normstat.log
    84137
    85138  task.exec
     
    191244
    192245  stdout NULL
    193   stderr $LOGDIR/detrend.norm.load.log
     246  stderr $LOGDIR/detrend.norm.log
    194247
    195248  task.exec
     
    264317    sprintf outroot "%s/%s.%s.%s/%s.%s.norm.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION
    265318
    266     stdout $LOGDIR/detrend.norm.run.log
    267     stderr $LOGDIR/detrend.norm.run.log
     319    stdout $LOGDIR/detrend.norm.log
     320    stderr $LOGDIR/detrend.norm.log
    268321
    269322    $run = detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE --outroot $outroot --redirect-output --verbose
     
    304357
    305358  stdout NULL
    306   stderr $LOGDIR/detrend.normexp.load.log
     359  stderr $LOGDIR/detrend.normexp.log
    307360
    308361  task.exec
     
    374427    sprintf outroot "%s/%s.%s.%s/%s.%s.normexp.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION
    375428
    376     stdout $LOGDIR/detrend.norm.run.log
    377     stderr $LOGDIR/detrend.norm.run.log
     429    stdout $LOGDIR/detrend.normexp.log
     430    stderr $LOGDIR/detrend.normexp.log
    378431
    379432    $run = detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE  --outroot $outroot
     
    401454  end
    402455end
     456
     457########## cleanup normstat ###########
     458task           detrend.cleanup.normstat.load
     459  host         local
     460
     461  periods      -poll $LOADPOLL
     462  periods      -exec $LOADEXEC
     463  periods      -timeout 30
     464  npending     1
     465  active       true
     466
     467  stdout NULL
     468  stderr $LOGDIR/detrend.cleanup.normstat.log
     469
     470  task.exec
     471    if ($DB:n == 0)
     472      option DEFAULT
     473      command dettool -pendingcleanup_normalizedstat -limit 20
     474    else
     475      # save the DB name for the exit tasks
     476      option $DB:$detCleanupNormStatImfile_DB
     477      command dettool -pendingcleanup_normalizedstat -limit 20 -dbname $DB:$detCleanupNormStatImfile_DB
     478      $detCleanupNormStatImfile_DB ++
     479      if ($detCleanupNormStatImfile_DB >= $DB:n) set detCleanupNormStatImfile_DB = 0
     480    end
     481  end
     482
     483  # success
     484  task.exit    0
     485    # convert 'stdout' to book format
     486    ipptool2book stdout detCleanupNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT
     487    if ($VERBOSE > 2)
     488      book listbook detCleanupNormStatImfile
     489    end
     490
     491    # delete existing entries in the appropriate pantaskStates
     492    process_cleanup detCleanupNormStatImfile
     493  end
     494
     495  # locked list
     496  task.exit    default
     497    showcommand failure
     498  end
     499
     500  # operation times out?
     501  task.exit    timeout
     502    showcommand timeout
     503  end
     504end
     505
     506# run the ipp_cleanup.pl script on pending images
     507task           detrend.cleanup.normstat.run
     508  periods      -poll $RUNPOLL
     509  periods      -exec $RUNEXEC
     510  periods      -timeout 60
     511  active       true
     512
     513  task.exec
     514    book npages detCleanupNormStatImfile -var N
     515    if ($N == 0) break
     516    if ($NETWORK == 0) break
     517   
     518    # look for new images in detCleanupNormStatImfile (pantaskState == INIT)
     519    book getpage detCleanupNormStatImfile 0 -var pageName -key pantaskState INIT
     520    if ("$pageName" == "NULL") break
     521
     522    book setword detCleanupNormStatImfile $pageName pantaskState RUN
     523    book getword detCleanupNormStatImfile $pageName det_id   -var DET_ID   
     524    book getword detCleanupNormStatImfile $pageName iteration -var ITERATION
     525    book getword detCleanupNormStatImfile $pageName camera -var CAMERA
     526    book getword detCleanupNormStatImfile $pageName state -var CLEANUP_MODE
     527    book getword detCleanupNormStatImfile $pageName dbname -var DBNAME
     528
     529    # specify choice of local or remote host based on camera and chip (class_id)
     530    set.host.for.camera $CAMERA FPA
     531
     532    stdout $LOGDIR/detrend.cleanup.normstat.log
     533    stderr $LOGDIR/detrend.cleanup.normstat.log
     534
     535    # XXX is everything listed here needed?
     536    $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE
     537    add_standard_args run
     538
     539    # save the pageName for future reference below
     540    options $pageName
     541
     542    # create the command line
     543    if ($VERBOSE > 1)
     544      echo command $run
     545    end
     546    command $run
     547  end
     548
     549  # default exit status
     550  task.exit    default
     551    process_exit detCleanupNormStatImfile $options:0 $JOB_STATUS
     552  end
     553
     554  # operation timed out?
     555  task.exit    timeout
     556    showcommand timeout
     557    book setword detCleanupNormStatImfile $options:0 pantaskState TIMEOUT
     558  end
     559end
     560 
     561########## cleanup norm (normalized.imfile) ###########
     562task           detrend.cleanup.norm.load
     563  host         local
     564
     565  periods      -poll $LOADPOLL
     566  periods      -exec $LOADEXEC
     567  periods      -timeout 30
     568  npending     1
     569  active       true
     570
     571  stdout NULL
     572  stderr $LOGDIR/detrend.cleanup.norm.log
     573
     574  task.exec
     575    if ($DB:n == 0)
     576      option DEFAULT
     577      command dettool -pendingcleanup_normalizedimfile -limit 20
     578    else
     579      # save the DB name for the exit tasks
     580      option $DB:$detCleanupNormImfile_DB
     581      command dettool -pendingcleanup_normalizedimfile -limit 20 -dbname $DB:$detCleanupNormImfile_DB
     582      $detCleanupNormImfile_DB ++
     583      if ($detCleanupNormImfile_DB >= $DB:n) set detCleanupNormImfile_DB = 0
     584    end
     585  end
     586
     587  # success
     588  task.exit    0
     589    # convert 'stdout' to book format
     590    ipptool2book stdout detCleanupNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     591    if ($VERBOSE > 2)
     592      book listbook detCleanupNormImfile
     593    end
     594
     595    # delete existing entries in the appropriate pantaskStates
     596    process_cleanup detCleanupNormImfile
     597  end
     598
     599  # locked list
     600  task.exit    default
     601    showcommand failure
     602  end
     603
     604  # operation times out?
     605  task.exit    timeout
     606    showcommand timeout
     607  end
     608end
     609
     610# run the ipp_cleanup.pl script on pending images
     611task           detrend.cleanup.norm.run
     612  periods      -poll $RUNPOLL
     613  periods      -exec $RUNEXEC
     614  periods      -timeout 60
     615  active       true
     616
     617  task.exec
     618    book npages detCleanupNormImfile -var N
     619    if ($N == 0) break
     620    if ($NETWORK == 0) break
     621   
     622    # look for new images in detCleanupNormImfile (pantaskState == INIT)
     623    book getpage detCleanupNormImfile 0 -var pageName -key pantaskState INIT
     624    if ("$pageName" == "NULL") break
     625
     626    book setword detCleanupNormImfile $pageName pantaskState RUN
     627    book getword detCleanupNormImfile $pageName det_id   -var DET_ID   
     628    book getword detCleanupNormImfile $pageName iteration -var ITERATION
     629    book getword detCleanupNormImfile $pageName class_id -var CLASS_ID
     630    book getword detCleanupNormImfile $pageName camera -var CAMERA
     631    book getword detCleanupNormImfile $pageName state -var CLEANUP_MODE
     632    book getword detCleanupNormImfile $pageName dbname -var DBNAME
     633
     634    # specify choice of local or remote host based on camera and chip (class_id)
     635    set.host.for.camera $CAMERA FPA
     636
     637    stdout $LOGDIR/detrend.cleanup.norm.log
     638    stderr $LOGDIR/detrend.cleanup.norm.log
     639
     640    # XXX is everything listed here needed?
     641    $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE
     642    add_standard_args run
     643
     644    # save the pageName for future reference below
     645    options $pageName
     646
     647    # create the command line
     648    if ($VERBOSE > 1)
     649      echo command $run
     650    end
     651    command $run
     652  end
     653
     654  # default exit status
     655  task.exit    default
     656    process_exit detCleanupNormImfile $options:0 $JOB_STATUS
     657  end
     658
     659  # operation timed out?
     660  task.exit    timeout
     661    showcommand timeout
     662    book setword detCleanupNormImfile $options:0 pantaskState TIMEOUT
     663  end
     664end
     665 
     666########## cleanup normexp ###########
     667task           detrend.cleanup.normexp.load
     668  host         local
     669
     670  periods      -poll $LOADPOLL
     671  periods      -exec $LOADEXEC
     672  periods      -timeout 30
     673  npending     1
     674  active       true
     675
     676  stdout NULL
     677  stderr $LOGDIR/detrend.normexp.cleanup.log
     678
     679  task.exec
     680    if ($DB:n == 0)
     681      option DEFAULT
     682      command dettool -pendingcleanup_normalizedexp -limit 20
     683    else
     684      # save the DB name for the exit tasks
     685      option $DB:$detCleanupNormExp_DB
     686      command dettool -pendingcleanup_normalizedexp -limit 20 -dbname $DB:$detCleanupNormExp_DB
     687      $detCleanupNormExp_DB ++
     688      if ($detCleanupNormExp_DB >= $DB:n) set detCleanupNormExp_DB = 0
     689    end
     690  end
     691
     692  # success
     693  task.exit    0
     694    # convert 'stdout' to book format
     695    ipptool2book stdout detCleanupNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT
     696    if ($VERBOSE > 2)
     697      book listbook detCleanupNormExp
     698    end
     699
     700    # delete existing entries in the appropriate pantaskStates
     701    process_cleanup detCleanupNormExp
     702  end
     703
     704  # locked list
     705  task.exit    default
     706    showcommand failure
     707  end
     708
     709  # operation times out?
     710  task.exit    timeout
     711    showcommand timeout
     712  end
     713end
     714
     715# run the ipp_cleanup.pl script on pending images
     716task           detrend.cleanup.normexp.run
     717  periods      -poll $RUNPOLL
     718  periods      -exec $RUNEXEC
     719  periods      -timeout 60
     720  active       true
     721
     722  task.exec
     723    book npages detCleanupNormExp -var N
     724    if ($N == 0) break
     725    if ($NETWORK == 0) break
     726   
     727    # look for new images in detCleanupNormExp (pantaskState == INIT)
     728    book getpage detCleanupNormExp 0 -var pageName -key pantaskState INIT
     729    if ("$pageName" == "NULL") break
     730
     731    book setword detCleanupNormExp $pageName pantaskState RUN
     732    book getword detCleanupNormExp $pageName det_id   -var DET_ID   
     733    book getword detCleanupNormExp $pageName iteration -var ITERATION
     734    book getword detCleanupNormExp $pageName camera -var CAMERA
     735    book getword detCleanupNormExp $pageName state -var CLEANUP_MODE
     736    book getword detCleanupNormExp $pageName dbname -var DBNAME
     737
     738    # specify choice of local or remote host based on camera and chip (class_id)
     739    set.host.for.camera $CAMERA FPA
     740
     741    stdout $LOGDIR/detrend.cleanup.normexp.log
     742    stderr $LOGDIR/detrend.cleanup.normexp.log
     743
     744    # XXX is everything listed here needed?
     745    $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE
     746    add_standard_args run
     747
     748    # save the pageName for future reference below
     749    options $pageName
     750
     751    # create the command line
     752    if ($VERBOSE > 1)
     753      echo command $run
     754    end
     755    command $run
     756  end
     757
     758  # default exit status
     759  task.exit    default
     760    process_exit detCleanupNormExp $options:0 $JOB_STATUS
     761  end
     762
     763  # operation timed out?
     764  task.exit    timeout
     765    showcommand timeout
     766    book setword detCleanupNormExp $options:0 pantaskState TIMEOUT
     767  end
     768end
     769 
Note: See TracChangeset for help on using the changeset viewer.