IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Oct 5, 2006, 5:20:43 PM (20 years ago)
Author:
eugene
Message:

substantial work to use current db tables

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/detrend.norm.pro

    r9104 r9338  
    1414if ($?parallel == 0)
    1515  $parallel = 0
     16end
     17
     18# select images ready for copy
     19# new entries are added to queue DetrendNormStat
     20# compare the new list with the ones already selected
     21task           dettool.normstat.load
     22  command      dettool -tonormalizedstat -simple
     23  host         local
     24
     25  periods      -poll 1
     26  periods      -exec 5
     27  periods      -timeout 5
     28
     29  stdout dettool.normstat.log
     30  stderr dettool.normstat.log
     31
     32  # success
     33  task.exit    0
     34    # XXX is it necessary for these to be local?
     35    # drop after this task macro is done?
     36    local i Nqueue
     37
     38    # compare output with newImage queue
     39    # only add entries which don't exist in queue
     40    queuesize stdout -var Nqueue
     41    for i 0 $Nqueue
     42      queuepop stdout -var line
     43      list word -split $line
     44      $DET_ID   = $word:0
     45      $DET_TYPE = $word:1
     46      $ITER     = $word:2
     47      $CAMERA   = $word:3
     48      queuepush DetrendNormStat -uniq -key 1:3 "NEW $DET_ID $DET_TYPE $ITER $CAMERA"
     49    end
     50
     51    # delete existing entries which are DONE
     52    queuedrop DetrendNormStat -key 0 DONE
     53  end
     54
     55  # locked list
     56  # default exit status
     57  task.exit    default
     58    echo       "detrend.norm: failure"
     59  end
     60
     61  # operation times out?
     62  task.exit    timeout
     63    echo       "detrend.norm: timeout"
     64  end
     65end
     66
     67# copy new images, sending job to desired host
     68task           dettool.normstat.process
     69  periods      -poll 0.5
     70  periods      -exec 5.0
     71  periods      -timeout 30
     72
     73  task.exec
     74    queuesize DetrendNormStat -var N
     75   
     76    if ($N == 0) break
     77    if ($network == 0) break
     78   
     79    # look for new images on the internal DetrendNormStat queue
     80    # caution with these 'if' statements: syntax errors
     81    # will make the task fail without given a good status
     82    queuepop DetrendNormStat -var line -key 0 NEW
     83    if ("$line" == "NULL") break
     84
     85    strpop line state
     86    queuepush DetrendNormStat -replace -key 1:3 "RUN $line"
     87
     88    # the DetrendNormStat queue contains:
     89    # STATE DET_ID ITER
     90    list word -split $line
     91    $DET_ID   = $word:0
     92    $DET_TYPE = $word:1
     93    $ITER     = $word:2
     94    $CAMERA   = $word:3
     95
     96    # XXX add $WORKDIR/$LOG_DIR
     97    stdout $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
     98    stderr $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
     99
     100    # specify choice of remote host:
     101    if ($parallel)
     102      host anyhost
     103    else
     104      host local
     105    end
     106
     107    # create example job options as a demonstration
     108    options "$line"
     109    echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE
     110    command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER --det_type $DET_TYPE
     111  end
     112
     113  # success
     114  task.exit 0
     115    queuepush DetrendNormStat -replace -key 1:3 "DONE $options:0"
     116  end
     117
     118  # default exit status
     119  task.exit    default
     120    queuepush DetrendNormStat -replace -key 1:3 "FAIL $options:0"
     121  end
     122
     123  # operation times out?
     124  task.exit    timeout
     125    queuepush DetrendNormStat -replace -key 1:3 "TIMEOUT $options:0"
     126  end
    16127end
    17128
    … …  
    41152    for i 0 $Nqueue
    42153      queuepop stdout -var line
    43       queuepush DetrendNormalize -uniq -key 1:2 "NEW $line"
     154      list word -split $line
     155      $DET_TYPE = $word:0
     156      $CAMERA   = $word:1
     157      $URI      = $word:2
     158      $DET_ID   = $word:3
     159      $ITER     = $word:4
     160      $CLASS_ID = $word:5
     161      $NORM     = $word:6
     162      queuepush DetrendNormalize -uniq -key 4:5 "NEW $DET_TYPE $CAMERA $URI $DET_ID $ITER $CLASS_ID $NORM"
    44163    end
    45164
    … …  
    79198
    80199    strpop line state
    81     queuepush DetrendNormalize -replace -key 1:2 "RUN $line"
     200    queuepush DetrendNormalize -replace -key 4:5 "RUN $line"
    82201
    83202    # the DetrendNormalize queue contains:
    84203    # STATE DET_ID ITER
    85204    list word -split $line
    86     $DET_ID   = $word:0
    87     $ITER     = $word:1
    88 
    89     # XXX we need to get CAMERA and TYPE from this query
    90     # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log
    91     # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log
    92     stdout isp.flat.$DET_ID.$ITER.log
    93     stderr isp.flat.$DET_ID.$ITER.log
     205    $DET_TYPE = $word:0
     206    $CAMERA   = $word:1
     207    $URI      = $word:2
     208    $DET_ID   = $word:3
     209    $ITER     = $word:4
     210    $CLASS_ID = $word:5
     211    $NORM     = $word:6
     212
     213    $LOG_DIR = `dirname $URI`
     214
     215    stdout $WORKDIR/$LOG_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
     216    stderr $WORKDIR/$LOG_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.log
    94217
    95218    # specify choice of remote host:
    … …  
    102225    # create example job options as a demonstration
    103226    options "$line"
    104     echo command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER
    105     command detrend_norm_calc.pl --det_id $DET_ID --iteration $ITER
     227    echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
     228    command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
    106229  end
    107230
    108231  # success
    109232  task.exit 0
    110     queuepush DetrendNormalize -replace -key 1:2 "DONE $options:0"
    111   end
    112 
    113   # default exit status
    114   task.exit    default
    115     queuepush DetrendNormalize -replace -key 1:2 "FAIL $options:0"
    116   end
    117 
    118   # operation times out?
    119   task.exit    timeout
    120     queuepush DetrendNormalize -replace -key 1:2 "TIMEOUT $options:0"
    121   end
    122 end
    123 
    124 # select images ready for copy
    125 # new entries are added to queue DetrendNormStat
    126 # compare the new list with the ones already selected
    127 task           dettool.normstat.load
    128   command      dettool -normstat -simple
    129   host         local
    130 
    131   periods      -poll 1
    132   periods      -exec 5
    133   periods      -timeout 5
    134 
    135   stdout dettool.normstat.log
    136   stderr dettool.normstat.log
    137 
    138   # success
    139   task.exit    0
    140     # XXX is it necessary for these to be local?
    141     # drop after this task macro is done?
    142     local i Nqueue
    143 
    144     # compare output with newImage queue
    145     # only add entries which don't exist in queue
    146     queuesize stdout -var Nqueue
    147     for i 0 $Nqueue
    148       queuepop stdout -var line
    149       queuepush DetrendNormStat -uniq -key 1:2 "NEW $line"
    150     end
    151 
    152     # delete existing entries which are DONE
    153     queuedrop DetrendNormStat -key 0 DONE
    154   end
    155 
    156   # locked list
    157   # default exit status
    158   task.exit    default
    159     echo       "detrend.norm: failure"
    160   end
    161 
    162   # operation times out?
    163   task.exit    timeout
    164     echo       "detrend.norm: timeout"
    165   end
    166 end
    167 
    168 # copy new images, sending job to desired host
    169 task           dettool.normstat.process
    170   periods      -poll 0.5
    171   periods      -exec 5.0
    172   periods      -timeout 30
    173 
    174   task.exec
    175     queuesize DetrendNormStat -var N
    176    
    177     if ($N == 0) break
    178     if ($network == 0) break
    179    
    180     # look for new images on the internal DetrendNormStat queue
    181     # caution with these 'if' statements: syntax errors
    182     # will make the task fail without given a good status
    183     queuepop DetrendNormStat -var line -key 0 NEW
    184     if ("$line" == "NULL") break
    185 
    186     strpop line state
    187     queuepush DetrendNormStat -replace -key 1:2 "RUN $line"
    188 
    189     # the DetrendNormStat queue contains:
    190     # STATE DET_ID ITER
    191     list word -split $line
    192     $DET_ID   = $word:0
    193     $ITER     = $word:1
    194 
    195     # XXX we need to get CAMERA and TYPE from this query
    196     # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log
    197     # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log
    198     stdout isp.flat.$DET_ID.$ITER.log
    199     stderr isp.flat.$DET_ID.$ITER.log
    200 
    201     # specify choice of remote host:
    202     if ($parallel)
    203       host anyhost
    204     else
    205       host local
    206     end
    207 
    208     # create example job options as a demonstration
    209     options "$line"
    210     echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
    211     command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
    212   end
    213 
    214   # success
    215   task.exit 0
    216     queuepush DetrendNormStat -replace -key 1:2 "DONE $options:0"
    217   end
    218 
    219   # default exit status
    220   task.exit    default
    221     queuepush DetrendNormStat -replace -key 1:2 "FAIL $options:0"
    222   end
    223 
    224   # operation times out?
    225   task.exit    timeout
    226     queuepush DetrendNormStat -replace -key 1:2 "TIMEOUT $options:0"
     233    queuepush DetrendNormalize -replace -key 4:5 "DONE $options:0"
     234  end
     235
     236  # default exit status
     237  task.exit    default
     238    queuepush DetrendNormalize -replace -key 4:5 "FAIL $options:0"
     239  end
     240
     241  # operation times out?
     242  task.exit    timeout
     243    queuepush DetrendNormalize -replace -key 4:5 "TIMEOUT $options:0"
    227244  end
    228245end
    … …  
    253270    for i 0 $Nqueue
    254271      queuepop stdout -var line
    255       queuepush DetrendNormalizeExposures -uniq -key 1:2 "NEW $line"
     272      list word -split $line
     273      $DET_ID   = $word:0
     274      $ITER     = $word:1
     275      $DET_TYPE = $word:2
     276      $CAMERA   = $word:3
     277      #$TELESCOPE= $word:4
     278      #$EXP_TYPE = $word:5
     279      #$IMFILES  = $word:6
     280      queuepush DetrendNormalizeExposures -uniq -key 1:2 "NEW $DET_ID $ITER $DET_TYPE $CAMERA"
    256281    end
    257282
    … …  
    298323    $DET_ID   = $word:0
    299324    $ITER     = $word:1
    300 
    301     # XXX we need to get CAMERA and TYPE from this query
    302     # stdout $CAMERA.$TYPE.$DET_ID.$ITER.log
    303     # stderr $CAMERA.$TYPE.$DET_ID.$ITER.log
    304     stdout isp.flat.$DET_ID.$ITER.log
    305     stderr isp.flat.$DET_ID.$ITER.log
     325    $DET_TYPE = $word:2
     326    $CAMERA   = $word:3
     327    #$TELESCOPE= $word:4
     328    #$EXP_TYPE = $word:5
     329    #$IMFILES  = $word:6
     330
     331    # XXX add $WORKDIR/$LOG_DIR
     332    stdout $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
     333    stderr $CAMERA.$DET_TYPE.$DET_ID.$ITER.log
    306334
    307335    # specify choice of remote host:
    … …  
    314342    # create example job options as a demonstration
    315343    options "$line"
    316     echo command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
    317     command detrend_norm_apply.pl --det_id $DET_ID --iteration $ITER
     344    echo command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE
     345    command detrend_norm_exp.pl --det_id $DET_ID --iteration $ITER --camera $CAMERA --det_type $DET_TYPE
    318346  end
    319347
Note: See TracChangeset for help on using the changeset viewer.