IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jan 19, 2007, 5:24:59 PM (20 years ago)
Author:
eugene
Message:

converting to book version

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/detrend.resid.pro

    r10745 r11210  
    1 
    21## this file contains the tasks for running the detrend processing stage
    3 
    4 # dettool -toresidimfile -simple
    5 # 0    bias 1      850131b mc       o_850131b.mc.fit PPIMAGE_O 1.492870 2.848350 292.565582
    6 # ITER TYPE DET_ID EXP_TAG CLASS_ID URI-IN           
    7 
    8 # detrend_create_resid.pl 1      0    850131b mc       bias bias_mc_1_0.fit o_850131b.mc.fit 850131b
    9 # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET)       (URL-IN)         (EXP_TAG)
    10 # URL-DET: $TYPE\_$CLASS_ID\_$DET_ID\_$ITER.fit
    11 
    12 ## these tasks use the queue DetrendResidImfiles
    13 ## the DetrendResidImfiles queue contains:
    14 ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN
    15 
    16 queueinit DetrendResidImfiles
    17 queueinit DetrendResidExposures
     2## these tasks use the books detPendingResidImfile and detPendingResidExp
     3
    184if ($?network == 0)
    195  $network = 1
    … …  
    228  $parallel = 0
    239end
    24 
    25 $DetrendResidImfileFail = 0
    26 $DetrendResidExpFail = 0
    27 
    2810if ($?VERBOSE == 0)
    2911  echo "VERBOSE not defined: load pantasks.pro first"
    … …  
    3416  break
    3517end
     18
    3619$LOGSUBDIR = $LOGDIR/detrend
    3720exec mkdir -p $LOGSUBDIR
    3821
     22book init detPendingResidImfile
     23book init detPendingResidExp
     24
    3925macro detresid.status
    40   queueprint DetrendResidImfiles
    41   queueprint DetrendResidExposures
    42   echo "DetrendResidImfileFail: $DetrendResidImfileFail"
    43   echo "DetrendResidExpFail: $DetrendResidExpFail"
     26  book listbook detPendingResidImfile
     27  book listbook detPendingResidExp
    4428end
    4529
    4630# select images ready for copy
    47 # new entries are added to queue DetrendResidImfiles
     31# new entries are added to detPendingResidImfile
    4832# compare the new list with the ones already selected
    4933task           dettool.resid.load
    … …  
    5438  periods      -timeout 30
    5539
    56   stdout $LOGSUBDIR/detresid.log
     40  stdout NULL
    5741  stderr $LOGSUBDIR/detresid.log
    5842
    5943  task.exec
    60     command dettool -toresidimfile -simple -limit {$DetrendResidImfileFail + 20}
     44    command dettool -toresidimfile -limit 20
    6145  end
    6246
    6347  # success
    6448  task.exit    0
    65     # XXX is it necessary for these to be local?
    66     # drop after this task macro is done?
    67     local i Nqueue
    68 
    69     # compare output with newImage queue
    70     # only add entries which don't exist in queue
    71     queuesize stdout -var Nqueue
    72     for i 0 $Nqueue
    73       queuepop stdout -var line
    74       if ($VERBOSE > 2)
    75         echo $line
    76       end
    77       list word -split $line
    78       $DET_ID   = $word:0
    79       $ITER     = $word:1
    80       $DET_TYPE = $word:2
    81       $MODE     = $word:3
    82       $EXP_TAG  = $word:4
    83       $CLASS_ID = $word:5
    84       $URI      = $word:6
    85       $DET_URI  = $word:7
    86       $CAMERA   = $word:8
    87       ## must include the detID and iter in the key (if we have two running at the same time...)
    88       ## Also needed to be in the output file (or path)
    89       queuepush DetrendResidImfiles -uniq -key 1:2:5:6 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $CLASS_ID $URI $DET_URI $CAMERA"
     49    # convert 'stdout' to book format
     50    ipptool2book stdout detPendingResidImfile -key det_id:iteration:exp_tag:class_id -uniq
     51    if ($VERBOSE > 2)
     52      book listbook detPendingResidImfile
    9053    end
    9154
    9255    # delete existing entries which are DONE
    93     queuedrop DetrendResidImfiles -key 0 DONE
    94   end
    95 
    96   # locked list
     56    book delpage detPendingResidImfile -key state DONE
     57  end
     58
    9759  # default exit status
    9860  task.exit    default
    … …  
    11375
    11476  task.exec
    115     queuesize DetrendResidImfiles -var N
     77    book npages detPendingResidImfile -var N
    11678    if ($N == 0) break
    11779    if ($network == 0) break
    11880   
    119     # look for new images on the internal DetrendResidImfiles queue
    120     # caution with these 'if' statements: syntax errors
    121     # will make the task fail without given a good status
    122     queuepop DetrendResidImfiles -var line -key 0 NEW
    123     if ("$line" == "NULL") break
    124 
    125     strpop line state
    126     queuepush DetrendResidImfiles -replace -key 1:2:5:6 "RUN $line"
    127 
    128     ## the DetrendResidImfiles queue contains:
    129     ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN
    130     ## NOTE: the queue has the STATE prepended
    131     list word -split $line
    132     $DET_ID   = $word:0
    133     $ITER     = $word:1
    134     $DET_TYPE = $word:2
    135     $MODE     = $word:3
    136     $EXP_TAG  = $word:4
    137     $CLASS_ID = $word:5
    138     $URI      = $word:6
    139     $DET_URI  = $word:7
    140     $CAMERA   = $word:8
    141 
    142     $word = `basename $EXP_TAG | tr '.' ' '`
    143     list word -split $word
    144     $base = $word:0
    145     stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
    146     stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.$CLASS_ID.log
    147     exec mkdir -p $LOGSUBDIR/$base
     81    # look for new images in detPendingResidImfile
     82    book getpage detPendingResidImfile 0 -var pageName -key state NULL
     83    if ($pageName == NULL) break
     84
     85    book setword detPendingResidImfile $pageName state RUN
     86    book getword detPendingResidImfile $pageName det_id    -var DET_ID   
     87    book getword detPendingResidImfile $pageName iteration -var ITERATION     
     88    book getword detPendingResidImfile $pageName det_type  -var DET_TYPE
     89    book getword detPendingResidImfile $pageName mode      -var MODE     
     90    book getword detPendingResidImfile $pageName exp_tag   -var EXP_TAG 
     91    book getword detPendingResidImfile $pageName class_id  -var CLASS_ID
     92    book getword detPendingResidImfile $pageName uri       -var URI     
     93    book getword detPendingResidImfile $pageName det_uri   -var DET_URI 
     94    book getword detPendingResidImfile $pageName camera    -var CAMERA   
    14895
    14996    # specify choice of remote host:
    … …  
    154101    end
    155102
    156     # create example job options as a demonstration
    157     options "$line"
    158     # detrend_create_resid.pl DET_ID ITER EXP_TAG CLASS_ID TYPE (URI-DET) (URL-IN) (EXP_TAG ???)
    159     # need to parse the subdir path for the detrend URL
    160     # $DET_URL = $DET_DIR/$CAMERA.$DET_TYPE.$DET_ID.$ITER.$CLASS_ID.fits
    161     echo command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE
    162     command detrend_resid.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE
    163   end
    164 
    165   # success
    166   task.exit 0
    167     queuepush DetrendResidImfiles -replace -key 1:2:5:6 "DONE $options:0"
    168   end
    169 
    170   # default exit status
    171   task.exit    default
    172     showcommand failure
    173     queuepush DetrendResidImfiles -replace -key 1:2:5:6 "FAIL $options:0"
    174     $DetrendResidImfileFail ++
    175   end
    176 
    177   # operation times out?
    178   task.exit    timeout
    179     showcommand timeout
    180     queuepush DetrendResidImfiles -replace -key 1:2:6:6 "TIMEOUT $options:0"
    181     $DetrendResidImfileFail ++
     103    # XXX use ipp_filename.pl to lookup output file names
     104    $word = `basename $EXP_TAG | tr '.' ' '`
     105    list word -split $word
     106    $base = $word:0
     107    stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.$CLASS_ID.log
     108    stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.$CLASS_ID.log
     109    exec mkdir -p $LOGSUBDIR/$base
     110
     111    # save the pageName for future reference below
     112    options $pageName
     113
     114    # create command
     115    if ($VERBOSE > 1)
     116      echo command detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE
     117    end
     118    command detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE
     119  end
     120
     121  # default exit status
     122  task.exit    default
     123    process_exit detPendingResidImfile $options:0 $JOB_STATUS
     124  end
     125
     126  # operation times out?
     127  task.exit    timeout
     128    showcommand timeout
     129    book setword detPendingResidImfile $options:0 state TIMEOUT
    182130  end
    183131end
    184132
    185133# select images ready for copy
    186 # new entries are added to queue DetrendResidExposures
     134# new entries are added to detPendingResidExp
    187135# compare the new list with the ones already selected
    188136task           dettool.residexp.load
    … …  
    193141  periods      -timeout 30
    194142
    195   stdout $LOGSUBDIR/detresid.exp.log
     143  stdout NULL
    196144  stderr $LOGSUBDIR/detresid.exp.log
    197145
    198146  task.exec
    199     command      dettool -toresidexp -simple -limit {$DetrendResidExpFail + 20}
     147    command dettool -toresidexp -limit 20
    200148  end
    201149
    202150  # success
    203151  task.exit    0
    204     # XXX is it necessary for these to be local?
    205     # drop after this task macro is done?
    206     local i Nqueue
    207 
    208     # compare output with newImage queue
    209     # only add entries which don't exist in queue
    210     queuesize stdout -var Nqueue
    211     for i 0 $Nqueue
    212       queuepop stdout -var line
    213       if ($VERBOSE > 2)
    214         echo $line
    215       end
    216       list word -split $line
    217       $DET_ID   = $word:0
    218       $ITER     = $word:1
    219       $DET_TYPE = $word:2
    220       $MODE     = $word:3
    221       $EXP_TAG  = $word:4
    222       $INCLUDE  = $word:5
    223       $CAMERA   = $word:6
    224       queuepush DetrendResidExposures -uniq -key 1:2:5 "NEW $DET_ID $ITER $DET_TYPE $MODE $EXP_TAG $INCLUDE $CAMERA"
     152    # convert 'stdout' to book format
     153    ipptool2book stdout detPendingResidExp -key det_id:iteration:exp_tag -uniq
     154    if ($VERBOSE > 2)
     155      book listbook detPendingResidExp
    225156    end
    226157
    227158    # delete existing entries which are DONE
    228     queuedrop DetrendResidExposures -key 0 DONE
    229   end
    230 
    231   # locked list
     159    book delpage detPendingResidExp -key state DONE
     160  end
     161
    232162  # default exit status
    233163  task.exit    default
    … …  
    248178
    249179  task.exec
    250     queuesize DetrendResidExposures -var N
     180    book npages detPendingResidExp -var N
    251181    if ($N == 0) break
    252182    if ($network == 0) break
    253183   
    254     # look for new images on the internal DetrendResidExposures queue
    255     # caution with these 'if' statements: syntax errors
    256     # will make the task fail without given a good status
    257     queuepop DetrendResidExposures -var line -key 0 NEW
    258     if ("$line" == "NULL") break
    259 
    260     strpop line state
    261     queuepush DetrendResidExposures -replace -key 1:2:5 "RUN $line"
    262 
    263     ## the DetrendResidExposures queue contains:
    264     ## STATE ITER TYPE DET_ID EXP_TAG CLASS_ID URI_IN
    265     ## NOTE: the queue has the STATE prepended
    266     list word -split $line
    267     $DET_ID   = $word:0
    268     $ITER     = $word:1
    269     $DET_TYPE = $word:2
    270     $MODE     = $word:3
    271     $EXP_TAG  = $word:4
    272     $INCLUDE  = $word:5
    273     $CAMERA   = $word:6
    274 
    275     # XXX this is still a lame rule
    276     $word = `basename $EXP_TAG | tr '.' ' '`
    277     list word -split $word
    278     $base = $word:0
    279 
    280     stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log
    281     stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITER.log
    282     exec mkdir -p $LOGSUBDIR/$base
     184    # look for new images in detPendingResidExp
     185    book getpage detPendingResidExp 0 -var pageName -key state NULL
     186    if ($pageName == NULL) break
     187
     188    book setword detPendingResidExp $pageName state RUN
     189    book getword detPendingResidExp $pageName det_id    -var DET_ID 
     190    book getword detPendingResidExp $pageName iteration -var ITERATION   
     191    book getword detPendingResidExp $pageName det_type  -var DET_TYPE
     192    book getword detPendingResidExp $pageName mode      -var MODE   
     193    book getword detPendingResidExp $pageName exp_tag   -var EXP_TAG
     194    book getword detPendingResidExp $pageName include   -var INCLUDE
     195    book getword detPendingResidExp $pageName camera    -var CAMERA 
    283196
    284197    # specify choice of remote host:
    … …  
    289202    end
    290203
     204    # XXX use ipp_filename.pl to lookup output file names
     205    $word = `basename $EXP_TAG | tr '.' ' '`
     206    list word -split $word
     207    $base = $word:0
     208    stdout $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.log
     209    stderr $LOGSUBDIR/$base/$EXP_TAG.detresid.$DET_ID.$ITERATION.log
     210    exec mkdir -p $LOGSUBDIR/$base
     211
     212    # save the pageName for future reference below
     213    options $pageName
     214
    291215    # create example job options as a demonstration
    292     options "$line"
    293216    if ($VERBOSE > 1)
    294       echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
    295     end
    296     command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITER --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
    297   end
    298 
    299   # success
    300   task.exit 0
    301     queuepush DetrendResidExposures -replace -key 1:2:5 "DONE $options:0"
    302   end
    303 
    304   # default exit status
    305   task.exit    default
    306     showcommand failure
    307     queuepush DetrendResidExposures -replace -key 1:2:5 "FAIL $options:0"
    308     $DetrendResidExpFail ++
    309   end
    310 
    311   # operation times out?
    312   task.exit    timeout
    313     showcommand timeout
    314     queuepush DetrendResidExposures -replace -key 1:2:5 "TIMEOUT $options:0"
    315     $DetrendResidExpFail ++
    316   end
    317 end
     217      echo command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
     218    end
     219    command detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
     220  end
     221
     222  # default exit status
     223  task.exit    default
     224    process_exit detPendingResidExp $options:0 $JOB_STATUS
     225  end
     226
     227  # operation times out?
     228  task.exit    timeout
     229    showcommand timeout
     230    book setword detPendingResidExp $options:0 state TIMEOUT
     231  end
     232end
Note: See TracChangeset for help on using the changeset viewer.