IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 42440


Ignore:
Timestamp:
Mar 28, 2023, 2:43:59 PM (3 years ago)
Author:
eugene
Message:

cleanup logging and automatic advance

Location:
trunk/tools/eam/rawfix.20230221/src
Files:
14 edited

Legend:

Unmodified
Added
Removed

  • trunk/tools/eam/rawfix.20230221/src/advance.rawfix.sh

    r42430 r42440  
    22
    33if ($#argv < 1) then
    4   echo "USAGE: advance.rawfix.sh all"
     4  echo "USAGE: advance.rawfix.sh all (Nmax)"
    55  echo "USAGE: advance.rawfix.sh night (YYYYMMDD)"
    66  exit 2
    … …  
    99set mode = $1
    1010
     11## include a max number of active programs
    1112if ($mode == "all") then
     13  if ($#argv != 2) then
     14    echo "USAGE: advance.rawfix.sh all (Nmax)"
     15    exit 2
     16  endif
     17
     18  set nMax = $2
     19  set nActive  = 0
     20  set nAdvance = 0
    1221  set nights = `./check.rawfix.sh all | awk '(NR > 1){print $1}'`
    1322  foreach f ($nights)
    14     echo -n "$f "
    1523    ./advance.rawfix.sh night $f
     24    set myResult = $status
     25    if ($myResult == 1) continue
     26    if ($myResult == 2) continue
     27    if ($myResult == 3) continue
     28    if ($myResult == 4) then
     29      @ nActive ++
     30    endif
     31    if ($myResult == 5) then
     32      @ nActive ++
     33      @ nAdvance ++
     34    endif
     35    if ($nActive >= $nMax) then
     36      echo "advance $nAdvance nights, total of $nActive active nights"
     37      exit 0
     38    endif
    1639  end
     40  echo "advance $nAdvance nights, total of $nActive active nights"
    1741  exit 0
    1842endif
    1943
     44# advance STAGE0.done to STAGE1.new
     45# if already in STAGEx.new, return 4
     46# if setting to STAGEx.new, return 5
     47# otherwise, return 3
    2048if ($mode == "night") then
    2149  if ($#argv != 2) then
    … …  
    2957
    3058  set tgtstate = "NONE"
     59  if ($mystate == "new")             set tgtstate = "ckchip.new"
    3160  if ($mystate == "ckchip.done")     set tgtstate = "fixchip.new"
    3261  if ($mystate == "fixchip.done")    set tgtstate = "md5sum.new"   
    … …  
    3766  if ($mystate == "md5chk_s2.done")  set tgtstate = "fixchip_s3.new"   
    3867  if ($mystate == "fixchip_s3.done") set tgtstate = "ckchip_s3.new"   
     68  if ($mystate == "ckchip_s3.done")  set tgtstate = "DONE"
    3969
    40   if ($tgtstate == "NONE") then
    41     echo "nothing to advance"
    42     exit 0
    43   endif
     70  if ($mystate == "ckchip.new")      set tgtstate = "ACTIVE"
     71  if ($mystate == "fixchip.new")     set tgtstate = "ACTIVE"
     72  if ($mystate == "md5sum.new")      set tgtstate = "ACTIVE"
     73  if ($mystate == "md5chk.new")      set tgtstate = "ACTIVE"
     74  if ($mystate == "fixchip_s2.new")  set tgtstate = "ACTIVE"
     75  if ($mystate == "md5sum_s2.new")   set tgtstate = "ACTIVE"
     76  if ($mystate == "md5chk_s2.new")   set tgtstate = "ACTIVE"
     77  if ($mystate == "fixchip_s3.new")  set tgtstate = "ACTIVE"
     78  if ($mystate == "ckchip_s3.new")   set tgtstate = "ACTIVE"
    4479
     80  echo "$night : $mystate : $tgtstate"
     81
     82  # do not set these two states
     83  if ($tgtstate == "NONE") exit 3
     84  if ($tgtstate == "ACTIVE") exit 4
     85
     86  # only set these states
    4587  ./update.rawfix.sh $night $tgtstate
    46   exit 0
     88
     89  if ($tgtstate == "DONE") exit 1
     90  exit 5
    4791endif
    4892
    49 echo "ERROR: unknown mode: $mode
     93echo "ERROR: unknown mode: $mode"
    5094exit 2

  • trunk/tools/eam/rawfix.20230221/src/create.rawfix.sh

    r42399 r42440  
    99  create database $DBNAME;
    1010  use $DBNAME;
    11   create table nights (night_id BIGINT AUTO_INCREMENT, dateobs char(64), state char(64), primary key (night_id));
     11  create table nights (night_id BIGINT AUTO_INCREMENT, dateobs char(64), state char(64), primary key (night_id), unique (dateobs));
    1212EOF
    1313

  • trunk/tools/eam/rawfix.20230221/src/queue.rawfix.sh

    r42399 r42440  
    11#!/bin/csh
    22
    3 if ($#argv != 1) then
    4   echo "USAGE: queue.rawfix.sh (mjddate)"
     3if ($#argv != 2) then
     4  echo "USAGE: queue.rawfix.sh (list) (filename)"
     5  echo "USAGE: queue.rawfix.sh (night) (YYYYMMDD)"
    56  exit 2
    67endif
     8
     9set mode = $1
    710
    811# make the rawfix database to track rawfix operations
    … …  
    1215set DBPASS = "dvo"
    1316
    14 ## XXX validate the format (YYYYMMDD)?
    15 set mynight = $1
     17if ($mode == "list") then
     18  set filename = $2
    1619
    17 mysql -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME -e "insert into nights (dateobs, state) values ('$mynight', 'new')"
     20  # just a big list of dates?
     21  foreach night (`cat $filename`)
     22    queue.rawfix.sh night $night
     23  end
     24  exit 0
     25endif
     26
     27if ($mode == "night") then
     28  ## XXX validate the format (YYYYMMDD)?
     29  set mynight = $2
     30 
     31  mysql -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME -e "insert into nights (dateobs, state) values ('$mynight', 'new')"
     32  exit 0
     33endif
     34
     35echo "ERROR: unknown mode $mode"
     36exit 2

  • trunk/tools/eam/rawfix.20230221/src/rawfix.advance.pt

    r42429 r42440  
     1
     2$MAX_ACTIVE_NIGHTS = 2
    13
    24## regularly run the command advance.rawfits.sh
    … …  
    1315
    1416  task.exec
    15     command advance.rawfix.sh all
     17    #
     18    command advance.rawfix.sh all $MAX_ACTIVE_NIGHTS
    1619  end
    1720
    … …  
    1922  # by already-running load tasks
    2023  task.exit 0
    21     echo "done with advance"   
    2224  end
    2325

  • trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt

    r42430 r42440  
    7878
    7979    # this is the first place in the proces that this location is known
    80     mkdir  $DATEOBS
    81     stdout $DATEOBS/log.ckchip.v0 
    82     stderr $DATEOBS/log.ckchip.v0
     80    mkdir  $mypath/$DATEOBS
     81    stdout $mypath/$DATEOBS/log.ckchip.runhosts.v0 
     82    stderr $mypath/$DATEOBS/log.ckchip.runhosts.v0 
    8383
    8484    substr $DATEOBS 0 4 YEAR

  • trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip_s3.pt

    r42430 r42440  
    6161  npending 1
    6262
    63   stdout $LOGDIR/ckchip_s3.run.stdout.log 
    64   stderr $LOGDIR/ckchip_s3.run.stderr.log
    65 
    6663  task.exec
    6764    # if we are unable to run the 'exec', use a long retry time
    … …  
    7572    book getpage ckchip_s3.book 0 -var pageName -key pantaskState INIT
    7673    if ("$pageName" == "NULL") break
     74
     75    stdout $mypath/$DATEOBS/log.ckchip_s3.runhosts.v0 
     76    stderr $mypath/$DATEOBS/log.ckchip_s3.runhosts.v0 
    7777
    7878    book setword ckchip_s3.book $pageName pantaskState RUN
    … …  
    8989  end
    9090
    91   ## XX delete the page from the book if done?
     91  # SUCCESS
    9292  task.exit 0
    93     queuepush stdout "------- end of stdout ---------"
    94     queuesave stdout $options:0/log.ckchip_s3.v1
    95     queuepush stderr "------- end of stderr ---------"
    96     queuesave stderr $options:0/log.ckchip_s3.v1
    9793    book setword ckchip_s3.book $options:0 pantaskState DONE
    9894    exec update.rawfix.sh $options:0 ckchip_s3.done

  • trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

    r42430 r42440  
    6060  npending 1
    6161
    62   stdout $LOGDIR/fixchip.run.stdout.log 
    63   stderr $LOGDIR/fixchip.run.stderr.log
    64 
    6562  task.exec
    6663    # if we are unable to run the 'exec', use a long retry time

  • trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s2.pt

    r42430 r42440  
    6060  npending 1
    6161
    62   stdout $LOGDIR/fixchip_s2.run.stdout.log 
    63   stderr $LOGDIR/fixchip_s2.run.stderr.log
    64 
    6562  task.exec
    6663    # if we are unable to run the 'exec', use a long retry time
    … …  
    8380    substr $DATEOBS 6 2 DAY
    8481    sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
     82
     83    # does the output go here?
     84    stdout $mypath/$DATEOBS/log.fixchip_s2.runhosts.v0 
     85    stderr $mypath/$DATEOBS/log.fixchip_s2.runhosts.v0 
    8586
    8687    command fix_chip_locations_stage2.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v0

  • trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s3.pt

    r42417 r42440  
    6060  npending 1
    6161
    62   stdout $LOGDIR/fixchip_s3.run.stdout.log 
    63   stderr $LOGDIR/fixchip_s3.run.stderr.log
    64 
    6562  task.exec
    6663    # if we are unable to run the 'exec', use a long retry time
    … …  
    8380    substr $DATEOBS 6 2 DAY
    8481    sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
     82
     83    # does the output go here?
     84    stdout $mypath/$DATEOBS/log.fixchip_s3.runhosts.v0 
     85    stderr $mypath/$DATEOBS/log.fixchip_s3.runhosts.v0 
    8586
    8687    command fix_chip_locations_stage3.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt

  • trunk/tools/eam/rawfix.20230221/src/rawfix.md5chk.pt

    r42411 r42440  
    5959  npending 1
    6060
    61   stdout $LOGDIR/md5chk.run.stdout.log 
    62   stderr $LOGDIR/md5chk.run.stderr.log
    63 
    6461  task.exec
    6562    # if we are unable to run the 'exec', use a long retry time
    … …  
    8380    sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
    8481
    85     # send output to a file in the DATEOBS directory
    86     stdout $DATEOBS/md5chk.log 
     82    # does the output go here?
     83    stdout $mypath/$DATEOBS/log.md5chk.v0 
     84    stderr $mypath/$DATEOBS/log.md5chk.v0 
    8785
    8886    command check_md5s_dateobs.pl --dateobs $FULLDATE

  • trunk/tools/eam/rawfix.20230221/src/rawfix.md5chk_s2.pt

    r42415 r42440  
    5959  npending 1
    6060
    61   stdout $LOGDIR/md5chk_s2.run.stdout.log 
    62   stderr $LOGDIR/md5chk_s2.run.stderr.log
    63 
    6461  task.exec
    6562    # if we are unable to run the 'exec', use a long retry time
    … …  
    8380    sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
    8481
    85     # send output to a file in the DATEOBS directory
    86     stdout $DATEOBS/md5chk_s2.log 
     82    # does the output go here?
     83    stdout $mypath/$DATEOBS/log.md5chk_s2.v0 
     84    stderr $mypath/$DATEOBS/log.md5chk_s2.v0 
    8785
    8886    command check_md5s_dateobs.pl --dateobs $FULLDATE

  • trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt

    r42430 r42440  
    8282
    8383    # write an entry point in the log for this night
    84     sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $DATEOBS
     84    sprintf tmpline "%s/%s/log.md5sum.runhosts.v0" $mypath $DATEOBS
    8585    date -var mydate
    8686    exec echo "# start runhosts $mydate" >> $tmpline
    … …  
    143143    # save DATEOBS so we can find the md5sum.book entry
    144144
     145    # does the output go here?
    145146    stdout $mypath/$DATEOBS/log.md5sum.runhosts.v0 
    146147    stderr $mypath/$DATEOBS/log.md5sum.runhosts.v0 
    … …  
    268269  if ("$1" != "OK")
    269270    book setword md5sum.book $options:1 pantasksQuality $1
    270     sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $options:1
     271    sprintf tmpline "%s/%s/log.md5sum.runhosts.v0" $mypath $options:1
    271272    exec echo "failure $1 $options:0" >> $tmpline
    272273  else

  • trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum_s2.pt

    r42430 r42440  
    8282
    8383    # write an entry point in the log for this night
    84     sprintf tmpline "%s/%s/md5sum_s2.runhosts.log" $mypath $DATEOBS
     84    sprintf tmpline "%s/%s/log.md5sum_s2.runhosts.v0" $mypath $DATEOBS
    8585    date -var mydate
    8686    exec echo "# start runhosts $mydate" >> $tmpline
    … …  
    124124  periods      -timeout 60
    125125
    126   stdout $LOGDIR/md5sum_s2.runhosts.stdout.log 
    127   stderr $LOGDIR/md5sum_s2.runhosts.stderr.log
    128 
    129126  task.exec
    130127    # if we are unable to run the 'exec', use a long retry time
    … …  
    145142    option $pageName $DATEOBS
    146143    # save DATEOBS so we can find the md5sum_s2.book entry
     144
     145    # does the output go here?
     146    stdout $mypath/$DATEOBS/log.md5sum_s2.runhosts.v0 
     147    stderr $mypath/$DATEOBS/log.md5sum_s2.runhosts.v0 
    147148
    148149    # XXX TEST: $USE_HOST = ipp060
    … …  
    270271  if ("$1" != "OK")
    271272    book setword md5sum_s2.book $options:1 pantasksQuality $1
    272     sprintf tmpline "%s/%s/md5sum_s2.runhosts.log" $mypath $options:1
     273    sprintf tmpline "%s/%s/log.md5sum_s2.runhosts.v0" $mypath $options:1
    273274    exec echo "failure $1 $options:0" >> $tmpline
    274275  else

  • trunk/tools/eam/rawfix.20230221/src/rawfix.pt

    r42430 r42440  
    2323# order is this file is not determinative, only the
    2424# change of state from STAGE1.new -> STAGE1.done -> STAGE2.new
    25 # input rawfix.advance.pt
     25input rawfix.advance.pt
    2626input rawfix.ckchip.pt
    27 # input rawfix.fixchip.pt
    28 # input rawfix.md5sum.pt
    29 # input rawfix.md5chk.pt
    30 # input rawfix.fixchip_s2.pt
    31 # input rawfix.md5sum_s2.pt
    32 # input rawfix.md5chk_s2.pt
    33 # input rawfix.fixchip_s3.pt
     27input rawfix.fixchip.pt
     28input rawfix.md5sum.pt
     29input rawfix.md5chk.pt
     30input rawfix.fixchip_s2.pt
     31input rawfix.md5sum_s2.pt
     32input rawfix.md5chk_s2.pt
     33input rawfix.fixchip_s3.pt
    3434
    3535# no need to run rsync if we use the b-node working directory:
Note: See TracChangeset for help on using the changeset viewer.