IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 42908


Ignore:
Timestamp:
Jun 26, 2025, 5:05:19 PM (13 months ago)
Author:
eugene
Message:

working on new burntool / rawfix / archive paradigm

Location:
trunk/tools/eam/burntool.20250520/src
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/tools/eam/burntool.20250520/src/burntool.pt

    r42878 r42908  
    1 ## pantasks scripts to manage the raw exposure data repair / cleanup
    2 ## this is for the initial chip instance checks
    3 ## use the top-level rawfix.pt to load
    4 
    5 if (not($?RAWFIX_BURNTOOL_INIT))
    6   book init burntool.book
    7   book init burnhost.book
    8   $RAWFIX_BURNTOOL_INIT = 1
    9 end
    10 
    11 $MAX_BURNHOST = 2
    12 
    13 # select nights in state burntool.new
    14 # output: burntool.book
    15 task nights.burntool.load
    16   host local
    17   periods -poll $TPOLL
    18   periods -exec $TEXEC
    19   periods -timeout 30
    20 
    21   stdout $LOGDIR/burntool.load.stdout.log 
    22   stderr $LOGDIR/burntool.load.stderr.log
    23 
    24   # only run one of these tasks at a time
    25   npending 1
    26 
    27   task.exec
    28     command check.rawfix.sh state burntool.new
    29   end
    30 
    31   task.exit 0
    32     # convert the stdout 'queue' to entries in a book
    33     # fields in the table will become words in the page
    34     queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout burntool.book
    35     book delpage burntool.book -key pantaskState DONE
    36   end
    37 
    38   # exit values other than 0:
    39   task.exit    default
    40     showcommand failure
    41   end
    42 
    43   task.exit    crash
    44     showcommand crash
    45   end
    46 
    47   # operation times out?
    48   task.exit    timeout
    49     showcommand timeout
    50   end
    51 end
    52 
    53 # output: burnhost.book
    54 task nights.burntool.loadhosts
     1## pantasks script to manage the burntool processing
     2
     3if (not($?BURNTOOL_INIT))
     4  book init XXX
     5  book init XXX
     6  $BURNTOOL_BOOKS:n = 0; # total number of books available
     7  $BURNTOOL_BOOKS:N = 0; # current book of interest
     8  $BURNTOOL_INIT = 1
     9  $MAX_BURNHOST = 2
     10end
     11
     12# burntool_night_chip.pl : run once per night/chip, output used to generate a book with name like YYYYMMDD.XXnn
     13
     14# we use the following lists
     15# BURNTOOL_BOOK_NAME  : list of the names of the night/chip books
     16# BURNTOOL_BOOK_PAGE  : current active item in this book
     17# BURNTOOL_BOOK_STATE : status of current active item (
     18
     19# run a single night/exposure/chip job
     20task burntool.onechip.run
    5521  periods      -poll $RPOLL
    5622  periods      -exec $REXEC
    57   periods      -timeout 60
    58   host          local
    59   npending 1
    60   # note: the tasks which return their results via stdout
    61   # seem to fail if run remotely. 
    62 
    63   stdout $LOGDIR/burntool.loadhosts.stdout.log 
    64   stderr $LOGDIR/burntool.loadhosts.stderr.log
     23  periods      -timeout 1800
    6524
    6625  task.exec
    … …  
    6827    periods -exec $REXEC
    6928
    70     book npages burntool.book -var N
     29    # if we have reached the end, restart the loop over books
     30    if ($BURNTOOL_BOOK_NVAL >= $BURNTOOL_BOOK_NAME:n)
     31      $BURNTOOL_BOOK_NVAL = 0
     32      break
     33    end
     34   
     35    # allowed state for a given book:
     36    # RUN : current entry is still running
     37    # STOP : current entry is done, ready for next entry
     38    # WAIT : current entry has not yet started
     39    # DONE : entry book is done
     40
     41    # only this task may manipulate $BURNTOOL_BOOK_NVAL or $BURNTOOL_BOOK_PAGE:i
     42
     43    # state of the current book
     44    $myState = $BURNTOOL_BOOK_STATE:$BURNTOOL_BOOK_NVAL
     45
     46    # if the current page in the current book is running, go to the next book
     47    if ("$myState" == "RUN")
     48      $BURNTOOL_BOOK_NVAL ++
     49      periods -exec 0.05
     50      break
     51    end
     52
     53    # if the current page in the current book is done, go to the next page
     54    # (advance page counter and set state to WAIT)
     55    if ("$myState" == "STOP")
     56      # advance to the next entry
     57      $BURNTOOL_BOOK_STATE:$BURNTOOL_BOOK_NVAL = WAIT
     58      $BURNTOOL_BOOK_PAGE:$BURNTOOL_BOOK_NVAL ++
     59      $BURNTOOL_BOOK_NVAL ++
     60      periods -exec 0.05
     61      break
     62    end
     63
     64    # if the current page in the current book is done, go to the next book
     65    if ("$myState" == "DONE")
     66      $BURNTOOL_BOOK_NVAL ++
     67      periods -exec 0.05
     68      break
     69    end
     70
     71    $myBook = $BURNTOOL_BOOK_NAME:$BURNTOOL_BOOK_NVAL
     72    $myPage = $BURNTOOL_BOOK_PAGE:$BURNTOOL_BOOK_NVAL
     73
     74    # if the current page in the current book is done, go to the next book
     75    if ("$myState" != "WAIT")
     76      echo "unexpected state for book $myBook ($BURNTOOL_BOOK_NVAL)
     77      $BURNTOOL_BOOK_NVAL ++
     78      break
     79    end
     80
     81    book npages $myBook -var N
    7182    if ($N == 0) break
    7283    if ($NETWORK == 0) break
    7384   
    74     # look for new nights in burntool.book (pantaskState == INIT)
    75     book getpage burntool.book 0 -var pageName -key pantaskState INIT
     85    # select current page in book
     86    book getpage $myBook $myPage -var pageName
    7687    if ("$pageName" == "NULL") break
    7788
    78     book setword burntool.book $pageName pantaskState RUN
    79     book getword burntool.book $pageName dateobs -var DATEOBS
    80     book getword burntool.book $pageName iter    -var ITER
    81     option $pageName
    82     # save the page so we can use it below
    83 
    84     substr $DATEOBS 0 4 YEAR
    85 
    86     # write an entry point in the log for this night
    87     sprintf tmpline "%s/%s/%s/log.burntool.runhosts.v%d" $mypath $YEAR $DATEOBS $ITER
    88     date -var mydate
    89     exec echo "\\# start runhosts $mydate" >> $tmpline
    90 
    91     # get_hosts_fixchip.sh selects the most used machine at ITC
    92     # the command returns these fields: dateobs stage fixhost chipID
    93     # NOTE: stage == ITER
    94     command get_hosts_fixchip.sh $mypath $DATEOBS $ITER
    95   end
    96 
    97   task.exit 0
    98     # convert the stdout 'queue' to entries in a book
    99     # fields in the table will become words in the page
    100     # queueprint stdout
    101     queuesize stdout -var Nhosts
    102     queue2book -raw-columns -key dateobs:chipID -uniq -setword pantaskState INIT stdout burnhost.book
    103     book setword burntool.book $options:0 NHOST {$Nhosts - 1}
    104     book setword burntool.book $options:0 NDONE 0
    105     # book list
    106     # book listbook burntool.book
    107     # use the queuesize (Nhosts) to track how many jobs are done / to be done
    108   end
    109 
    110   # exit values other than 0:
    111   task.exit    default
    112     showcommand failure
    113     book setword burntool.book $options:0 pantasksQuality FAIL
    114   end
    115 
    116   task.exit    crash
    117     showcommand crash
    118     book setword burntool.book $options:0 pantasksQuality CRASH
    119   end
    120 
    121   # operation times out?
    122   task.exit    timeout
    123     showcommand timeout
    124     book setword burntool.book $options:0 pantasksQuality TIMEOUT
    125   end
    126 end
    127 
    128 task nights.burntool.run
    129   periods      -poll $RPOLL
    130   periods      -exec $REXEC
    131   periods      -timeout 1800
    132 
    133   task.exec
    134     # if we are unable to run the 'exec', use a long retry time
    135     periods -exec $REXEC
    136 
    137     book npages burnhost.book -var N
    138     if ($N == 0) break
    139     if ($NETWORK == 0) break
    140    
    141     # look for new nights in burnhost.book (pantaskState == INIT)
    142     book getpage burnhost.book 0 -var pageName -key pantaskState INIT
    143     if ("$pageName" == "NULL") break
    144 
    145     book setword burnhost.book $pageName pantaskState RUN
    146     book getword burnhost.book $pageName dateobs -var DATEOBS
    147     book getword burnhost.book $pageName iter    -var ITER
    148     book getword burnhost.book $pageName fixhost -var BURNHOST
    149     book getword burnhost.book $pageName chipID  -var CHIP_ID
    150 
    151     substr $DATEOBS 0 4 YEAR
    152     substr $DATEOBS 4 2 MONTH
    153     substr $DATEOBS 6 2 DAY
    154     sprintf STARTDATE "%s/%s/%s" $YEAR $MONTH $DAY
    155     sprintf ENDDATE "%s/%s/%s,23:59:59" $YEAR $MONTH $DAY
    156 
    157     option $pageName $DATEOBS $YEAR
    158     # save DATEOBS so we can find the md5sum.book entry
    159     # also save YEAR for output log
    160 
    161     stdout $mypath/$YEAR/$DATEOBS/log.burntool.run.$CHIP_ID.v$ITER
    162     stderr $mypath/$YEAR/$DATEOBS/log.burntool.run.$CHIP_ID.v$ITER
    163 
    164    
    165     host $BURNHOST
    166 #   host -required $BURNHOST
    167 #   too many jobs wanted the same machines so the processing was clogged.
    168 #   allow them to go elsewhere if needed.
     89    book getword burnhost.book $pageName rawImfileReal -var rawImfileReal
     90    book getword burnhost.book $pageName outTableReal  -var outTableReal
     91    book getword burnhost.book $pageName exposure      -var exposure     
     92    book getword burnhost.book $pageName class_id      -var class_id     
     93    book getword burnhost.book $pageName fullname      -var fullname     
     94    book getword burnhost.book $pageName datahost      -var BURNHOST
     95
     96    # XXX choose something sensible:
     97    # stdout $mypath/$YEAR/$DATEOBS/log.burntool.run.$CHIP_ID.v$ITER
     98    # stderr $mypath/$YEAR/$DATEOBS/log.burntool.run.$CHIP_ID.v$ITER
     99
     100    host -required $BURNHOST
    169101   
    170102    ## check if this host needs to be launched
    … …  
    179111
    180112    sprintf psvidOutput "%s/%s/%s" $mypath $YEAR $DATEOBS
     113    if ("$previousTable" != "NULL")
     114      $addtable = --prevtable=$previousTable
     115    else
     116      $addtable =
     117    end
    181118
    182119    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
    183120    periods -exec 0.05
    184     command ipp_apply_burntool_psvideo.pl --rerun --dateobs_begin $STARTDATE --dateobs_end $ENDDATE --class_id $CHIP_ID --dbname gpc1 --psvidOutput $psvidOutput
    185     ## NOTE the rerun option is set here
     121    command burntool_one_chip.pl --rerun --imfile $rawImfileReal --outtable=$outTableReal $addtable
    186122  end
    187123
    … …  
    332268end
    333269
     270
     271
     272
     273
     274if (0)
     275  # select nights in state burntool.new
     276  # output: burntool.book
     277  task nights.burntool.load
     278    host local
     279    periods -poll $TPOLL
     280    periods -exec $TEXEC
     281    periods -timeout 30
     282 
     283    stdout $LOGDIR/burntool.load.stdout.log 
     284    stderr $LOGDIR/burntool.load.stderr.log
     285 
     286    # only run one of these tasks at a time
     287    npending 1
     288 
     289    task.exec
     290      command check.rawfix.sh state burntool.new
     291    end
     292 
     293    task.exit 0
     294      # convert the stdout 'queue' to entries in a book
     295      # fields in the table will become words in the page
     296      queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout burntool.book
     297      book delpage burntool.book -key pantaskState DONE
     298    end
     299 
     300    # exit values other than 0:
     301    task.exit    default
     302      showcommand failure
     303    end
     304 
     305    task.exit    crash
     306      showcommand crash
     307    end
     308 
     309    # operation times out?
     310    task.exit    timeout
     311      showcommand timeout
     312    end
     313  end
     314end
     315

  • trunk/tools/eam/burntool.20250520/src/burntool_night_chip.pl

    r42878 r42908  
    2222my $date = `date`;
    2323print "\n\n";
    24 print "Starting script $0 on $host at $date\n\n";
     24# print "Starting script $0 on $host at $date\n\n";
     25print "data MULTI\n\n";
    2526
    2627my ( $class_id, $dateobs, $dbname, $rerun, $verbose, $save_temps);
    … …  
    5354my $missing_tools;
    5455my $regtool      = can_run('regtool')         or (warn "Can't find regtool"         and $missing_tools = 1);
    55 my $funpack      = can_run('funpack')         or (warn "Can't find funpack"         and $missing_tools = 1);
    56 my $ppConfigDump = can_run('ppConfigDump')    or (warn "Can't find ppConfigDump"    and $missing_tools = 1);
    5756if ($missing_tools) {
    5857    warn("Can't find required tools.");
    … …  
    7271$command .= " -ordered_by_date";
    7372$command .= " -dbname $dbname" if defined $dbname;
     73$command .= " -limit 10";
    7474
    7575my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    111111foreach my $file (@$files) {
    112112    my $exp_id     = $file->{exp_id};
    113     my $exp_name   = $file->{exp_id};
     113    my $exp_name   = $file->{exp_name};
    114114    my $state      = $file->{burntool_state};
    115115    my $data_state = $file->{data_state};
    … …  
    126126        next;
    127127    }
     128
     129    # extract the storage host name
     130    my ($datahost) = $rawImfileReal =~ m|/data/(ipp[a-z0-9]+).[0123]|;
     131    # print STDERR "datahost: $datahost\n";
     132    # next;
    128133
    129134    # instantiate the burntable instance for use by the actual analysis script
    … …  
    137142
    138143    my $process = 1;
    139     my $previousTableStyle = 0;
    140144
    141145    if ($state == -2) { &my_die("Aborting as $rawImfile has modified pixel data!"); }
    142     if ($state == +1 * $burntoolStateGood) {
    143         $process = 0;
    144         $previousTableStyle = 1;
    145     }
     146    if ($state == +1 * $burntoolStateGood) { &my_die("Unexpected burntool State $state"); }
     147
     148    # $state = -15, $burntoolStateGood = +15
    146149    if ($state == -1 * $burntoolStateGood) {
    147150        $process = 0;
    148         $previousTableStyle = -1;
    149151    }
    150152    if ($rerun) { $process = 1; }
    … …  
    156158
    157159    if (($process == 1) || ($processNext == 1)) {
    158         my $status;
     160        # we now have $rawImfileReal, $previousTable, $outTableReal
     161        print STDOUT "data METADATA\n";
     162        print STDOUT "     rawImfileReal STR $rawImfileReal\n";
     163        print STDOUT "     outTableReal  STR $outTableReal\n";
     164        print STDOUT "     exposure      STR $exp_name\n";
     165        print STDOUT "     class_id      STR $class_id\n";
     166        print STDOUT "     fullname      STR $exp_name.$class_id\n";
     167        print STDOUT "     datahost      STR $datahost\n";
     168        if ($previousTable ne "") {
     169            print STDOUT "     previousTable STR $previousTable\n";
     170        }
     171        print STDOUT "END\n\n";
     172
    159173        $processNext = 1;
    160 
    161         # we now have $rawImfileReal, $previousTable, $outTableReal
    162         print STDERR "$rawImfileReal $previousTable $outTableReal\n";
    163 
    164         $previousTableStyle = -1;
    165174    }
    166175
    167176    ### XXX I think I can make the assumption that we are saving the tables as separate files since we
    168177    ### want to get to burntool state -15 (v15)
    169     $previousTable = "in=$outTableReal";
     178    $previousTable = "$outTableReal";
    170179}
    171180
Note: See TracChangeset for help on using the changeset viewer.