IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 42430


Ignore:
Timestamp:
Mar 23, 2023, 2:05:41 PM (3 years ago)
Author:
eugene
Message:

adding advance task, fix the logs

Location:
trunk/tools/eam/rawfix.20230221/src
Files:
2 added
14 edited

Legend:

Unmodified
Added
Removed

  • trunk/tools/eam/rawfix.20230221/src/advance.rawfix.sh

    r42429 r42430  
    3636  if ($mystate == "md5sum_s2.done")  set tgtstate = "md5chk_s2.new"   
    3737  if ($mystate == "md5chk_s2.done")  set tgtstate = "fixchip_s3.new"   
     38  if ($mystate == "fixchip_s3.done") set tgtstate = "ckchip_s3.new"   
    3839
    3940  if ($tgtstate == "NONE") then
    … …  
    4647endif
    4748
     49echo "ERROR: unknown mode: $mode
     50exit 2

  • trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh

    r42416 r42430  
    8989  if ($mystate == "fixchip_s3.done")  goto good_state
    9090  if ($mystate == "fixchip_s3.fail")  goto good_state
     91  if ($mystate == "ckchip_s3.new")    goto good_state
     92  if ($mystate == "ckchip_s3.done")   goto good_state
     93  if ($mystate == "ckchip_s3.fail")   goto good_state
    9194
    9295# disable the rsyncs stages: run in directory from b-node

  • trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl

    r42415 r42430  
    1313use Getopt::Long qw( GetOptions :config auto_help auto_version );
    1414
    15 # USAGE: check_chip_locations.pl --dateobs (night) --chiplist (file)
    16 # USAGE: check_chip_locations.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt
    17 
    18 my ($fulldate, $dateword, $chiplist) = &parse_cmdopts;
     15# USAGE: check_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc)
     16# USAGE: check_chip_locations.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt --stage v0
     17
     18my ($fulldate, $dateword, $chiplist, $stage) = &parse_cmdopts;
    1919my ($neb_dbh, $gpc_dbh) = &parse_db_config ('nebulous.config');
    2020
    … …  
    3030
    3131# output files:
    32 # instance lists by host: YYYYMMDD/HOSTNAME.inst.txt
     32# instance lists by host: YYYYMMDD/HOSTNAME.inst.STAGE.txt
    3333# chip states by chip: YYYYMMDD/CHIPNAME.uris.txt
    3434
    … …  
    7979    my @hostnames = keys %hostcount;
    8080    foreach my $myname (@hostnames) {
    81         print $hostfh "host $myname = $hostcount{$myname}\n";
     81        print $hostfh "$myname $hostcount{$myname}\n";
    8282    }
    8383    close ($hostfh);
    … …  
    231231
    232232        if (not defined $hostfile{$hostname}) {
    233             open ($hostfile{$hostname}, ">$dateword/$hostname.inst.txt");
     233            open ($hostfile{$hostname}, ">$dateword/$hostname.inst.$stage.txt");
    234234            ## XXX deal with open failure here
    235235        }
    … …  
    330330sub parse_cmdopts {
    331331
    332     my ($dateobs, $chiplist);
    333     GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2);
     332    my ($dateobs, $chiplist, $stage);
     333    GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'stage=s' => \$stage) || pod2usage(2);
    334334   
    335335    pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs;
    336336    pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist;
     337    pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage;
    337338   
    338339    # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?)
    … …  
    343344    pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12);
    344345    pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31);
     346
    345347    my $fulldate = "$date_year/$date_month/$date_day";
    346    
    347348    my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day);
    348349
    349     return ($fulldate, $dateword, $chiplist)
     350    return ($fulldate, $dateword, $chiplist, $stage)
    350351}
    351352

  • trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl

    r42415 r42430  
    1212use Getopt::Long qw( GetOptions :config auto_help auto_version );
    1313
    14 # USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file)
     14# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc)
    1515
    1616# we need to select targets for ITC copies of files currently only at ATRC:
    … …  
    1818# for each chip, target a single host?
    1919
    20 my ($fulldate, $dateword, $chiplist) = &parse_cmdopts;
     20my ($fulldate, $dateword, $chiplist, $stage) = &parse_cmdopts;
    2121my ($neb_dbh) = &parse_db_config ('nebulous.config');
    2222
    … …  
    7171            my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id);
    7272            unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; }
    73             open (OUTFILE, ">>$dateword/$newhost.inst.txt");
     73            open (OUTFILE, ">>$dateword/$newhost.inst.$stage.txt");
    7474            print OUTFILE "$newfile $datast $md5sum\n";
    7575            close (OUTFILE);
    … …  
    8585            my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id);
    8686            unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; }
    87             open (OUTFILE, ">>$dateword/$newhost.inst.txt");
     87            open (OUTFILE, ">>$dateword/$newhost.inst.$stage.txt");
    8888            print OUTFILE "$newfile $datast $md5sum\n";
    8989            close (OUTFILE);
    … …  
    189189sub parse_cmdopts {
    190190
    191     my ($dateobs, $chiplist);
    192     GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2);
     191    my ($dateobs, $chiplist, $stage);
     192    GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'stage=s' => \$stage) || pod2usage(2);
    193193   
    194194    pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs;
    195195    pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist;
     196    pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage;
    196197   
    197198    # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?)
    … …  
    206207    my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day);
    207208
    208     return ($fulldate, $dateword, $chiplist);
     209    return ($fulldate, $dateword, $chiplist, $stage);
    209210}
    210211

  • trunk/tools/eam/rawfix.20230221/src/fix_chip_locations_stage2.pl

    r42415 r42430  
    1212use Getopt::Long qw( GetOptions :config auto_help auto_version );
    1313
    14 # USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file)
     14# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc)
    1515
    1616# we need to select targets for ITC copies of files currently only at ATRC:
    … …  
    1818# for each chip, target a single host?
    1919
    20 my ($fulldate, $dateword, $chiplist) = &parse_cmdopts;
     20my ($fulldate, $dateword, $chiplist, $stage) = &parse_cmdopts;
    2121my ($neb_dbh) = &parse_db_config ('nebulous.config');
    2222
    … …  
    7575            my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id);
    7676            unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; }
    77             open (OUTFILE, ">>$dateword/$newhost.inst.txt");
     77            open (OUTFILE, ">>$dateword/$newhost.inst.$stage.txt");
    7878            print OUTFILE "$newfile $datast $md5sum\n";
    7979            close (OUTFILE);
    … …  
    189189sub parse_cmdopts {
    190190
    191     my ($dateobs, $chiplist);
    192     GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2);
     191    my ($dateobs, $chiplist, $stage);
     192    GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'stage=s' => \$stage) || pod2usage(2);
    193193   
    194194    pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs;
    195195    pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist;
     196    pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage;
    196197   
    197198    # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?)
    … …  
    206207    my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day);
    207208
    208     return ($fulldate, $dateword, $chiplist);
     209    return ($fulldate, $dateword, $chiplist, $stage);
    209210}
    210211

  • trunk/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh

    r42404 r42430  
    11#!/bin/csh
    22
    3 if ($#argv != 1) then
    4   echo "USAGE: get_hosts_md5s.sh (dateobs)"
     3if ($#argv != 2) then
     4  echo "USAGE: get_hosts_md5s.sh (dateobs) (stage)"
    55  exit 2
    66endif
    77
    88set dirname = $1
     9set stage   = $2
    910
    1011if (! -d $dirname) then
    … …  
    1415
    1516# report both the dateobs and the hostnames
    16 ls $dirname/*.inst.txt | awk -v dateobs=$dirname -F/ '(NR == 1){print "dateobs md5host"}{print dateobs, $2}' | sed s/.inst.txt//
     17ls $dirname/*.inst.$stage.txt | awk -v dateobs=$dirname -F/ '(NR == 1){print "dateobs md5host"}{print dateobs, $2}' | sed s/.inst.$stage.txt//
    1718exit 0

  • trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl

    r42411 r42430  
    1111use Getopt::Long qw( GetOptions :config auto_help auto_version );
    1212
    13 # USAGE: get_md5s_instances.pl --hostname (hostname) --input (file)
     13# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (v0,v1,etc)
    1414
    15 my ($myhost, $input) = &parse_cmdopts;
     15my ($myhost, $input, $stage) = &parse_cmdopts;
    1616
    1717# check that we are running this script on 'myhost'
    … …  
    2323
    2424my $output = $input;
    25 $output =~ s/inst/md5s/;
     25$output =~ s/inst.$stage/md5s/;
    2626
    2727if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; }
    … …  
    8484sub parse_cmdopts {
    8585
    86     my ($myhost, $input);
    87     GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input) || pod2usage(2);
     86    my ($myhost, $input, $stage);
     87    GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input, 'stage=s' => \$stage) || pod2usage(2);
    8888   
    8989    pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input;
    9090    pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost;
     91    pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage;
    9192   
    92     return ($myhost, $input);
     93    return ($myhost, $input, $stage);
    9394}
    9495

  • trunk/tools/eam/rawfix.20230221/src/get_md5s_instances_stage2.pl

    r42415 r42430  
    1111use Getopt::Long qw( GetOptions :config auto_help auto_version );
    1212
    13 # USAGE: get_md5s_instances.pl --hostname (hostname) --input (file)
     13# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (v0,v1,etc)
    1414
    15 my ($myhost, $input) = &parse_cmdopts;
     15my ($myhost, $input, $stage) = &parse_cmdopts;
    1616
    1717# check that we are running this script on 'myhost'
    … …  
    2323
    2424my $output = $input;
    25 $output =~ s/inst/md5s/;
     25$output =~ s/inst.$stage/md5s/;
    2626
    2727if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; }
    … …  
    117117sub parse_cmdopts {
    118118
    119     my ($myhost, $input);
    120     GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input) || pod2usage(2);
     119    my ($myhost, $input, $stage);
     120    GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input, 'stage=s' => \$stage) || pod2usage(2);
    121121   
    122122    pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input;
    123123    pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost;
     124    pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage;
    124125   
    125     return ($myhost, $input);
     126    return ($myhost, $input, $stage);
    126127}
    127128

  • trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt

    r42411 r42430  
    6161  npending 1
    6262
    63   stdout $LOGDIR/ckchip.run.stdout.log 
    64   stderr $LOGDIR/ckchip.run.stderr.log
    65 
    6663  task.exec
    6764    # if we are unable to run the 'exec', use a long retry time
    … …  
    8077    option $pageName
    8178
     79    # this is the first place in the proces that this location is known
     80    mkdir  $DATEOBS
     81    stdout $DATEOBS/log.ckchip.v0 
     82    stderr $DATEOBS/log.ckchip.v0
     83
    8284    substr $DATEOBS 0 4 YEAR
    8385    substr $DATEOBS 4 2 MONTH
    … …  
    8688    sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
    8789
    88     command check_chip_locations.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt
     90    command check_chip_locations.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v0
    8991  end
    9092
    91   ## XX delete the page from the book if done?
     93  # mark the entry done, will be deleted above in .load
    9294  task.exit 0
    93     # convert the stdout 'queue' to entries in a book
    94     # fields in the table will become words in the page
    95     # use the queuesize (Nhosts) to track how many jobs are done / to be done
    9695    book setword ckchip.book $options:0 pantaskState DONE
    9796    exec update.rawfix.sh $options:0 ckchip.done
    … …  
    119118end
    120119
     120# option for writing stdout / stderr to a file
     121# queuepush stdout "------- end of stdout ---------"
     122# queuesave stdout $options:0/log.ckchip.v0
     123# queuepush stderr "------- end of stderr ---------"
     124# queuesave stderr $options:0/log.ckchip.v0

  • trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

    r42417 r42430  
    7979    option $pageName
    8080
     81    stdout $pageName/log.fixchip.v0 
     82    stderr $pageName/log.fixchip.v0
     83
    8184    substr $DATEOBS 0 4 YEAR
    8285    substr $DATEOBS 4 2 MONTH
    … …  
    8487    sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
    8588
    86     command fix_chip_locations.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt
     89    command fix_chip_locations.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v0
    8790  end
    8891
    8992  ## XX delete the page from the book if done?
    9093  task.exit 0
    91     # convert the stdout 'queue' to entries in a book
    92     # fields in the table will become words in the page
    93     # use the queuesize (Nhosts) to track how many jobs are done / to be done
    9494    book setword fixchip.book $options:0 pantaskState DONE
    9595    exec update.rawfix.sh $options:0 fixchip.done

  • trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s2.pt

    r42417 r42430  
    8484    sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
    8585
    86     command fix_chip_locations_stage2.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt
     86    command fix_chip_locations_stage2.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v0
    8787  end
    8888

  • trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt

    r42417 r42430  
    8686    exec echo "# start runhosts $mydate" >> $tmpline
    8787
    88     command get_hosts_md5s.sh $DATEOBS
     88    command get_hosts_md5s.sh $DATEOBS v0
    8989  end
    9090
    … …  
    124124  periods      -timeout 60
    125125
    126   stdout $LOGDIR/md5sum.runhosts.stdout.log 
    127   stderr $LOGDIR/md5sum.runhosts.stderr.log
    128 
    129126  task.exec
    130127    # if we are unable to run the 'exec', use a long retry time
    … …  
    146143    # save DATEOBS so we can find the md5sum.book entry
    147144
     145    stdout $mypath/$DATEOBS/log.md5sum.runhosts.v0 
     146    stderr $mypath/$DATEOBS/log.md5sum.runhosts.v0 
     147
    148148    # XXX TEST: $USE_HOST = ipp060
    149 
    150149    $USE_HOST = $MD5HOST
    151150    host -required $USE_HOST
    … …  
    161160
    162161    ## even when testing, use the original hostname here:
    163     $input = $mypath/$DATEOBS/$MD5HOST.inst.txt
     162    $input = $mypath/$DATEOBS/$MD5HOST.inst.v0.txt
    164163
    165164    # need to make sure this will work on the ipp b-nodes:
    166165    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
    167166    periods -exec 0.05
    168     command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input
     167    command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input --stage v0
    169168  end
    170169

  • trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum_s2.pt

    r42417 r42430  
    8686    exec echo "# start runhosts $mydate" >> $tmpline
    8787
    88     command get_hosts_md5s.sh $DATEOBS
     88    command get_hosts_md5s.sh $DATEOBS v0
    8989  end
    9090
    … …  
    161161
    162162    ## even when testing, use the original hostname here:
    163     $input = $mypath/$DATEOBS/$MD5HOST.inst.txt
     163    $input = $mypath/$DATEOBS/$MD5HOST.inst.v0.txt
    164164
    165165    # need to make sure this will work on the ipp b-nodes:
    166166    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
    167167    periods -exec 0.05
    168     command $mypath/get_md5s_instances_stage2.pl --hostname $USE_HOST --input $input
     168    command $mypath/get_md5s_instances_stage2.pl --hostname $USE_HOST --input $input --stage v0
    169169  end
    170170

  • trunk/tools/eam/rawfix.20230221/src/rawfix.pt

    r42417 r42430  
    2323# order is this file is not determinative, only the
    2424# change of state from STAGE1.new -> STAGE1.done -> STAGE2.new
     25# input rawfix.advance.pt
    2526input rawfix.ckchip.pt
    26 input rawfix.fixchip.pt
    27 input rawfix.md5sum.pt
    28 input rawfix.md5chk.pt
    29 input rawfix.fixchip_s2.pt
    30 input rawfix.md5sum_s2.pt
    31 input rawfix.md5chk_s2.pt
    32 input rawfix.fixchip_s3.pt
     27# input rawfix.fixchip.pt
     28# input rawfix.md5sum.pt
     29# input rawfix.md5chk.pt
     30# input rawfix.fixchip_s2.pt
     31# input rawfix.md5sum_s2.pt
     32# input rawfix.md5chk_s2.pt
     33# input rawfix.fixchip_s3.pt
    3334
    3435# no need to run rsync if we use the b-node working directory:
Note: See TracChangeset for help on using the changeset viewer.