IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 42451


Ignore:
Timestamp:
Apr 14, 2023, 12:02:11 PM (3 years ago)
Author:
eugene
Message:

new instances from fix_chip_locations need to go to unique files by chip; get_md5s_instances needs to load the additional instance lists

Location:
trunk/tools/eam/rawfix.20230221/src
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl

    r42450 r42451  
    5555    open ($outfh, ">$topdir/$dateword/$chip_id.uris.$stage.txt");
    5656
    57     print "open $topdir/$dateword/$chip_id.uris.$stage.txt\n";
     57    # print "open $topdir/$dateword/$chip_id.uris.$stage.txt\n";
    5858
    5959    # get the list of chip neb paths for one chip

  • trunk/tools/eam/rawfix.20230221/src/create.rawfix.sh

    r42445 r42451  
    66
    77# add in the iteration and the state summary
     8
     9mysql -h localhost -u root <<EOF
     10  use $DBNAME;
     11  ALTER TABLE nights MODIFY Ngood INT, MODIFY Ndead INT, MODIFY Ngone INT;
     12  ALTER TABLE nights MODIFY Nsing INT, MODIFY Ndups INT, MODIFY Nsbck INT, MODIFY Nxbck INT, MODIFY Nfail INT;
     13  ALTER TABLE nights MODIFY Ncul1 INT, MODIFY Ncul2 INT, MODIFY Ncul3 INT;
     14EOF
     15
     16exit 0
    817
    918mysql -h localhost -u root <<EOF

  • trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl

    r42450 r42451  
    145145            my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id);
    146146            unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; }
    147             open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt");
     147            open (OUTFILE, ">>$topdir/$dateword/$newhost.$chip_id.inst.$stage.txt");
    148148            print OUTFILE "$newfile $datast $md5sum\n";
    149149            close (OUTFILE);
    … …  
    160160            my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id);
    161161            unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; }
    162             open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt");
     162            open (OUTFILE, ">>$topdir/$dateword/$newhost.$chip_id.inst.$stage.txt");
    163163            print OUTFILE "$newfile $datast $md5sum\n";
    164164            close (OUTFILE);
    … …  
    175175                my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id);
    176176                unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; }
    177                 open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt");
     177                open (OUTFILE, ">>$topdir/$dateword/$newhost.$chip_id.inst.$stage.txt");
    178178                print OUTFILE "$newfile $datast $md5sum\n";
    179179                close (OUTFILE);
    … …  
    197197                my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id);
    198198                unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; }
    199                 open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt");
     199                open (OUTFILE, ">>$topdir/$dateword/$newhost.$chip_id.inst.$stage.txt");
    200200                print OUTFILE "$newfile $datast $md5sum\n";
    201201                close (OUTFILE);

  • trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl

    r42444 r42451  
    1313# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (0,1,etc)
    1414
    15 my ($myhost, $input, $stage) = &parse_cmdopts;
     15my $input;
     16my ($myhost, $dateword, $topdir, $stage) = &parse_cmdopts;
    1617
    1718# check that we are running this script on 'myhost'
    … …  
    2021
    2122# read the list of instances:
    22 my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ($input);
     23my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ();
    2324
    24 my $output = $input;
    25 $output =~ s/inst.$stage/md5s/;
     25my $output = "$topdir/$dateword/$myhost.md5s.txt";
    2626
    27 if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; }
    28 
    29 # read in both instance
     27# read in existing md5s
    3028my $out_hash_r;
    3129if (-e $output) {
    … …  
    6058    }
    6159
    62     my $line;
    63     if (0) {
    64         $line = "fakevalue $filename"; # XXX TEST: disable calculation:
    65     } else {
    66         $line = `md5sum $filename`;
    67     }
     60    my $line = `md5sum $filename`;
    6861
    6962    my ($md5sum, $rawfile) = split (" ", $line);
    … …  
    7366exit 0;
    7467
     68# use global $topdir, $dateword, $myhost, $stage
     69sub load_instance_list {
     70    my (@ins_list, @dst_list, @md5_list);
     71
     72    # load the master list
     73    my $input = "$topdir/$dateword/$myhost.inst.$stage.txt";
     74    &load_instance_list_onefile ($input, \@ins_list, \@dst_list, \@md5_list);
     75
     76    # there may be additional instances in files with names like HOSTNAME.XYnn.inst.STAGE
     77    my @instlist = glob ("$topdir/$dateword/$myhost.XY??.inst.$stage.txt");
     78    foreach my $infile  (@instlist) {
     79        &load_instance_list_onefile ($infile, \@ins_list, \@dst_list, \@md5_list);
     80    }
     81    return (\@ins_list, \@dst_list, \@md5_list);
     82}
     83
    7584# read the instance info
    76 sub load_instance_list {
     85sub load_instance_list_onefile {
    7786    my $infile = $_[0];
    78 
    79     my (@ins_list, @dst_list, @md5_list);
     87    my $ins_list_r = $_[1]; # reference to the instance list array
     88    my $dst_list_r = $_[2]; # reference to the instance list array
     89    my $md5_list_r = $_[3]; # reference to the instance list array
    8090
    8191    open(FILE,$infile) or die "cannot open file $infile\n";
    … …  
    8999
    90100        if (@words != 3) { die "error in input $infile : $line\n"; }
    91         push (@ins_list, $words[0]);
    92         push (@dst_list, $words[1]);
    93         push (@md5_list, $words[2]);
     101        push (@$ins_list_r, $words[0]);
     102        push (@$dst_list_r, $words[1]);
     103        push (@$md5_list_r, $words[2]);
    94104    }
    95105    close (FILE);
    96 
    97     return (\@ins_list, \@dst_list, \@md5_list);
    98106}
    99107
    … …  
    123131sub parse_cmdopts {
    124132
    125     my ($myhost, $input, $stage);
    126     GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input, 'stage=s' => \$stage) || pod2usage(2);
     133    my ($myhost, $topdir, $dateobs, $stage);
     134    GetOptions( 'hostname=s' => \$myhost, 'topdir=s' => \$topdir, 'dateobs=s' => \$dateobs, 'stage=s' => \$stage) || pod2usage(2);
    127135   
    128     pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input;
     136    pod2usage( -msg => "Cannot determine target date, use --dateobs YYYYMMDD", -exitval => 2) unless defined $dateobs;
     137#   pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input;
    129138    pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost;
    130139    pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage;
     140    pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir;
    131141   
    132142    # check that stage is an integer
    … …  
    134144    unless ($stage >= 0) { die "stage must be a non-negative integer\n"; }
    135145   
    136     return ($myhost, $input, $stage);
     146    return ($myhost, $dateobs, $topdir, $stage);
    137147}
    138148

  • trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

    r42449 r42451  
    149149    option $pageName $DATEOBS
    150150
    151     stdout $pageName/log.fixchip.run.v$ITER
    152     stderr $pageName/log.fixchip.run.v$ITER
     151    stdout $DATEOBS/log.fixchip.run.$CHIP_ID.v$ITER
     152    stderr $DATEOBS/log.fixchip.run.$CHIP_ID.v$ITER
    153153
    154154    substr $DATEOBS 0 4 YEAR

  • trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt

    r42449 r42451  
    177177    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
    178178    periods -exec 0.05
    179     command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input --stage $ITER
     179    command $mypath/get_md5s_instances.pl --hostname $USE_HOST --topdir $mypath --dateobs $DATEOBS --stage $ITER
     180#   command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input --stage $ITER
    180181  end
    181182

  • trunk/tools/eam/rawfix.20230221/src/rawfix.pt

    r42447 r42451  
    1919# required host
    2020
    21 $HOST_CKCHIP  = ipp060
    22 $HOST_FIXCHIP = ipp070
     21# run check_chip_locations.pl on this machine:
     22$HOST_CKCHIP  = ippc180
    2323
    2424if (not($?LOGDIR))
    … …  
    3838if (not($?INIT_RAWFIX))
    3939  control host add $HOST_CKCHIP
    40   control host add $HOST_FIXCHIP
    4140  $INIT_RAWFIX = 1
    4241end
Note: See TracChangeset for help on using the changeset viewer.