IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Aug 17, 2023, 4:41:34 PM (3 years ago)
Author:
eugene
Message:

merging from trunk

Location:
branches/eam_branches/ipp-20230313/tools
Files:
2 deleted
16 edited
21 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20230313/tools

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check.rawfix.sh

    r42406 r42531  
    1414
    1515# make the rawfix database to track rawfix operations
    16 set DBHOST = "ipp060"
     16set DBHOST = "ippc190"
    1717set DBNAME = "rawfix"
    18 set DBUSER = "dvo"
    19 set DBPASS = "dvo"
     18# set DBUSER = "dvo"
     19# set DBPASS = "dvo"
    2020
    2121# set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME"
    22 set DBOPTS = "-B -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME"
     22set DBOPTS = "--defaults-extra-file=~/.my.cnf.rawfix -B -h $DBHOST $DBNAME"
    2323
    2424if ($mode == "all") then
    … …  
    2828  endif
    2929
    30   mysql $DBOPTS -e "select dateobs, state from nights"
     30  mysql $DBOPTS -e "select dateobs, iter, state from nights"
    3131  exit 0
    3232endif
    … …  
    4949
    5050  set mynight = $2
    51   mysql $DBOPTS -e "select dateobs, state from nights where (dateobs = '$mynight')"
     51  mysql $DBOPTS -E -e "select * from nights where (dateobs = '$mynight')"
    5252  exit 0
    5353endif
    … …  
    6161  set mystate = $2
    6262
    63   if ($mystate == "new") goto good_state
    64   if ($mystate == "rsync") goto good_state
    65   if ($mystate == "md5sum.new") goto good_state
    66   if ($mystate == "md5sum.done") goto good_state
    67   if ($mystate == "md5sum.fail") goto good_state
    68   if ($mystate == "ckchip.new") goto good_state
    69   if ($mystate == "ckchip.done") goto good_state
    70   if ($mystate == "ckchip.fail") goto good_state
    71   if ($mystate == "fixchip.new") goto good_state
    72   if ($mystate == "fixchip.done") goto good_state
    73   if ($mystate == "fixchip.fail") goto good_state
     63  if ($mystate == "ckchip.new")      goto good_state
     64  if ($mystate == "ckchip.done")     goto good_state
     65  if ($mystate == "ckchip.fail")     goto good_state
     66  if ($mystate == "fixchip.new")     goto good_state
     67  if ($mystate == "fixchip.done")    goto good_state
     68  if ($mystate == "fixchip.fail")    goto good_state
     69  if ($mystate == "md5sum.new")      goto good_state
     70  if ($mystate == "md5sum.done")     goto good_state
     71  if ($mystate == "md5sum.fail")     goto good_state
     72  if ($mystate == "md5chk.new")      goto good_state
     73  if ($mystate == "md5chk.done")     goto good_state
     74  if ($mystate == "md5chk.fail")     goto good_state
     75  if ($mystate == "burntool.new")    goto good_state
     76  if ($mystate == "burntool.done")   goto good_state
     77  if ($mystate == "burntool.fail")   goto good_state
    7478
    7579  echo "ERROR: unknown mode $mystate"
    … …  
    7781
    7882  good_state:
    79   mysql $DBOPTS -e "select dateobs from nights where (state = '$mystate')"
     83  set t = `date +%T..%N`
     84  mysql $DBOPTS -e "select dateobs, iter from nights where (state = '$mystate')" | awk -v T=$t '(NR == 1){print $0, "timestamp"}(NR > 1){print $1, $2, T}'
    8085  exit 0
    8186endif

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_chip_locations.pl

    r42406 r42531  
    1313use Getopt::Long qw( GetOptions :config auto_help auto_version );
    1414
    15 # USAGE: check_chip_locations.pl --dateobs (night) --chiplist (file)
    16 # USAGE: check_chip_locations.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt
    17 
    18 my ($fulldate, $dateword, $chiplist) = &parse_cmdopts;
    19 my ($neb_dbh, $gpc_dbh) = &parse_db_config ('nebulous.config');
     15# USAGE: check_chip_locations.pl --dateobs (night) --chiplist (file) --stage (0,1,etc)
     16# USAGE: check_chip_locations.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt --stage 0
     17
     18my ($fulldate, $dateword, $chiplist, $stage, $topdir, $basedir) = &parse_cmdopts;
     19my ($neb_dbh, $gpc_dbh, $raw_dbh) = &parse_db_config ("$topdir/nebulous.config");
    2020
    2121# read the list of chip_ids:
    … …  
    2727
    2828# put output files in a directory defined by the night
    29 mkdir $dateword;
     29mkdir "$basedir/$dateword";
    3030
    3131# output files:
    32 # instance lists by host: YYYYMMDD/HOSTNAME.inst.txt
    33 # chip states by chip: YYYYMMDD/CHIPNAME.uris.txt
     32# instance lists by host: YYYYMMDD/HOSTNAME.inst.STAGE.txt
     33# chip states by chip: YYYYMMDD/CHIPNAME.uris.STAGE.txt
     34
     35my $Nchip = 0;
     36
     37my $Ngood = 0;
     38my $Ndead = 0;
     39my $Ngone = 0;
     40my $Nsing = 0;
     41my $Ndups = 0;
     42my $Nsbck = 0;
     43my $Nxbck = 0;
     44
     45my $Ncul1 = 0;
     46my $Ncul2 = 0;
     47my $Ncul3 = 0;
     48my $Nfail = 0;
     49
     50my ($goodchips_r, $failchips_r) = &get_good_or_fail_chiplists ($basedir, $dateword, $stage);
    3451
    3552foreach my $chip_id (@chip_ids) {
    3653
    37     my $outfile = "$dateword/$chip_id.uris.txt";
    3854    my $outfh;
    39     open ($outfh, ">$outfile");
     55    open ($outfh, ">$basedir/$dateword/$chip_id.uris.$stage.txt");
     56
     57    # print "open $basedir/$dateword/$chip_id.uris.$stage.txt\n";
    4058
    4159    # get the list of chip neb paths for one chip
    4260    my ($neb_paths_r, $ds_array_r, $md_array_r) = &get_chip_paths ($fulldate, $chip_id);
    43 
    44     # I think this makes a copy of the arrays, which is inefficient
    45     # my @neb_paths = @$neb_paths_r;
    46     # my @ds_array  = @$ds_array_r;
    47     # my @md_array  = @$md_array_r;
    4861
    4962    # reset the hostcount for this chip
    … …  
    6578
    6679        # determine the status of the instances
    67         my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum);
    68 
    69         &check_instance_state ($outfh, $ext_id, $Nmain, $Nback, $Ndead);
     80        my ($Nmain, $Nback, $Nlost) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum);
     81
     82        &check_instance_state ($outfh, $ext_id, $data_st, $md5_sum, $Nmain, $Nback, $Nlost);
    7083    }
    7184
    … …  
    7487    # now save the host count for this chip
    7588    my $hostfh;
    76     open ($hostfh, ">$dateword/$chip_id.hosts.txt");
     89    open ($hostfh, ">$basedir/$dateword/$chip_id.hosts.txt");
    7790
    7891    # save the host count for this chip
    7992    my @hostnames = keys %hostcount;
    8093    foreach my $myname (@hostnames) {
    81         print $hostfh "host $myname = $hostcount{$myname}\n";
     94        print $hostfh "$myname $hostcount{$myname}\n";
    8295    }
    8396    close ($hostfh);
    … …  
    91104}
    92105
    93 # run neb-replicate for SING entries?
    94 # run neb-replicate for SBCK entries?  (does this put a second copy at ATRC or ITC?)
    95 # save the list of instances on each host
    96 ## run md5sum on the instances for each host
    97 ## only note inconsistent md5sums?  (need to save the gpc1 md5sum values)
    98 # rerun to generate a list of the host with max instances per chip?
    99 
    100 ### when to cull entries?
    101 ## neb-replicate puts the new entry at ATRC by default
    102 ## neb-replicate --volume VOL puts the new entry at the specified location
    103 ### in order to use this, I need to choose the target volume
     106print "Nchip: $Nchip\n";
     107print "Ngood: $Ngood\n";
     108print "Ngone: $Ngone\n";
     109print "Ndead: $Ndead\n";
     110print "Nsing: $Nsing\n";
     111print "Ndups: $Ndups\n";
     112print "Nsbck: $Nsbck\n";
     113print "Nxbck: $Nxbck\n";
     114print "Ncul1: $Ncul1\n";
     115print "Ncul2: $Ncul2\n";
     116print "Ncul3: $Ncul3\n";
     117print "Nfail: $Nfail\n";
     118
     119my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3 + $Nfail;
     120if ($Ntotal != $Nchip) { die "ERROR: inconsistent count of chips\n"; }
     121
     122# update the stats in the database
     123my $sql_update_stats = "UPDATE nights set";
     124$sql_update_stats .= "  Nchip = $Nchip";
     125$sql_update_stats .= ", Ngood = $Ngood";
     126$sql_update_stats .= ", Ndead = $Ndead";
     127$sql_update_stats .= ", Ngone = $Ngone";
     128$sql_update_stats .= ", Nsing = $Nsing";
     129$sql_update_stats .= ", Ndups = $Ndups";
     130$sql_update_stats .= ", Nsbck = $Nsbck";
     131$sql_update_stats .= ", Nxbck = $Nxbck";
     132$sql_update_stats .= ", Ncul1 = $Ncul1";
     133$sql_update_stats .= ", Ncul2 = $Ncul2";
     134$sql_update_stats .= ", Ncul3 = $Ncul3";
     135$sql_update_stats .= ", Nfail = $Nfail";
     136$sql_update_stats .= " WHERE dateobs = $dateword";
     137
     138my $query = $raw_dbh->prepare($sql_update_stats);
     139$query->execute () or die "invalid response from RAW database\n";
    104140
    105141exit 0;
    … …  
    121157    my @chip_ids;
    122158
    123     open(CHIPS,$chiplist) or die "cannot open file $chiplist\n";
     159    open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n";
    124160
    125161    my @lines = <CHIPS>;
    … …  
    160196        if ($Nvalues != 3) { print "ERROR: invalid response?\n"; next; }
    161197       
     198        # data_state and possibly md5sum can be NULL, need to handle specially
     199        my $dstate = $values[1]; if (not defined $dstate) { $dstate = "NULL"; }
     200        my $md5sum = $values[2]; if (not defined $md5sum) { $md5sum = "XXXX"; }
     201       
    162202        push (@nebarray, $values[0]);
    163         push (@ds_array, $values[1]);
    164         push (@md_array, $values[2]);
    165         if ($DEBUG > 4) { print "$values[0]\n $values[1]\n $values[2]\n"; }
     203        push (@md_array, $md5sum);
     204        push (@ds_array, $dstate);
     205
     206        if ($DEBUG > 5) { print "V0: $values[0]\n"; print "V1: $values[1]\n"; print "V2: $values[2]\n"; }
     207        if ($DEBUG > 4) { print "PRINT: $values[0]\n $dstate\n $md5sum\n"; }
    166208    }
    167209
    … …  
    178220
    179221    # my ($ext_id) = $nebname =~ m|neb://\w*/(\S*)|; ---> what if we do not have the host.volnum portion? (this attempt fails)
    180     my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z]*\.[0-9]/(\S*)|;
     222    my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z\.]*/(\S*)|;
    181223    if ($DEBUG > 5) { print "nebname: $ext_id\n"; }
    182224   
    183225    my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'";
    184226    my $rowref   = $neb_dbh->selectall_arrayref( $sql_chip_instances );
    185  
     227   
    186228    return ($rowref, $ext_id);
    187229}
    … …  
    205247    my $Nmain = 0;
    206248    my $Nback = 0;
    207     my $Ndead = 0;
     249    my $Nlost = 0;
    208250
    209251    # xattr = 0 means instance @ ITC  (main)
    … …  
    226268        my $xattr    = $values[3];
    227269       
     270        if ($xattr  > 1) { $Nlost ++; next; } # do not save instances on dead partitions
     271        if ($xattr == 0) { $Nmain ++; }
     272        if ($xattr == 1) { $Nback ++; }
     273
    228274        # count the number of each host by incrementing the hash element corresponding to the hostname:
    229275        # NOTE: hostcount is global
    … …  
    231277
    232278        if (not defined $hostfile{$hostname}) {
    233             open ($hostfile{$hostname}, ">$dateword/$hostname.inst.txt");
     279            open ($hostfile{$hostname}, ">$basedir/$dateword/$hostname.inst.$stage.txt");
    234280            ## XXX deal with open failure here
    235281        }
     282
    236283        my $fh = $hostfile{$hostname};
    237284        print $fh "$instance $data_st $md5_sum\n";
    238 
    239         if ($xattr == 0) { $Nmain ++; }
    240         if ($xattr == 1) { $Nback ++; }
    241         if ($xattr  > 1) { $Ndead ++; }
    242     }
    243 
    244     return ($Nmain, $Nback, $Ndead);
     285    }
     286
     287    return ($Nmain, $Nback, $Nlost);
    245288}
    246289
    … …  
    249292    my $outfh = $_[0];
    250293    my $extID = $_[1];
    251     my $Nmain = $_[2];
    252     my $Nback = $_[3];
    253     my $Ndead = $_[4];
    254 
    255     # SKIP : already fixed
     294    my $data_st = $_[2];
     295    my $md5_sum = $_[3];
     296    my $Nmain = $_[4];
     297    my $Nback = $_[5];
     298    my $Nlost = $_[6];
     299
     300    my $outinfo = "$extID $data_st $md5_sum";
     301    $Nchip ++;
     302
     303    my $haveMD5 = 0;
     304    my $goodMD5 = 0;
     305    if ($goodchips_r->{$extID}) { $haveMD5 = 1; $goodMD5 = 1; }
     306    if ($failchips_r->{$extID}) { $haveMD5 = 1; $goodMD5 = 0; }
     307
     308    if ($extID eq "gpc1/20100503/o5319g0003d/o5319g0003d.ota46.fits") {
     309        print STDERR "$outinfo $Nmain $Nback $haveMD5 $goodMD5\n";
     310    }
     311
     312    # GOOD : already fixed
    256313    if (($Nmain == 1) && ($Nback == 1)) {
    257         print $outfh "GOOD: $extID\n";
    258         return;
    259     }
    260 
    261     # single copy @ ITC, no backup
     314        print $outfh "GOOD: $outinfo\n";
     315        &update_counts (\$Ngood, \$Nfail, $haveMD5, $goodMD5);
     316        return;
     317    }
     318
     319    # SING : single copy @ ITC, no backup
    262320    # make a backup
    263321    if (($Nmain == 1) && ($Nback == 0)) {
    264         print $outfh "SING: $extID\n";
    265         return;
    266     }
    267 
    268     # ITC files with a single backup copies need to be culled
    269     # cull Nmain - 1
    270     if (($Nmain > 1) && ($Nback == 1)) {
    271         print $outfh "CULL: $extID\n";
    272         return;
    273     }
    274 
    275     # multiple copies @ ITC
     322        print $outfh "SING: $outinfo\n";
     323        &update_counts (\$Nsing, \$Nfail, $haveMD5, $goodMD5);
     324        return;
     325    }
     326
     327    # SBCK : single copy @ ATRC, no main copy
     328    # make a backup @ ITC
     329    if (($Nmain == 0) && ($Nback == 1)) {
     330        print $outfh "SBCK: $outinfo\n";
     331        &update_counts (\$Nsbck, \$Nfail, $haveMD5, $goodMD5);
     332        return;
     333    }
     334
     335    # DUPS : multiple copies @ ITC, none at ATRC
    276336    # make a backup and cull Nmain - 1
    277337    if (($Nmain > 1) && ($Nback == 0)) {
    278         print $outfh "DUPS: $extID\n";
    279         return;
    280     }
    281 
    282     # no more valid instances
    283     if (($Nmain == 0) && ($Nback == 0) && $Ndead) {
    284         print $outfh "DEAD: $extID\n";
    285         return;
    286     }
    287     if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) {
    288         print $outfh "GONE: $extID\n";
    289         return;
    290     }
    291 
    292     # single copy @ ATRC
    293     # make a backup @ ITC
    294     if (($Nmain == 0) && ($Nback == 1)) {
    295         print $outfh "SBCK: $extID\n";
    296         return;
    297     }
    298 
    299     # multiple copies @ ATRC, but none at ITC
     338        print $outfh "DUPS: $outinfo\n";
     339        &update_counts (\$Ndups, \$Nfail, $haveMD5, $goodMD5);
     340        return;
     341    }
     342
     343    # XBCK : multiple copies @ ATRC, but none at ITC
    300344    # make a backup at ITC, cull ATRC extras
    301345    if (($Nmain == 0) &&  ($Nback > 1)) {
    302         print $outfh "XBCK: $extID\n";
     346        print $outfh "XBCK: $outinfo\n";
     347        &update_counts (\$Nxbck, \$Nfail, $haveMD5, $goodMD5);
     348        return;
     349    }
     350
     351    # DEAD : no more valid instances
     352    if (($Nmain == 0) && ($Nback == 0) && $Nlost) {
     353        print $outfh "DEAD: $outinfo\n";
     354        &update_counts (\$Ndead, \$Nfail, $haveMD5, $goodMD5);
     355        return;
     356    }
     357    # GONE : no more valid instances
     358    if (($Nmain == 0) && ($Nback == 0) && ($Nlost == 0)) {
     359        print $outfh "GONE: $outinfo\n";
     360        &update_counts (\$Ngone, \$Nfail, $haveMD5, $goodMD5);
     361        return;
     362    }
     363
     364    # CUL1 : multiple ITC files, single backup
     365    # cull only ITC
     366    if (($Nmain > 1) && ($Nback == 1)) {
     367        print $outfh "CUL1: $outinfo\n";
     368        &update_counts (\$Ncul1, \$Nfail, $haveMD5, $goodMD5);
     369        return;
     370    }
     371
     372    # CUL2 : multiple ATRC files, single ITC
     373    # cull only ATRC
     374    if (($Nback > 1) && ($Nmain == 1)) {
     375        print $outfh "CUL2: $outinfo\n";
     376        &update_counts (\$Ncul2, \$Nfail, $haveMD5, $goodMD5);
     377        return;
     378    }
     379
     380    # CUL3 : excess ATRC & ITC files, cull from both?
     381    # cull Nmain - 1
     382    if (($Nback > 1) && ($Nmain > 1)) {
     383        print $outfh "CUL3: $outinfo\n";
     384        &update_counts (\$Ncul3, \$Nfail, $haveMD5, $goodMD5);
    303385        return;
    304386    }
    305387
    306388    # unexpected state:
    307     print $outfh "OOPS: $extID\n";
     389    print $outfh "OOPS: $outinfo\n";
    308390    return;
    309391}
    310392
     393sub update_counts {
     394
     395    my $Nstat_r = $_[0];
     396    my $Nfail_r = $_[1];
     397    my $haveMD5 = $_[2];
     398    my $goodMD5 = $_[3];
     399
     400    ## if we know the MD5 is bad, add to Nfail, otherwise add to Ndups
     401    if ($haveMD5 && !$goodMD5) {
     402        $$Nfail_r ++;
     403    } else {
     404        $$Nstat_r ++;
     405    }
     406}
     407
     408sub get_good_or_fail_chiplists {
     409
     410    my $basedir   = $_[0];
     411    my $dateword = $_[1];
     412    my $stage    = $_[2];
     413
     414    # load the lists of good and fail chips
     415    my $prev = $stage - 1;
     416    my $goodfile = "$basedir/$dateword/goodchips.$prev.txt";
     417    my %goodchips;
     418    if (-e $goodfile) {
     419        open (GOODFILE,  $goodfile);
     420        my @lines = <GOODFILE>;
     421        close (GOODFILE);
     422       
     423        foreach my $line (@lines) {
     424            chomp $line;
     425            $goodchips{$line} = 1;
     426        }
     427    }
     428    my $failfile = "$basedir/$dateword/failchips.$prev.txt";
     429    my %failchips;
     430    if (-e $failfile) {
     431        open (FAILFILE,  $failfile);
     432        my @lines = <FAILFILE>;
     433        close (FAILFILE);
     434       
     435        foreach my $line (@lines) {
     436            chomp $line;
     437            $failchips{$line} = 1;
     438        }
     439    }
     440    return (\%goodchips, \%failchips);
     441}
     442
    311443sub parse_cmdopts {
    312444
    313     my ($dateobs, $chiplist);
    314     GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2);
     445    my ($dateobs, $chiplist, $stage, $topdir);
     446    GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2);
    315447   
    316448    pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs;
    317449    pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist;
     450    pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage;
     451    pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir;
    318452   
     453    # check that stage is an integer
     454    unless (&is_integer ($stage)) { die "stage must be a non-negative integer\n"; }
     455    unless ($stage >= 0) { die "stage must be a non-negative integer\n"; }
     456
    319457    # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?)
    320458
    … …  
    324462    pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12);
    325463    pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31);
     464
    326465    my $fulldate = "$date_year/$date_month/$date_day";
    327    
    328466    my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day);
    329467
    330     return ($fulldate, $dateword, $chiplist)
     468    my $yearobs = substr ($dateword, 0, 4);
     469    my $basedir = "$topdir/$yearobs";
     470
     471    return ($fulldate, $dateword, $chiplist, $stage, $topdir, $basedir)
    331472}
    332473
    … …  
    338479    my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user );
    339480    my ( $gpcdb_name, $gpcdb_host, $gpcdb_port, $gpcdb_pass, $gpcdb_user );
     481    my ( $rawdb_name, $rawdb_host, $rawdb_port, $rawdb_pass, $rawdb_user );
    340482
    341483    if (-e $conffile) {
    … …  
    355497            if ($_ =~ /GPC1_PASS /) { $gpcdb_pass = (split /\s+/,$_)[2]; }
    356498            if ($_ =~ /GPC1_PORT /) { $gpcdb_port = (split /\s+/,$_)[2]; }
     499
     500            if ($_ =~ /RAW_NAME /)  { $rawdb_name = (split /\s+/,$_)[2]; }
     501            if ($_ =~ /RAW_HOST /)  { $rawdb_host = (split /\s+/,$_)[2]; }
     502            if ($_ =~ /RAW_USER /)  { $rawdb_user = (split /\s+/,$_)[2]; }
     503            if ($_ =~ /RAW_PASS /)  { $rawdb_pass = (split /\s+/,$_)[2]; }
     504            if ($_ =~ /RAW_PORT /)  { $rawdb_port = (split /\s+/,$_)[2]; }
    357505        }
    358506        close(IN);
    359507        print STDERR  "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n";
    360508        print STDERR  "gpc1 dbh config: $gpcdb_name $gpcdb_host $gpcdb_port $gpcdb_user $gpcdb_pass\n";
     509        print STDERR  "raw  dbh config: $rawdb_name $rawdb_host $rawdb_port $rawdb_user $rawdb_pass\n";
    361510    }
    362511
    … …  
    373522    pod2usage( -msg => "Cannot configure gpc1 database (dbpass)", -exitval => 2) unless defined $gpcdb_pass;
    374523
     524    pod2usage( -msg => "Cannot configure raw database (dbname)", -exitval => 2) unless defined $rawdb_name;
     525    pod2usage( -msg => "Cannot configure raw database (dbhost)", -exitval => 2) unless defined $rawdb_host;
     526    pod2usage( -msg => "Cannot configure raw database (dbuser)", -exitval => 2) unless defined $rawdb_user;
     527    pod2usage( -msg => "Cannot configure raw database (dbport)", -exitval => 2) unless defined $rawdb_port;
     528    pod2usage( -msg => "Cannot configure raw database (dbpass)", -exitval => 2) unless defined $rawdb_pass;
     529
    375530    my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host";
    376531    if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; }
    … …  
    381536    my $gpc_dbh = DBI->connect($gpcdb_dsn, $gpcdb_user, $gpcdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n";
    382537
    383     return ($neb_dbh, $gpc_dbh);
    384 }
    385 
    386    
     538    my $rawdb_dsn = "DBI:mysql:database=$rawdb_name:host=$rawdb_host";
     539    if ($rawdb_port ne "NONE") { $rawdb_dsn .= ":port=$rawdb_port"; }
     540    my $raw_dbh = DBI->connect($rawdb_dsn, $rawdb_user, $rawdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n";
     541
     542    return ($neb_dbh, $gpc_dbh, $raw_dbh);
     543}
     544
     545sub is_integer {
     546    defined $_[0] && $_[0] =~ /^[+-]?\d+$/;
     547}
     548
    387549# example of using the return value of arrayref:
    388550

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl

    r42404 r42531  
    11#! /usr/bin/env perl
    22
    3 my $DEBUG = 4;
     3my $DEBUG = 1;
    44my $time = time();
    55
    … …  
    1212use Getopt::Long qw( GetOptions :config auto_help auto_version );
    1313
    14 # USAGE: check_md5s_dateobs.pl --dateobs (night)
    15 
    16 my ($dateword) = &parse_cmdopts;
    17 
    18 my @instlist = glob ("$dateword/*.inst.txt");
    19 my @md5slist = glob ("$dateword/*.md5s.txt");
    20 
    21 foreach my $instfile (@instlist) {
    22 
    23     # find the matching md5sum file for this host
    24 
    25     my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.txt|;
    26     # print STDERR "host: $host\n";
    27 
    28     my $md5match = "NONE";
    29     foreach my $md5sfile (@md5slist) {
    30        
    31         my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|;
    32         # print STDERR "md5host: $md5shost\n";
    33        
    34         if ($host ne $md5shost) { next; }
    35 
    36         $md5match = $md5sfile;
    37         last;
    38     }
    39 
    40     if ($md5match eq "NONE") {
    41         print STDERR "missing md5sum file for $instfile\n";
    42         next;
    43     }
     14# USAGE: check_md5s_dateobs.pl --dateobs (night) --topdir (topdir)
     15
     16my ($dateword, $stage, $topdir, $basedir) = &parse_cmdopts;
     17# basedir used only for night-level files
     18
     19my $instlist_r = &load_instlist_filenames ($basedir, $dateword, $stage);
     20my $md5slist_r = &load_md5slist_filenames ($basedir, $dateword, $stage, $instlist_r);
     21
     22my $Nfail = 0;
     23my %failchips;
     24my %goodchips;
     25
     26# make a list of the master instance files:
     27
     28foreach my $instfile (@$instlist_r) {
     29
     30    # print "inst: $instfile\n";
     31
     32    my $md5sfile = $md5slist_r->{$instfile};
    4433
    4534    ## read in the md5sums
    46     open (FILE, $md5match);
     35    open (FILE, $md5sfile);
    4736    my @md5data = <FILE>;
    4837    close (FILE);
    4938
    50     ## read in the list of instances
    51     open (FILE, $instfile);
    52     my @instdata = <FILE>;
    53     close (FILE);
     39    my $instdata_r = &load_instance_list ($basedir, $dateword, $stage, $instfile);
    5440
    5541    # if these do not match, complain and move on
     42    # we are allowed to have more MD5s than instances: some instances
     43    # may have been culled during this process (but the MD5s stay behind)
    5644    my $N1 = @md5data;
    57     my $N2 = @instdata;
    58     if ($N1 != $N2) {
    59         print STDERR "inconsistent number of lines in $md5match : $N1 vs $N2\n";
     45    my $N2 = @$instdata_r;
     46    if ($N1 < $N2) {
     47        print STDOUT "not enough MD5s $md5sfile : $N1 vs $N2\n";
     48        $Nfail ++;
    6049        next;
    61     }
    62 
    63     # check for mismatched MD5sums
    64     foreach my $line (@md5data) {
    65 
     50    } else {
     51        # print STDOUT "checking MD5s $md5sfile : $N1 vs $N2\n";
     52    }
     53
     54    # set up an empty hash for the instance names:
     55    my %instfound = ();
     56    foreach my $line (@$instdata_r) {
    6657        chomp $line;
    6758        my @words = split (/\s+/,$line);
     59        my $inst   = $words[0];
     60        $instfound{$inst} = 0;
     61    }
     62
     63    # check for mismatched MD5sums
     64    # we are not checking that all instances in the inst file are in the MD5 file
     65    foreach my $line (@md5data) {
     66
     67        chomp $line;
     68        my @words = split (/\s+/,$line);
    6869
    6970        # we expect lines of the form: INST STATE MDtru MDobs
    70         if (@words != 4) { die "error in MD5 list $md5match, line: $line\n"; }
     71        if (@words != 4) { die "error in MD5 list $md5sfile, line: $line\n"; }
    7172
    7273        my $inst   = $words[0];
    … …  
    7576        my $md5obs = $words[3];
    7677
     78        unless (defined $instfound{$inst}) { print "skip md5 for removed instance $inst\n"; next; }
     79        $instfound{$inst} = 1;
     80
    7781        if ($md5tru ne $md5obs) {
    78             print STDERR "FAIL: $inst $state $md5tru $md5obs\n";
     82            &save_failure ($inst);
     83            if ($DEBUG > 2) { print STDOUT "FAIL: $inst $state $md5tru $md5obs\n"; }
     84        } else {
     85            &save_goodchip ($inst);
     86            if ($DEBUG > 2) { print STDOUT "$inst $state $md5tru = $md5obs\n"; }
    7987        }
    8088    }
     89
     90    my @instfoundKEYS = keys %instfound;
     91    foreach my $inst (@instfoundKEYS) {
     92        unless ($instfound{$inst}) { die "missing MD5 for instance $inst\n"; }
     93    }
     94}
     95
     96# save both good and fail chip lists
     97open (OUTFILE, ">$basedir/$dateword/goodchips.$stage.txt");
     98my @goodchipURIs = keys %goodchips;
     99foreach my $uri (@goodchipURIs) {
     100    print OUTFILE "$uri\n";
     101}
     102close (OUTFILE);
     103
     104open (OUTFILE, ">$basedir/$dateword/failchips.$stage.txt");
     105my @failchipURIs = keys %failchips;
     106foreach my $uri (@failchipURIs) {
     107    print OUTFILE "$uri\n";
     108}
     109close (OUTFILE);
     110
     111
     112## we only report an error if we could not find the md5sum files or if the inst and md5s
     113## files are inconsistent.  In that case, we cannot check the md5s for all instances
     114## if the md5sum is wrong for a chip, mark that chip and save it.  It will be skipped in fix_chip_locations
     115if ($Nfail) {
     116    print STDOUT "ERROR: $Nfail errors\n";
     117    exit 1;
     118} else {
     119    print STDOUT "SUCCESS: $Nfail errors\n";
     120    exit 0;
     121}
     122
     123# use global $basedir, $dateword, $myhost, $stage
     124sub load_instance_list {
     125    my $basedir   = $_[0];
     126    my $dateword = $_[1];
     127    my $stage    = $_[2];
     128    my $instfile = $_[3];
     129
     130    my @instdata_list;
     131
     132    # load the master list
     133    &load_instance_list_onefile ($instfile, \@instdata_list);
     134
     135    # there may be additional instances in files with names like HOSTNAME.XYnn.inst.STAGE
     136    my ($myhost) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|;
     137    my @instlist = glob ("$basedir/$dateword/$myhost.XY??.inst.$stage.txt");
     138    foreach my $infile  (@instlist) {
     139        &load_instance_list_onefile ($infile, \@instdata_list);
     140    }
     141    return (\@instdata_list);
     142}
     143
     144# read the instance info
     145sub load_instance_list_onefile {
     146    my $infile = $_[0];
     147    my $instdata_list_r = $_[1]; # reference to the file data
     148
     149    open(FILE,$infile) or die "cannot open file $infile\n";
     150    my @lines = <FILE>;
     151    close (FILE);
     152
     153    foreach my $line (@lines) {
     154        chomp $line;
     155        if ($line =~ /^\s*#/) { next; } # skip lines with leading #
     156        push (@$instdata_list_r, $line);
     157    }
     158}
     159
     160# return only the master instlist filenames (ignore ones with the chipnames in the name)
     161sub load_instlist_filenames {
     162   
     163    my $basedir   = $_[0];
     164    my $dateword = $_[1];
     165    my $stage    = $_[2];
     166
     167    my @instlist_all = glob ("$basedir/$dateword/*.inst.$stage.txt");
     168
     169    my @instlist = ();
     170
     171    foreach my $instfile (@instlist_all) {
     172
     173        # skip the per-chip inst files (these should probably be merged into the master)
     174        if ($instfile =~ m|XY[0-9][0-9].inst|) { next; }
     175
     176        push (@instlist, $instfile);
     177    }
     178    return (\@instlist);
     179}
     180
     181# return a hash of the matched md5 file for each inst file
     182sub load_md5slist_filenames {
     183
     184    my $basedir     = $_[0];
     185    my $dateword   = $_[1];
     186    my $stage      = $_[2];
     187    my $instlist_r = $_[3];
     188
     189    my @md5slist = glob ("$basedir/$dateword/*.md5s.txt");
     190
     191    my %md5shash;
     192
     193    my $Nfail = 0;
     194
     195    foreach my $instfile (@$instlist_r) {
     196
     197        # find the matching md5sum file for this host
     198
     199        my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|;
     200        # print STDERR "host: $host\n";
     201
     202        my $md5match = "NONE";
     203        foreach my $md5sfile (@md5slist) {
     204       
     205            my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|;
     206       
     207            if ($host ne $md5shost) { next; }
     208
     209            # print STDERR "md5host: $md5shost : $md5sfile = $instfile\n";
     210
     211            $md5match = $md5sfile;
     212            last;
     213        }
     214
     215        if ($md5match eq "NONE") {
     216            print STDOUT "missing md5sum file for $instfile\n";
     217            $Nfail ++;
     218            next;
     219        }
     220        $md5shash{$instfile} = $md5match;
     221
     222        # print "match: $instfile = $md5match\n";
     223
     224    }
     225    if ($Nfail > 0) {
     226        print STDOUT "ERROR: $Nfail errors\n";
     227        exit 1;
     228    }
     229    return (\%md5shash);
    81230}
    82231
    83232sub parse_cmdopts {
    84233
    85     my ($dateobs);
    86     GetOptions( 'dateobs=s' => \$dateobs) || pod2usage(2);
     234    my ($dateobs, $stage, $topdir);
     235    GetOptions( 'dateobs=s' => \$dateobs, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2);
    87236   
    88237    pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs;
     238    pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir;
     239    pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage;
     240   
     241    # check that stage is an integer
     242    unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; }
     243    unless ($stage >= 0) { die "stage must be a non-negative integer\n"; }
    89244   
    90245    # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?)
    … …  
    98253    my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day);
    99254
    100     return ($dateword);
     255    my $yearobs = substr ($dateword, 0, 4);
     256    my $basedir = "$topdir/$yearobs";
     257
     258    return ($dateword, $stage, $topdir, $basedir);
    101259}
    102260
    … …  
    107265    return $status;
    108266}
     267
     268sub neb_reverse {
     269
     270    my (@neb_path) = split /\//, $_[0];
     271    my $neb_key = $neb_path[-1];
     272
     273    # Remove ins_id
     274    $neb_key =~ s/^\d+\.//;
     275
     276    # Convert colons to slashes.
     277    $neb_key =~ s%:%/%g;
     278
     279    # strip ending .fz added for compressed .fits images
     280    $neb_key =~ s%.fz$%%;
     281    return $neb_key;
     282}
     283
     284sub save_failure {
     285    my $nebfile = &neb_reverse($_[0]);
     286    if (defined $failchips{$nebfile}) {
     287        $failchips{$nebfile} ++;
     288    } else {
     289        $failchips{$nebfile} = 1;
     290    }
     291    return;
     292}
     293
     294sub save_goodchip {
     295    my $nebfile = &neb_reverse($_[0]);
     296    if (defined $goodchips{$nebfile}) {
     297        $goodchips{$nebfile} ++;
     298    } else {
     299        $goodchips{$nebfile} = 1;
     300    }
     301    return;
     302}
     303
     304sub is_integer {
     305   defined $_[0] && $_[0] =~ /^[+-]?\d+$/;
     306}

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/create.rawfix.sh

    r42399 r42531  
    11#!/bin/csh
     2
     3echo "do not use without caution"
     4exit 2
    25
    36# make the rawfix database to track rawfix operations
    … …  
    58set DBUSER = "dvo"
    69
     10# add in the iteration and the state summary
     11
     12mysql -h localhost -u root <<EOF
     13  use $DBNAME;
     14  ALTER TABLE nights MODIFY Ngood INT, MODIFY Ndead INT, MODIFY Ngone INT;
     15  ALTER TABLE nights MODIFY Nsing INT, MODIFY Ndups INT, MODIFY Nsbck INT, MODIFY Nxbck INT, MODIFY Nfail INT;
     16  ALTER TABLE nights MODIFY Ncul1 INT, MODIFY Ncul2 INT, MODIFY Ncul3 INT;
     17EOF
     18
     19exit 0
     20
     21mysql -h localhost -u root <<EOF
     22  use $DBNAME;
     23  ALTER TABLE nights ADD COLUMN iter SMALLINT, ADD COLUMN Ngood SMALLINT, ADD COLUMN Ndead SMALLINT, ADD COLUMN Ngone SMALLINT;
     24  ALTER TABLE nights ADD COLUMN Nsing SMALLINT, ADD COLUMN Ndups SMALLINT, ADD COLUMN Nsbck SMALLINT, ADD COLUMN Nxbck SMALLINT;
     25  ALTER TABLE nights ADD COLUMN Ncul1 SMALLINT, ADD COLUMN Ncul2 SMALLINT, ADD COLUMN Ncul3 SMALLINT;
     26EOF
     27
     28exit 0
     29
    730mysql -h localhost -u root <<EOF
    831  drop database if exists $DBNAME;
    932  create database $DBNAME;
    1033  use $DBNAME;
    11   create table nights (night_id BIGINT AUTO_INCREMENT, dateobs char(64), state char(64), primary key (night_id));
     34  create table nights (night_id BIGINT AUTO_INCREMENT, dateobs char(64), state char(64), primary key (night_id), unique (dateobs));
    1235EOF
    1336

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/fix_chip_locations.pl

    r42406 r42531  
    1212use Getopt::Long qw( GetOptions :config auto_help auto_version );
    1313
    14 # USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file)
     14# USAGE: fix_chip_locations.pl --dateobs (night) --stage (0,1,etc) [--chiplist (file) | --chipID (ID)]
    1515
    1616# we need to select targets for ITC copies of files currently only at ATRC:
    … …  
    1818# for each chip, target a single host?
    1919
    20 my ($fulldate, $dateword, $chiplist) = &parse_cmdopts;
    21 my ($neb_dbh) = &parse_db_config ('nebulous.config');
    22 
    23 # read the list of chip_ids:
    24 my @chip_ids = &load_chip_list ($chiplist);
     20my ($fulldate, $dateword, $chiplist, $chipID, $stage, $topdir, $basedir) = &parse_cmdopts;
     21my ($neb_dbh, $raw_dbh) = &parse_db_config ("$topdir/nebulous.config");
     22# topdir used for nebulous.config and chiplist.txt, basedir used for night-level data
    2523
    2624# choose ITC machines on which to place backup copies:
    27 my @volarray = &get_neb_volumes;
    28 my $volnum = 0;
     25my @vol_copy = &get_neb_volumes_copy; # array of neb hosts to copy data to at ITC
     26my %vol_good = &get_neb_volumes_good; # hash of valid neb hosts
     27
     28my $Nchip = 0; # counts the number of instances
    2929
    3030my $Ngood = 0;
     31my $Ndead = 0;
    3132my $Ngone = 0;
    32 my $Ndead = 0;
    3333my $Nsing = 0;
    3434my $Ndups = 0;
    … …  
    3636my $Nxbck = 0;
    3737
     38my $Ncul1 = 0;
     39my $Ncul2 = 0;
     40my $Ncul3 = 0;
     41my $Nfail = 0;
     42
     43my ($goodchips_r, $failchips_r) = &get_good_or_fail_chiplists ($basedir, $dateword, $stage);
     44
     45# read the list of chip_ids:
     46my @chip_ids = &load_chip_list ($chiplist);
     47
     48my $volnum = 0; # select the volume to be used from the list of @vol_copy
     49
     50# loop over all chip_ids so we can distribute the target volumes
    3851foreach my $chip_id (@chip_ids) {
    39 
    40     my $urifile = "$dateword/$chip_id.uris.txt";
    41     open (FILE, "$urifile");
    42     my @lines = <FILE>;
    43     close (FILE);
     52    if (not defined $chipID or ($chipID eq $chip_id)) {
     53        &fix_chip_locations_chipID ($chip_id, $volnum);
     54    }
     55    $volnum ++;
     56    if ($volnum > @vol_copy) { $volnum = 0; }
     57}
     58
     59my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3 + $Nfail;
     60if ($Ntotal != $Nchip) { die "ERROR: inconsistent count of chips\n"; }
     61
     62if (not defined $chipID) {
     63    # update the stats in the database
     64    my $sql_update_stats = "UPDATE nights set";
     65    $sql_update_stats .= "  Nchip = $Nchip";
     66    $sql_update_stats .= ", Ngood = $Ngood";
     67    $sql_update_stats .= ", Ndead = $Ndead";
     68    $sql_update_stats .= ", Ngone = $Ngone";
     69    $sql_update_stats .= ", Nsing = $Nsing";
     70    $sql_update_stats .= ", Ndups = $Ndups";
     71    $sql_update_stats .= ", Nsbck = $Nsbck";
     72    $sql_update_stats .= ", Nxbck = $Nxbck";
     73    $sql_update_stats .= ", Ncul1 = $Ncul1";
     74    $sql_update_stats .= ", Ncul2 = $Ncul2";
     75    $sql_update_stats .= ", Ncul3 = $Ncul3";
     76    $sql_update_stats .= ", Nfail = $Nfail";
     77    $sql_update_stats .= " WHERE dateobs = $dateword";
     78   
     79    my $query = $raw_dbh->prepare($sql_update_stats);
     80    $query->execute () or die "invalid response from RAW database\n";
     81} else {
     82
     83    open (OUTFILE, ">$basedir/$dateword/$chipID.stats.$stage.txt");
     84
     85    print OUTFILE "Nchip: $Nchip\n";
     86    print OUTFILE "Ngood: $Ngood\n";
     87    print OUTFILE "Ngone: $Ngone\n";
     88    print OUTFILE "Ndead: $Ndead\n";
     89    print OUTFILE "Nsing: $Nsing\n";
     90    print OUTFILE "Ndups: $Ndups\n";
     91    print OUTFILE "Nsbck: $Nsbck\n";
     92    print OUTFILE "Nxbck: $Nxbck\n";
     93    print OUTFILE "Ncul1: $Ncul1\n";
     94    print OUTFILE "Ncul2: $Ncul2\n";
     95    print OUTFILE "Ncul3: $Ncul3\n";
     96    print OUTFILE "Nfail: $Nfail\n";
     97
     98    close (OUTFILE);
     99}
     100
     101exit 0;
     102
     103sub fix_chip_locations_chipID {
     104
     105    my $chip_id = $_[0];
     106    my $volnum  = $_[1];
     107
     108    # read from chip_id file, update to next stage number
     109    my $urifile = "$basedir/$dateword/$chip_id.uris.$stage.txt";
     110    if (! -e $urifile) { die "missing file $urifile\n"; }
     111    open (URIFILE,  $urifile);
     112    my @lines = <URIFILE>;
     113    close (URIFILE);
    44114
    45115    foreach my $line (@lines) {
    … …  
    49119
    50120        # we expect lines of the form: STATE PATH
    51         # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits
    52         if (@words != 2) { die "error in URI list $urifile, line: $line\n"; }
     121        # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum
     122        if (@words != 4) { die "error in URI list $urifile, line: $line\n"; }
    53123
    54124        my $state  = $words[0];
    55125        my $ext_id = $words[1];
    56 
     126        my $datast = $words[2];
     127        my $md5sum = $words[3];
     128
     129        my $haveMD5 = 0;
     130        my $goodMD5 = 0;
     131        if ($goodchips_r->{$ext_id}) { $haveMD5 = 1; $goodMD5 = 1; }
     132        if ($failchips_r->{$ext_id}) { $haveMD5 = 1; $goodMD5 = 0; }
     133
     134        $Nchip ++;
     135
     136        # nothing to be done, or need be done for these:
    57137        if ($state eq "GOOD:") { $Ngood++; next; }
    58138        if ($state eq "DEAD:") { $Ndead++; next; }
    59139        if ($state eq "GONE:") { $Ngone++; next; }
    60         if ($state eq "DUPS:") { $Ndups++; next; }
    61         if ($state eq "XBCK:") { $Nxbck++; next; }
    62140
    63141        # one copy at ITC: neb-replicate will create a copy at ATRC
     142        # does not depend on existence of MD5sum
    64143        if ($state eq "SING:") {
    65144            my $status = &vsystem ("neb-replicate $ext_id");
    … …  
    67146            my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id);
    68147            unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; }
    69             open (OUTFILE, ">>$dateword/$newhost.inst.txt");
    70             print OUTFILE "$newfile\n";
     148            open (OUTFILE, ">>$basedir/$dateword/$newhost.$chip_id.inst.$stage.txt");
     149            print OUTFILE "$newfile $datast $md5sum\n";
    71150            close (OUTFILE);
    72151            $Nsing ++;
    … …  
    75154
    76155        # one copy at ATRC: neb-replicate will create a copy at ITC iff we supply the target volume
     156        # does not depend on existence of MD5sum
    77157        if ($state eq "SBCK:") {
    78             my $tgtvol = $volarray[$volnum];
     158            my $tgtvol = $vol_copy[$volnum];
    79159            my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id");
    80160            if ($status) { print STDERR "FAIL SBCK: neb-replicate : $ext_id on $tgtvol\n"; next; }
    81161            my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id);
    82162            unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; }
    83             open (OUTFILE, ">>$dateword/$newhost.inst.txt");
    84             print OUTFILE "$newfile\n";
     163            open (OUTFILE, ">>$basedir/$dateword/$newhost.$chip_id.inst.$stage.txt");
     164            print OUTFILE "$newfile $datast $md5sum\n";
    85165            close (OUTFILE);
    86166            $Nsbck ++;
    87167            next;
    88168        }
    89     }
    90     $volnum ++;
    91 }
    92 
    93 print "Ngood: $Ngood\n";
    94 print "Ngone: $Ngone\n";
    95 print "Ndead: $Ndead\n";
    96 print "Nsing: $Nsing\n";
    97 print "Ndups: $Ndups\n";
    98 print "Nsbck: $Nsbck\n";
    99 print "Nxbck: $Nxbck\n";
    100 
    101 exit 0;
     169
     170        # multiple copies @ ITC, but none at ATRC. use neb-replicate to make copies
     171        # skip if we do not know the state of the md5sum
     172        if ($state eq "DUPS:") {
     173            if ($haveMD5 && $goodMD5) {
     174                my $status = &vsystem ("neb-replicate $ext_id");
     175                if ($status) { print STDERR "FAIL DUPS: neb-replicate : $ext_id\n"; next; }
     176                my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id);
     177                unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; }
     178                open (OUTFILE, ">>$basedir/$dateword/$newhost.$chip_id.inst.$stage.txt");
     179                print OUTFILE "$newfile $datast $md5sum\n";
     180                close (OUTFILE);
     181            }
     182            ## if we know the MD5 is bad, add to Nfail, otherwise add to Ndups
     183            if ($haveMD5 && !$goodMD5) {
     184                $Nfail ++;
     185            } else {
     186                $Ndups++;
     187            }
     188            next;
     189        }
     190
     191        # multiple copies @ ATRC, but none @ ITC: neb-replicate will create a copy at ITC iff we supply the target volume
     192        # skip if we do not know the state of the md5sum
     193        if ($state eq "XBCK:") {
     194            if ($haveMD5 && $goodMD5) {
     195                my $tgtvol = $vol_copy[$volnum];
     196                my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id");
     197                if ($status) { print STDERR "FAIL XBCK: neb-replicate : $ext_id on $tgtvol\n"; next; }
     198                my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id);
     199                unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; }
     200                open (OUTFILE, ">>$basedir/$dateword/$newhost.$chip_id.inst.$stage.txt");
     201                print OUTFILE "$newfile $datast $md5sum\n";
     202                close (OUTFILE);
     203            }
     204            ## if we know the MD5 is bad, add to Nfail, otherwise add to Nxbck:
     205            if ($haveMD5 && !$goodMD5) {
     206                $Nfail ++;
     207            } else {
     208                $Nxbck++;
     209            }
     210            next;
     211        }
     212
     213        # one copy at ITC: neb-replicate will create a copy at ATRC
     214        # skip if we do not know the state of the md5sum
     215        if (($state eq "CUL1:") || ($state eq "CUL2:") || ($state eq "CUL3:")) {
     216
     217            if ($haveMD5 && $goodMD5) {
     218                # get info for all instances of this chip
     219                # returns a reference to the array of instance data
     220                # also strips neb://hostname.volnum/ from nebname and returns as ext_id
     221                my ($inforows, $ext_id) =  &get_chip_instances ($ext_id);
     222
     223                my ($main_keep, $back_keep, $cull_list_r) = &check_instance_xattr ($inforows, $ext_id);
     224
     225                if ($main_keep eq "") { print STDERR "FAIL $state: missing ITC copy for $ext_id\n";  next; }
     226                if ($back_keep eq "") { print STDERR "FAIL $state: missing ATRC copy for $ext_id\n"; next; }
     227
     228                my $NcullTotal = @$cull_list_r;
     229                foreach my $instance (@$cull_list_r) {
     230                    print "state $state : checking instance $instance ... ";
     231                    my ($volname) = $instance =~ m|file:///data/([0-9a-zA-Z]*\.[0-9])/\S*|;
     232                    print "volname $volname\n";
     233                    unless (defined $vol_good{$volname}) { die "ERROR: invalid volume name $volname\n"; }
     234                    # print STDERR "neb-cull $state: $ext_id $volname ($main_keep $back_keep)\n";
     235                    my $status = &vsystem ("neb-cull --volume $volname $ext_id");
     236                    if ($status) { print STDERR "FAIL CULL: neb-replicate : $ext_id\n"; next; }
     237                }
     238                # print STDERR "Ncull Total $NcullTotal\n";
     239            }
     240
     241            ## if we know the MD5 is bad, add to Nfail, otherwise add to NculX
     242            if ($haveMD5 && !$goodMD5) {
     243                $Nfail ++;
     244            } else {
     245                if ($state eq "CUL1:") { $Ncul1++; next; }
     246                if ($state eq "CUL2:") { $Ncul2++; next; }
     247                if ($state eq "CUL3:") { $Ncul3++; next; }
     248            }
     249        }
     250    }
     251}
    102252
    103253sub get_new_instance {
    … …  
    120270}
    121271
    122 # get the list of chip neb paths for one chip
    123 sub get_neb_volumes {
     272sub get_chip_instances {
     273
     274    my $ext_id = $_[0];
     275   
     276    my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'";
     277    my $rowref   = $neb_dbh->selectall_arrayref( $sql_chip_instances );
     278   
     279    return ($rowref, $ext_id);
     280}
     281
     282# select 1 instance with xattr = 0, 1 with xattr = 1.
     283# the rest should be culled.
     284
     285sub check_instance_xattr {
     286
     287    my $rowref  = $_[0];
     288    my $ext_id  = $_[1];
     289
     290    my @inforows = @$rowref;
     291    my $Nrows    = @inforows;
     292
     293    if ($DEBUG > 5) { print "found $Nrows instances\n"; }
     294
     295    # get user_copies for this storage_object
     296    my $main_keep = "";
     297    my $back_keep = "";
     298    my @cull_list = ();
     299
     300    # xattr = 0 means instance @ ITC  (main)
     301    # xattr = 1 means instance @ ATRC (back)
     302    for (my $j = 0; $j < $Nrows; $j ++) {
     303        my $valref = $inforows[$j];
     304        my @values = @$valref;
     305       
     306        # the number of returned values per row must match the query above (in get_chip_instances)
     307        my $Nvalue = @values;
     308        if ($Nvalue != 5) {
     309            print STDERR "ERROR : INVALID DB XATTR RESPONSE: $ext_id\n";
     310            next;
     311        }
     312
     313        my $instance = $values[0];
     314        my $hostname = $values[1];
     315        my $xattr    = $values[3];
     316       
     317        if (($xattr == 0) && ($main_keep eq "")) { $main_keep = $instance; next; }
     318        if (($xattr == 1) && ($back_keep eq "")) { $back_keep = $instance; next; }
     319
     320        if ($xattr > 1) { print "ignore lost instance $instance\n"; next; }
     321
     322        # if we make it here, we already have an instance at this location
     323        push (@cull_list, $instance);
     324    }
     325    return ($main_keep, $back_keep, \@cull_list);
     326}
     327
     328sub get_good_or_fail_chiplists {
     329
     330    my $basedir   = $_[0];
     331    my $dateword = $_[1];
     332    my $stage    = $_[2];
     333
     334    # load the lists of good and fail chips
     335    my $prev = $stage - 1;
     336    my $goodfile = "$basedir/$dateword/goodchips.$prev.txt";
     337    my %goodchips;
     338    if (-e $goodfile) {
     339        open (GOODFILE,  $goodfile);
     340        my @lines = <GOODFILE>;
     341        close (GOODFILE);
     342       
     343        foreach my $line (@lines) {
     344            chomp $line;
     345            $goodchips{$line} = 1;
     346        }
     347    }
     348    my $failfile = "$basedir/$dateword/failchips.$prev.txt";
     349    my %failchips;
     350    if (-e $failfile) {
     351        open (FAILFILE,  $failfile);
     352        my @lines = <FAILFILE>;
     353        close (FAILFILE);
     354       
     355        foreach my $line (@lines) {
     356            chomp $line;
     357            $failchips{$line} = 1;
     358        }
     359    }
     360    return (\%goodchips, \%failchips);
     361}
     362
     363# get the list of neb hosts to copy data to at ITC
     364sub get_neb_volumes_copy {
    124365
    125366    my @volarray;
    … …  
    148389}
    149390
     391# get the list of valid neb hosts (ITC or ATRC)
     392sub get_neb_volumes_good {
     393
     394    my %volarray;
     395
     396    my $sql_neb_volumes = "select name from mountedvol where ((xattr = 0) or (xattr = 1))";
     397    if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; }
     398
     399    my $rowref   = $neb_dbh->selectall_arrayref( $sql_neb_volumes );
     400    my @tmparray = @$rowref;
     401    my $Nrows    = @tmparray;
     402
     403    if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; }
     404
     405    foreach my $valref (@tmparray) {
     406        my @values = @$valref;
     407        # there should only be a single value (only name selected above)
     408        my $Nvalues = @values;
     409        if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; }
     410        if ($Nvalues  > 1) { print "ERROR: too many uri entries?\n"; next; }
     411       
     412        $volarray{$values[0]} = 1;
     413        if ($DEBUG > 4) { print "$values[0]\n"; }
     414    }
     415   
     416    return %volarray;
     417}
     418
    150419# read the list of chip_ids:
    151420sub load_chip_list {
    … …  
    154423    my @chip_ids;
    155424
    156     open(CHIPS,$chiplist) or die "cannot open file $chiplist\n";
     425    open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n";
    157426
    158427    my @lines = <CHIPS>;
    … …  
    173442sub parse_cmdopts {
    174443
    175     my ($dateobs, $chiplist);
    176     GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2);
     444    my ($dateobs, $chiplist, $chipID, $stage, $topdir);
     445    GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'chipID=s' => \$chipID, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2);
    177446   
    178447    pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs;
     448    pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage;
     449    pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir;
    179450    pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist;
    180    
     451
     452    # check that stage is an integer
     453    unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; }
     454    unless ($stage >= 0) { die "stage must be a non-negative integer\n"; }
     455
    181456    # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?)
    182457
    … …  
    190465    my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day);
    191466
    192     return ($fulldate, $dateword, $chiplist);
     467    my ($yearobs) = substr ($dateword, 0, 4);
     468    my $basedir = "$topdir/$yearobs";
     469
     470    return ($fulldate, $dateword, $chiplist, $chipID, $stage, $topdir, $basedir);
    193471}
    194472
    … …  
    199477
    200478    my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user );
     479    my ( $rawdb_name, $rawdb_host, $rawdb_port, $rawdb_pass, $rawdb_user );
    201480
    202481    if (-e $conffile) {
    … …  
    210489            if ($_ =~ /NEB_PASS /)  { $nebdb_pass = (split /\s+/,$_)[2]; }
    211490            if ($_ =~ /NEB_PORT /)  { $nebdb_port = (split /\s+/,$_)[2]; }
     491
     492            if ($_ =~ /RAW_NAME /)  { $rawdb_name = (split /\s+/,$_)[2]; }
     493            if ($_ =~ /RAW_HOST /)  { $rawdb_host = (split /\s+/,$_)[2]; }
     494            if ($_ =~ /RAW_USER /)  { $rawdb_user = (split /\s+/,$_)[2]; }
     495            if ($_ =~ /RAW_PASS /)  { $rawdb_pass = (split /\s+/,$_)[2]; }
     496            if ($_ =~ /RAW_PORT /)  { $rawdb_port = (split /\s+/,$_)[2]; }
    212497        }
    213498        close(IN);
    214499        print STDERR  "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n";
     500        print STDERR  "raw  dbh config: $rawdb_name $rawdb_host $rawdb_port $rawdb_user $rawdb_pass\n";
    215501    }
    216502
    … …  
    221507    pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass;
    222508
     509    pod2usage( -msg => "Cannot configure raw database (dbname)", -exitval => 2) unless defined $rawdb_name;
     510    pod2usage( -msg => "Cannot configure raw database (dbhost)", -exitval => 2) unless defined $rawdb_host;
     511    pod2usage( -msg => "Cannot configure raw database (dbuser)", -exitval => 2) unless defined $rawdb_user;
     512    pod2usage( -msg => "Cannot configure raw database (dbport)", -exitval => 2) unless defined $rawdb_port;
     513    pod2usage( -msg => "Cannot configure raw database (dbpass)", -exitval => 2) unless defined $rawdb_pass;
     514
    223515    my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host";
    224516    if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; }
    225517    my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n";
    226518
    227     return ($neb_dbh);
     519    my $rawdb_dsn = "DBI:mysql:database=$rawdb_name:host=$rawdb_host";
     520    if ($rawdb_port ne "NONE") { $rawdb_dsn .= ":port=$rawdb_port"; }
     521    my $raw_dbh = DBI->connect($rawdb_dsn, $rawdb_user, $rawdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n";
     522
     523    return ($neb_dbh, $raw_dbh);
    228524}
    229525
    230526sub vsystem {
    231527    print STDERR "@_\n";
    232     #return 0;
     528    # return 0;
    233529    my $status = system ("@_");
    234530    return $status;
    235531}
     532
     533sub is_integer {
     534   defined $_[0] && $_[0] =~ /^[+-]?\d+$/;
     535}
     536   

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh

    r42404 r42531  
    11#!/bin/csh
    22
    3 if ($#argv != 1) then
    4   echo "USAGE: get_hosts_md5s.sh (dateobs)"
     3if ($#argv != 3) then
     4  echo "USAGE: get_hosts_md5s.sh (topdir) (dateobs) (stage)"
    55  exit 2
    66endif
    77
    8 set dirname = $1
     8set topdir  = $1
     9set dateobs = $2
     10set stage   = $3
    911
    10 if (! -d $dirname) then
    11   echo "ERROR: missing dateobs directory $dirname"
     12if (! -d $topdir) then
     13  echo "ERROR: missing top-level directory $topdir"
     14  exit 3
     15endif
     16
     17set yearobs = `echo $dateobs | awk '{print substr($1,1,4)}'`
     18set basedir = $topdir/$yearobs
     19
     20if (! -d $basedir/$dateobs) then
     21  echo "ERROR: missing dateobs directory $basedir/$dateobs"
    1222  exit 3
    1323endif
    1424
    1525# report both the dateobs and the hostnames
    16 ls $dirname/*.inst.txt | awk -v dateobs=$dirname -F/ '(NR == 1){print "dateobs md5host"}{print dateobs, $2}' | sed s/.inst.txt//
     26ls $basedir/$dateobs/*.inst.$stage.txt | grep -v "XY...inst" | awk -v dateobs=$dateobs -v stage=$stage -F/ '(NR == 1){print "dateobs stage md5host"}{print dateobs, stage, $NF}' | sed s/.inst.$stage.txt//
    1727exit 0

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_md5s_instances.pl

    r42404 r42531  
    44my $time = time();
    55
    6 use DBI;
    76use Carp;
    87use Pod::Usage qw( pod2usage );
    … …  
    1211use Getopt::Long qw( GetOptions :config auto_help auto_version );
    1312
    14 # USAGE: get_md5s_instances.pl --hostname (hostname) --input (file)
     13# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (0,1,etc)
    1514
    16 my ($myhost, $input) = &parse_cmdopts;
     15my $input;
     16my ($myhost, $dateword, $topdir, $basedir, $stage) = &parse_cmdopts;
    1717
    1818# check that we are running this script on 'myhost'
    … …  
    2121
    2222# read the list of instances:
    23 my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ($input);
     23my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ();
    2424
    25 my $output = $input;
    26 $output =~ s/inst/md5s/;
     25my $output = "$basedir/$dateword/$myhost.md5s.txt";
    2726
    28 if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; }
     27# read in existing md5s
     28my $out_hash_r;
    2929if (-e $output) {
    30     # check that all instances are in the output file?
    31     print "$output exists, skipping\n";
    32     exit 0;
     30    $out_hash_r = &load_output_list ($output);
    3331}
    3432
    35 open (OUTFILE, ">$output");
     33open (OUTFILE, ">>$output");
    3634
    3735# foreach my $instance (@$ins_list_r) {
    … …  
    4240    my $md5sum_tru = $md5_list_r->[$i];
    4341
     42    # if the instance is already in the output file, skip it
     43    if (defined $out_hash_r) {
     44        my $name = $out_hash_r->{$instance};
     45        if (defined $name) {
     46            print STDERR "skipping $instance (already in output)\n";
     47            next;
     48        }
     49    }
     50
    4451    my $filename = $instance;
    4552    $filename =~ s|file://||;
    4653
    47     my $line;
    48     if (0) {
    49         $line = "fakevalue $filename"; # XXX TEST: disable calculation:
    50     } else {
    51         $line = `md5sum $filename`;
     54    # check for the existence of the file first:
     55    unless (-e $filename) {
     56        print OUTFILE "$instance $data_state $md5sum_tru XXX\n";
     57        next;
    5258    }
     59
     60    my $line = `md5sum $filename`;
    5361
    5462    my ($md5sum, $rawfile) = split (" ", $line);
    … …  
    5866exit 0;
    5967
    60 # read the list of chip_ids:
     68# use global $basedir, $dateword, $myhost, $stage
    6169sub load_instance_list {
     70    my (@ins_list, @dst_list, @md5_list);
     71
     72    # load the master list
     73    my $input = "$basedir/$dateword/$myhost.inst.$stage.txt";
     74    &load_instance_list_onefile ($input, \@ins_list, \@dst_list, \@md5_list);
     75
     76    # there may be additional instances in files with names like HOSTNAME.XYnn.inst.STAGE
     77    my @instlist = glob ("$basedir/$dateword/$myhost.XY??.inst.$stage.txt");
     78    foreach my $infile  (@instlist) {
     79        &load_instance_list_onefile ($infile, \@ins_list, \@dst_list, \@md5_list);
     80    }
     81    return (\@ins_list, \@dst_list, \@md5_list);
     82}
     83
     84# read the instance info
     85sub load_instance_list_onefile {
    6286    my $infile = $_[0];
    63 
    64     my (@ins_list, @dst_list, @md5_list);
     87    my $ins_list_r = $_[1]; # reference to the instance list array
     88    my $dst_list_r = $_[2]; # reference to the instance list array
     89    my $md5_list_r = $_[3]; # reference to the instance list array
    6590
    6691    open(FILE,$infile) or die "cannot open file $infile\n";
    … …  
    7499
    75100        if (@words != 3) { die "error in input $infile : $line\n"; }
    76         push (@ins_list, $words[0]);
    77         push (@dst_list, $words[1]);
    78         push (@md5_list, $words[2]);
     101        push (@$ins_list_r, $words[0]);
     102        push (@$dst_list_r, $words[1]);
     103        push (@$md5_list_r, $words[2]);
     104    }
     105    close (FILE);
     106}
     107
     108# read the instance list in the output file
     109sub load_output_list {
     110    my $infile = $_[0];
     111
     112    my (%out_hash);
     113
     114    open(FILE,$infile) or die "cannot open file $infile\n";
     115
     116    my @lines = <FILE>;
     117
     118    foreach my $line (@lines) {
     119        chomp $line;
     120        if ($line =~ /^\s*#/) { next; } # skip lines with leading #
     121        my @words = split (/\s+/,$line);
     122
     123        if (@words != 4) { die "error in input $infile : $line\n"; }
     124        $out_hash{$words[0]} = 1;
    79125    }
    80126    close (FILE);
    81127
    82     return (\@ins_list, \@dst_list, \@md5_list);
     128    return (\%out_hash);
    83129}
    84130
    85131sub parse_cmdopts {
    86132
    87     my ($myhost, $input);
    88     GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input) || pod2usage(2);
     133    my ($myhost, $topdir, $dateobs, $stage);
     134    GetOptions( 'hostname=s' => \$myhost, 'topdir=s' => \$topdir, 'dateobs=s' => \$dateobs, 'stage=s' => \$stage) || pod2usage(2);
    89135   
    90     pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input;
     136    pod2usage( -msg => "Cannot determine target date, use --dateobs YYYYMMDD", -exitval => 2) unless defined $dateobs;
     137#   pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input;
    91138    pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost;
     139    pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage;
     140    pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir;
    92141   
    93     return ($myhost, $input);
     142    # check that stage is an integer
     143    unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; }
     144    unless ($stage >= 0) { die "stage must be a non-negative integer\n"; }
     145   
     146    my $yearobs = substr ($dateobs, 0, 4);
     147    my $basedir = "$topdir/$yearobs";
     148
     149    return ($myhost, $dateobs, $topdir, $basedir, $stage);
    94150}
    95151
    … …  
    100156    return $status;
    101157}
     158
     159sub is_integer {
     160   defined $_[0] && $_[0] =~ /^[+-]?\d+$/;
     161}
     162   

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/queue.rawfix.sh

    r42399 r42531  
    11#!/bin/csh
    22
    3 if ($#argv != 1) then
    4   echo "USAGE: queue.rawfix.sh (mjddate)"
     3if ($#argv != 2) then
     4  echo "USAGE: queue.rawfix.sh (list) (filename)"
     5  echo "USAGE: queue.rawfix.sh (night) (YYYYMMDD)"
    56  exit 2
    67endif
    78
     9set mode = $1
     10
    811# make the rawfix database to track rawfix operations
    9 set DBHOST = "ipp060"
     12set DBHOST = "ippc190"
    1013set DBNAME = "rawfix"
    11 set DBUSER = "dvo"
    12 set DBPASS = "dvo"
     14# set DBUSER = "dvo"
     15# set DBPASS = "dvo"
    1316
    14 ## XXX validate the format (YYYYMMDD)?
    15 set mynight = $1
     17set DBOPTS = "--defaults-extra-file=~/.my.cnf.rawfix -h $DBHOST $DBNAME"
    1618
    17 mysql -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME -e "insert into nights (dateobs, state) values ('$mynight', 'new')"
     19if ($mode == "list") then
     20  set filename = $2
     21
     22  # validate the list: could be DATE STATE or just DATE (prcol 1)?
     23
     24  # just a big list of dates?
     25  foreach night (`cat $filename`)
     26    queue.rawfix.sh night $night
     27  end
     28  exit 0
     29endif
     30
     31if ($mode == "night") then
     32  ## XXX validate the format (YYYYMMDD)?
     33  set mynight = $2
     34 
     35  set year = `echo $mynight | awk '{print substr($0, 1, 4)}'`
     36  if (($year < 2005) || ($year > 2030)) then
     37    echo "invalid date $mynight"
     38    exit 2
     39  endif
     40
     41  set month = `echo $mynight | awk '{print 1*substr($0, 5, 2)}'`
     42  if (($month < 1) || ($month > 12)) then
     43    echo "invalid date $mynight"
     44    exit 2
     45  endif
     46
     47  set day = `echo $mynight | awk '{print 1*substr($0, 7, 2)}'`
     48  if (($day < 1) || ($day > 31)) then
     49    echo "invalid date $mynight"
     50    exit 2
     51  endif
     52
     53  mysql $DBOPTS -e "insert into nights (dateobs, state) values ('$mynight', 'new')"
     54  exit 0
     55endif
     56
     57echo "ERROR: unknown mode $mode"
     58exit 2

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt

    r42406 r42531  
    3030  end
    3131
     32  # need to delete DONE entries here so the book is not repopulated
     33  # by already-running load tasks
    3234  task.exit 0
    3335    # convert the stdout 'queue' to entries in a book
    3436    # fields in the table will become words in the page
    3537    queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout ckchip.book
     38    book delpage ckchip.book -key pantaskState DONE
    3639  end
    3740
    … …  
    5255
    5356task nights.ckchip.run
    54 
    5557  periods      -poll $RPOLL
    5658  periods      -exec $REXEC
    5759  periods      -timeout 60
    58   npending 1
    59 
    60   stdout $LOGDIR/ckchip.run.stdout.log 
    61   stderr $LOGDIR/ckchip.run.stderr.log
     60  host         -required $HOST_CKCHIP
     61  npending 2
    6262
    6363  task.exec
    … …  
    7575    book setword ckchip.book $pageName pantaskState RUN
    7676    book getword ckchip.book $pageName dateobs -var DATEOBS
     77    book getword ckchip.book $pageName iter    -var ITER
    7778    option $pageName
    7879
    … …  
    8182    substr $DATEOBS 6 2 DAY
    8283
     84    # this is the first place in the proces that this location is known
     85    mkdir  $mypath/$YEAR/$DATEOBS
     86    stdout $mypath/$YEAR/$DATEOBS/log.ckchip.runhosts.v$ITER
     87    stderr $mypath/$YEAR/$DATEOBS/log.ckchip.runhosts.v$ITER 
     88
    8389    sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
    8490
    85     command check_chip_locations.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt
     91    # this command accesses the Nebulous and GPC databases, so it must run at ITC.
     92    command check_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage $ITER
    8693  end
    8794
    88   ## XX delete the page from the book if done?
     95  # mark the entry done, will be deleted above in .load
    8996  task.exit 0
    90     # convert the stdout 'queue' to entries in a book
    91     # fields in the table will become words in the page
    92     # use the queuesize (Nhosts) to track how many jobs are done / to be done
     97    book setword ckchip.book $options:0 pantaskState DONE
    9398    exec update.rawfix.sh $options:0 ckchip.done
    94     book delpage ckchip.book $options:0
    9599  end
    96100
    … …  
    116120end
    117121
     122# option for writing stdout / stderr to a file
     123# queuepush stdout "------- end of stdout ---------"
     124# queuesave stdout $options:0/log.ckchip.v$ITER
     125# queuepush stderr "------- end of stderr ---------"
     126# queuesave stderr $options:0/log.ckchip.v$ITER

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

    r42406 r42531  
    99if (not($?RAWFIX_FIXCHIP_INIT))
    1010  book init fixchip.book
     11  book init fixhost.book
    1112  $RAWFIX_FIXCHIP_INIT = 1
    1213end
     14
     15$MAX_FIXHOST = 2
    1316
    1417# select nights in state fixchip.new
    … …  
    2730
    2831  task.exec
    29     # command check.rawfix.sh state ckchip.done
    3032    command check.rawfix.sh state fixchip.new
    3133  end
    … …  
    3537    # fields in the table will become words in the page
    3638    queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip.book
    37   end
    38 
    39   # exit values other than 0:
    40   task.exit    default
    41     showcommand failure
    42   end
    43 
    44   task.exit    crash
    45     showcommand crash
    46   end
    47 
    48   # operation times out?
    49   task.exit    timeout
    50     showcommand timeout
    51   end
    52 end
    53 
    54 task nights.fixchip.run
    55 
     39    book delpage fixchip.book -key pantaskState DONE
     40  end
     41
     42  # exit values other than 0:
     43  task.exit    default
     44    showcommand failure
     45  end
     46
     47  task.exit    crash
     48    showcommand crash
     49  end
     50
     51  # operation times out?
     52  task.exit    timeout
     53    showcommand timeout
     54  end
     55end
     56
     57# output: fixhost.book
     58task nights.fixchip.loadhosts
    5659  periods      -poll $RPOLL
    5760  periods      -exec $REXEC
    5861  periods      -timeout 60
     62  host          local
    5963  npending 1
    60 
    61   stdout $LOGDIR/fixchip.run.stdout.log 
    62   stderr $LOGDIR/fixchip.run.stderr.log
     64  # note: the tasks which return their results via stdout
     65  # seem to fail if run remotely. 
     66
     67  stdout $LOGDIR/fixchip.loadhosts.stdout.log 
     68  stderr $LOGDIR/fixchip.loadhosts.stderr.log
    6369
    6470  task.exec
    … …  
    7682    book setword fixchip.book $pageName pantaskState RUN
    7783    book getword fixchip.book $pageName dateobs -var DATEOBS
     84    book getword fixchip.book $pageName iter    -var ITER
    7885    option $pageName
     86    # save the page so we can use it below
     87
     88    substr $DATEOBS 0 4 YEAR
     89
     90    # write an entry point in the log for this night
     91    sprintf tmpline "%s/%s/%s/log.fixchip.runhosts.v%d" $mypath $YEAR $DATEOBS $ITER
     92    date -var mydate
     93    exec echo "# start runhosts $mydate" >> $tmpline
     94
     95    # the command returns these fields: dateobs stage fixhost chipID
     96    # NOTE: stage == ITER
     97    command get_hosts_fixchip.sh $mypath $DATEOBS $ITER
     98  end
     99
     100  task.exit 0
     101    # convert the stdout 'queue' to entries in a book
     102    # fields in the table will become words in the page
     103    # queueprint stdout
     104    queuesize stdout -var Nhosts
     105    queue2book -raw-columns -key dateobs:chipID -uniq -setword pantaskState INIT stdout fixhost.book
     106    book setword fixchip.book $options:0 NHOST {$Nhosts - 1}
     107    book setword fixchip.book $options:0 NDONE 0
     108    # book list
     109    # book listbook fixchip.book
     110    # use the queuesize (Nhosts) to track how many jobs are done / to be done
     111  end
     112
     113  # exit values other than 0:
     114  task.exit    default
     115    showcommand failure
     116    book setword fixchip.book $options:0 pantasksQuality FAIL
     117  end
     118
     119  task.exit    crash
     120    showcommand crash
     121    book setword fixchip.book $options:0 pantasksQuality CRASH
     122  end
     123
     124  # operation times out?
     125  task.exit    timeout
     126    showcommand timeout
     127    book setword fixchip.book $options:0 pantasksQuality TIMEOUT
     128  end
     129end
     130
     131task nights.fixchip.run
     132  periods      -poll $RPOLL
     133  periods      -exec $REXEC
     134  periods      -timeout 1800
     135
     136  task.exec
     137    # if we are unable to run the 'exec', use a long retry time
     138    periods -exec $REXEC
     139
     140    book npages fixhost.book -var N
     141    if ($N == 0) break
     142    if ($NETWORK == 0) break
     143   
     144    # look for new nights in fixhost.book (pantaskState == INIT)
     145    book getpage fixhost.book 0 -var pageName -key pantaskState INIT
     146    if ("$pageName" == "NULL") break
     147
     148    book setword fixhost.book $pageName pantaskState RUN
     149    book getword fixhost.book $pageName dateobs -var DATEOBS
     150    book getword fixhost.book $pageName iter    -var ITER
     151    book getword fixhost.book $pageName fixhost -var FIXHOST
     152    book getword fixhost.book $pageName chipID  -var CHIP_ID
    79153
    80154    substr $DATEOBS 0 4 YEAR
    … …  
    83157    sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY
    84158
    85     command fix_chip_locations.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt
    86   end
    87 
    88   ## XX delete the page from the book if done?
    89   task.exit 0
    90     # convert the stdout 'queue' to entries in a book
    91     # fields in the table will become words in the page
    92     # use the queuesize (Nhosts) to track how many jobs are done / to be done
    93     exec update.rawfix.sh $options:0 fixchip.done
    94     book delpage fixchip.book $options:0
     159    option $pageName $DATEOBS $YEAR
     160    # save DATEOBS so we can find the md5sum.book entry
     161    # also save YEAR for output log
     162
     163    stdout $mypath/$YEAR/$DATEOBS/log.fixchip.run.$CHIP_ID.v$ITER
     164    stderr $mypath/$YEAR/$DATEOBS/log.fixchip.run.$CHIP_ID.v$ITER
     165
     166    host -required $FIXHOST
     167    ## check if this host needs to be launched
     168    if (not($?host:$FIXHOST))
     169      echo "host not defined, creating : $FIXHOST"
     170      $host:$FIXHOST = 0
     171    end
     172    if ($host:$FIXHOST < $MAX_FIXHOST)
     173      control host add $FIXHOST
     174      $host:$FIXHOST ++
     175    end
     176
     177    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
     178    periods -exec 0.05
     179    command fix_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --chipID $CHIP_ID --stage $ITER
     180  end
     181
     182  task.exit 0
     183    nights.fixchip.exit OK
     184  end
     185
     186  # exit values other than 0:
     187  task.exit    default
     188    nights.fixchip.exit FAIL
     189    showcommand failure
     190  end
     191
     192  task.exit    crash
     193    nights.fixchip.exit CRASH
     194    showcommand crash
     195  end
     196
     197  # operation times out?
     198  task.exit    timeout
     199    nights.fixchip.exit TIMEOUT
     200    showcommand timeout
     201  end
     202end
     203
     204# output: fixhost.book
     205task nights.fixchip.clean
     206
     207  periods      -poll $RPOLL
     208  periods      -exec $REXEC
     209  periods      -timeout 60
     210  npending 1
     211
     212  stdout $LOGDIR/fixchip.clean.stdout.log 
     213  stderr $LOGDIR/fixchip.clean.stderr.log
     214
     215  task.exec
     216    # if we are unable to run the 'exec', use a long retry time
     217    periods -exec $REXEC
     218
     219    book npages fixchip.book -var N
     220    if ($N == 0) break
     221    if ($NETWORK == 0) break
     222   
     223    # look for completed nights in fixchip.book (pantaskState == STOP)
     224    book getpage fixchip.book 0 -var pageName -key pantaskState STOP
     225    if ("$pageName" == "NULL") break
     226
     227    # book listpage fixchip.book $pageName
     228
     229    # book setword fixchip.book $pageName pantaskState RUN
     230    book getword fixchip.book $pageName dateobs -var DATEOBS
     231    book getword fixchip.book $pageName pantasksQuality -var QUALITY
     232
     233    option $pageName
     234    # save the page so we can use it below
     235
     236    if ("$QUALITY" == "OK")
     237      command update.rawfix.sh $DATEOBS fixchip.done
     238    else
     239      command update.rawfix.sh $DATEOBS fixchip.fail
     240    end
     241  end
     242
     243  task.exit 0
     244    # now that the entry is set to done in the db, remove the pantasks book entry
     245    echo "cleaning this page: $options:0"
     246    book setword fixchip.book $options:0 pantaskState DONE
    95247  end
    96248
    … …  
    99251    showcommand failure
    100252    book setword fixchip.book $options:0 pantasksQuality FAIL
    101     exec update.rawfix.sh $options:0 fixchip.fail
    102253  end
    103254
    … …  
    105256    showcommand crash
    106257    book setword fixchip.book $options:0 pantasksQuality CRASH
    107     exec update.rawfix.sh $options:0 fixchip.fail
    108258  end
    109259
    … …  
    112262    showcommand timeout
    113263    book setword fixchip.book $options:0 pantasksQuality TIMEOUT
    114     exec update.rawfix.sh $options:0 fixchip.fail
    115   end
    116 end
    117 
     264  end
     265end
     266
     267######################### UTILITIES #################################
     268
     269macro nights.fixchip.exit
     270  if ($0 != 2)
     271    echo "USAGE: nights.fixchip.exit (quality)"
     272    break
     273  end
     274
     275  # options are : pagename DATEOBS YEAR
     276
     277  book setword fixhost.book $options:0 pantaskState DONE
     278  book getword fixchip.book $options:1 NHOST -var Nhosts
     279  book getword fixchip.book $options:1 NDONE -var Ndone
     280  book getword fixchip.book $options:1 stage -var ITER
     281  $Ndone ++
     282  book setword fixchip.book $options:1 NDONE $Ndone
     283
     284  sprintf tmpline "%s/%s/%s/log.fixchip.runhosts.v%d" $mypath $options:2 $options:1 $ITER
     285  if ("$1" != "OK")
     286    book setword fixchip.book $options:1 pantasksQuality $1
     287    exec echo "failure $1 $options:0" >> $tmpline
     288  else
     289    exec echo "success $1 $options:0" >> $tmpline
     290  end
     291  if ($Ndone == $Nhosts)
     292    book setword fixchip.book $options:1 pantaskState STOP
     293  end
     294  book delpage fixhost.book $options:0
     295end
     296

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt

    r42405 r42531  
    1212  $RAWFIX_MD5SUM_INIT = 1
    1313end
     14
     15$MAX_MD5HOST = 2
    1416
    1517# select nights which are ready for md5sum calcs
    … …  
    3537    # fields in the table will become words in the page
    3638    queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum.book
     39    book delpage md5sum.book -key pantaskState DONE
    3740  end
    3841
    … …  
    5457# output: md5host.book
    5558task nights.md5sum.loadhosts
    56 
    5759  periods      -poll $RPOLL
    5860  periods      -exec $REXEC
    5961  periods      -timeout 60
     62  host          local
    6063  npending 1
     64  # note: the tasks which return their results via stdout
     65  # seem to fail if run remotely. 
    6166
    6267  stdout $LOGDIR/md5sum.loadhosts.stdout.log 
    … …  
    7782    book setword md5sum.book $pageName pantaskState RUN
    7883    book getword md5sum.book $pageName dateobs -var DATEOBS
     84    book getword md5sum.book $pageName iter    -var ITER
    7985    option $pageName
    8086    # save the page so we can use it below
    8187
     88    substr $DATEOBS 0 4 YEAR
     89
    8290    # write an entry point in the log for this night
    83     sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $DATEOBS
     91    sprintf tmpline "%s/%s/%s/log.md5sum.runhosts.v%d" $mypath $YEAR $DATEOBS $ITER
    8492    date -var mydate
    8593    exec echo "# start runhosts $mydate" >> $tmpline
    8694
    87     command get_hosts_md5s.sh $DATEOBS
     95    # the command returns these fields: dateobs stage md5host
     96    # NOTE: stage == ITER
     97    command get_hosts_md5s.sh $mypath $DATEOBS $ITER
    8898  end
    8999
    … …  
    91101    # convert the stdout 'queue' to entries in a book
    92102    # fields in the table will become words in the page
     103    # queueprint stdout
    93104    queuesize stdout -var Nhosts
    94105    queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book
    95106    book setword md5sum.book $options:0 NHOST {$Nhosts - 1}
    96107    book setword md5sum.book $options:0 NDONE 0
     108    # book list
     109    # book listbook md5sum.book
    97110    # use the queuesize (Nhosts) to track how many jobs are done / to be done
    98111  end
    … …  
    116129end
    117130
     131# runs on the hosts identified by get_hosts_md5s.sh
    118132# output: md5host.book
    119 task nights.md5sum.runhosts
     133task nights.md5sum.run
    120134
    121135  periods      -poll $RPOLL
    122136  periods      -exec $REXEC
    123137  periods      -timeout 60
    124 
    125   stdout $LOGDIR/md5sum.runhosts.stdout.log 
    126   stderr $LOGDIR/md5sum.runhosts.stderr.log
    127138
    128139  task.exec
    … …  
    138149    if ("$pageName" == "NULL") break
    139150
     151    # XXX NOTE: the output field is called 'stage'
    140152    book setword md5host.book $pageName pantaskState RUN
    141153    book getword md5host.book $pageName dateobs -var DATEOBS
     154    book getword md5host.book $pageName stage   -var ITER
    142155    book getword md5host.book $pageName md5host -var MD5HOST
    143156
    144     option $pageName $DATEOBS
     157    substr $DATEOBS 0 4 YEAR
     158
     159    option $pageName $DATEOBS $YEAR
    145160    # save DATEOBS so we can find the md5sum.book entry
    146 
    147     # XXX TEST: $USE_HOST = ipp060
     161    # also save YEAR for output log
     162
     163    # does the output go here?
     164    stdout $mypath/$YEAR/$DATEOBS/log.md5sum.$MD5HOST.v$ITER 
     165    stderr $mypath/$YEAR/$DATEOBS/log.md5sum.$MD5HOST.v$ITER 
    148166
    149167    $USE_HOST = $MD5HOST
    … …  
    154172      $host:$USE_HOST = 0
    155173    end
    156     if (not($host:$USE_HOST))
     174    if ($host:$USE_HOST < $MAX_MD5HOST)
    157175      control host add $USE_HOST
    158       $host:$USE_HOST = 1
     176      $host:$USE_HOST ++
    159177    end
    160178
    161179    ## even when testing, use the original hostname here:
    162     $input = $mypath/$DATEOBS/$MD5HOST.inst.txt
     180    $input = $mypath/$DATEOBS/$MD5HOST.inst.$ITER.txt
    163181
    164182    # need to make sure this will work on the ipp b-nodes:
    165183    # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
    166184    periods -exec 0.05
    167     command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input
     185    command get_md5s_instances.pl --hostname $USE_HOST --topdir $mypath --dateobs $DATEOBS --stage $ITER
    168186  end
    169187
    170188  task.exit 0
    171     nights.mdsum.exit OK
     189    nights.md5sum.exit OK
    172190  end
    173191
    174192  # exit values other than 0:
    175193  task.exit    default
    176     nights.mdsum.exit FAIL
     194    nights.md5sum.exit FAIL
    177195    showcommand failure
    178196  end
    179197
    180198  task.exit    crash
    181     nights.mdsum.exit CRASH
     199    nights.md5sum.exit CRASH
    182200    showcommand crash
    183201  end
    … …  
    185203  # operation times out?
    186204  task.exit    timeout
    187     nights.mdsum.exit TIMEOUT
     205    nights.md5sum.exit TIMEOUT
    188206    showcommand timeout
    189207  end
    … …  
    210228   
    211229    # look for completed nights in md5sum.book (pantaskState == DONE)
    212     book getpage md5sum.book 0 -var pageName -key pantaskState DONE
     230    book getpage md5sum.book 0 -var pageName -key pantaskState STOP
    213231    if ("$pageName" == "NULL") break
    214232
    … …  
    232250    # now that the entry is set to done in the db, remove the pantasks book entry
    233251    echo "cleaning this page: $options:0"
    234     book delpage md5sum.book $options:0
     252    book setword md5sum.book $options:0 pantaskState DONE
    235253  end
    236254
    … …  
    255273######################### UTILITIES #################################
    256274
    257 macro nights.mdsum.exit
     275macro nights.md5sum.exit
    258276  if ($0 != 2)
    259     echo "USAGE: nights.mdsum.exit (quality)"
     277    echo "USAGE: nights.md5sum.exit (quality)"
    260278    break
    261279  end
     280
     281  # options are : pagename DATEOBS YEAR
    262282
    263283  book setword md5host.book $options:0 pantaskState DONE
    264284  book getword md5sum.book $options:1 NHOST -var Nhosts
    265285  book getword md5sum.book $options:1 NDONE -var Ndone
     286  book getword md5sum.book $options:1 stage -var ITER
    266287  $Ndone ++
    267288  book setword md5sum.book $options:1 NDONE $Ndone
     289
     290  sprintf tmpline "%s/%s/%s/log.md5sum.runhosts.v%d" $mypath $options:2 $options:1 $ITER
    268291  if ("$1" != "OK")
    269292    book setword md5sum.book $options:1 pantasksQuality $1
    270     sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $options:1
    271293    exec echo "failure $1 $options:0" >> $tmpline
    272294  else
    … …  
    274296  end
    275297  if ($Ndone == $Nhosts)
    276     book setword md5sum.book $options:1 pantaskState DONE
     298    book setword md5sum.book $options:1 pantaskState STOP
    277299  end
    278300  book delpage md5host.book $options:0

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.pt

    r42406 r42531  
    11## pantasks scripts to manage the raw exposure data repair / cleanup
    2 
    3 $TPOLL = 5
    4 $TEXEC = 5
    5 $RPOLL = 5
    6 $REXEC = 5
    7 $VERBOSE = 1
    8 $NETWORK = 1
    9 
    10 # use this to find the files in this local directory
    11 $mypath = /home/panstarrs/eugene/rawfix.20230221
    12 
    13 if (not($?LOGDIR))
    14   $LOGDIR = $mypath/pantasks_logs
    15   mkdir $LOGDIR
    16 end
    17 
    18 input rawfix.ckchip.pt
    19 # input rawfix.md5sum.pt
    202
    213######################### UTILITIES #################################

  • branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/update.rawfix.sh

    r42400 r42531  
    22
    33if ($#argv < 1) then
    4   echo "USAGE: update.rawfix.sh (night) (state) [quality]"
     4  echo "USAGE: update.rawfix.sh (night) (state) [iter]"
    55  echo "   EG: update.rawfix.sh 20100501 new"
    66  exit 2
    … …  
    1010set state   = $2
    1111
     12set iter = ""
     13if ($#argv == 3) then
     14  set iter = ", iter = $3"
     15endif
     16
    1217# make the rawfix database to track rawfix operations
    13 set DBHOST = "ipp060"
     18set DBHOST = "ippc190"
    1419set DBNAME = "rawfix"
    15 set DBUSER = "dvo"
    16 set DBPASS = "dvo"
     20# set DBUSER = "dvo"
     21# set DBPASS = "dvo"
    1722
    18 set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME"
     23# set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME"
     24set DBOPTS = "--defaults-extra-file=~/.my.cnf.rawfix -B --skip-column-names -h $DBHOST $DBNAME"
    1925
    20 mysql $DBOPTS -e "update nights set state = '$state' where (dateobs = '$mynight')"
     26mysql $DBOPTS -e "update nights set state = '$state' $iter where (dateobs = '$mynight')"
    2127exit 0

  • branches/eam_branches/ipp-20230313/tools/eam/rebalance/launch.rsync.sh

    r42314 r42531  
    11#!/bin/csh
    22
    3 # set list = "ipp100 ipp101 ipp102 ipp104 ipp105 ipp108 ipp112 ipp114 ipp115 ipp117 ipp123 ipp124 ipp125 ipp126 ipp134 ipp136 ipp137 ipp139"
    4 # set list = "ipp105 ipp108 ipp112 ipp114 ipp115 ipp123 ipp124 ipp125 ipp126 ipp134 ipp136 ipp137 ipp139"
    5   set list = "ipp100 ipp101 ipp102 ipp104 ipp117"
    6 set mydir = /data/ipphome1.0/home/eugene/rebalance.20220922
     3set program = `basename $0`
    74
    8 if ($#argv == 1) then
    9   if ("$1" != "launch") then
    10     echo "USAGE: launch.find.sh launch"
    11     exit 2
    12   endif
    13   foreach f ($list)
    14     echo ssh -f $f "cd $mydir && ./launch.rsync.sh"
    15          ssh -f $f "cd $mydir && ./launch.rsync.sh"
    16 #     echo ssh $f ps ux
    17 #          ssh $f ps ux
    18 #      ssh $f killall mana
    19 #      ssh $f killall find
     5set run = v0
     6set version = 20230802
     7set mode = rsync
     8
     9set srchost = (ipp100 ipp108 ipp112 ipp114)
     10set srcvol  = (     0      0      1      0)
     11set tgthost = (ipp161 ipp161 ipp162 ipp162)
     12set tgtvol  = (     0      1      0      1)
     13
     14set mydir = /data/ipphome1.0/home/eugene/rebalance.20230802
     15
     16if ($#argv == 0) then
     17  echo "USAGE: $program launch"
     18  echo "USAGE: $program run (Nentry)"
     19  echo "   eg: $program run 1"
     20  exit 2
     21endif
     22
     23if ("$1" == "launch") then
     24
     25  set i = 1
     26  while ($i <= $#srchost)
     27    echo ssh -f $srchost[$i] "cd $mydir && ./$program run $i"
     28         ssh -f $srchost[$i] "cd $mydir && ./$program run $i"
     29    @ i++
    2030  end
    2131  exit 0
    2232endif
    2333
    24 set run = v0
    25 set version = 20220922
     34if ("$1" == "run") then
     35  if ($#argv != 2) then
     36    echo "USAGE: $program run (Nentry)"
     37    echo "   eg: $program run 1"
     38    exit 2
     39  endif
    2640
    27 set myhost = `hostname`
    28 set Nhost = `hostname | sed s/ipp//`
     41  set N = $2
    2942
    30 set target = NONE
    31 switch ($myhost)
    32   # total of 23TB
    33   case ipp100
    34   case ipp101
    35   case ipp123
    36   set target = ipp153
    37   breaksw;
     43  set hn = $srchost[$N]
     44  set hv = $srcvol[$N]
     45  set tn = $tgthost[$N]
     46  set tv = $tgtvol[$N]
    3847
    39   # total of 35TB
    40   case ipp102
    41   case ipp104
    42   case ipp124
    43   case ipp136
    44   set target = ipp154
    45   breaksw;
     48  set myhost = `hostname`
     49  if ($hn != $myhost) then
     50    echo "ERROR: need to run $mode on the source host ($hn vs $myhost)"
     51    exit 3
     52  endif
    4653
    47   # total of 35TB
    48   case ipp105
    49   case ipp108
    50   case ipp125
    51   case ipp137
    52   set target = ipp155
    53   breaksw;
    54 
    55   # total of 23TB
    56   case ipp112
    57   case ipp114
    58   case ipp126
    59   set target = ipp156
    60   breaksw;
    61 
    62   # total of 38TB
    63   case ipp115
    64   case ipp117
    65   case ipp134
    66   case ipp139
    67   set target = ipp157
    68   breaksw;
    69 endsw
    70 
    71 if ("$target" == "NONE") then
    72   echo "need target"
    73   exit 2
     54  echo "running on $myhost for source $hn.$hv to target $tn.$tv in mode $mode, version $version"
     55  mkdir -p $hn.$hv
     56  echo "./rebalance.mana $mode $hn $hv $tn $tv $version > & $hn.$hv/log.$mode.$version.$run &"
     57        ./rebalance.mana $mode $hn $hv $tn $tv $version > & $hn.$hv/log.$mode.$version.$run &
     58  exit 0
    7459endif
    75 
    76 set v = 0 ; set h = `hostname` ; set t = $target; ./rebalance.mana rsync $h $v $t $v $version > & $h.$v/log.rsync.$version.$run &
    77 set v = 1 ; set h = `hostname` ; set t = $target; ./rebalance.mana rsync $h $v $t $v $version > & $h.$v/log.rsync.$version.$run &
    78 
    79 # e.g.: ssh -f ipp127 "cd `pwd` && ./launch.rsync.sh"

  • branches/eam_branches/ipp-20230313/tools/eam/skycoverage/coverage.mana.sh

    r41436 r42531  
    319319  cplot -pt 0 -sz 0.4 $RCOLOR el eb
    320320 end
     321
     322 $RCOLOR = -c grey70
     323 DR1.boundary
     324end
     325
     326macro DR1.boundary
     327 if (1)
     328  if (0)
     329    data EuclidDR1_polygon.csv
     330    read -skip 1 -csv Re 1 De 2
     331    cplot -pt 0 -sz 0.5 -c green Re De
     332  end
     333
     334  # north boundary:
     335  # corner 105, 80 -> 94.7, 56.9 in ecliptic
     336  create Rn 105.2 285 0.1
     337  set Dn = zero(Rn) + 80
     338  cplot $RCOLOR -pt box -sz 0.4 Rn Dn
     339
     340  # west boundary
     341  create elat 50 56.9 0.1
     342  set elon = zero(elat) + 95
     343  csystem E C elon elat
     344  cplot $RCOLOR -pt box -sz 0.4 elon elat
     345
     346  # south boundary west
     347  create elon 95 150 0.1
     348  set elat = zero(elon) + 50
     349  csystem E C elon elat
     350  cplot $RCOLOR -pt box -sz 0.4 elon elat
     351 
     352  # south boundary east
     353  create elat 50 80 0.1
     354  set elon = zero(elat) + 150
     355  csystem E C elon elat
     356  cplot $RCOLOR -pt box -sz 0.4 elon elat
     357
     358  # NEP circle
     359  create elon3 150 296 0.1
     360  set elat3 = zero(elon3) + 80
     361  csystem E C elon3 elat3
     362  cplot $RCOLOR elon3 elat3 -pt box -sz 0.4
     363
     364  # galactic plane boundary east
     365  create glon 86.5 103 0.1
     366  set glat = zero(glon) + 25
     367  csystem G C glon glat
     368  cplot $RCOLOR -pt box -sz 0.4 glon glat
     369
     370  # east boundary
     371  create Dn 72 80 0.1
     372  set Rn = zero(Dn) + 285
     373  csystem E C elon elat
     374  cplot $RCOLOR -pt box -sz 0.4 Rn Dn
     375
     376  # DR2 south boundary
     377  create elon 100.3 189.5 0.1
     378  set elat = zero(elon) + 37
     379  csystem E C elon elat
     380  cplot $RCOLOR -pt box -sz 0.4 elon elat
     381 end
    321382end
    322383
Note: See TracChangeset for help on using the changeset viewer.