IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 41172 for trunk/Nebulous


Ignore:
Timestamp:
Nov 27, 2019, 12:04:14 PM (7 years ago)
Author:
eugene
Message:

merging updates to Nebulous from EAM dev branch ipp-20191011

Location:
trunk/Nebulous
Files:
1 deleted
29 edited
2 copied

Legend:

Unmodified
Added
Removed

  • trunk/Nebulous/Build.PL

    r32473 r41172  
    9696        'Test::Nebulous'            => 0,
    9797        'Test::Cmd'                 => '1.05',
    98         'Apache2::SOAP'             => 0,
     98# gentoo: 'Apache2::SOAP'             => 0,
     99        'Apache::SOAP'             => 0, # needed instead for ubuntu 18.04
    99100        'Apache::DBI'               => '1.05',
    100101        'Apache::Test'              => '1.27',  # bundles Apache::TestMB

  • trunk/Nebulous/bin/neb-cat

    r13080 r41172  
    1818use constant CHUNK_SIZE => 1024;
    1919
    20 my ($server);
     20my ($server, $help);
    2121
    2222$server = $ENV{'NEB_SERVER'} unless $server;
    … …  
    2424GetOptions(
    2525    'server|s=s'    => \$server,
     26    'help|h'        => \$help,
    2627) || pod2usage( 2 );
    2728
    28 pod2usage( -msg => "Required options: --server <key...>", -exitval => 2 )
    29         unless defined $server;
     29# explicitly defined -h / --help:
     30if ($help) { pod2usage( -verbose => 1 ); }
     31
     32pod2usage( -msg => "Required options: --server <key...>", -exitval => 2 ) unless defined $server;
    3033
    3134my $neb = Nebulous::Client->new(
    … …  
    9598=over 4
    9699
    97 =item * --server|-s <URL>
     100=item --server|-s <URL>
    98101
    99102URL of the Nebulous server to connect to.

  • trunk/Nebulous/bin/neb-cp

    r16182 r41172  
    1616use Pod::Usage qw( pod2usage );
    1717
    18 my ($volume, $server);
     18my ($volume, $server, $help);
    1919
    2020$server = $ENV{'NEB_SERVER'} unless $server;
    … …  
    2323    'volume=s'      => \$volume,
    2424    'server|s=s'    => \$server,
     25    'help|h'        => \$help,
    2526) || pod2usage( 2 );
     27
     28# explicitly defined -h / --help:
     29if ($help) { pod2usage( -verbose => 1 ); }
    2630
    2731my $src = shift;

  • trunk/Nebulous/bin/neb-cull

    r30236 r41172  
    1717
    1818my ($min_copies, $server, $volume, $one_only);
    19 my ($soft_volume);
     19my ($soft_volume, $help);
    2020$server = $ENV{'NEB_SERVER'} unless $server;
    2121
    … …  
    2626    'one_only|o'    => \$one_only,
    2727    'soft_volume'   => \$soft_volume,
     28    'help|h'        => \$help,
    2829) || pod2usage( 2 );
     30
     31# explicitly defined -h / --help:
     32if ($help) { pod2usage( -verbose => 1 ); }
    2933
    3034my $key = shift;
    … …  
    5963# print "@cull_args\n";
    6064
    61 
    6265if ($one_only) {
    6366    my $status = $neb->there_can_be_only_one(@cull_args);
     67    if (not defined $status) {
     68        die "not enough instances of Nebulous key: $key";
     69    }
    6470    if ($status == 0) {
    6571        die "not enough instances of Nebulous key: $key";

  • trunk/Nebulous/bin/neb-insert

    r25939 r41172  
    9595=head1 SYNOPSIS
    9696
    97     neb-insert --server <neb-server> <neb-key> <local-file> [--copies <n>] [--destroy]
     97    neb-insert [--server <neb-server>] <neb-key> <local-file> [--copies <n>] [--destroy]
    9898
    9999#=head1 DESCRIPTION

  • trunk/Nebulous/bin/neb-ls

    r28581 r41172  
    1919    $ls,
    2020    $column,
     21    $help
    2122);
    2223
    2324$server = $ENV{'NEB_SERVER'} unless $server;
    24 
    2525
    2626GetOptions(
    … …  
    2929    'l'             => \$ls,
    3030    'column|c'      => \$column,
     31    'help|h'        => \$help,
    3132) || pod2usage( 2 );
     33
     34# explicitly defined -h / --help:
     35if ($help) { pod2usage( -verbose => 1 ); }
    3236
    3337# twiddle column so that it does it by default.
    … …  
    3539$column = not $column;
    3640
     41if ($ls) { $path = 1; }
     42
    3743my $pattern = shift;
    3844
    3945pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    40 pod2usage( -msg => "Required options: --server", -exitval => 2 )
    41     unless $server;
     46pod2usage( -msg => "Required options: --server", -exitval => 2 ) unless $server;
    4247
    4348my $neb = Nebulous::Client->new(
    … …  
    96101=head1 SYNOPSIS
    97102
    98     neb-ls [--server <URL>] [-c] [-p] <pattern>
     103    neb-ls [--server <URL>] [-c] [-p] [-l] <pattern>
    99104
    100105=head1 DESCRIPTION
    101106
    102107This program list Nebulous keys matched by C<<pattern>>.  Call it with no
    103 arguments is equivalent to searching with the pattern C<neb://>.  SQL like
     108arguments is equivalent to searching with the pattern C<neb://>.  SQL-like
    104109wildcards are supported to select out certain files.
    105110
    … …  
    108113=over 4
    109114
    110 =item * --path|-p
     115=item --path|-p
    111116
    112117Find the disk files corresponding to the keys found.
    113118
    114 =item * --column|-c
     119=item --column|-c
    115120
    116121Disable column formatting, and output results one to a line.
    117122
     123=item --l|-l
     124
     125Give disk file information using ls (implies -p).
     126
    118127Optional
    119128
    120 =item * --server|-s <URL>
     129=item --server|-s <URL>
    121130
    122131URL of the Nebulous server to connect to.

  • trunk/Nebulous/bin/neb-migrate

    r28446 r41172  
    2323
    2424$server = $ENV{'NEB_SERVER'} unless $server;
     25
     26die "this tool is probably too inefficient unless used very carefully";
    2527
    2628GetOptions(

  • trunk/Nebulous/bin/neb-repair

    r39926 r41172  
    4545);
    4646
    47 die "can't connected to Nebulous Server: $server"
     47die "can't connect to Nebulous Server: $server"
    4848    unless defined $neb;
    4949
    5050my $stat = $neb->stat($key);
    5151die "nebulous key: $key not found" unless $stat;   
     52
    5253my $instances = $neb->find_instances($key, 'any');
    5354die "no instances found" unless $instances;   
    … …  
    7172
    7273for (my $i = 0; $i <= $#files; $i++) {
     74    # count existing files and get the MD5 sums
    7375    if (-e $files[$i]) {
    7476        $existance[$i] = 1;
    … …  
    8991        push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i];
    9092        $last_f = $files[$i];
    91     }
    92     else {
     93    } else {
    9394        $existance[$i] = 0;
    9495        $md5sums[$i] = 'NON-EXISTANT';
    … …  
    9899}
    99100
     101# check number of copies and number of unique MD5 sum values
    100102my $N_mds = scalar(keys(%md5sum_uniq));
    101103my $N_files = scalar(@files);
     104
     105# seems like we should not call neb-replicate until *after* we have
     106# removed 0-byte copies
    102107if ($N_files < $user_copies) {
    103108    print "Insufficient copies ($N_files) of $key, replicating!\n";
    104109    system("neb-replicate $key");
    105110}
     111
    106112if ($N_mds == 1) {
    107113    print "Nebulous key $key is consistent.\n";
    108 }
    109 else {
     114} else {
    110115    my $valid_sum = '';
    111116    my $valid_file = '';
    112117    my $m;
    113118    foreach $m (keys %md5sum_valid) {
    114         if (($m ne 'd41d8cd98f00b204e9800998ecf8427e')&&($m ne 'NON-EXISTANT')) {
     119        if (($m ne 'd41d8cd98f00b204e9800998ecf8427e') && ($m ne 'NON-EXISTANT')) {
    115120            if ($valid_sum ne '') {
    116121                die "Too many acceptable files for key $key\n";
    … …  
    122127    if ($valid_sum eq '') {
    123128        die "No acceptable file for key $key\n";
    124     }
    125     else {
     129    } else {
    126130        print "Repairing instances\n";
    127131        foreach $m (keys %md5sum_valid) {
    … …  
    130134                    print "\tcp $valid_file $f\n";
    131135                    system("cp $valid_file $f");
     136
     137                    # if we failed to make a copy, the target ($f) could be inaccessible
    132138                    if ($? >> 8 != 0) {
    133139                        if ($existance[0] == 1) {
    134                             # We can fix this with culls and replications.
     140
     141                            # The target is a valid file 0-byte file
     142                            # but we cannot replace it with the
     143                            # correct file.  We can fix this with
     144                            # culls and replications.
     145
    135146                            print "Attempting to fix this.\n";
    136147                            print("neb-replicate $key\n");
    137148                            system("neb-replicate $key");
    138149                            if ($? >> 8 == 0) {
     150                                # if the neb-replicate succeeded, remove the bad copy with neb-cull
    139151                                my $bad_volume = (split /\//, $f)[2];
    140152                                print("neb-cull --volume $bad_volume $key\n");
    141153                                system("neb-cull --volume $bad_volume $key");
    142                             }
    143                             else {
     154                            } else {
    144155                                die "Failed to copy file $valid_file to $f, and subsequent replication also failed!\n";
    145156                            }
    146                         }
    147                         else {
     157                        } else {
    148158                            die "Failed to copy file $valid_file to $f!\n";
    149159                        }
    … …  
    187197=head1 DESCRIPTION
    188198
    189 This program repairs instances and corrects instance counts for a nebulous key. 
     199This program repairs instances and corrects instance counts for a nebulous key:
     200
     201If some of the existing instances have the MD5 sum corresponding to a
     2020-byte file, but the rest have the same, non-zero byte md5sum, then
     203replace the 0-byte files with the non-zero byte version.
     204
     205At the end, if the number of instances is less than the value of
     206user.copies, then additional copies are generated.
    190207
    191208=head1 OPTIONS

  • trunk/Nebulous/bin/neb-replicate

    r29476 r41172  
    4242);
    4343
     44# Nebulous::Client programs can use this to set the log level:
     45# $neb->set_log_level ('DEBUG');
     46
    4447die "can't connected to Nebulous Server: $server"
    4548    unless defined $neb;
    … …  
    5053    for (my $i = 0; $i < $copies; $i++) {
    5154        if (defined $volume) {
     55            # this is coded so that all copies end up on the specified volume which seems wrong
    5256            $neb->replicate($key, $volume)
    5357                or die "failed to replicate Nebulous key: $key";

  • trunk/Nebulous/bin/neb-shift

    r31964 r41172  
    1717
    1818my ($volume, $server);
     19
     20die "this tool may fail since find_instances will return an instance which is not on the hard volume";
    1921
    2022$server = $ENV{'NEB_SERVER'} unless $server;

  • trunk/Nebulous/bin/neb-stat

    r32045 r41172  
    1919use URI;
    2020
    21 my ($validate,$server,$nouris);
     21my ($validate,$server,$nouris,$help);
    2222
    2323$server = $ENV{'NEB_SERVER'} unless $server;
    2424
    2525GetOptions(
    26     'server|s=s'     => \$server,
    27     'validate|v'         => \$validate,
    28     'files'          => \$nouris,
     26    'server|s=s'    => \$server,
     27    'validate|v'    => \$validate,
     28    'files|f'       => \$nouris,
     29    'help|h'        => \$help,
    2930) || pod2usage( 2 );
    3031
     32# explicitly defined -h / --help:
     33if ($help) { pod2usage( -verbose => 1 ); }
     34
    3135my $key = shift;
    3236
    3337pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    34 pod2usage( -msg => "Required options: --server", -exitval => 2 )
    35     unless $server;
    36 pod2usage( -msg => "missing key", exitval => 2 )
    37     unless defined $key;
     38pod2usage( -msg => "Required options: --server", -exitval => 2 ) unless $server;
     39pod2usage( -msg => "missing key", exitval => 2 ) unless defined $key;
    3840
    3941my $neb = Nebulous::Client->new(
    … …  
    5254eval {
    5355    $user_copies = $neb->getxattr($key, "user.copies");
     56    die ("user.copies not found") unless $user_copies;
    5457};
    55 unless(defined($user_copies)) {
     58unless(defined($user_copies) and $user_copies ne "") {
    5659    $user_copies = 1;
    5760}
    … …  
    7477            $md5sum_uniq{$md5sums[$i]} = 1;
    7578            close($fh);
    76         }
    77         else {
     79        } else {
    7880            $existance[$i] = 0;
    7981            $md5sums[$i] = 'NON-EXISTANT';
    8082            $md5sum_uniq{$md5sums[$i]} = 1;
    8183        }
     84
    8285        if (defined($nouris)) {
    8386            $validation[$i] = sprintf("% 3d %32s %s",
    … …  
    8588                                      $md5sums[$i],
    8689                                      $files[$i]);
    87         }
    88         else {
     90        } else {
    8991            $validation[$i] = sprintf("% 3d %32s %s",
    9092                                      $existance[$i],
    … …  
    112114        "instance location:\n", " " x 4,
    113115        join("\n" . " " x 4, @validation), "\n";
     116} else {
     117   
     118    if (defined($nouris)) {
     119        my @files = map {URI->new($_)->file if $_} @$instances;
     120
     121        print
     122            "object id:             ", @$stat[0], "\n",
     123            "key:                   ", @$stat[1], "\n",
     124            "read lock:             ", @$stat[2], "\n",
     125            "write lock:            ", @$stat[3], "\n",
     126            "epoch:                 ", @$stat[4], "\n",
     127            "mtime:                 ", @$stat[5], "\n",
     128            "available instances:   ", @$stat[6], "\n",
     129            "requested instances:   ", $user_copies, "\n",
     130            "total instances:       ", @$stat[7], "\n",
     131           
     132            "instance location:\n", " " x 4,
     133            join("\n" . " " x 4, @files), "\n";
     134    } else {
     135        print
     136            "object id:             ", @$stat[0], "\n",
     137            "key:                   ", @$stat[1], "\n",
     138            "read lock:             ", @$stat[2], "\n",
     139            "write lock:            ", @$stat[3], "\n",
     140            "epoch:                 ", @$stat[4], "\n",
     141            "mtime:                 ", @$stat[5], "\n",
     142            "available instances:   ", @$stat[6], "\n",
     143            "requested instances:   ", $user_copies, "\n",
     144            "total instances:       ", @$stat[7], "\n",
     145           
     146            "instance location:\n", " " x 4,
     147            join("\n" . " " x 4, @$instances), "\n";
     148    }
     149
    114150}
    115 else {
    116     print
    117         "object id:             ", @$stat[0], "\n",
    118         "key:                   ", @$stat[1], "\n",
    119         "read lock:             ", @$stat[2], "\n",
    120         "write lock:            ", @$stat[3], "\n",
    121         "epoch:                 ", @$stat[4], "\n",
    122         "mtime:                 ", @$stat[5], "\n",
    123         "available instances:   ", @$stat[6], "\n",
    124         "requested instances:   ", $user_copies, "\n",
    125         "total instances:       ", @$stat[7], "\n",
    126        
    127         "instance location:\n", " " x 4,
    128         join("\n" . " " x 4, @$instances), "\n";
    129 }
    130151
    131152
    … …  
    150171=over 4
    151172
    152 =item * --server|-s <URL>
     173=item --server|-s <URL>
    153174
    154175URL of the Nebulous server to connect to.
    … …  
    156177Optional if the appropriate environment variable is set.
    157178
    158 =item * --validate|-v
     179=item --validate|-v
    159180
    160181Optional - prints out additional information including md5sum.
    161182
    162 =item * --files
     183=item --files
    163184
    164185Optional - prints out disk files instead of uris during validate stage.
    … …  
    174195=over 4
    175196
    176 =item * C<NEB_SERVER>
     197=item C<NEB_SERVER>
    177198
    178199Equivalent to --server|-s

  • trunk/Nebulous/lib/Nebulous/Client.pm

    r40680 r41172  
    209209        },
    210210    );
    211 
     211   
    212212    $log->debug( "entered - @_" );
    213213
    … …  
    310310    my $stats = $self->stat($key);
    311311    my $instances = $stats->[6];
     312
    312313    if ((not defined $instances) or ($instances == 0)) {
    313314        $self->set_err("can not cull - no instances");
    … …  
    315316        return;
    316317    }
    317     if ($instances < $min_copies) {
     318    if ($instances <= $min_copies) {
    318319        $self->set_err("can not cull - not enough instances");
    319320        $log->debug("leaving");
    320 
    321321        return;
    322322    }
    … …  
    334334
    335335    unless (defined $locations) {
    336         $self->set_err( "no instances" );
    337336        $log->debug( "leaving" );
    338 
    339337        return;
    340338    }
    … …  
    351349        }
    352350    }
     351
    353352    # If we don't have two copies on one volume, see if we have two copies on a single
    354353    # cabinet. If so, delete the first of those copies.
    … …  
    362361        }
    363362    }
     363
    364364    # Fail-safe. We didn't have any duplicates (the instances are "well-mixed"), so
    365365    # delete the first.
    … …  
    403403    eval {
    404404        # first - strip off any inaccesible instances
     405        # these are not counted in the 'removed' number
    405406        if (not defined $self->prune($key)) {
    406407            # use the prune() method to also determine if $key is valid
    407408            die "invalid key: $key";
    408409        }
    409 
    410         # check to see if there is an instance on $vol_name that should be the
    411         # sole survivor
     410       
     411        # check to see if there is an instance on $vol_name that
     412        # should be the sole survivor.  This is a bit dangerous : what
     413        # if the only instance is on an invalid volume?  what if an
     414        # invalid volume is specified?
     415
    412416        if (defined $vol_name) {
    413417            $locations = $self->find_instances($key, $vol_name);
     418            if (not defined $locations) {
     419                die "no instances on requested node";
     420            }
     421            if (scalar @$locations == 0) {
     422                die "no instances on requested node";
     423            }
    414424        }
    415425
    … …  
    418428
    419429        if (defined $locations) {
    420             my $instances = $self->find_instances($key, undef, 'find them all');
    421             if (scalar @$instances == 1) {
    422                 # only one instance nothing to do
    423                 return 0;
    424             }
    425             if (scalar @$instances == 0) {
     430            print "dev: looking for undef volume instances\n";
     431            # first delete the valid instances (except the one we own)
     432            my $instances_valid = $self->find_instances($key, undef, 0); # find all valid instances
     433            if (not defined $instances_valid) {
     434                # there should have been at least one valid instance?
    426435                die "no instances";
    427436            }
    428             foreach my $victim (@$instances) {
    429                 next if $victim eq $locations->[0];
    430                 $self->delete_instance($key, $victim);
    431                 $removed++;
     437            if (scalar @$instances_valid == 0) {
     438                # there should have been at least one valid instance?
     439                die "no instances";
    432440            }
     441
     442            # we have more than one instance, delete the others
     443            if (scalar @$instances_valid > 1) {
     444                foreach my $victim (@$instances_valid) {
     445                    next if $victim eq $locations->[0];
     446                    $self->delete_instance($key, $victim);
     447                    $removed ++;
     448                }
     449            }
     450            # the prune above should have deleted any invalid instances, so we only
     451            # need to worry about the valid instances;
    433452        } else {
    434453            # nuke whatever
    … …  
    448467            }
    449468        }
    450 
    451469    };
    452470    if ($@) {
    … …  
    663681    if ($response->fault) {
    664682        $self->set_err($response->faultstring);
     683
     684        ## invalid object key
     685        if ($response->faultstring =~ /is valid object key/) {
     686            $log->debug( "leaving" );
     687            die "invalid key $key, fails 'is valid object key' test\n";
     688        }
     689
     690        ## invalid object key
     691        if ($response->faultstring =~ /xattr is in user. namespace/) {
     692            $log->debug( "leaving" );
     693            die "invalid xattr key $name : fails 'xattr is in user. namespace' test\n";
     694        }
     695
    665696        $log->logdie("unhandled fault - ", $self->err);
    666697    }
    … …  
    690721    if ( $response->fault ) {
    691722        $self->set_err($response->faultstring);
     723
     724        ## invalid object key
     725        if ($response->faultstring =~ /is valid object key/) {
     726            $log->debug( "leaving" );
     727            die "invalid key $key, fails 'is valid object key' test\n";
     728        }
     729
     730        ## invalid xattr key (not in user. namespace)
     731        if ($response->faultstring =~ /xattr is in user. namespace/) {
     732            $log->debug( "leaving" );
     733            die "xattr $name is invalid : xattr is in user. namespace\n";
     734        }
     735
     736        ## invalid xattr key
     737        if ($response->faultstring =~ /user\..*? does not exist/) {
     738            $log->debug( "leaving" );
     739            die "xattr key $name not found : xattr $name does not exist\n";
     740        }
    692741        $log->logdie("unhandled fault - ", $self->err);
    693742    }
    … …  
    747796    if ( $response->fault ) {
    748797        $self->set_err($response->faultstring);
     798
     799        ## invalid object key
     800        if ($response->faultstring =~ /is valid object key/) {
     801            $log->debug( "leaving" );
     802            die "invalid key $key, fails 'is valid object key' test\n";
     803        }
     804
     805        ## invalid xattr key (not in user. namespace)
     806        if ($response->faultstring =~ /xattr is in user. namespace/) {
     807            $log->debug( "leaving" );
     808            die "xattr $name is invalid : xattr is in user. namespace\n";
     809        }
     810
     811        ## invalid xattr key
     812        if ($response->faultstring =~ /user\..*? does not exist/) {
     813            $log->debug( "leaving" );
     814            die "xattr key $name not found : xattr $name does not exist\n";
     815        }
     816
    749817        $log->logdie("unhandled fault - ", $self->err);
    750818    }
    … …  
    865933    unless(defined($params[0])) {
    866934        $params[0] = hostname() . ".0";
    867 #       print STDERR "Setting host to $params[0]\n";
     935        # print STDERR "Setting host to $params[0]\n";
    868936    }
    869937
    … …  
    871939    if ( $response->fault ) {
    872940        $self->set_err($response->faultstring);
     941
    873942        # check to see if this failure is because $key doesn't exist
    874943        if ($response->faultstring =~ /is valid object key/) {
    … …  
    878947        # key is valid but no instances are on the specified volume
    879948        if ($response->faultstring =~ /no instances on storage volume/) {
     949            $log->debug( "leaving" );
     950            return;
     951        }
     952        # check to see if this failure is volume is unknown
     953        if ($response->faultstring =~ /is not a valid volume name/) {
     954            $log->debug( "leaving" );
     955            return;
     956        }
     957        # key is valid but no instances are on the specified volume
     958        if ($response->faultstring =~ /no instances available for key/) {
    880959            $log->debug( "leaving" );
    881960            return;
    … …  
    9491028        $self->set_err($response->faultstring);
    9501029        # check to see if this failure is because $key doesn't exist
     1030        my $string = $response->faultstring;
    9511031        if ($response->faultstring =~ /is valid object key/) {
     1032            $self->set_err("invalid object key");
    9521033            $log->debug( "leaving" );
    9531034            return;
    … …  
    9551036        # key is valid but no instances are on the specified volume
    9561037        if ($response->faultstring =~ /no instances on storage volume/) {
     1038            $self->set_err("no instances on storage volume or volume is not available");
    9571039            $log->debug( "leaving" );
    9581040            return;
    … …  
    9721054
    9731055
     1056## this is used by PS:IPP:Config nebulous functions
    9741057sub find
    9751058{
    … …  
    10141097    return $path;
    10151098}
    1016 
    10171099
    10181100sub open
    … …  
    11241206    $log->debug( "entered - @_" );
    11251207
    1126 
    11271208    my $locations;
    11281209    if ($invalid) {
    1129         $locations = $self->find_instances( $key, 'any', 'find them all');
     1210        # delete the invalid instances:
     1211        $locations = $self->find_instances( $key, 'any', 1);
    11301212    }
    11311213    else {
    … …  
    13151397        }
    13161398
     1399        if ($response->faultstring =~ /is valid object key/) {
     1400            $log->logdie("parameter #1 to delete_instance did not pass the 'is valid object key' callback");
     1401        }
     1402
    13171403        $log->logdie("unhandled fault - ", $self->err);
    13181404    }
    … …  
    14771563}
    14781564
     1565sub set_log_level
     1566{
     1567    my $self = shift;
     1568
     1569    my $log_level = shift;
     1570
     1571    $log->level($log_level);
     1572
     1573    return $self;
     1574}
     1575
    14791576
    148015771;

  • trunk/Nebulous/lib/Nebulous/Util.pm

    r23934 r41172  
    7878    # The instance file attempting to be opened should always exist as it was
    7979    # created by the Nebulous server.
     80
    8081    my $fh;
    8182    for (my $i = 0; $i < 60; $i++) {
    … …  
    160161        die "can not process $arg because it is not in name:value form"
    161162            unless defined $name and defined $value;
    162         die "xattr name: $name is not in to the form user.name"
     163        die "xattr name: $name is not in the form user.name"
    163164            unless $name =~ /^user\./;
    164165        $neb->setxattr($key, $name, $value, "replace")

  • trunk/Nebulous/nebclient

    • Property svn:ignore

      •  

        old new  
        2121stamp-h1
        2222test-driver
         23INSTALL
         24COPYING

  • trunk/Nebulous/nebclient/tests/tap

    • Property svn:ignore

      •  

        old new  
        1919src/.deps
        2020src/Makefile
         21INSTALL
         22COPYING

  • trunk/Nebulous/nebclient/tests/tests.c

    r24999 r41172  
    1616    if (foo) nebFree(foo)
    1717
     18int show_instances (nebServer *server, char *key) {
     19
     20  nebObjectInstances *locations = NULL;
     21
     22  locations = nebFindInstances(server, key, NULL);
     23
     24  for (int i = 0; i < locations->n; i++) {
     25    fprintf (stderr, "# loc %d : %s\n", i, locations->URI[i]);
     26  }
     27  nebObjectInstancesFree(locations);
     28  return 1;
     29}
     30
     31int count_instances (nebServer *server, char *key) {
     32
     33  nebObjectInstances *locations = NULL;
     34  locations = nebFindInstances(server, key, NULL);
     35  int Ninstances = locations ? locations->n : 0;
     36  nebObjectInstancesFree(locations);
     37  return Ninstances;
     38}
     39
    1840int main (int argc, char **argv) {
    1941    nebServer       *server = NULL;
    2042    char            *key = "foobarbaz";
    2143
    22     plan_tests(31);
     44    plan_tests(37);
    2345
    2446    if (getenv("NEB_SERVER")) {
    … …  
    3759        filename = nebCreate(server, key, NULL, &URI);
    3860        diag("filename is %s", filename);
     61
     62        if (!filename) {
     63          fprintf (stdout, "not ok: no files in test server, exiting\n");
     64          exit (2);
     65        }
     66
     67        show_instances (server, key);
     68        ok (count_instances (server, key) == 1, "correct instance count");
     69
    3970        neb_ok(server, filename, "create object");
    4071        ok(URI, "URI not NULL");
    … …  
    5384        neb_ok(server, fh > -1, "create new object filehandle");
    5485        ok(URI, "URI not NULL");
     86        ok (count_instances (server, key) == 1, "correct instance count");
    5587
    5688        tst_nebFree(URI);
    5789        close(fh);
    5890    }
     91
     92    show_instances (server, key);
    5993
    6094    {
    … …  
    80114    }
    81115
    82 
    83116    neb_ok(server, nebReplicate(server, key, NULL, NULL), "replicate object");
     117    show_instances (server, key);
     118    ok (count_instances (server, key) == 2, "correct instance count");
    84119
    85120    neb_ok(server, nebOpen(server, key, NEB_WRITE) < 0, "write to object with multiple instances");
     121    show_instances (server, key);
    86122
    87123    neb_ok(server, nebCull(server, key), "cull object");
     124    show_instances (server, key);
     125    ok (count_instances (server, key) == 1, "correct instance count");
     126
     127    neb_ok(server, nebCull(server, key) == 0, "cannot cull object with one instance");
     128    show_instances (server, key);
     129    ok (count_instances (server, key) == 1, "correct instance count");
    88130
    89131    neb_ok(server, nebStat(server, key), "stat object");
     132    show_instances (server, key);
    90133
    91134    neb_ok(server, nebLock(server, key, NEB_WRITE), "lock object write");

  • trunk/Nebulous/t/50_client_create.t

    r24357 r41172  
    1010use Apache::Test qw( -withtestmore );
    1111
    12 plan tests => 9;
     12plan tests => 13;
    1313
    1414use lib qw( ./t ./lib );
    … …  
    1919use URI::Split qw( uri_split );
    2020
     21# this returns the test apache server location
    2122my $hostport = Apache::Test->config->{ 'hostport' };
    2223
    … …  
    3031
    3132    my $uri = $neb->create("foo");
     33    if (not defined $uri) { die "failure to create basic file (does it already exist?)"; }
     34
    3235    my ($scheme, $auth, $path, $query, $frag) = uri_split($uri);
    3336    ok(-e $path, "good filename");
     37
    3438    uri_scheme_ok($uri, 'file');
     39   
     40    # need to remove the test entry we made above (create fails if file exists)
     41    ok( $neb->delete( "foo" ), "delete object" );
    3542}
    3643
    … …  
    4855    ok(-e $path, "good filename");
    4956    uri_scheme_ok($uri, 'file');
     57
     58    # need to remove the test entry we made above (create fails if file exists)
     59    ok( $neb->delete( "foo" ), "delete object" );
    5060}
    5161
    … …  
    6373    ok(-e $path, "good filename");
    6474    uri_scheme_ok($uri, 'file');
     75
     76    # need to remove the test entry we made above (create fails if file exists)
     77    ok( $neb->delete( "foo" ), "delete object" );
    6578}
    6679
    … …  
    7487    $neb->create("foo");
    7588    is($neb->create("foo"), undef, "object already exists");
     89
     90    # need to remove the test entry we made above (create fails if file exists)
     91    ok( $neb->delete( "foo" ), "delete object" );
    7692}
    7793

  • trunk/Nebulous/t/53_client_cull.t

    r25121 r41172  
    1010use Apache::Test qw( -withtestmore );
    1111
    12 plan tests => 15;
     12plan tests => 18;
    1313
    1414use lib qw( ./t ./lib );
    … …  
    2121
    2222Test::Nebulous->setup;
    23 
    2423{
    2524    # key
    … …  
    3736
    3837Test::Nebulous->setup;
    39 
    4038{
    4139    # key, $volume
    … …  
    4341        proxy => "http://$hostport/nebulous",
    4442    );
    45     $neb->create("foo");
    46     $neb->replicate("foo", "node01");
    47 
    48     my $uri = $neb->cull("foo", "node01");
     43    $neb->create("foo", "node01");
     44    $neb->replicate("foo", "node02");
     45   
     46    my $uri = $neb->cull("foo", "node02");
    4947
    5048    ok($uri, "good cull");
    5149    ok(! -e _get_file_path($uri), "file exists");
     50}
     51
     52Test::Nebulous->setup;
     53{
     54    # key, $volume
     55    my $neb = Nebulous::Client->new(
     56        proxy => "http://$hostport/nebulous",
     57    );
     58    $neb->create("foo", "node01");
     59    $neb->replicate("foo", "node02");
     60
     61    # cull with soft volume should succeed even if node does not have instance
     62    my $uri = $neb->cull("foo", "node03");
     63
     64    ok($uri, "good cull");
     65    ok(! -e _get_file_path($uri), "file exists");
     66}
     67
     68Test::Nebulous->setup;
     69{
     70    # key, $volume
     71    my $neb = Nebulous::Client->new(
     72        proxy => "http://$hostport/nebulous",
     73    );
     74    $neb->create("foo", "node01");
     75    $neb->replicate("foo", "node02");
     76
     77    # cull with hard volume should not succeed if node does not have instance
     78    my $uri = $neb->cull("foo", "~node03");
     79
     80    is($uri, undef, "nothing to cull on this node");
    5281}
    5382
    … …  
    112141        proxy => "http://$hostport/nebulous",
    113142    );
    114     $neb->cull();
     143    my $uri = $neb->cull();
     144
     145    is ($uri, undef, "no params given to cull");
    115146};
    116147like( $@, qr/1 - 3 were expected/, "no params" );
    … …  
    122153        proxy => "http://$hostport/nebulous",
    123154    );
    124     $neb->cull( 1, 2, 3, 4 );
     155    my $uri = $neb->cull( 1, 2, 3, 4 );
     156
     157    is ($uri, undef, "no params given to cull");
    125158};
    126159like( $@, qr/1 - 3 were expected/, "too many params" );

  • trunk/Nebulous/t/56_client_find_instances.t

    r19469 r41172  
    1010use Apache::Test qw( -withtestmore );
    1111
    12 plan tests => 29;
     12plan tests => 35;
    1313
    1414use lib qw( ./t ./lib );
    … …  
    2020my $hostport = Apache::Test->config->{ 'hostport' };
    2121
    22 Test::Nebulous->setup;
    23 
     22### XXX EAM : this test suite finds real errors in the implementation. 
     23
     24## the API is now slightly different (what is the state of the API in the ipp ops code?)
     25## find_instances ("key", "volume", find_invalid)
     26
     27## this returns a result even if we request a hard volume unless the hard volume is not valid
     28
     29Test::Nebulous->setup;
     30Test::Nebulous->cleanup;
     31
     32### TEST BLOCK for find_invalid
     33## Test::Nebulous->setup;
     34## {
     35##     # volume/key, volume override
     36##     my $neb = Nebulous::Client->new(
     37##         proxy => "http://$hostport/nebulous",
     38##     );
     39##     $neb->create( "foo", "~node02" );
     40##     $neb->replicate( "foo", "~node01" );
     41##
     42##     Test::Nebulous->switch_node_state;
     43##     my $tmphost;
     44##
     45## #   my $locations = $neb->find_instances( "neb:///foo", "~node02", 1); # result : no instance
     46## #   my $locations = $neb->find_instances( "neb:///foo", "~node02", 0); # result : no instance
     47## #   my $locations = $neb->find_instances( "neb:///foo", "~node02"); # result : no instance
     48##
     49## #   my $locations = $neb->find_instances( "neb:///foo", "node02", 1); # result : no instance
     50## #   my $locations = $neb->find_instances( "neb:///foo", "node02", 0); # result : no instance
     51##     my $locations = $neb->find_instances( "neb:///foo", "node02"); # result : no instances
     52##
     53## #   my $locations = $neb->find_instances( "neb:///foo", undef, 1);
     54## #   my $locations = $neb->find_instances( "neb:///foo", undef, 0);
     55## #   my $locations = $neb->find_instances( "neb:///foo", $tmphost, 0);
     56##
     57## #   my $locations = $neb->find_instances( "neb:///foo", "bebaz", 1);
     58## #   my $locations = $neb->find_instances( "neb:///foo", "bebaz", 0);
     59##
     60## #   my $locations = $neb->find_instances( "neb:///foo");
     61## #   my $locations = $neb->find_instances( "neb://bebaz/foo");
     62##
     63## #   my $locations = $neb->find_instances( "neb:///foo");
     64## #   my $locations = $neb->find_instances( "neb:///foo", undef, 0);
     65##
     66##     foreach my $f (@$locations) {
     67##      print "location: $f\n";
     68##     }
     69##
     70##     is( scalar @$locations, 1, "found 1" );
     71##     like( @$locations[0], qr/file:/, "URIs match" );
     72##     ok( -e _get_file_path( @$locations[0] ), "URI matches file" );
     73## }
     74## die "quit";
     75
     76## Test::Nebulous->setup;
     77## eval {
     78##     # volume/key, volume override
     79##     my $neb = Nebulous::Client->new(
     80##         proxy => "http://$hostport/nebulous",
     81##     );
     82##     $neb->create( "foo", "~node01" );
     83##
     84##     my $locations_1 = $neb->find_instances( "neb://~invalid/foo", "~invalid");
     85##     is($locations_1, undef, "no instances in invalid node" );
     86##
     87##     # EAM: we fail this test because the logic of find_instances has
     88##     # changed: as long as there is an instance on a valid node, the
     89##     # instance gets returned
     90##     my $locations = $neb->find_instances( "neb://node01/foo", "~node02" );
     91##     is($locations, undef, "no instances on specified volume" );
     92## };
     93##print "result: $@\n";
     94##like( $@, qr/result message/, "my message" );
     95
     96# find a single instance by raw name (foo)
     97Test::Nebulous->setup;
    2498{
    2599    # key
    … …  
    36110}
    37111
    38 Test::Nebulous->setup;
    39 
     112# find a both instances of a duplicated object
     113Test::Nebulous->setup;
    40114{
    41115    # key
    … …  
    56130
    57131Test::Nebulous->setup;
    58 
    59132{
    60133    # key, volume
    … …  
    79152    );
    80153    $neb->create( "foo" );
     154    $neb->set_log_level ('DEBUG');
    81155
    82156    my $locations = $neb->find_instances( "foo", undef );
    … …  
    103177}
    104178
    105 Test::Nebulous->setup;
    106 
     179# find instance using hard_volume for wrong node
     180# (should still return a valid instance)
     181Test::Nebulous->setup;
    107182{
    108183    # volume/key, volume override
    … …  
    111186    );
    112187    $neb->create( "foo", "~node01" );
     188    $neb->set_log_level ('DEBUG');
    113189
    114190    my $locations = $neb->find_instances( "neb://node01/foo", "~node02" );
    115191
     192    is( scalar @$locations, 1, "found 1" );
     193    like( @$locations[0], qr/file:/, "URIs match" );
     194    ok( -e _get_file_path( @$locations[0] ), "URI matches file" );
     195#   the API used to return undef if we used a hard_volume but it was not there
     196#   is($locations, undef, "no instances on specified volume" );
     197}
     198
     199# find instance using hard_volume for unknown node
     200# (should fail)
     201Test::Nebulous->setup;
     202{
     203    # volume/key, volume override
     204    my $neb = Nebulous::Client->new(
     205        proxy => "http://$hostport/nebulous",
     206    );
     207    $neb->create( "foo", "~node01" );
     208    $neb->set_log_level ('DEBUG');
     209
     210    my $locations = $neb->find_instances( "neb://node01/foo", "~nobody" );
    116211    is($locations, undef, "no instances on specified volume" );
     212#   is( scalar @$locations, 0, "found 0" );
     213#   under Ubuntu (Perl 5.26.1), @$locations evaluates to 0 if $locations is undef,
     214#   but not under Gentoo (Perl 5.6.6)
     215}
     216
     217# find instance using hard_volume for unavailable node
     218# (should fail)
     219Test::Nebulous->setup;
     220{
     221    # volume/key, volume override
     222    my $neb = Nebulous::Client->new(
     223        proxy => "http://$hostport/nebulous",
     224    );
     225    $neb->create( "foo", "~node01" );
     226    $neb->set_log_level ('DEBUG');
     227
     228    my $locations = $neb->find_instances( "neb://node01/foo", "~node04" );
     229
     230    is( scalar @$locations, 1, "found 1" );
     231    like( @$locations[0], qr/file:/, "URIs match" );
     232    ok( -e _get_file_path( @$locations[0] ), "URI matches file" );
     233
     234#   the API used to return undef if we used a hard_volume but it was not there
     235#   is($locations, undef, "no instances on specified volume" );
    117236}
    118237
    … …  
    153272
    154273Test::Nebulous->setup;
    155 
    156274{
    157275    my $neb = Nebulous::Client->new(
    … …  
    171289    $neb->find_instances();
    172290};
    173 like( $@, qr/1 - 2 were expected/, "no params" );
     291like( $@, qr/1 - 3 were expected/, "no params" );
    174292
    175293Test::Nebulous->setup;
    … …  
    180298    );
    181299
    182     $neb->find_instances( "foo", 'read', 3 );
     300    $neb->find_instances( "foo", 'read', 3, 5 );
    183301};
    184 like( $@, qr/1 - 2 were expected/, "too many params" );
     302like( $@, qr/1 - 3 were expected/, "too many params" );
    185303
    186304Test::Nebulous->cleanup;

  • trunk/Nebulous/t/59_client_delete.t

    r24440 r41172  
    1010use Apache::Test qw( -withtestmore );
    1111
    12 plan tests => 13;
     12plan tests => 18;
    1313
    1414use lib qw( ./t ./lib );
    … …  
    2020my $hostport = Apache::Test->config->{ 'hostport' };
    2121
    22 Test::Nebulous->setup;
    23 
     22# ensure fresh start:
     23Test::Nebulous->setup;
     24Test::Nebulous->cleanup;
     25
     26Test::Nebulous->setup;
    2427{
    2528    my $neb = Nebulous::Client->new(
    … …  
    7881}
    7982
    80 # force flag
    81 Test::Nebulous->setup;
    82 
     83# test passing in undefined force flag
     84Test::Nebulous->setup;
    8385{
    8486    my $neb = Nebulous::Client->new(
    … …  
    9092}
    9193
    92 Test::Nebulous->setup;
    93 
     94# test passing in force flag = 1
     95Test::Nebulous->setup;
    9496{
    9597    my $neb = Nebulous::Client->new(
    … …  
    101103}
    102104
     105# test passing in force flag = 0
     106Test::Nebulous->setup;
     107{
     108    my $neb = Nebulous::Client->new(
     109        proxy => "http://$hostport/nebulous",
     110    );
     111    $neb->create( "foo" );
     112
     113    is( $neb->delete("foo", 0), 1, "force flag false" );
     114}
     115
     116# test passing in force flag = 0 where disk file is removed
     117Test::Nebulous->setup;
     118eval {
     119    my $neb = Nebulous::Client->new(
     120        proxy => "http://$hostport/nebulous",
     121    );
     122    $neb->create( "foo" );
     123    my $locations = $neb->find_instances( "foo" );
     124    my $diskfile = _get_file_path(@$locations[0]);
     125    unlink $diskfile;
     126    $neb->delete("foo", 0);
     127};
     128like( $@, qr/can't unlink file/, "no force flag, cannot delete inconsistent file" );
     129
     130# test passing in force flag = 1 where disk file is removed
     131Test::Nebulous->setup;
     132{
     133    my $neb = Nebulous::Client->new(
     134        proxy => "http://$hostport/nebulous",
     135    );
     136    $neb->create( "foo" );
     137    my $locations = $neb->find_instances( "foo" );
     138    my $diskfile = _get_file_path(@$locations[0]);
     139    unlink $diskfile;
     140    is( $neb->delete("foo", 1), 1, "force flag true deletes file" );
     141}
     142
     143# test passing in undef force flag where disk file is removed
     144Test::Nebulous->setup;
     145eval {
     146    my $neb = Nebulous::Client->new(
     147        proxy => "http://$hostport/nebulous",
     148    );
     149    $neb->create( "foo" );
     150    my $locations = $neb->find_instances( "foo" );
     151    my $diskfile = _get_file_path(@$locations[0]);
     152    unlink $diskfile;
     153    $neb->delete("foo", undef);
     154};
     155like( $@, qr/can't unlink file/, "no force flag, cannot delete inconsistent file" );
     156
     157# test passing in undefined force flag where disk file is removed
     158Test::Nebulous->setup;
     159eval {
     160    my $neb = Nebulous::Client->new(
     161        proxy => "http://$hostport/nebulous",
     162    );
     163    $neb->create( "foo" );
     164    my $locations = $neb->find_instances( "foo" );
     165    my $diskfile = _get_file_path(@$locations[0]);
     166    unlink $diskfile;
     167    $neb->delete("foo");
     168};
     169like( $@, qr/can't unlink file/, "no force flag, cannot delete inconsistent file" );
     170
    103171Test::Nebulous->setup;
    104172
    … …  
    109177    $neb->delete();
    110178};
    111 like( $@, qr/1 - 2 were expected/, "no params" );
     179like( $@, qr/1 - 3 were expected/, "no params" );
    112180
    113181Test::Nebulous->setup;
    … …  
    127195        proxy => "http://$hostport/nebulous",
    128196    );
    129     $neb->delete( "foo", 0, 2 );
    130 };
    131 like( $@, qr/1 - 2 were expected/, "too many params" );
     197    $neb->delete( "foo", 0, 1, 2 );
     198};
     199like( $@, qr/1 - 3 were expected/, "too many params" );
    132200
    133201Test::Nebulous->cleanup;

  • trunk/Nebulous/t/64_client_find_objects.t

    r17693 r41172  
    2121
    2222Test::Nebulous->setup;
    23 
    2423{
    2524    # key

  • trunk/Nebulous/t/65_client_mounts.t

    r19494 r41172  
    2929    my $mounts = $neb->mounts();
    3030
    31     is(scalar @$mounts, 7, "number of rows");
     31    is(scalar @$mounts, 8, "number of rows");
    3232
    3333    my %row;

  • trunk/Nebulous/t/66_client_xattr.t

    r24301 r41172  
    2121);
    2222
    23 # 1 key / xattr
    24 
    25 Test::Nebulous->setup;
    26 
     23# setxattr
     24Test::Nebulous->setup;
    2725{
    2826    my $uri = $neb->create('foo');
    … …  
    4644
    4745# multiple xattrs
    48 
    49 Test::Nebulous->setup;
    50 
     46Test::Nebulous->setup;
    5147{
    5248    my $uri = $neb->create('foo');
    … …  
    184180    $neb->getxattr('foo', 'user.bar');
    185181};
    186 like($@, qr|xattr neb:///foo:user.bar does not exist|,
     182like($@, qr|xattr user.bar does not exist|,
    187183    "get xattr from non-existant xattr key");
    188184
    … …  
    256252    $neb->removexattr('foo', 'user.bar');
    257253};
    258 like($@, qr|xattr neb:///foo:user.bar does not exist|,
     254like($@, qr|xattr user.bar does not exist|,
    259255    "remove xattr from non-existant xattr key");
    260256

  • trunk/Nebulous/t/68_client_chmod.t

    r24281 r41172  
    4141    is($mode, 0440, "returned mode");
    4242    is($neb->getxattr($key, 'user.mode'), 0440, "xattr user.mode");
     43
     44    # reset to 600 so the user can delete the file and /tmp entry
     45    chmod(0600, $path);
    4346}
    4447
    … …  
    5356    my $mode = $neb->chmod($key, 0440);
    5457
    55     my $st1 = stat(URI->new($uri1)->path);
    56     my $st2 = stat(URI->new($uri2)->path);
     58    my $path1 = URI->new($uri1)->path;
     59    my $path2 = URI->new($uri2)->path;
     60
     61    my $st1 = stat($path1);
     62    my $st2 = stat($path2);
    5763
    5864    is($st1->mode &07777, 0440, "chmod() first instance");
    … …  
    6066    is($mode, 0440, "returned mode");
    6167    is($neb->getxattr($key, 'user.mode'), 0440, "xattr user.mode");
     68
     69    # reset to 600 so the user can delete the file and /tmp entry
     70    chmod(0600, $path1);
     71    chmod(0600, $path2);
    6272}
    6373

  • trunk/Nebulous/t/70_neb-ls.t

    r24560 r41172  
    2323use Test::Cmd;
    2424use Nebulous::Client;
     25use Nebulous::Util qw( :standard );
    2526use Test::Nebulous;
    2627
    … …  
    2829my $neb_url  = "http://$hostport/nebulous";
    2930
    30 
    3131my $cmd = 'bin/neb-ls';
    3232
    33 # last ditch effort to make sure dsget is executable
     33# last ditch effort to make sure neb-ls is executable
    3434chmod 0755, 'bin/neb-ls';
    3535
    … …  
    3737isa_ok($test, 'Test::Cmd');
    3838
    39 # NEB_SERVER env var not set
     39# test if NEB_SERVER env var not set
    4040undef $ENV{'NEB_SERVER'} if defined $ENV{'NEB_SERVER'};
    4141Test::Nebulous->setup;
    42 
    4342{
    4443    $test->run(args => '');
    … …  
    4645}
    4746
    48 # NEB_SERVER set
    49 Test::Nebulous->setup;
    50 
     47# NEB_SERVER set (and used in the following tests)
     48Test::Nebulous->setup;
    5149{
    5250    $ENV{NEB_SERVER} = $neb_url;
    … …  
    5755
    5856Test::Nebulous->setup;
    59 
    60 {
    61     my $neb = Nebulous::Client->new(
    62         proxy => $neb_url,
    63     );
    64     $neb->create('foo');
    65 
    66     $test->run(args => '');
    67 
     57{
     58    my $neb = Nebulous::Client->new(
     59        proxy => $neb_url,
     60    );
     61    $neb->create('foo');
     62
     63    $test->run(args => '');
    6864    is($? >> 8, 0, "exit code");
    6965    like($test->stdout, qr/^foo$/,      "stdout");
    … …  
    7268
    7369Test::Nebulous->setup;
    74 
    7570{
    7671    my $neb = Nebulous::Client->new(
    … …  
    8378
    8479    is($? >> 8, 0, "exit code");
    85     like($test->stdout, qr/^bar\nfoo$/,  "stdout");
     80    like($test->stdout, qr/^bar\s+foo$/,  "stdout");
    8681    like($test->stderr, qr/^$/,         "stderr");
    8782}
    8883
    8984Test::Nebulous->setup;
    90 
     85{
     86    my $neb = Nebulous::Client->new(
     87        proxy => $neb_url,
     88    );
     89    $neb->create('foo');
     90
     91    my $locations = $neb->find_instances( "foo" );
     92    my $diskfile = _get_file_path( @$locations[0] );
     93    my $filestats = `ls -l $diskfile`;
     94
     95    $test->run(args => '-l');
     96
     97    is($? >> 8, 0, "exit code");
     98    like($test->stdout, qr/^$filestats$/,  "stdout");
     99    like($test->stderr, qr/^$/,         "stderr");
     100}
     101
     102Test::Nebulous->setup;
    91103{
    92104    my $neb = Nebulous::Client->new(
    … …  
    96108    $neb->create('bar');
    97109
    98     $test->run(args => '-l');
     110    $test->run(args => '-c');
    99111
    100112    is($? >> 8, 0, "exit code");
    … …  
    109121        proxy => $neb_url,
    110122    );
    111     $neb->create('foo');
    112     $neb->create('bar');
    113 
    114     $test->run(args => '-1');
    115 
    116     is($? >> 8, 0, "exit code");
    117     like($test->stdout, qr/^bar\nfoo\n$/,   "stdout");
    118     like($test->stderr, qr/^$/,             "stderr");
    119 }
    120 
    121 Test::Nebulous->setup;
    122 
    123 {
    124     my $neb = Nebulous::Client->new(
    125         proxy => $neb_url,
    126     );
    127     $neb->create('a/foo');
    128 
    129     $test->run(args => '');
    130 
    131     is($? >> 8, 0, "exit code");
    132     like($test->stdout, qr||,               "stdout");
     123    $neb->create('a/foo');
     124
     125    $test->run(args => '');
     126
     127    is($? >> 8, 0, "exit code");
     128    like($test->stdout, qr|^a/$|,           "stdout");
    133129    like($test->stderr, qr/^$/,             "stderr");
    134130}
    … …  
    161157
    162158    is($? >> 8, 0, "exit code");
    163     like($test->stdout, qr|^a/bar\na/foo$|,  "stdout");
     159    like($test->stdout, qr|^a/bar\s+a/foo$|,  "stdout");
    164160    like($test->stderr, qr/^$/,             "stderr");
    165161}
    … …  
    193189
    194190    is($? >> 8, 0, "exit code");
    195     like($test->stdout, qr|^a/b/\na/foo$|,        "stdout");
     191    like($test->stdout, qr|^a/b/\s+a/foo$|,        "stdout");
    196192    like($test->stderr, qr/^$/,             "stderr");
    197193}

  • trunk/Nebulous/t/71_client_there_can_be_only_one.t

    r24930 r41172  
    1010use Apache::Test qw( -withtestmore );
    1111
    12 plan tests => 8;
     12plan tests => 13;
    1313
    1414use lib qw( ./t ./lib );
    … …  
    2121
    2222Test::Nebulous->setup;
     23Test::Nebulous->cleanup;
    2324
     25## volume is defined, but it is not valid (should exit)
     26## Test::Nebulous->setup;
     27## {
     28##     my $key = "foo";
     29##     my $neb = Nebulous::Client->new(
     30##         proxy => "http://$hostport/nebulous",
     31##     );
     32##
     33##     my $uri = $neb->create($key, "node01");
     34##
     35##     $neb->replicate($key, "node02");
     36##
     37##     ## set node02 to an invalid state:
     38##     Test::Nebulous->switch_node_state;
     39##
     40##     # if requested node is invalid, the other instance is kept and the invalid one removed
     41##     # but since invalid instances are removed with a prune call, they are not counted
     42##     # in the number of items removed (return by the function call)
     43##     is($neb->there_can_be_only_one($key, "node02"), 0, "there can be only one!");
     44##     my $locations = $neb->find_instances( "foo" );
     45##
     46##     is( scalar @$locations, 1, "found 1" );
     47##     is($locations->[0], $uri, "instance on correct volume" );
     48## };
     49## is($@, "", "there_can_be_only_one with supplied volume should succeed");
     50## die "asdf";
     51
     52#######
     53
     54## volume not defined
     55Test::Nebulous->setup;
    2456{
    2557    my $key = "foo";
    … …  
    3769}
    3870
     71## volume is defined, only valid instances
    3972Test::Nebulous->setup;
    40 
    41 {
     73eval {
    4274    my $key = "foo";
    4375    my $neb = Nebulous::Client->new(
    … …  
    4577    );
    4678    $neb->create($key, "node01");
     79
    4780    my $uri = $neb->replicate($key, "node02");
    48 
    4981    is($neb->there_can_be_only_one($key, "node02"), 1, "there can be only one!");
    5082
    … …  
    5385    is( scalar @$locations, 1, "found 1" );
    5486    is($locations->[0], $uri, "instance on correct volume" );
    55 }
     87};
     88is($@, "", "there_can_be_only_one with supplied volume should succeed");
     89
     90## volume is defined, but it is not valid (should exit)
     91Test::Nebulous->setup;
     92eval {
     93    my $key = "foo";
     94    my $neb = Nebulous::Client->new(
     95        proxy => "http://$hostport/nebulous",
     96    );
     97    my $uri = $neb->create($key, "node01");
     98
     99    $neb->replicate($key, "node02");
     100
     101    ## set node02 to an invalid state:
     102    Test::Nebulous->switch_node_state;
     103
     104    # if requested node is invalid, nothing is removed
     105    is($neb->there_can_be_only_one($key, "node02"), 0, "there can be only one!");
     106
     107    my $locations = $neb->find_instances( "foo" );
     108
     109    is( scalar @$locations, 1, "found 1" );
     110    is($locations->[0], $uri, "instance on correct volume" );
     111};
     112is($@, "", "there_can_be_only_one with supplied volume should succeed");
    56113
    57114Test::Nebulous->setup;

  • trunk/Nebulous/t/72_neb-cull.t

    r25121 r41172  
    3939# NEB_SERVER env var not set
    4040undef $ENV{'NEB_SERVER'} if defined $ENV{'NEB_SERVER'};
    41 Test::Nebulous->setup;
    42 
     41
     42## problem test
     43## Test::Nebulous->setup;
     44## {
     45##     $ENV{NEB_SERVER} = $neb_url;
     46##
     47##     my $key = 'foo';
     48##
     49##     my $neb = Nebulous::Client->new(
     50##         proxy => $neb_url,
     51##     );
     52##     $neb->create($key);
     53##
     54##     $test->run(args => "--one_only " . $key);
     55##
     56##     ## my $line1 = $test->stdout;
     57##     ## print "stdout: $line1\n";
     58##     ##
     59##     ## my $line2 = $test->stderr;
     60##     ## print "stderr: $line2\n";
     61##
     62##     is($neb->stat($key)->[6], 1, "correct # of instances");
     63##     is($? >> 8, 255, "exit code");
     64##     like($test->stdout, qr/^$/, "stdout");
     65##     like($test->stderr, qr/no instances/, "stderr");
     66## }
     67## die "stop";
     68
     69## Test::Nebulous->setup;
     70## {
     71##     my $key = 'foo';
     72##
     73##     my $neb = Nebulous::Client->new(
     74##         proxy => $neb_url,
     75##     );
     76##     $neb->create($key);
     77##     $neb->replicate($key);
     78##
     79##     $test->run(args => "--server $neb_url $key");
     80##
     81##     my $line = $test->stderr;
     82##     print "stderr: $line\n";
     83##
     84##     is($neb->stat($key)->[6], 1, "correct # of instances");
     85##     is($? >> 8, 0, "exit code");
     86##     like($test->stdout, qr/^$/, "stdout");
     87##     like($test->stderr, qr/^$/, "stderr");
     88## }
     89## die "TEST";
     90
     91Test::Nebulous->setup;
    4392{
    4493    $test->run(args => '');
    … …  
    4897# NEB_SERVER set
    4998Test::Nebulous->setup;
    50 
    5199{
    52100    $ENV{NEB_SERVER} = $neb_url;
    … …  
    57105
    58106Test::Nebulous->setup;
    59 
    60 {
    61     my $key = 'foo';
    62 
    63     my $neb = Nebulous::Client->new(
    64         proxy => $neb_url,
    65     );
    66     $neb->create($key);
     107{
     108    my $key = 'foo';
     109
     110    my $neb = Nebulous::Client->new(
     111        proxy => $neb_url,
     112    );
     113    $neb->create($key);
     114
     115    $test->run(args => "--server $neb_url $key");
     116#   $test->run(args => $key);
     117
     118    is($neb->stat($key)->[6], 1, "correct # of instances");
     119    is($? >> 8, 255, "exit code");
     120    like($test->stdout, qr/^$/, "stdout");
     121    like($test->stderr, qr/failed to cull Nebulous key/, "stderr");
     122}
     123
     124Test::Nebulous->setup;
     125
     126{
     127    my $key = 'foo';
     128
     129    my $neb = Nebulous::Client->new(
     130        proxy => $neb_url,
     131    );
     132    $neb->create($key);
     133    $neb->replicate($key);
    67134
    68135    $test->run(args => $key);
    69136
    70     is($neb->stat($key)->[6], 1, "correct # of instances");
    71     is($? >> 8, 255, "exit code");
    72     like($test->stdout, qr/^$/, "stdout");
    73     like($test->stderr, qr/failed to cull Nebulous key/, "stderr");
    74 }
    75 
    76 Test::Nebulous->setup;
    77 
    78 {
    79     my $key = 'foo';
    80 
    81     my $neb = Nebulous::Client->new(
    82         proxy => $neb_url,
    83     );
    84     $neb->create($key);
     137    my $line = $test->stderr;
     138
     139    # the default min_copies is 2:
     140    is($neb->stat($key)->[6], 2, "correct # of instances");
     141    is($? >> 8, 255, "exit code");
     142    like($test->stdout, qr/^$/, "stdout");
     143    like($test->stderr, qr/not enough instances/, "stderr");
     144}
     145
     146Test::Nebulous->setup;
     147
     148{
     149    my $key = 'foo';
     150
     151    my $neb = Nebulous::Client->new(
     152        proxy => $neb_url,
     153    );
     154    $neb->create($key);
     155    $neb->replicate($key);
    85156    $neb->replicate($key);
    86157
    87158    $test->run(args => $key);
    88159
    89     is($neb->stat($key)->[6], 1, "correct # of instances");
    90     is($? >> 8, 0, "exit code");
    91     like($test->stdout, qr/^$/, "stdout");
    92     like($test->stderr, qr/^$/, "stderr");
    93 }
    94 
    95 Test::Nebulous->setup;
    96 
    97 {
    98     my $key = 'foo';
    99 
    100     my $neb = Nebulous::Client->new(
    101         proxy => $neb_url,
    102     );
    103     $neb->create($key);
    104     $neb->replicate($key);
    105     $neb->replicate($key);
    106 
    107     $test->run(args => $key);
    108 
    109160    is($neb->stat($key)->[6], 2, "correct # of instances");
    110161    is($? >> 8, 0, "exit code");
    … …  
    113164}
    114165
     166## problem test
    115167Test::Nebulous->setup;
    116168
    … …  
    202254
    203255    is($neb->stat($key)->[6], 1, "correct # of instances");
    204     is($? >> 8, 0, "exit code");
     256
     257    is($? >> 8, 255, "exit code");
    205258    like($test->stdout, qr/^$/, "stdout");
    206259    like($test->stderr, qr/not enough instances/, "stderr");
    … …  
    221274    $test->run(args => "--min_copies 2 " . $key);
    222275
    223     is($neb->stat($key)->[6], 1, "correct # of instances");
     276    is($neb->stat($key)->[6], 2, "correct # of instances");
    224277    is($? >> 8, 0, "exit code");
    225278    like($test->stdout, qr/^$/, "stdout");

  • trunk/Nebulous/t/conf/extra.conf.in

    r4911 r41172  
    33# this file will be Include-d by @ServerRoot@/httpd.conf
    44
     5# need to add MPM?
     6
     7# Conflicts: mpm_worker mpm_prefork
     8#LoadModule mpm_event_module /usr/lib/apache2/modules/mod_mpm_event.so
     9#LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so
     10LoadModule perl_module /usr/lib/apache2/modules/mod_perl.so
     11#LoadModule cgid_module /usr/lib/apache2/modules/mod_cgid.so
     12LoadModule cgi_module /usr/lib/apache2/modules/mod_cgi.so
     13LoadModule alias_module /usr/lib/apache2/modules/mod_alias.so
     14
     15# event MPM
     16# StartServers: initial number of server processes to start
     17# MinSpareThreads: minimum number of worker threads which are kept spare
     18# MaxSpareThreads: maximum number of worker threads which are kept spare
     19# ThreadsPerChild: constant number of worker threads in each server process
     20# MaxRequestWorkers: maximum number of worker threads
     21# MaxConnectionsPerChild: maximum number of requests a server process serves
     22<IfModule mpm_event_module>
     23        StartServers                     2
     24        MinSpareThreads          25
     25        MaxSpareThreads          75
     26        ThreadLimit                      64
     27        ThreadsPerChild          25
     28        MaxRequestWorkers         150
     29        MaxConnectionsPerChild   0
     30</IfModule>
     31
     32<IfModule mod_alias.c>
     33        <IfModule mod_cgi.c>
     34#       Define ENABLE_USR_LIB_CGI_BIN
     35        </IfModule>
     36
     37        <IfModule mod_cgid.c>
     38#       Define ENABLE_USR_LIB_CGI_BIN
     39        </IfModule>
     40
     41        <IfDefine ENABLE_USR_LIB_CGI_BIN>
     42                ScriptAlias /cgi-bin/ /usr/lib/cgi-bin/
     43                <Directory "/usr/lib/cgi-bin">
     44                        AllowOverride None
     45                        Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
     46                        Require all granted
     47                </Directory>
     48        </IfDefine>
     49</IfModule>
     50
     51# nebulous location:
    552<Location /nebulous>
    653    SetHandler perl-script
    7     PerlResponseHandler Apache2::SOAP
     54    PerlResponseHandler Apache::SOAP
    855    PerlSetVar dispatch_to "PerlHandler Nebulous::Server::SOAP"
    956    PerlSetVar options "compress_threshold => 10000"

  • trunk/Nebulous/t/conf/startup.pl.in

    r24372 r41172  
    55use lib qw( @ServerRoot@ );
    66
     7use ModPerl::Util (); #for CORE::GLOBAL::exit
     8
     9use Apache2::RequestRec ();
     10use Apache2::RequestIO ();
     11use Apache2::RequestUtil ();
     12
     13use Apache2::ServerRec ();
     14use Apache2::ServerUtil ();
     15use Apache2::Connection ();
     16use Apache2::Log ();
     17
     18use APR::Table ();
     19
     20use ModPerl::Registry ();
     21
     22use Apache2::Const -compile => ':common';
     23use APR::Const -compile => ':common';
     24
     25use Apache::SOAP;
     26
    727use Apache::DBI;
    828use DBI;
    … …  
    1030use Nebulous::Server::Apache;
    1131use Nebulous::Server;
    12 use Test::Nebulous;
     32
     33# do not inherit here, let's be explicit
     34# use Test::Nebulous;
     35
     36# EAM : making this match Nebulous-Server/docs/apache2-mod_perl-startup.pl:
     37
     38# use the real mysql server:
     39#my $dsn         = 'DBI:mysql:database=nebulous:host=localhost,mysql_socket=8500';
     40#my $dbuser      = 'nebulous';
     41#my $dbpasswd    = '';
     42
     43# use the test mysql server (defined in Test::Nebulous)
     44my $dsn         = 'DBI:mysql:database=test:host=localhost:mysql_socket=/var/run/mysqld/mysqld.sock';
     45my $dbuser      = 'test';
     46my $dbpasswd    = '';
    1347
    1448#$Apache::DBI::DEBUG = 1;
    15 Apache::DBI->connect_on_init( $NEB_DB, $NEB_USER, $NEB_PASS );
    16 Apache::DBI->setPingTimeOut($NEB_DB, 10);
     49#Apache::DBI->connect_on_init( $NEB_DB, $NEB_USER, $NEB_PASS );
     50Apache::DBI->setPingTimeOut($dsn, 10);
    1751
    1852my $config = Nebulous::Server::Config->new(
    … …  
    2256$config->add_db(
    2357    dbindex     => 0,
    24     dsn         => $NEB_DB,
    25     dbuser      => $NEB_USER,
    26     dbpasswd    => $NEB_PASS,
     58    dsn         => $dsn,
     59    dbuser      => $dbuser,
     60    dbpasswd    => $dbpasswd,
    2761);
    28 #$config->add_db(
    29 #    dbindex     => 1,
    30 #    dsn         => 'DBI:mysql:database=nebulous1:host=localhost',
    31 #    dbuser      => $dbuser,
    32 #    dbpasswd    => $dbpasswd,
    33 #);
    3462
    3563Nebulous::Server::SOAP->new_on_init($config);
Note: See TracChangeset for help on using the changeset viewer.