IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Apr 3, 2009, 3:14:24 PM (17 years ago)
Author:
jhoblitt
Message:

merge everything from head except Server.pm

Location:
branches/neb_distrib_20081210/Nebulous-Server
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • branches/neb_distrib_20081210/Nebulous-Server/Build.PL

    r20157 r23708  
    1515        'DBI'                   => '1.53',
    1616        'Digest::SHA1'          => 0,
     17        'File::Basename'        => 0,
    1718        'File::ExtAttr'         => '1.03',
    1819        'File::Mountpoint'      => '0.01',
    1920        'File::Path'            => '1.08',
    2021        'File::Spec'            => 0,
    21         'Filesys::Df'           => '0.92',
    2222        'File::Spec::Functions' => 0,
    2323        'File::Temp'            => 0,
     24        'Filesys::Df'           => '0.92',
     25        'Log::Dispatch::Email::MailSend' => 0,
    2426        'Log::Log4perl'         => '0.48',
    2527        'Net::Server::Daemonize'=> '0.05',
    2628        'Params::Validate'      => '0.73',
    2729        'SOAP::Lite'            => '0.69',
     30        'SQL::Interp'           => '1.06',
    2831        'URI'                   => '1.30',
    29         'SQL::Interp'           => '1.06',
    3032    },
    3133    build_requires      => {

  • branches/neb_distrib_20081210/Nebulous-Server/Changes

    r23537 r23708  
    330.17
    44    - retry database transactions when a deadlock is detected
    5 
     5    - add log4perl logging to nebdiskd
     6    - base the on disk directory hashing of files only on the dirname()
     7      component of keys
     8    - add email logging of events to nebdiskd
     9    - attempt to optimize _is_valid_object_key() by eliminating an unused join
     10     
    6110.16
    712    - add so_id/name idxs to storage_object_xattr table

  • branches/neb_distrib_20081210/Nebulous-Server/bin/nebdiskd

    r20206 r23708  
    1616use File::Spec;
    1717use Filesys::Df;
     18use Log::Log4perl;
    1819use Nebulous::Server::SQL;
    1920use Net::Server::Daemonize qw( daemonize unlink_pid_file );
    … …  
    8586    unless $db && $dbuser && $dbpass;
    8687
     88# start up logging
     89my $conf = '
     90    log4perl.category.nebdiskd = WARN, Screen, SERVERLOGFILE, Mailer
     91
     92    log4perl.appender.Screen        = Log::Log4perl::Appender::Screen
     93    log4perl.appender.Screen.stderr = 1
     94    log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout
     95    log4perl.appender.Screen.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
     96
     97    log4perl.appender.SERVERLOGFILE           = Log::Log4perl::Appender::File
     98    log4perl.appender.SERVERLOGFILE.filename  = /tmp/nebdiskd.log
     99    log4perl.appender.SERVERLOGFILE.mode      = append
     100    log4perl.appender.SERVERLOGFILE.layout    = Log::Log4perl::Layout::PatternLayout
     101    log4perl.appender.SERVERLOGFILE.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
     102
     103    log4perl.filter.MatchWarn  = Log::Log4perl::Filter::LevelMatch
     104    log4perl.filter.MatchWarn.LevelToMatch  = WARN
     105    log4perl.filter.MatchWarn.AcceptOnMatch = off
     106
     107    log4perl.appender.Mailer         = Log::Dispatch::Email::MailSend
     108    log4perl.appender.Mailer.to      = ps-ipp-ops@ifa.hawaii.edu
     109    log4perl.appender.Mailer.subject = nebdiskd alert
     110    log4perl.appender.AppError.Filter= MatchWarn
     111    log4perl.appender.Mailer.layout = Log::Log4perl::Layout::PatternLayout
     112    log4perl.appender.Mailer.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
     113';
     114Log::Log4perl::init(\$conf);
     115my $log = Log::Log4perl::get_logger("nebdiskd");
     116$log->level('WARN');
     117$log->level('DEBUG') if $debug;
     118
    87119daemonize(
    88120    $user,     # User
    … …  
    104136                poll_interval   => $poll_interval,
    105137                debug           => $debug,
    106         ) or die "poll_mounts() should not have returned";
     138        );
    107139    };
    108     if ($!) {
    109         warn $@;
     140    if ($@) {
     141        $log->warn($@);
    110142    }
    111143
    … …  
    113145}
    114146
    115 die "poll loop exited -- THIS SHOULD NOT HAPPEN";
     147$log->logdie("poll loop exited -- THIS SHOULD NOT HAPPEN");
    116148
    117149
    … …  
    141173        # determine valid mountpoints
    142174        foreach my $mnt (@$mounts) {
    143             print "checking $mnt\n" if $debug;
     175            $log->debug("checking $mnt");
    144176            # this /SHOULD/ fail if the mount point is handled by the
    145177            # automounter and it fails to mount
    146178            eval {
    147179                unless (is_mountpoint($mnt)) {
    148                     die "$mnt is not a valid mountpoint\n";
     180                    $log->warn("$mnt is not a valid mountpoint");
    149181                }
    150182            };
    151183            if ($@) {
    152                 print $@ if $debug;
     184                $log->warn($@);
    153185                $d_query->execute($mnt);
    154186                next;
    … …  
    160192            my $dev_info = df($mnt, 1024);
    161193            unless (defined $dev_info) {
    162                 print "can't find device info for $mnt\n" if $debug;
     194                $log->error("can't find device info for $mnt");
    163195                next;
    164196            }
    165197
    166198            $r_query->execute($mnt, @$dev_info{qw( blocks used )});
    167             print "adding $mnt to db\n" if $debug;
     199            $log->debug("adding $mnt to db");
    168200
    169201        }
    … …  
    172204
    173205        $dbh->commit;
    174         print "commited to database\n" if $debug;
     206        $log->debug("commited to database");
    175207    };
    176208    if ($@) {
    177209        $dbh->rollback;
    178         print "rolledback transaction\n" if $debug;
    179         warn $@;
     210        $log->debug("rolledback transaction");
     211        $log->logdie($@);
    180212    }
    181213}
    … …  
    188220
    189221    if (!-f $rcfile) {
    190         open(my $fh, '>', $rcfile) or die "can't open file: $!";
    191         close($fh) or die "can't close file:$!";
     222        open(my $fh, '>', $rcfile) or $log->logdie("can't open file: $!");
     223        close($fh) or $log->logdie("can't close file:$!");
    192224    }
    193225
    … …  
    226258    };
    227259    if ($@) {
    228         $db->rollback;
    229         die $@;
     260        $dbh->rollback;
     261        $log->logdie($@);
    230262    }
    231263
    … …  
    242274    foreach my $path (@$mounts) {
    243275        if (stat File::Spec->canonpath($path)) {
    244             print "stated $path\n" if $debug;
     276            $log->debug("stated $path");
    245277        } else {
    246             warn "can not stat path: $path";
     278            $log->warn("can not stat path: $path");
    247279        }
    248280    }
    … …  
    263295  ### get the currently listed pid
    264296  if( ! open(_PID,$pid_file) ){
    265     die "Couldn't open existant pid_file \"$pid_file\" [$!]\n";
     297    $log->logdie("Couldn't open existant pid_file \"$pid_file\" [$!]");
    266298  }
    267299  my $_current_pid = <_PID>;
    268300  close _PID;
    269   my $current_pid = $_current_pid =~ /^(\d{1,10})/ ? $1 : die "Couldn't find pid in existing pid_file";
     301  my $current_pid = $_current_pid =~ /^(\d{1,10})/ ? $1 : $log->logdie("Couldn't find pid in existing pid_file");
    270302
    271303  my $exists = undef;
    … …  
    289321
    290322        if( $current_pid == $$ ){
    291             warn "Pid_file created by this same process. Doing nothing.\n";
     323            $log->warn("Pid_file created by this same process. Doing nothing.");
    292324            return 1;
    293325        }else{
    294326            kill 'TERM', $current_pid
    295                 or die "Failed to signal process ($current_pid)";
    296             unlink $pid_file || die "Couldn't remove pid_file \"$pid_file\" [$!]\n";
     327                or $log->logdie("Failed to signal process ($current_pid)");
     328            unlink $pid_file || $log->logdie("Couldn't remove pid_file \"$pid_file\" [$!]");
    297329        }
    298330    }

  • branches/neb_distrib_20081210/Nebulous-Server/docs/install.txt

    r23537 r23708  
    5656
    5757    neb-initdb
    58     neb-addvol --name ipp000.0 --uri file:///data/ipp000.0/nebulous
    59     neb-addvol --name ipp000.1 --uri file:///data/ipp000.1/nebulous
    60     neb-addvol --name ipp002.0 --uri file:///data/ipp002.0/nebulous
    61     neb-addvol --name ipp003.0 --uri file:///data/ipp003.0/nebulous
    62 
    63 #    perl -e 'for (1..24) { $i = sprintf "po%02d", $_ ; system "neb-addvol --name $i --uri file:/$i/nebulous" }'
     58
     59    neb-voladd --vhost ipp000 --vname ipp000.0 --uri file:///data/ipp000.0/nebulous
     60    neb-voladd --vhost ipp001 --vname ipp000.1 --uri file:///data/ipp000.1/nebulous
     61    neb-voladd --vhost ipp002 --vname ipp002.0 --uri file:///data/ipp002.0/nebulous
     62    neb-voladd --vhost ipp003 --vname ipp003.0 --uri file:///data/ipp003.0/nebulous
     63    neb-voladd --vhost ipp022 --vname ipp022.0 --uri file:///data/ipp022.0/nebulous
     64
     65    # or in bulk:
     66   
     67    for i in ipp005 ipp006 ipp007 ipp009 ipp010 ipp011 ipp012 ipp013 ipp014 ipp015 ipp016 ipp017 ipp018 ipp020 ipp021 ipp022 ipp023 ipp024 ipp025 ipp026 ipp027 ipp028 ipp029 ipp030 ipp031 ipp032 ipp033 ipp034 ipp035
     68    do
     69        /usr/bin/neb-voladd --vname ${i}.0 --vhost $i --uri file:///data/${i}.0/nebulous_beta
     70    done
     71
     72#    perl -e 'for (1..24) { $i = sprintf "po%02d", $_ ; system "neb-voladd
     73#    --vname $i --vhost $i --uri file:/$i/nebulous" }'
    6474    .
    6575    .

  • branches/neb_distrib_20081210/Nebulous-Server/lib/Nebulous/Server/SQL.pm

    r23537 r23708  
    4747        (so_id, vol_id, uri)
    4848        VALUES (?, ?, 'error')
    49     },
    50     get_all_instances   => qq{
    51         SELECT * FROM INSTANCE
    52         WHERE so_id = ?
    5349    },
    5450    get_object          => qq{
    … …  
    6359        JOIN storage_object_attr
    6460        USING (so_id)
     61        WHERE ext_id = ?
     62    },
     63    check_object_name => qq{
     64        SELECT
     65            so_id,
     66            ext_id
     67        FROM storage_object
    6568        WHERE ext_id = ?
    6669    },
    … …  
    351354        limit 5;
    352355    },
     356    find_objects_with_extra_instances => qq{
     357        SELECT
     358            storage_object.so_id,
     359            ext_id,
     360            count(ins_id) as instances,
     361            volume.name as volume_name,
     362            volume.host as volume_host,
     363            count(mymountedvol.vol_id) as available_instances,
     364            count(mymountedvol.vol_id) > 0 as recoverable,
     365            storage_object_xattr.value as copies
     366        FROM storage_object
     367        JOIN instance
     368            USING(so_id)
     369        JOIN volume
     370            USING(vol_id)
     371        LEFT JOIN storage_object_xattr
     372            ON storage_object.so_id = storage_object_xattr.so_id
     373        JOIN mymountedvol
     374            USING(vol_id)
     375        WHERE
     376            mymountedvol.available = 1
     377            AND storage_object_xattr.name = 'user.copies'
     378        GROUP BY so_id
     379        HAVING available_instances > copies
     380    },
    353381    get_mounted_volumes => qq{
    354382        SELECT * FROM mountedvol ORDER BY host, name
    … …  
    408436    type enum('REG_FILE'),
    409437    PRIMARY KEY(so_id),
    410     KEY(ext_id(64)),
    411438    KEY(type)
    412439) ENGINE=innodb DEFAULT CHARSET=latin1;
    … …  
    443470    type ENUM( 'read', 'write' ) NOT NULL,
    444471    epoch TIMESTAMP,
    445     KEY(so_ID)
     472    KEY(so_id)
    446473) ENGINE=innodb DEFAULT CHARSET=latin1;
    447474
    … …  
    457484    xattr BOOLEAN DEFAULT FALSE,
    458485    PRIMARY KEY(vol_id),
    459     KEY(name(16)),
    460486    KEY(host(16)),
    461487    KEY(path(255)),
    … …  
    472498    vol_id INT NOT NULL,
    473499    FOREIGN KEY(vol_id) REFERENCES volume(vol_id),
    474     uri VARCHAR(255) NOT NULL UNIQUE,
     500    uri VARCHAR(255) NOT NULL,
    475501    epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    476502    mtime TIMESTAMP,
    … …  
    478504    KEY(so_id),
    479505    KEY(vol_id),
    480     KEY(uri(64))
     506    KEY(uri(40))
    481507) ENGINE=innodb DEFAULT CHARSET=latin1;
    482508
    … …  
    517543    available BOOLEAN DEFAULT FALSE,
    518544    xattr BOOLEAN DEFAULT FALSE,
     545    PRIMARY KEY(mountpoint),
    519546    KEY(vol_id),
     547    KEY(name),
     548    KEY(host),
     549    KEY(path),
    520550    KEY(allocate),
    521     KEY(available)
     551    KEY(available),
     552    KEY(xattr)
    522553) ENGINE=innodb DEFAULT CHARSET=latin1;
    523554
Note: See TracChangeset for help on using the changeset viewer.