Index: /trunk/DataStore/Changes =================================================================== --- /trunk/DataStore/Changes (revision 17636) +++ /trunk/DataStore/Changes (revision 17637) @@ -2,7 +2,9 @@ 0.07 + - add --timeout option to all ds*ls utilties and other minor doc changes + - add the ability to all DataStore::* classes that handle HTTP to pass + options directly to LWP::UserAgent - add support for checksumming compressed fits files - Change DateStore::File to handle passing args to LWP::UserAgent - - add dsget --timeout option - change dsget to return the HTTP status code on failure - remove use of parse_neb_key() from dsget Index: /trunk/DataStore/lib/DataStore/FileSet.pm =================================================================== --- /trunk/DataStore/lib/DataStore/FileSet.pm (revision 17636) +++ /trunk/DataStore/lib/DataStore/FileSet.pm (revision 17637) @@ -1,5 +1,5 @@ -# Copyright (C) 2006 Joshua Hoblitt +# Copyright (C) 2006-2008 Joshua Hoblitt # -# $Id: FileSet.pm,v 1.14 2007-09-25 23:52:46 jhoblitt Exp $ +# $Id: FileSet.pm,v 1.15 2008-05-12 22:04:53 jhoblitt Exp $ package DataStore::FileSet; @@ -9,5 +9,5 @@ use vars qw($VERSION); -$VERSION = '0.03'; +$VERSION = '0.04'; use base qw( DataStore::Record ); @@ -186,9 +186,19 @@ my $self = shift; - # no params - validate(@_, {}); + my %p = validate(@_, + { + ua_args => { + optional => 1, + }, + }, + ); # make request - my $ua = LWP::UserAgent->new; + my $ua; + if (defined $p{ua_args}) { + $ua = LWP::UserAgent->new(%{$p{ua_args}}); + } else { + $ua = LWP::UserAgent->new; + } my $request = HTTP::Request->new(GET => $self->uri); my $response = $ua->request($request); Index: /trunk/DataStore/lib/DataStore/Product.pm =================================================================== --- /trunk/DataStore/lib/DataStore/Product.pm (revision 17636) +++ /trunk/DataStore/lib/DataStore/Product.pm (revision 17637) @@ -1,5 +1,5 @@ -# Copyright (C) 2006 Joshua Hoblitt +# Copyright (C) 2006-2008 Joshua Hoblitt # -# $Id: Product.pm,v 1.10 2007-09-25 23:50:34 jhoblitt Exp $ +# $Id: Product.pm,v 1.11 2008-05-12 22:04:53 jhoblitt Exp $ package DataStore::Product; @@ -9,5 +9,5 @@ use vars qw($VERSION); -$VERSION = '0.01'; +$VERSION = '0.02'; use base qw( DataStore::Record ); @@ -167,9 +167,20 @@ my $self = shift; - # no params - validate(@_, {}); + my %p = validate(@_, + { + ua_args => { + optional => 1, + }, + }, + ); # make request - my $ua = LWP::UserAgent->new; + my $ua; + if (defined $p{ua_args}) { + $ua = LWP::UserAgent->new(%{$p{ua_args}}); + } else { + $ua = LWP::UserAgent->new; + } + my $request; if ($self->last_fileset) { Index: /trunk/DataStore/lib/DataStore/Root.pm =================================================================== --- /trunk/DataStore/lib/DataStore/Root.pm (revision 17636) +++ /trunk/DataStore/lib/DataStore/Root.pm (revision 17637) @@ -1,7 +1,7 @@ # Adapted from Product.pm # -# Copyright (C) 2006 Joshua Hoblitt +# Copyright (C) 2006-2008 Joshua Hoblitt # -# $Id: Root.pm,v 1.4 2007-09-26 00:13:29 jhoblitt Exp $ +# $Id: Root.pm,v 1.5 2008-05-12 22:04:53 jhoblitt Exp $ package DataStore::Root; @@ -11,5 +11,5 @@ use vars qw($VERSION); -$VERSION = '0.01'; +$VERSION = '0.02'; use base qw( DataStore::Record ); @@ -131,9 +131,20 @@ my $self = shift; - # no params - validate(@_, {}); + my %p = validate(@_, + { + ua_args => { + optional => 1, + }, + }, + ); # make request - my $ua = LWP::UserAgent->new; + my $ua; + if (defined $p{ua_args}) { + $ua = LWP::UserAgent->new(%{$p{ua_args}}); + } else { + $ua = LWP::UserAgent->new; + } + my $request = HTTP::Request->new(GET => $self->uri); my $response = $ua->request($request); Index: /trunk/DataStore/scripts/dsfilesetls =================================================================== --- /trunk/DataStore/scripts/dsfilesetls (revision 17636) +++ /trunk/DataStore/scripts/dsfilesetls (revision 17637) @@ -1,7 +1,7 @@ #!/usr/bin/env perl -# Copyright (C) 2006 Joshua Hoblitt +# Copyright (C) 2006-2008 Joshua Hoblitt # -# $Id: dsfilesetls,v 1.6 2008-04-18 23:33:13 jhoblitt Exp $ +# $Id: dsfilesetls,v 1.7 2008-05-12 22:04:53 jhoblitt Exp $ use strict; @@ -16,8 +16,9 @@ use Pod::Usage qw( pod2usage ); -my ($uri); +my ($uri, $timeout); GetOptions( 'uri|u=s' => \$uri, + 'timeout|t' => \$timeout, ) or pod2usage( 2 ); @@ -28,5 +29,10 @@ ) unless defined $uri; -my $response = DataStore::FileSet->new( uri => $uri )->request; +# default http request timeout is 30s +$timeout ||= 30; + +my $response = DataStore::FileSet->new( uri => $uri )->request( + ua_args => { timeout => $timeout }, + ); die "request failed" unless defined $response; @@ -76,4 +82,11 @@ The URI of the file to be retrieved. +=item * --timeout|-t + +The ammount of time (in seconds) to wait for a response from the DataStore +after making an HTTP request. The default is 30s. + +Optional. + =back @@ -92,5 +105,5 @@ =head1 COPYRIGHT -Copyright (C) 2006 Joshua Hoblitt. All rights reserved. +Copyright (C) 2006-2008 Joshua Hoblitt. All rights reserved. This program is free software; you can redistribute it and/or modify it under @@ -112,5 +125,5 @@ =head1 SEE ALSO -L, L, L +L, L, L, L, L, L =cut Index: /trunk/DataStore/scripts/dsget =================================================================== --- /trunk/DataStore/scripts/dsget (revision 17636) +++ /trunk/DataStore/scripts/dsget (revision 17637) @@ -3,5 +3,5 @@ # Copyright (C) 2006-2008 Joshua Hoblitt # -# $Id: dsget,v 1.23 2008-05-07 03:05:03 jhoblitt Exp $ +# $Id: dsget,v 1.24 2008-05-12 22:04:53 jhoblitt Exp $ use strict; @@ -126,7 +126,7 @@ # request the file from the remote data store server (save in tmpfilename) my $response = DataStore::File->new(%p)->request( - filename => $tmpfilename, - ua_args => { timeout => $timeout}, -); + filename => $tmpfilename, + ua_args => { timeout => $timeout }, + ); die "request failed" unless defined $response; @@ -282,5 +282,5 @@ =head1 SEE ALSO -L, L +L, L, L, L, L, L =cut Index: /trunk/DataStore/scripts/dsleech =================================================================== --- /trunk/DataStore/scripts/dsleech (revision 17636) +++ /trunk/DataStore/scripts/dsleech (revision 17637) @@ -1,7 +1,7 @@ #!/usr/bin/env perl -# Copyright (C) 2006 Joshua Hoblitt +# Copyright (C) 2006-2008 Joshua Hoblitt # -# $Id: dsleech,v 1.9 2008-02-22 01:50:43 jhoblitt Exp $ +# $Id: dsleech,v 1.10 2008-05-12 22:04:53 jhoblitt Exp $ use strict; @@ -18,5 +18,5 @@ use Pod::Usage qw( pod2usage ); -my ($dir, $uri, $last_fileset, $overwrite, $recall, $remember, $verbose); +my ($dir, $uri, $last_fileset, $overwrite, $recall, $remember, $verbose, $timeout); GetOptions( @@ -28,4 +28,5 @@ 'remember' => \$remember, 'verbose|v' => \$verbose, + 'timeout|t' => \$timeout, ) or pod2usage( 2 ); @@ -35,4 +36,7 @@ -exitval => 3, ) unless defined $uri; + +# default http request timeout is 30s +$timeout ||= 30; my %p = ( @@ -59,5 +63,8 @@ } -my $response = DataStore::Product->new(%p)->request; +my $response = DataStore::Product->new(%p)->request( + ua_args => { timeout => $timeout }, +); + unless (defined $response->is_success) { die "request failed: ", $response->status_line; @@ -96,5 +103,7 @@ print "processing fileset: $fileset->fileset $n/@$filesets\n" if $verbose; - my $response = $fileset->request; + my $response = $fileset->request( + ua_args => { timeout => $timeout }, + ); unless (defined $response->is_success) { warn "request failed: ", $response->status_line; @@ -155,5 +164,8 @@ print "fetching ", $file->fileid, "..." if $verbose; - my $response = $file->request( filename => $filename ); + my $response = $file->request( + filename => $filename, + ua_args => { timeout => $timeout }, + ); unless (defined $response->is_success) { warn "request failed: ", $response->status_line; @@ -261,4 +273,11 @@ This flag is optional. +=item * --timeout|-t + +The ammount of time (in seconds) to wait for a response from the DataStore +after making an HTTP request. The default is 30s. + +Optional. + =back @@ -277,5 +296,5 @@ =head1 COPYRIGHT -Copyright (C) 2006 Joshua Hoblitt. All rights reserved. +Copyright (C) 2006-2008 Joshua Hoblitt. All rights reserved. This program is free software; you can redistribute it and/or modify it under @@ -297,5 +316,5 @@ =head1 SEE ALSO -L, L, L, L +L, L, L, L, L, L =cut Index: /trunk/DataStore/scripts/dsproductls =================================================================== --- /trunk/DataStore/scripts/dsproductls (revision 17636) +++ /trunk/DataStore/scripts/dsproductls (revision 17637) @@ -1,7 +1,7 @@ #!/usr/bin/env perl -# Copyright (C) 2006 Joshua Hoblitt +# Copyright (C) 2006-2008 Joshua Hoblitt # -# $Id: dsproductls,v 1.3 2006-08-30 22:45:53 jhoblitt Exp $ +# $Id: dsproductls,v 1.4 2008-05-12 22:04:53 jhoblitt Exp $ use strict; @@ -16,9 +16,10 @@ use Pod::Usage qw( pod2usage ); -my ($uri, $last_fileset); +my ($uri, $last_fileset, $timeout); GetOptions( 'uri|u=s' => \$uri, 'last_fileset|l=s' => \$last_fileset, + 'timeout|t' => \$timeout, ) or pod2usage( 2 ); @@ -29,4 +30,7 @@ ) unless defined $uri; +# default http request timeout is 30s +$timeout ||= 30; + my %p = ( uri => $uri, @@ -35,5 +39,7 @@ $p{last_fileset} = $last_fileset if defined $last_fileset; -my $response = DataStore::Product->new(%p)->request; +my $response = DataStore::Product->new(%p)->request( + ua_args => { timeout => $timeout }, + ); die "request failed" unless defined $response; @@ -82,4 +88,11 @@ This flag is optional. +=item * --timeout|-t + +The ammount of time (in seconds) to wait for a response from the DataStore +after making an HTTP request. The default is 30s. + +Optional. + =back @@ -98,5 +111,5 @@ =head1 COPYRIGHT -Copyright (C) 2006 Joshua Hoblitt. All rights reserved. +Copyright (C) 2006-2008 Joshua Hoblitt. All rights reserved. This program is free software; you can redistribute it and/or modify it under @@ -118,5 +131,5 @@ =head1 SEE ALSO -L, L, L +L, L, L, L, L, L =cut Index: /trunk/DataStore/scripts/dsrootls =================================================================== --- /trunk/DataStore/scripts/dsrootls (revision 17636) +++ /trunk/DataStore/scripts/dsrootls (revision 17637) @@ -1,7 +1,7 @@ #!/usr/bin/env perl -# Copyright (C) 2006-2007 Joshua Hoblitt +# Copyright (C) 2006-2008 Joshua Hoblitt # -# $Id: dsrootls,v 1.1 2007-09-18 04:15:09 jhoblitt Exp $ +# $Id: dsrootls,v 1.2 2008-05-12 22:04:53 jhoblitt Exp $ use strict; @@ -16,8 +16,9 @@ use Pod::Usage qw( pod2usage ); -my ($uri); +my ($uri, $timeout); GetOptions( - 'uri|u=s' => \$uri, + 'uri|u=s' => \$uri, + 'timeout|t' => \$timeout, ) or pod2usage( 2 ); @@ -28,9 +29,14 @@ ) unless defined $uri; +# default http request timeout is 30s +$timeout ||= 30; + my %p = ( uri => $uri, ); -my $response = DataStore::Root->new(%p)->request; +my $response = DataStore::Root->new(%p)->request( + ua_args => { timeout => $timeout }, + ); die "request failed" unless defined $response; @@ -78,4 +84,11 @@ The URI of the file to be retrieved. +=item * --timeout|-t + +The ammount of time (in seconds) to wait for a response from the DataStore +after making an HTTP request. The default is 30s. + +Optional. + =back @@ -94,5 +107,5 @@ =head1 COPYRIGHT -Copyright (C) 2006-2007 Joshua Hoblitt. All rights reserved. +Copyright (C) 2006-2008 Joshua Hoblitt. All rights reserved. This program is free software; you can redistribute it and/or modify it under @@ -114,5 +127,5 @@ =head1 SEE ALSO -L, L, L +L, L, L, L, L, L =cut