Permalink
Browse files

Make file uploads use data chunks

Instead of slurps. Thanks to @keedi for the original
patch.
  • Loading branch information...
1 parent 88c32f1 commit 5aef136acb7ae2c6c8b14491890b01ac8c7023aa @mrallen1 committed Feb 2, 2012
Showing with 360 additions and 24 deletions.
  1. +4 −0 Changes
  2. +1 −0 MANIFEST
  3. +1 −1 README
  4. +8 −3 lib/Net/API/Gett.pm
  5. +39 −14 lib/Net/API/Gett/File.pm
  6. +28 −4 lib/Net/API/Gett/Request.pm
  7. +1 −1 lib/Net/API/Gett/Share.pm
  8. +1 −1 lib/Net/API/Gett/User.pm
  9. +277 −0 t/04-chunked_upload.t
View
@@ -1,5 +1,9 @@
Revision history for Net-API-Gett
+1.04 2012-02-01
+ Reworked a patch from @keedi to chunk file uploads dynamically. This cuts
+ RAM usage substantially.
+
1.03 2012-01-10
Patch from @keedi to fix the getturl attribute on file objects.
Set MooX::Types::MooseLike to be version 0.02 (unreleased version 0.03 breaks API)
View
@@ -13,6 +13,7 @@ t/00-load.t
t/01-basic.t
t/02-use_auth.t
t/03-many_file_share.t
+t/04-chunked_upload.t
t/manifest.t
t/pod.t
Makefile.PL
View
2 README
@@ -5,7 +5,7 @@ share up to 2 GB of files for free. They recently implemented a REST API; this
binding for the API. See http://ge.tt/developers for full details and how to get an
API key.
-Version: 1.03
+Version: 1.04
INSTALLATION
View
@@ -24,11 +24,11 @@ Net::API::Gett - Perl bindings for Ge.tt API
=head1 VERSION
-Version 1.03
+Version 1.04
=cut
-our $VERSION = '1.03';
+our $VERSION = '1.04';
=head1 SYNOPSIS
@@ -359,6 +359,10 @@ C<contents> as a buffer and uploads that data.
An encoding scheme for the file content. By default it uses C<:raw>. See C<perldoc -f binmode>
for more information about encodings.
+=item * chunk_size
+
+The chunk_size in bytes to use when uploading a file. Defaults to 1 MB.
+
=back
Returns a L<Net::API::Gett:File> object representing the uploaded file.
@@ -403,7 +407,8 @@ sub upload_file {
if ( $file->readystate eq "remote" ) {
my $put_upload_url = $file->put_upload_url;
croak "Didn't get put upload URL from $endpoint" unless $put_upload_url;
- if ( $file->send_file($put_upload_url, $opts->{'contents'}, $opts->{'encoding'}) ) {
+ if ( $file->send_file($put_upload_url, $opts->{'contents'},
+ $opts->{'encoding'}, $opts->{'chunk_size'}) ) {
return $file;
}
else {
View
@@ -7,12 +7,10 @@ Net::API::Gett::File - Gett file object
=cut
use Moo;
-use Sub::Quote;
use Carp qw(croak);
-use File::Slurp qw(read_file);
use MooX::Types::MooseLike qw(Int Str);
-our $VERSION = '1.03';
+our $VERSION = '1.04';
=head1 PURPOSE
@@ -21,17 +19,17 @@ its own, as the library will create and return this object as appropriate.
=head1 ATTRIBUTES
-These are read only attributes.
+These are read only attributes unless otherwise noted.
=over
=item filename
-Scalar string
+Scalar string.
=item fileid
-Scalar string
+Scalar string.
=item downloads
@@ -73,6 +71,11 @@ attribute is only populated during certain times.)
Scalar string. This url to use to upload the contents of this file using the POST method. (This
attribute is only populated during certain times.)
+=item chunk_size
+
+Scalar integer. This is the chunk size to use for file uploads. It defaults to
+1,048,576 bytes (1 MB). This attribute is read-only.
+
=back
=cut
@@ -132,6 +135,12 @@ has 'post_upload_url' => (
isa => Str,
);
+has 'chunk_size' => (
+ is => 'rw',
+ isa => Int,
+ default => sub { 1_048_576; },
+);
+
=over
=item user
@@ -182,13 +191,18 @@ the following parameters:
=item * data
-a scalar representing the file contents which can be one of: a buffer, an L<IO::Handle> object, a FILEGLOB, or a
+a scalar representing the file contents which can be one of: a buffer, an L<IO::Handle> object, or a
file pathname.
=item * encoding
an encoding scheme. By default, it uses C<:raw>.
+=item * chunk_size
+
+The maximum chunksize to load into to memory at one time. If the file to transmit is larger than
+this size, it will be dynamically streamed.
+
=back
Returns a true value on success.
@@ -202,25 +216,36 @@ sub send_file {
my $url = shift;
my $contents = shift;
my $encoding = shift || ":raw";
+ my $chunk_size = shift || $self->chunk_size || 1_048_576; # 1024 * 1024 = 1 MB
- my $data;
+ my $fh;
+ my $length;
if ( ! ref($contents) ) {
# $contents is scalar
if ( ! -e $contents ) {
# $contents doesn't point to a valid filename,
# assume it's a buffer
+ $contents .= "";
# Make sure this data is stringified.
- $data = $contents . "";
+ open($fh, "<", \$contents) or croak "Couldn't open a filehandle on the content buffer\n";
+ binmode($fh, $encoding);
+ $length = length($contents);
+ }
+ else {
+ open($fh, "<", $contents) or croak "Couldn't open a filehandle on $contents: $!";
+ binmode($fh, $encoding);
+ $length = -s $contents;
}
}
-
- unless ( $data ) {
- $data = read_file($contents, { binmode => $encoding });
+ else {
+ $fh = $contents if ref($contents) =~ /IO/;
+ $length = -s $fh;
}
- return 0 unless $data;
- my $response = $self->request->put($url, $data);
+ return 0 unless $fh;
+
+ my $response = $self->request->put($url, $fh, $chunk_size, $length);
if ( $response ) {
return 1;
@@ -6,8 +6,9 @@ use Carp qw(croak);
use JSON;
use LWP::UserAgent;
use HTTP::Request::Common;
+use HTTP::Headers;
-our $VERSION = '1.03';
+our $VERSION = '1.04';
=head1 NAME
@@ -189,7 +190,11 @@ Input:
=item * Full endpoint
-=item * Data
+=item * Data filehandle
+
+=item * A chunksize
+
+=item * the length of the data in bytes
=back
@@ -212,10 +217,29 @@ This method will die under any error condition.
sub put {
my $self = shift;
my $url = shift;
- my $data = shift;
+ my $fh = shift;
+ my $chunk_size = shift;
+ my $length = shift;
+
+ local $HTTP::Request::Common::DYNAMIC_FILE_UPLOAD = 1;
- my $response = $self->ua->request(PUT $url, Content => $data);
+ my $header = HTTP::Headers->new;
+ $header->content_length($length);
+
+ my $req = HTTP::Request->new(
+ 'PUT',
+ $url,
+ $header,
+ sub {
+ my $ret = read($fh, my $chunk, $chunk_size);
+ return $ret ? $chunk : ();
+ },
+ );
+
+ my $response = $self->ua->request($req);
+ close $fh;
+
if ( $response->is_success ) {
return 1;
}
@@ -11,7 +11,7 @@ use Carp qw(croak);
use Array::Iterator;
use MooX::Types::MooseLike qw(Int Str);
-our $VERSION = '1.02';
+our $VERSION = '1.04';
=head1 PURPOSE
View
@@ -6,7 +6,7 @@ use MooX::Types::MooseLike qw(Str Int);
use Net::API::Gett::Request;
-our $VERSION = '1.03';
+our $VERSION = '1.04';
=head1 NAME
Oops, something went wrong.

0 comments on commit 5aef136

Please sign in to comment.