Make file uploads use data chunks

Instead of slurps. Thanks to @keedi for the original
patch.

Changes

@@ -1,5 +1,9 @@
 Revision history for Net-API-Gett
 
+1.04    2012-02-01
+        Reworked a patch from @keedi to chunk file uploads dynamically. This cuts 
+        RAM usage substantially.
+
 1.03    2012-01-10
         Patch from @keedi to fix the getturl attribute on file objects.
         Set MooX::Types::MooseLike to be version 0.02 (unreleased version 0.03 breaks API)

MANIFEST

@@ -13,6 +13,7 @@ t/00-load.t
 t/01-basic.t
 t/02-use_auth.t
 t/03-many_file_share.t
+t/04-chunked_upload.t
 t/manifest.t
 t/pod.t
 Makefile.PL

README

@@ -5,7 +5,7 @@ share up to 2 GB of files for free.  They recently implemented a REST API; this
 binding for the API. See http://ge.tt/developers for full details and how to get an
 API key.
 
-Version: 1.03
+Version: 1.04
 
 INSTALLATION
 

lib/Net/API/Gett.pm

@@ -24,11 +24,11 @@ Net::API::Gett - Perl bindings for Ge.tt API
 
 =head1 VERSION
 
-Version 1.03
+Version 1.04
 
 =cut
 
-our $VERSION = '1.03';
+our $VERSION = '1.04';
 
 =head1 SYNOPSIS
 
@@ -359,6 +359,10 @@ C<contents> as a buffer and uploads that data.
 An encoding scheme for the file content. By default it uses C<:raw>. See C<perldoc -f binmode> 
 for more information about encodings.
 
+=item * chunk_size
+
+The chunk_size in bytes to use when uploading a file.  Defaults to 1 MB.
+
 =back
 
 Returns a L<Net::API::Gett:File> object representing the uploaded file.
@@ -403,7 +407,8 @@ sub upload_file {
         if ( $file->readystate eq "remote" ) {
             my $put_upload_url = $file->put_upload_url;
             croak "Didn't get put upload URL from $endpoint" unless $put_upload_url;
-            if ( $file->send_file($put_upload_url, $opts->{'contents'}, $opts->{'encoding'}) ) {
+            if ( $file->send_file($put_upload_url, $opts->{'contents'}, 
+                        $opts->{'encoding'}, $opts->{'chunk_size'}) ) {
                 return $file;
             }
             else {

lib/Net/API/Gett/File.pm

@@ -7,12 +7,10 @@ Net::API::Gett::File - Gett file object
 =cut
 
 use Moo;
-use Sub::Quote;
 use Carp qw(croak);
-use File::Slurp qw(read_file);
 use MooX::Types::MooseLike qw(Int Str);
 
-our $VERSION = '1.03';
+our $VERSION = '1.04';
 
 =head1 PURPOSE
 
@@ -21,17 +19,17 @@ its own, as the library will create and return this object as appropriate.
 
 =head1 ATTRIBUTES
 
-These are read only attributes. 
+These are read only attributes unless otherwise noted. 
 
 =over 
 
 =item filename
 
-Scalar string
+Scalar string.
 
 =item fileid
 
-Scalar string
+Scalar string.
 
 =item downloads
 
@@ -73,6 +71,11 @@ attribute is only populated during certain times.)
 Scalar string. This url to use to upload the contents of this file using the POST method. (This
 attribute is only populated during certain times.)
 
+=item chunk_size
+
+Scalar integer. This is the chunk size to use for file uploads. It defaults to 
+1,048,576 bytes (1 MB).  This attribute is read-only.
+
 =back
 
 =cut
@@ -132,6 +135,12 @@ has 'post_upload_url' => (
     isa => Str,
 );
 
+has 'chunk_size' => (
+    is => 'rw',
+    isa => Int,
+    default => sub { 1_048_576; },
+);
+
 =over
 
 =item user
@@ -182,13 +191,18 @@ the following parameters:
 
 =item * data
 
-a scalar representing the file contents which can be one of: a buffer, an L<IO::Handle> object, a FILEGLOB, or a 
+a scalar representing the file contents which can be one of: a buffer, an L<IO::Handle> object, or a 
 file pathname.
 
 =item * encoding
 
 an encoding scheme. By default, it uses C<:raw>.
 
+=item * chunk_size
+
+The maximum chunksize to load into to memory at one time.  If the file to transmit is larger than
+this size, it will be dynamically streamed.
+
 =back
 
 Returns a true value on success.
@@ -202,25 +216,36 @@ sub send_file {
     my $url = shift;
     my $contents = shift;
     my $encoding = shift || ":raw";
+    my $chunk_size = shift || $self->chunk_size || 1_048_576; # 1024 * 1024 = 1 MB
 
-    my $data;
+    my $fh;
+    my $length;
     if ( ! ref($contents) ) {
         # $contents is scalar
         if ( ! -e $contents ) {
             # $contents doesn't point to a valid filename, 
             # assume it's a buffer
 
+            $contents .= "";
             # Make sure this data is stringified.
-            $data = $contents . "";
+            open($fh, "<", \$contents) or croak "Couldn't open a filehandle on the content buffer\n";
+            binmode($fh, $encoding);
+            $length = length($contents);
+        }
+        else {
+            open($fh, "<", $contents) or croak "Couldn't open a filehandle on $contents: $!";
+            binmode($fh, $encoding);
+            $length = -s $contents;
         }
     }
-
-    unless ( $data ) {
-        $data = read_file($contents, { binmode => $encoding });
+    else {
+       $fh = $contents if ref($contents) =~ /IO/;
+       $length = -s $fh;
     }
-    return 0 unless $data;
 
-    my $response = $self->request->put($url, $data);
+    return 0 unless $fh;
+
+    my $response = $self->request->put($url, $fh, $chunk_size, $length);
 
     if ( $response ) {
         return 1;

lib/Net/API/Gett/Request.pm

@@ -6,8 +6,9 @@ use Carp qw(croak);
 use JSON;
 use LWP::UserAgent;
 use HTTP::Request::Common;
+use HTTP::Headers;
 
-our $VERSION = '1.03';
+our $VERSION = '1.04';
 
 =head1 NAME
 
@@ -189,7 +190,11 @@ Input:
 
 =item * Full endpoint
 
-=item * Data
+=item * Data filehandle
+
+=item * A chunksize
+
+=item * the length of the data in bytes
 
 =back
 
@@ -212,10 +217,29 @@ This method will die under any error condition.
 sub put {
     my $self = shift;
     my $url = shift;
-    my $data = shift;
+    my $fh = shift;
+    my $chunk_size = shift;
+    my $length = shift;
+
+    local $HTTP::Request::Common::DYNAMIC_FILE_UPLOAD = 1;
 
-    my $response = $self->ua->request(PUT $url, Content => $data);
+    my $header = HTTP::Headers->new;
+    $header->content_length($length);
+
+    my $req = HTTP::Request->new(
+        'PUT',
+        $url,
+        $header,
+        sub {
+            my $ret = read($fh, my $chunk, $chunk_size);
+            return $ret ? $chunk : ();
+        },
+    );
+
+    my $response = $self->ua->request($req);
 
+    close $fh;
+
     if ( $response->is_success ) {
         return 1;
     }

lib/Net/API/Gett/Share.pm

@@ -11,7 +11,7 @@ use Carp qw(croak);
 use Array::Iterator;
 use MooX::Types::MooseLike qw(Int Str);
 
-our $VERSION = '1.02';
+our $VERSION = '1.04';
 
 =head1 PURPOSE
 

lib/Net/API/Gett/User.pm

@@ -6,7 +6,7 @@ use MooX::Types::MooseLike qw(Str Int);
 
 use Net::API::Gett::Request;
 
-our $VERSION = '1.03';
+our $VERSION = '1.04';
 
 =head1 NAME