From 5aef136acb7ae2c6c8b14491890b01ac8c7023aa Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Wed, 1 Feb 2012 22:56:59 -0600 Subject: [PATCH] Make file uploads use data chunks Instead of slurps. Thanks to @keedi for the original patch. --- Changes | 4 + MANIFEST | 1 + README | 2 +- lib/Net/API/Gett.pm | 11 +- lib/Net/API/Gett/File.pm | 53 +++++-- lib/Net/API/Gett/Request.pm | 32 ++++- lib/Net/API/Gett/Share.pm | 2 +- lib/Net/API/Gett/User.pm | 2 +- t/04-chunked_upload.t | 277 ++++++++++++++++++++++++++++++++++++ 9 files changed, 360 insertions(+), 24 deletions(-) create mode 100644 t/04-chunked_upload.t diff --git a/Changes b/Changes index 2d24375..2137dc2 100644 --- a/Changes +++ b/Changes @@ -1,5 +1,9 @@ Revision history for Net-API-Gett +1.04 2012-02-01 + Reworked a patch from @keedi to chunk file uploads dynamically. This cuts + RAM usage substantially. + 1.03 2012-01-10 Patch from @keedi to fix the getturl attribute on file objects. Set MooX::Types::MooseLike to be version 0.02 (unreleased version 0.03 breaks API) diff --git a/MANIFEST b/MANIFEST index 708d630..281b84e 100644 --- a/MANIFEST +++ b/MANIFEST @@ -13,6 +13,7 @@ t/00-load.t t/01-basic.t t/02-use_auth.t t/03-many_file_share.t +t/04-chunked_upload.t t/manifest.t t/pod.t Makefile.PL diff --git a/README b/README index c981f3d..940495f 100644 --- a/README +++ b/README @@ -5,7 +5,7 @@ share up to 2 GB of files for free. They recently implemented a REST API; this binding for the API. See http://ge.tt/developers for full details and how to get an API key. -Version: 1.03 +Version: 1.04 INSTALLATION diff --git a/lib/Net/API/Gett.pm b/lib/Net/API/Gett.pm index 7afcab0..04d50f7 100644 --- a/lib/Net/API/Gett.pm +++ b/lib/Net/API/Gett.pm @@ -24,11 +24,11 @@ Net::API::Gett - Perl bindings for Ge.tt API =head1 VERSION -Version 1.03 +Version 1.04 =cut -our $VERSION = '1.03'; +our $VERSION = '1.04'; =head1 SYNOPSIS @@ -359,6 +359,10 @@ C as a buffer and uploads that data. An encoding scheme for the file content. By default it uses C<:raw>. See C for more information about encodings. +=item * chunk_size + +The chunk_size in bytes to use when uploading a file. Defaults to 1 MB. + =back Returns a L object representing the uploaded file. @@ -403,7 +407,8 @@ sub upload_file { if ( $file->readystate eq "remote" ) { my $put_upload_url = $file->put_upload_url; croak "Didn't get put upload URL from $endpoint" unless $put_upload_url; - if ( $file->send_file($put_upload_url, $opts->{'contents'}, $opts->{'encoding'}) ) { + if ( $file->send_file($put_upload_url, $opts->{'contents'}, + $opts->{'encoding'}, $opts->{'chunk_size'}) ) { return $file; } else { diff --git a/lib/Net/API/Gett/File.pm b/lib/Net/API/Gett/File.pm index 5965e95..de31609 100644 --- a/lib/Net/API/Gett/File.pm +++ b/lib/Net/API/Gett/File.pm @@ -7,12 +7,10 @@ Net::API::Gett::File - Gett file object =cut use Moo; -use Sub::Quote; use Carp qw(croak); -use File::Slurp qw(read_file); use MooX::Types::MooseLike qw(Int Str); -our $VERSION = '1.03'; +our $VERSION = '1.04'; =head1 PURPOSE @@ -21,17 +19,17 @@ its own, as the library will create and return this object as appropriate. =head1 ATTRIBUTES -These are read only attributes. +These are read only attributes unless otherwise noted. =over =item filename -Scalar string +Scalar string. =item fileid -Scalar string +Scalar string. =item downloads @@ -73,6 +71,11 @@ attribute is only populated during certain times.) Scalar string. This url to use to upload the contents of this file using the POST method. (This attribute is only populated during certain times.) +=item chunk_size + +Scalar integer. This is the chunk size to use for file uploads. It defaults to +1,048,576 bytes (1 MB). This attribute is read-only. + =back =cut @@ -132,6 +135,12 @@ has 'post_upload_url' => ( isa => Str, ); +has 'chunk_size' => ( + is => 'rw', + isa => Int, + default => sub { 1_048_576; }, +); + =over =item user @@ -182,13 +191,18 @@ the following parameters: =item * data -a scalar representing the file contents which can be one of: a buffer, an L object, a FILEGLOB, or a +a scalar representing the file contents which can be one of: a buffer, an L object, or a file pathname. =item * encoding an encoding scheme. By default, it uses C<:raw>. +=item * chunk_size + +The maximum chunksize to load into to memory at one time. If the file to transmit is larger than +this size, it will be dynamically streamed. + =back Returns a true value on success. @@ -202,25 +216,36 @@ sub send_file { my $url = shift; my $contents = shift; my $encoding = shift || ":raw"; + my $chunk_size = shift || $self->chunk_size || 1_048_576; # 1024 * 1024 = 1 MB - my $data; + my $fh; + my $length; if ( ! ref($contents) ) { # $contents is scalar if ( ! -e $contents ) { # $contents doesn't point to a valid filename, # assume it's a buffer + $contents .= ""; # Make sure this data is stringified. - $data = $contents . ""; + open($fh, "<", \$contents) or croak "Couldn't open a filehandle on the content buffer\n"; + binmode($fh, $encoding); + $length = length($contents); + } + else { + open($fh, "<", $contents) or croak "Couldn't open a filehandle on $contents: $!"; + binmode($fh, $encoding); + $length = -s $contents; } } - - unless ( $data ) { - $data = read_file($contents, { binmode => $encoding }); + else { + $fh = $contents if ref($contents) =~ /IO/; + $length = -s $fh; } - return 0 unless $data; - my $response = $self->request->put($url, $data); + return 0 unless $fh; + + my $response = $self->request->put($url, $fh, $chunk_size, $length); if ( $response ) { return 1; diff --git a/lib/Net/API/Gett/Request.pm b/lib/Net/API/Gett/Request.pm index 9d65e57..2430526 100644 --- a/lib/Net/API/Gett/Request.pm +++ b/lib/Net/API/Gett/Request.pm @@ -6,8 +6,9 @@ use Carp qw(croak); use JSON; use LWP::UserAgent; use HTTP::Request::Common; +use HTTP::Headers; -our $VERSION = '1.03'; +our $VERSION = '1.04'; =head1 NAME @@ -189,7 +190,11 @@ Input: =item * Full endpoint -=item * Data +=item * Data filehandle + +=item * A chunksize + +=item * the length of the data in bytes =back @@ -212,10 +217,29 @@ This method will die under any error condition. sub put { my $self = shift; my $url = shift; - my $data = shift; + my $fh = shift; + my $chunk_size = shift; + my $length = shift; + + local $HTTP::Request::Common::DYNAMIC_FILE_UPLOAD = 1; - my $response = $self->ua->request(PUT $url, Content => $data); + my $header = HTTP::Headers->new; + $header->content_length($length); + + my $req = HTTP::Request->new( + 'PUT', + $url, + $header, + sub { + my $ret = read($fh, my $chunk, $chunk_size); + return $ret ? $chunk : (); + }, + ); + + my $response = $self->ua->request($req); + close $fh; + if ( $response->is_success ) { return 1; } diff --git a/lib/Net/API/Gett/Share.pm b/lib/Net/API/Gett/Share.pm index e149213..9ddb0f8 100644 --- a/lib/Net/API/Gett/Share.pm +++ b/lib/Net/API/Gett/Share.pm @@ -11,7 +11,7 @@ use Carp qw(croak); use Array::Iterator; use MooX::Types::MooseLike qw(Int Str); -our $VERSION = '1.02'; +our $VERSION = '1.04'; =head1 PURPOSE diff --git a/lib/Net/API/Gett/User.pm b/lib/Net/API/Gett/User.pm index e49298c..4eaa102 100644 --- a/lib/Net/API/Gett/User.pm +++ b/lib/Net/API/Gett/User.pm @@ -6,7 +6,7 @@ use MooX::Types::MooseLike qw(Str Int); use Net::API::Gett::Request; -our $VERSION = '1.03'; +our $VERSION = '1.04'; =head1 NAME diff --git a/t/04-chunked_upload.t b/t/04-chunked_upload.t new file mode 100644 index 0000000..eb7301c --- /dev/null +++ b/t/04-chunked_upload.t @@ -0,0 +1,277 @@ +#!/usr/bin/perl + +use strict; +use Test::More; +use File::Temp; + +if (!eval { require Socket; Socket::inet_aton('open.ge.tt') }) { + plan skip_all => "Cannot connect to the API server"; +} +elsif ( ! $ENV{GETT_API_KEY} || ! $ENV{GETT_EMAIL} || ! $ENV{GETT_PASSWORD} ) { + plan skip_all => "API credentials required for these tests"; +} +else { + plan tests => 11; +} + +# untaint environment variables +# They will be validated for correctness in the User.pm module, so just match anything here. + +my @params = map {my ($v) = $ENV{uc "GETT_$_"} =~ /\A(.*)\z/; $_ => $v} qw(api_key email password); + +use Net::API::Gett; + +my $gett = Net::API::Gett->new( @params ); + +isa_ok($gett, 'Net::API::Gett', "Gett object constructed"); +isa_ok($gett->request, 'Net::API::Gett::Request', "Gett request constructed"); + +isa_ok($gett->user, 'Net::API::Gett::User', "Gett User object constructed"); +is($gett->user->has_access_token, '', "Has no access token"); + +$gett->user->login or die $!; + +is($gett->user->has_access_token, 1, "Has access token now"); + +my $tmp = File::Temp->new(); +open my $fh, ">", $tmp->filename; +print $fh ; +close $fh; + +# Upload a file, download its contents, then destroy the share and the file +my $file = $gett->upload_file( + filename => "queen_mab.txt", + contents => $tmp->filename, + title => "shakespeare", + chunk_size => 1024, +); + +isa_ok($file, 'Net::API::Gett::File', "File uploaded"); + +is($file->filename, "queen_mab.txt", "Got right filename"); + +my $content = $file->contents(); + +like($content, qr/Queen Mab/, "Got right file content"); + +my $share = $gett->get_share( $file->sharename ); + +is($share->title, "shakespeare", "Got right share title"); + +my $file1 = ($share->files)[0]; + +is($file1->size, -s $tmp->filename, "Got right filesize"); + +is($share->destroy(), 1, "Share destroyed"); + +__DATA__ +SCENE IV. A street. + + Enter ROMEO, MERCUTIO, BENVOLIO, with five or six Maskers, Torch-bearers, and others + +ROMEO + + What, shall this speech be spoke for our excuse? + Or shall we on without a apology? + +BENVOLIO + + The date is out of such prolixity: + We'll have no Cupid hoodwink'd with a scarf, + Bearing a Tartar's painted bow of lath, + Scaring the ladies like a crow-keeper; + Nor no without-book prologue, faintly spoke + After the prompter, for our entrance: + But let them measure us by what they will; + We'll measure them a measure, and be gone. + +ROMEO + + Give me a torch: I am not for this ambling; + Being but heavy, I will bear the light. + +MERCUTIO + + Nay, gentle Romeo, we must have you dance. + +ROMEO + + Not I, believe me: you have dancing shoes + With nimble soles: I have a soul of lead + So stakes me to the ground I cannot move. + +MERCUTIO + + You are a lover; borrow Cupid's wings, + And soar with them above a common bound. + +ROMEO + + I am too sore enpierced with his shaft + To soar with his light feathers, and so bound, + I cannot bound a pitch above dull woe: + Under love's heavy burden do I sink. + +MERCUTIO + + And, to sink in it, should you burden love; + Too great oppression for a tender thing. + +ROMEO + + Is love a tender thing? it is too rough, + Too rude, too boisterous, and it pricks like thorn. + +MERCUTIO + + If love be rough with you, be rough with love; + Prick love for pricking, and you beat love down. + Give me a case to put my visage in: + A visor for a visor! what care I + What curious eye doth quote deformities? + Here are the beetle brows shall blush for me. + +BENVOLIO + + Come, knock and enter; and no sooner in, + But every man betake him to his legs. + +ROMEO + + A torch for me: let wantons light of heart + Tickle the senseless rushes with their heels, + For I am proverb'd with a grandsire phrase; + I'll be a candle-holder, and look on. + The game was ne'er so fair, and I am done. + +MERCUTIO + + Tut, dun's the mouse, the constable's own word: + If thou art dun, we'll draw thee from the mire + Of this sir-reverence love, wherein thou stick'st + Up to the ears. Come, we burn daylight, ho! + +ROMEO + + Nay, that's not so. + +MERCUTIO + + I mean, sir, in delay + We waste our lights in vain, like lamps by day. + Take our good meaning, for our judgment sits + Five times in that ere once in our five wits. + +ROMEO + + And we mean well in going to this mask; + But 'tis no wit to go. + +MERCUTIO + + Why, may one ask? + +ROMEO + + I dream'd a dream to-night. + +MERCUTIO + + And so did I. + +ROMEO + + Well, what was yours? + +MERCUTIO + + That dreamers often lie. + +ROMEO + + In bed asleep, while they do dream things true. + +MERCUTIO + + O, then, I see Queen Mab hath been with you. + She is the fairies' midwife, and she comes + In shape no bigger than an agate-stone + On the fore-finger of an alderman, + Drawn with a team of little atomies + Athwart men's noses as they lie asleep; + Her wagon-spokes made of long spiders' legs, + The cover of the wings of grasshoppers, + The traces of the smallest spider's web, + The collars of the moonshine's watery beams, + Her whip of cricket's bone, the lash of film, + Her wagoner a small grey-coated gnat, + Not so big as a round little worm + Prick'd from the lazy finger of a maid; + Her chariot is an empty hazel-nut + Made by the joiner squirrel or old grub, + Time out o' mind the fairies' coachmakers. + And in this state she gallops night by night + Through lovers' brains, and then they dream of love; + O'er courtiers' knees, that dream on court'sies straight, + O'er lawyers' fingers, who straight dream on fees, + O'er ladies ' lips, who straight on kisses dream, + Which oft the angry Mab with blisters plagues, + Because their breaths with sweetmeats tainted are: + Sometime she gallops o'er a courtier's nose, + And then dreams he of smelling out a suit; + And sometime comes she with a tithe-pig's tail + Tickling a parson's nose as a' lies asleep, + Then dreams, he of another benefice: + Sometime she driveth o'er a soldier's neck, + And then dreams he of cutting foreign throats, + Of breaches, ambuscadoes, Spanish blades, + Of healths five-fathom deep; and then anon + Drums in his ear, at which he starts and wakes, + And being thus frighted swears a prayer or two + And sleeps again. This is that very Mab + That plats the manes of horses in the night, + And bakes the elflocks in foul sluttish hairs, + Which once untangled, much misfortune bodes: + This is the hag, when maids lie on their backs, + That presses them and learns them first to bear, + Making them women of good carriage: + This is she-- + +ROMEO + + Peace, peace, Mercutio, peace! + Thou talk'st of nothing. + +MERCUTIO + + True, I talk of dreams, + Which are the children of an idle brain, + Begot of nothing but vain fantasy, + Which is as thin of substance as the air + And more inconstant than the wind, who wooes + Even now the frozen bosom of the north, + And, being anger'd, puffs away from thence, + Turning his face to the dew-dropping south. + +BENVOLIO + + This wind, you talk of, blows us from ourselves; + Supper is done, and we shall come too late. + +ROMEO + + I fear, too early: for my mind misgives + Some consequence yet hanging in the stars + Shall bitterly begin his fearful date + With this night's revels and expire the term + Of a despised life closed in my breast + By some vile forfeit of untimely death. + But He, that hath the steerage of my course, + Direct my sail! On, lusty gentlemen. + +BENVOLIO + + Strike, drum. + + Exeunt +