Permalink
Browse files

Add AWS.Request.createReadStream() for HTTP data

Update documentation and guides to reflect new API
  • Loading branch information...
1 parent ec875a4 commit 9808937bec19d08bf6286a9aea0c1c9c99e6e9c1 Loren Segal committed Jan 23, 2013
@@ -43,6 +43,22 @@ s3.client.createBucket({Bucket: 'myBucket'}, function() {
});
```
+### Amazon S3: Streaming Objects to Files on Disk (getObject)
+
+You can use the `createReadStream()` method on a request object to
+get a handle to a stream object which supports piping raw HTTP
+body data to a file. This is especially useful when streaming
+objects to streams like filesystem objects. The following example
+shows how you can stream an object from Amazon S3 directly to a file
+on disk:
+
+```js
+var s3 = new AWS.S3();
+var params = {Bucket: 'myBucket', Key: 'myImageFile.jpg'};
+var file = require('fs').createWriteStream('/path/to/file.jpg');
+s3.client.getObject(params).createReadStream().pipe(file);
+```
+
## Amazon DynamoDB
### Amazon DynamoDB: Listing Tables (listTables)
@@ -53,6 +53,10 @@ Feature: Working with Objects in S3
When I write file "testfile.txt" to the key "hello"
Then the object with the key "hello" should exist
And the object with the key "hello" should contain "CONTENTS OF FILE"
+
+ When I stream key "hello"
+ Then the streamed data should contain "CONTENTS OF FILE"
+
And I delete the object with the key "hello"
# final step here needs to happen to cleanup the shared bucket
@@ -16,7 +16,7 @@
module.exports = function () {
this.Before("@s3", function (callback) {
- this.s3 = new this.AWS.S3.Client();
+ this.client = this.s3 = new this.AWS.S3.Client();
callback();
});
@@ -81,6 +81,20 @@ module.exports = function () {
this.request('s3', 'putObject', params, next);
});
+ this.When(/^I stream key "([^"]*)"$/, function(key, callback) {
+ var params = {Bucket: this.sharedBucket, Key: key};
+ var world = this;
+ this.result = '';
+ this.client.getObject(params).createReadStream().
+ on('end', function() { callback(); }).
+ on('data', function(d) { world.result += d.toString(); });
+ });
+
+ this.Then(/^the streamed data should contain "([^"]*)"$/, function(data, callback) {
+ if (data === this.result.replace("\n", "")) callback();
+ else callback.fail("Expected " + data + ", got " + this.result);
+ });
+
// this scenario is a work around for not having an after all hook
this.Then(/^I delete the shared bucket$/, function(next) {
this.request('s3', 'deleteBucket', {Bucket:this.sharedBucket}, next);
View
@@ -15,6 +15,7 @@
var AWS = require('./core');
var inherit = AWS.util.inherit;
+var Stream = require('stream').Stream;
/**
* == Asynchronous Requests
@@ -230,6 +231,47 @@ AWS.Request = inherit({
},
/**
+ * Converts the request object into a readable stream that
+ * can be read from or piped into a writable stream.
+ *
+ * @note The data read from a readable stream contains only
+ * the raw HTTP body contents.
+ * @example Manually reading from a stream
+ * request.createReadStream().on('data', function(data) {
+ * console.log("Got data:", data.toString());
+ * });
+ * @example Piping a request body into a file
+ * var out = fs.createWriteStream('/path/to/outfile.jpg');
+ * s3.client.getObject(params).createReadStream().pipe(out);
+ */
+ createReadStream: function createReadStream() {
+ var req = this;
+ var stream = new Stream();
+
+ stream.readable = true;
+ stream.on('newListener', function(event) {
+ if (event === 'data') {
+ process.nextTick(function() { req.send(); });
+ }
+ });
+
+ this.on('httpData', function(data) {
+ stream.emit('data', data);
+ });
+
+ this.on('httpDone', function() {
+ stream.emit('end');
+ stream.readable = false;
+ });
+
+ this.on('error', function(err) {
+ stream.emit('error', err);
+ });
+
+ return stream;
+ },
+
+ /**
* @api private
*/
completeRequest: function completeRequest(response) {
@@ -0,0 +1,34 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+helpers = require('../helpers')
+AWS = helpers.AWS
+MockClient = helpers.MockClient
+
+describe 'AWS.Request', ->
+ client = new MockClient
+
+ describe 'createReadStream', ->
+ it 'streams data', ->
+ data = ''; done = false
+ helpers.mockHttpResponse 200, {}, ['FOO', 'BAR', 'BAZ', 'QUX']
+
+ runs ->
+ request = client.makeRequest('mockMethod')
+ s = request.createReadStream()
+ s.on 'end', -> done = true
+ s.on 'data', (c) -> data += c.toString()
+ waitsFor -> done == true
+ runs ->
+ expect(data).toEqual('FOOBARBAZQUX')
+

0 comments on commit 9808937

Please sign in to comment.