Skip to content
Use S3 as a source or destination of vinyl files.
Branch: master
Clone or download
Fetching latest commit…
Cannot retrieve the latest commit at this time.
Permalink
Type Name Latest commit message Commit time
Failed to load latest commit information.
example
lib
test
.editorconfig
.eslintrc
.gitignore
.travis.yml
README.md
package.json

README.md

vinyl-s3

Use S3 as a source or destination of vinyl files.

build status coverage license version downloads

Features:

  • Source with multi-globbing support,
  • Use either streaming or buffering,
  • Upload or download files,
  • Pass custom options to S3,
  • Smart Content-Type and Content-Encoding detection,
  • Works great with gulp.

Usage

var gulp = require('gulp'),
	s3 = require('vinyl-s3');

// Upload files to S3
gulp.task('upload', function() {
	return gulp.src('data/*.jpg', { buffer: false })
		.pipe(s3.dest('s3://my-bucket/prefix'));
});

// Download files from S3
gulp.task('download', function() {
	return s3.src('s3://my-bucket/prefix/*.jpg', { buffer: false })
		.pipe(gulp.dest('data'));
});

// Just print a list of files
var through2 = require('through2');
gulp.task('meta', function() {
	return s3.src('s3://my-bucket/foo/**/*.jpg', { read: false })
		.pipe(through2.obj(function(file, _, callback) {
			console.log(file.path);
			callback();
		}));
});

When working with large files you may find it useful to use streaming mode instead of buffering mode. You can enable this in the src() family of functions by setting { buffer: false }. The default mode is to use buffering as is the same with fs.src.

src

See getObject for a list of supported options.

// Specify custom attributes via S3 URL.
s3.src('s3://bucket/key/*?IfModifiedSince=123456789')
    .pipe(fs.dest('downloads'));
// Specify custom attributes by passing in an AWS options object.
s3.src({
    Bucket: 'bucket',
    Key: 'key/*',
    IfModifiedSince: Date.now()
}).pipe(fs.dest('downloads'));
// Use multiple source buckets and patterns.
s3.src(['s3://bucket1/*.jpg', 's3://bucket1/*.png', 's3://bucket2/*.gif'])
    .pipe(fs.dest('downloads'));

dest

See putObject and upload for a list of supported options. There is limited support for automatically detecting the correct Content-Type and correct Content-Encoding. Parallel uploads are supported by passing { queueSize: n } as a second parameter.

// Specify custom attributes via S3 URL.
fs.src('files/*.jpg')
    .pipe(s3.dest('s3://bucket/foo?ContentType=image/jpeg'));
// Specify custom attributes by passing in an AWS options object.
fs.src('files/*.jpg')
    .pipe(s3.dest({
        Bucket: 'bucket',
        Key: 'foo',
        ContentType: 'image/jpeg'
    }));
// Specify custom attributes per file.
fs.src('files/*.jpg')
    .pipe(through2.obj(function(file, enc, next) {
        // There are some non-standard properties on the file object that
        // are used to generate certain AWS options.
        file.contentType = 'image/jpeg';
        file.contentEncoding = 'gzip';

        // Setting the awsOptions property on a file causes the object to be
        // included in the command sent to S3. These options override any
        // previously set value.
        file.awsOptions = {
            ACL: 'private',
            CacheControl: 'max-age=1296000',
            ContentType: 'image/jpeg',
            Metadata: {
                color: 'red'
            }
        };
        this.push(file);
        next();
    }))
    .pipe(s3.dest('s3://bucket/foo'));
You can’t perform that action at this time.