Skip to content

Commit

Permalink
Merge 2dbc267 into b8dd365
Browse files Browse the repository at this point in the history
  • Loading branch information
@dschep
dschep committed Mar 14, 2019
2 parents b8dd365 + 2dbc267 commit c263078
Show file tree
Hide file tree
Showing 7 changed files with 295 additions and 32 deletions.
8 changes: 7 additions & 1 deletion docs/providers/aws/cli-reference/invoke-local.md
View file
Expand Up @@ -29,6 +29,8 @@ serverless invoke local --function functionName
- `--contextPath` or `-x`, The path to a json file holding input context to be passed to the invoked function. This path is relative to the root directory of the service.
- `--context` or `-c`, String data to be passed as a context to your function. Same like with `--data`, context included in `--contextPath` will overwrite the context you passed with `--context` flag.
* `--env` or `-e` String representing an environment variable to set when invoking your function, in the form `<name>=<value>`. Can be repeated for more than one environment variable.
* `--docker` Enable docker support for NodeJS/Python/Ruby/Java. Enabled by default for other
runtimes.

## Environment

Expand Down Expand Up @@ -107,7 +109,11 @@ serverless invoke local -f functionName -e VAR1=value1 -e VAR2=value2

### Limitations

Currently, `invoke local` only supports the NodeJs, Python, Java, & Ruby runtimes.
Use of the `--docker` flag and runtimes other than NodeJs, Python, Java, & Ruby depend on having
[Docker](https://www.docker.com/) installed. On MacOS & Windows, install
[Docker Desktop](https://www.docker.com/products/docker-desktop); On Linux install
[Docker engine](https://www.docker.com/products/docker-engine) and ensure your user is in the
`docker` group so that you can invoke docker without `sudo`.

**Note:** In order to get correct output when using Java runtime, your Response class must implement `toString()` method.

Expand Down
172 changes: 165 additions & 7 deletions lib/plugins/aws/invokeLocal/index.js
View file
Expand Up @@ -4,12 +4,19 @@ const BbPromise = require('bluebird');
const _ = require('lodash');
const os = require('os');
const fs = BbPromise.promisifyAll(require('fs'));
const fse = require('fs-extra');
const path = require('path');
const validate = require('../lib/validate');
const chalk = require('chalk');
const stdin = require('get-stdin');
const spawn = require('child_process').spawn;
const inspect = require('util').inspect;
const download = require('download');
const mkdirp = require('mkdirp');
const cachedir = require('cachedir');
const jszip = require('jszip');

const cachePath = path.join(cachedir('serverless'), 'invokeLocal');

class AwsInvokeLocal {
constructor(serverless, options) {
Expand All @@ -28,6 +35,12 @@ class AwsInvokeLocal {
};
}

getRuntime() {
return this.options.functionObj.runtime
|| this.serverless.service.provider.runtime
|| 'nodejs4.3';
}

validateFile(filePath, key) {
const absolutePath = path.isAbsolute(filePath) ?
filePath :
Expand Down Expand Up @@ -126,11 +139,13 @@ class AwsInvokeLocal {
}

invokeLocal() {
const runtime = this.options.functionObj.runtime
|| this.serverless.service.provider.runtime
|| 'nodejs4.3';
const runtime = this.getRuntime();
const handler = this.options.functionObj.handler;

if (this.options.docker) {
return this.invokeLocalDocker();
}

if (runtime.startsWith('nodejs')) {
const handlerPath = handler.split('.')[0];
const handlerName = handler.split('.')[1];
Expand Down Expand Up @@ -177,8 +192,151 @@ class AwsInvokeLocal {
this.options.context);
}

throw new this.serverless.classes
.Error('You can only invoke Node.js, Python, Java & Ruby functions locally.');
return this.invokeLocalDocker();
}

checkDockerDaemonStatus() {
return new BbPromise((resolve, reject) => {
const docker = spawn('docker', ['version']);
docker.on('exit', error => {
if (error) {
reject('Please start the Docker daemon to use the invoke local Docker integration.');
}
resolve();
});
});
}

checkDockerImage() {
const runtime = this.getRuntime();

return new BbPromise((resolve, reject) => {
const docker = spawn('docker', ['images', '-q', `lambci/lambda:${runtime}`]);
let stdout = '';
docker.stdout.on('data', (buf) => { stdout += buf.toString(); });
docker.on('exit', error => (error ? reject(error) : resolve(Boolean(stdout.trim()))));
});
}

pullDockerImage() {
const runtime = this.getRuntime();

this.serverless.cli.log('Downloading base Docker image...');

return new BbPromise((resolve, reject) => {
const docker = spawn('docker', ['pull', `lambci/lambda:${runtime}`]);
docker.on('exit', error => (error ? reject(error) : resolve()));
});
}

getLayerPaths() {
const layers = _.mapKeys(
this.serverless.service.layers,
(value, key) => this.provider.naming.getLambdaLayerLogicalId(key)
);

return BbPromise.all(
(this.options.functionObj.layers || this.serverless.service.provider.layers || [])
.map(layer => {
if (layer.Ref) {
return layers[layer.Ref].path;
}
const arnParts = layer.split(':');
const layerArn = arnParts.slice(0, -1).join(':');
const layerVersion = Number(arnParts.slice(-1)[0]);
const layerContentsPath = path.join(
'.serverless', 'layers', arnParts[6], arnParts[7]);
const layerContentsCachePath = path.join(
cachePath, 'layers', arnParts[6], arnParts[7]);
if (fs.existsSync(layerContentsPath)) {
return layerContentsPath;
}
let downloadPromise = BbPromise.resolve();
if (!fs.existsSync(layerContentsCachePath)) {
this.serverless.cli.log(`Downloading layer ${layer}...`);
mkdirp.sync(path.join(layerContentsCachePath));
downloadPromise = this.provider.request(
'Lambda', 'getLayerVersion', { LayerName: layerArn, VersionNumber: layerVersion })
.then(layerInfo => download(
layerInfo.Content.Location,
layerContentsPath,
{ extract: true }));
}
return downloadPromise
.then(() => fse.copySync(layerContentsCachePath, layerContentsPath))
.then(() => layerContentsPath);
}));
}

buildDockerImage(layerPaths) {
const runtime = this.getRuntime();


const imageName = 'sls-docker';

return new BbPromise((resolve, reject) => {
let dockerfile = `FROM lambci/lambda:${runtime}`;
for (const layerPath of layerPaths) {
dockerfile += `\nADD --chown=sbx_user1051:495 ${layerPath} /opt`;
}
mkdirp.sync(path.join('.serverless', 'invokeLocal'));
const dockerfilePath = path.join('.serverless', 'invokeLocal', 'Dockerfile');
fs.writeFileSync(dockerfilePath, dockerfile);
this.serverless.cli.log('Building Docker image...');
const docker = spawn('docker', ['build', '-t', imageName,
`${this.serverless.config.servicePath}`, '-f', dockerfilePath]);
docker.on('exit', error => (error ? reject(error) : resolve(imageName)));
});
}

extractArtifact() {
const artifact = _.get(this.options.functionObj, 'package.artifact', _.get(
this.serverless.service, 'package.artifact'
));
if (!artifact) {
return this.serverless.config.servicePath;
}
return fs.readFileAsync(artifact)
.then(jszip.loadAsync)
.then(zip => BbPromise.all(
Object.keys(zip.files)
.map(filename => zip.files[filename].async('nodebuffer').then(fileData => {
if (filename.endsWith(path.sep)) {
return BbPromise.resolve();
}
mkdirp.sync(path.join(
'.serverless', 'invokeLocal', 'artifact'));
return fs.writeFileAsync(path.join(
'.serverless', 'invokeLocal', 'artifact', filename), fileData, {
mode: zip.files[filename].unixPermissions,
});
}))))
.then(() => path.join(
this.serverless.config.servicePath, '.serverless', 'invokeLocal', 'artifact'));
}


invokeLocalDocker() {
const handler = this.options.functionObj.handler;

return BbPromise.all([
this.checkDockerDaemonStatus(),
this.checkDockerImage().then(exists => (exists ? {} : this.pullDockerImage())),
this.getLayerPaths().then(layerPaths => this.buildDockerImage(layerPaths)),
this.extractArtifact(),
])
.then((results) => new BbPromise((resolve, reject) => {
const imageName = results[2];
const artifactPath = results[3];
const dockerArgs = [
'run', '--rm', '-v', `${artifactPath}:/var/task`, imageName,
handler, JSON.stringify(this.options.data),
];
const docker = spawn('docker', dockerArgs);
docker.stdout.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString()));
docker.stderr.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString()));
docker.on('exit', error => (error ? reject(error) : resolve(imageName)));
}));
}

invokeLocalPython(runtime, handlerPath, handlerName, event, context) {
Expand Down Expand Up @@ -360,7 +518,7 @@ class AwsInvokeLocal {
this.serverless.cli.consoleLog(JSON.stringify(result, null, 4));
}

return new Promise((resolve) => {
return new BbPromise((resolve) => {
const callback = (err, result) => {
if (!hasResponded) {
hasResponded = true;
Expand Down Expand Up @@ -408,7 +566,7 @@ class AwsInvokeLocal {

const maybeThennable = lambda(event, context, callback);
if (!_.isUndefined(maybeThennable)) {
return Promise.resolve(maybeThennable)
return BbPromise.resolve(maybeThennable)
.then(
callback.bind(this, null),
callback.bind(this)
Expand Down
108 changes: 104 additions & 4 deletions lib/plugins/aws/invokeLocal/index.test.js
View file
Expand Up @@ -32,6 +32,7 @@ describe('AwsInvokeLocal', () => {
function: 'first',
};
serverless = new Serverless();
serverless.config.servicePath = 'servicePath';
serverless.cli = new CLI(serverless);
provider = new AwsProvider(serverless, options);
serverless.setProvider('aws', provider);
Expand Down Expand Up @@ -334,6 +335,7 @@ describe('AwsInvokeLocal', () => {
let invokeLocalPythonStub;
let invokeLocalJavaStub;
let invokeLocalRubyStub;
let invokeLocalDockerStub;

beforeEach(() => {
invokeLocalNodeJsStub =
Expand All @@ -344,6 +346,8 @@ describe('AwsInvokeLocal', () => {
sinon.stub(awsInvokeLocal, 'invokeLocalJava').resolves();
invokeLocalRubyStub =
sinon.stub(awsInvokeLocal, 'invokeLocalRuby').resolves();
invokeLocalDockerStub =
sinon.stub(awsInvokeLocal, 'invokeLocalDocker').resolves();

awsInvokeLocal.serverless.service.service = 'new-service';
awsInvokeLocal.provider.options.stage = 'dev';
Expand Down Expand Up @@ -468,10 +472,23 @@ describe('AwsInvokeLocal', () => {
});
});

it('throw error when using runtime other than Node.js, Python, Java or Ruby', () => {
awsInvokeLocal.options.functionObj.runtime = 'invalid-runtime';
expect(() => awsInvokeLocal.invokeLocal()).to.throw(Error);
delete awsInvokeLocal.options.functionObj.runtime;
it('should call invokeLocalDocker if using runtime provided', () => {
awsInvokeLocal.options.functionObj.runtime = 'provided';
awsInvokeLocal.options.functionObj.handler = 'handler.foobar';
return awsInvokeLocal.invokeLocal().then(() => {
expect(invokeLocalDockerStub.calledOnce).to.be.equal(true);
expect(invokeLocalDockerStub.calledWithExactly()).to.be.equal(true);
});
});

it('should call invokeLocalDocker if using --docker option with nodejs8.10', () => {
awsInvokeLocal.options.functionObj.runtime = 'nodejs8.10';
awsInvokeLocal.options.functionObj.handler = 'handler.foobar';
awsInvokeLocal.options.docker = true;
return awsInvokeLocal.invokeLocal().then(() => {
expect(invokeLocalDockerStub.calledOnce).to.be.equal(true);
expect(invokeLocalDockerStub.calledWithExactly()).to.be.equal(true);
});
});
});

Expand Down Expand Up @@ -1097,4 +1114,87 @@ describe('AwsInvokeLocal', () => {
});
});
});

describe('#invokeLocalDocker()', () => {
let awsInvokeLocalMocked;
let spawnStub;

beforeEach(() => {
awsInvokeLocal.provider.options.stage = 'dev';
awsInvokeLocal.options = {
function: 'first',
functionObj: {
handler: 'handler.hello',
name: 'hello',
timeout: 4,
},
data: {},
};

spawnStub = sinon.stub().returns({
stderr: new EventEmitter().on('data', () => {}),
stdout: new EventEmitter().on('data', () => {}),
stdin: {
write: () => {},
end: () => {},
},
on: (key, callback) => callback(),
});
mockRequire('child_process', { spawn: spawnStub });

// Remove Node.js internal "require cache" contents and re-require ./index.js
delete require.cache[require.resolve('./index')];
delete require.cache[require.resolve('child_process')];

const AwsInvokeLocalMocked = require('./index'); // eslint-disable-line global-require

serverless.setProvider('aws', new AwsProvider(serverless, options));
awsInvokeLocalMocked = new AwsInvokeLocalMocked(serverless, options);

awsInvokeLocalMocked.options = {
stage: 'dev',
function: 'first',
functionObj: {
handler: 'handler.hello',
name: 'hello',
timeout: 4,
runtime: 'nodejs8.10',
},
data: {},
};
});


afterEach(() => {
delete require.cache[require.resolve('./index')];
delete require.cache[require.resolve('child_process')];
});

it('calls docker', () =>
awsInvokeLocalMocked.invokeLocalDocker().then(() => {
expect(spawnStub.getCall(0).args).to.deep.equal(['docker', ['version']]);
expect(spawnStub.getCall(1).args).to.deep.equal(['docker',
['images', '-q', 'lambci/lambda:nodejs8.10']]);
expect(spawnStub.getCall(2).args).to.deep.equal(['docker',
['pull', 'lambci/lambda:nodejs8.10']]);
expect(spawnStub.getCall(3).args).to.deep.equal(['docker', [
'build',
'-t',
'sls-docker',
'servicePath',
'-f',
'.serverless/invokeLocal/Dockerfile',
]]);
expect(spawnStub.getCall(4).args).to.deep.equal(['docker', [
'run',
'--rm',
'-v',
'servicePath:/var/task',
'sls-docker',
'handler.hello',
'{}',
]]);
})
);
});
});
7 changes: 3 additions & 4 deletions lib/plugins/aws/package/compile/functions/index.js
View file
Expand Up @@ -361,10 +361,9 @@ class AwsCompileFunctions {

if (functionObject.layers && _.isArray(functionObject.layers)) {
newFunction.Properties.Layers = functionObject.layers;
/* TODO - is a DependsOn needed?
newLayer.DependsOn = [NEW LAYER??]
.concat(newLayer.DependsOn || []);
*/
} else if (this.serverless.service.provider.layers && _.isArray(
this.serverless.service.provider.layers)) {
newFunction.Properties.Layers = this.serverless.service.provider.layers;
}

const functionLogicalId = this.provider.naming
Expand Down

0 comments on commit c263078

Please sign in to comment.