From c71ae6808a7008bf43ecd02d81d99437a9284352 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Thu, 21 Feb 2019 16:49:10 -0500 Subject: [PATCH 01/18] WIP --- lib/plugins/aws/invokeLocal/index.js | 56 ++++------------------------ 1 file changed, 8 insertions(+), 48 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index bf32955ca42d..e153cd827cd9 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -131,54 +131,14 @@ class AwsInvokeLocal { || 'nodejs4.3'; const handler = this.options.functionObj.handler; - if (runtime.startsWith('nodejs')) { - const handlerPath = handler.split('.')[0]; - const handlerName = handler.split('.')[1]; - return this.invokeLocalNodeJs( - handlerPath, - handlerName, - this.options.data, - this.options.context); - } - - if (_.includes(['python2.7', 'python3.6', 'python3.7'], runtime)) { - const handlerComponents = handler.split(/\./); - const handlerPath = handlerComponents.slice(0, -1).join('.'); - const handlerName = handlerComponents.pop(); - return this.invokeLocalPython( - process.platform === 'win32' ? 'python.exe' : runtime, - handlerPath, - handlerName, - this.options.data, - this.options.context); - } - - if (runtime === 'java8') { - const className = handler.split('::')[0]; - const handlerName = handler.split('::')[1] || 'handleRequest'; - return this.invokeLocalJava( - 'java', - className, - handlerName, - this.serverless.service.package.artifact, - this.options.data, - this.options.context); - } - - if (runtime === 'ruby2.5') { - const handlerComponents = handler.split(/\./); - const handlerPath = handlerComponents[0]; - const handlerName = handlerComponents.slice(1).join('.'); - return this.invokeLocalRuby( - process.platform === 'win32' ? 'ruby.exe' : 'ruby', - handlerPath, - handlerName, - this.options.data, - this.options.context); - } - - throw new this.serverless.classes - .Error('You can only invoke Node.js, Python, Java & Ruby functions locally.'); + return new BbPromise(resolve => { + const docker = spawn('docker', + ['run', '--rm', '-v', `${this.serverless.config.servicePath}:/var/task`, + `lambci/lambda:${runtime}`, handler, JSON.stringify(this.options.data)]); + docker.stdout.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); + docker.stderr.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); + docker.on('close', () => resolve()); + }); } invokeLocalPython(runtime, handlerPath, handlerName, event, context) { From 0dc1a5f38f7962d232a6195b599738c02452e3b5 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Fri, 22 Feb 2019 16:11:11 -0500 Subject: [PATCH 02/18] support for layers!! --- lib/plugins/aws/invokeLocal/index.js | 68 +++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index e153cd827cd9..025b58167939 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -10,6 +10,9 @@ const chalk = require('chalk'); const stdin = require('get-stdin'); const spawn = require('child_process').spawn; const inspect = require('util').inspect; +const download = require('download'); +const jsZip = require('jszip'); +const mkdirp = require('mkdirp'); class AwsInvokeLocal { constructor(serverless, options) { @@ -125,20 +128,71 @@ class AwsInvokeLocal { return BbPromise.resolve(); } - invokeLocal() { + buildDockerImage() { const runtime = this.options.functionObj.runtime || this.serverless.service.provider.runtime || 'nodejs4.3'; + + const layers = _.mapKeys( + this.serverless.service.layers, + (value, key) => this.provider.naming.getLambdaLayerLogicalId(key) + ); + const layerPathsPromise = Promise.all((this.options.functionObj.layers || []).map(layer => { + if (layer.Ref) { + return layers[layer.Ref].path; + } + const arnParts = layer.split(':'); + const layerArn = arnParts.slice(0, -1).join(':'); + const layerVersion = Number(arnParts.slice(-1)[0]); + this.serverless.cli.consoleLog(`Downloading layer ${layer}...`); + return this.provider.request( + 'Lambda', 'getLayerVersion', { LayerName: layerArn, VersionNumber: layerVersion }) + .then(layerInfo => download(layerInfo.Content.Location)) + .then(jsZip.loadAsync) + .then(zip => Promise.all( + Object.keys(zip.files) + .map(filename => zip.files[filename].async('string').then(fileData => { + if (filename.endsWith('/')) { + return; + } + mkdirp.sync( + path.dirname(`.serverless/invokeLocal/layers/${arnParts[6]}/${filename}`)); + fs.writeFileAsync( + `.serverless/invokeLocal/layers/${arnParts[6]}/${filename}`, fileData); + })))) + .then(() => `.serverless/invokeLocal/layers/${arnParts[6]}`); + })); + + const imageName = 'sls-docker'; + + return layerPathsPromise.then(layerPaths => new BbPromise((resolve, reject) => { + let dockerfile = `FROM lambci/lambda:${runtime}`; + for (const layerPath of layerPaths) { + dockerfile += `\nADD --chown=sbx_user1051:495 ${layerPath} /opt`; + // fs.copySync(layerPath, `.serverless/invokeLocal/${layerPath}`); + } + mkdirp.sync(path.dirname('.serverless/invokeLocal')); + fs.writeFileSync('.serverless/invokeLocal/Dockerfile', dockerfile); + this.serverless.cli.consoleLog('Building docker image...'); + const docker = spawn('docker', ['build', '-t', imageName, + `${this.serverless.config.servicePath}`, '-f', '.serverless/invokeLocal/Dockerfile']); + docker.on('close', error => (error ? reject(error) : resolve(imageName))); + })); + } + + invokeLocal() { const handler = this.options.functionObj.handler; - return new BbPromise(resolve => { - const docker = spawn('docker', - ['run', '--rm', '-v', `${this.serverless.config.servicePath}:/var/task`, - `lambci/lambda:${runtime}`, handler, JSON.stringify(this.options.data)]); + return this.buildDockerImage().then(imageName => new BbPromise((resolve, reject) => { + const dockerArgs = [ + 'run', '--rm', '-v', `${this.serverless.config.servicePath}:/var/task`, imageName, + handler, JSON.stringify(this.options.data), + ]; + const docker = spawn('docker', dockerArgs); docker.stdout.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); docker.stderr.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); - docker.on('close', () => resolve()); - }); + docker.on('close', error => (error ? reject(error) : resolve(imageName))); + })); } invokeLocalPython(runtime, handlerPath, handlerName, event, context) { From 6046d9463aee471f56a829cfe127f96d33f26538 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Fri, 22 Feb 2019 17:00:25 -0500 Subject: [PATCH 03/18] fix executable & binary layer contents --- lib/plugins/aws/invokeLocal/index.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index 025b58167939..872b4218a74d 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -151,14 +151,16 @@ class AwsInvokeLocal { .then(jsZip.loadAsync) .then(zip => Promise.all( Object.keys(zip.files) - .map(filename => zip.files[filename].async('string').then(fileData => { + .map(filename => zip.files[filename].async('nodebuffer').then(fileData => { if (filename.endsWith('/')) { return; } mkdirp.sync( path.dirname(`.serverless/invokeLocal/layers/${arnParts[6]}/${filename}`)); - fs.writeFileAsync( - `.serverless/invokeLocal/layers/${arnParts[6]}/${filename}`, fileData); + return fs.writeFileAsync( + `.serverless/invokeLocal/layers/${arnParts[6]}/${filename}`, fileData, { + mode: zip.files[filename].unixPermissions, + }); })))) .then(() => `.serverless/invokeLocal/layers/${arnParts[6]}`); })); From ebf9eef91726698bc4b38580c4af7a81e94427a6 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Fri, 22 Feb 2019 17:06:07 -0500 Subject: [PATCH 04/18] support for service wide layer declaration. closes #5582 --- lib/plugins/aws/invokeLocal/index.js | 56 ++++++++++--------- .../aws/package/compile/functions/index.js | 7 +-- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index 872b4218a74d..6a75407df6b2 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -137,33 +137,35 @@ class AwsInvokeLocal { this.serverless.service.layers, (value, key) => this.provider.naming.getLambdaLayerLogicalId(key) ); - const layerPathsPromise = Promise.all((this.options.functionObj.layers || []).map(layer => { - if (layer.Ref) { - return layers[layer.Ref].path; - } - const arnParts = layer.split(':'); - const layerArn = arnParts.slice(0, -1).join(':'); - const layerVersion = Number(arnParts.slice(-1)[0]); - this.serverless.cli.consoleLog(`Downloading layer ${layer}...`); - return this.provider.request( - 'Lambda', 'getLayerVersion', { LayerName: layerArn, VersionNumber: layerVersion }) - .then(layerInfo => download(layerInfo.Content.Location)) - .then(jsZip.loadAsync) - .then(zip => Promise.all( - Object.keys(zip.files) - .map(filename => zip.files[filename].async('nodebuffer').then(fileData => { - if (filename.endsWith('/')) { - return; - } - mkdirp.sync( - path.dirname(`.serverless/invokeLocal/layers/${arnParts[6]}/${filename}`)); - return fs.writeFileAsync( - `.serverless/invokeLocal/layers/${arnParts[6]}/${filename}`, fileData, { - mode: zip.files[filename].unixPermissions, - }); - })))) - .then(() => `.serverless/invokeLocal/layers/${arnParts[6]}`); - })); + const layerPathsPromise = Promise.all( + (this.options.functionObj.layers || this.serverless.service.provider.layers || []) + .map(layer => { + if (layer.Ref) { + return layers[layer.Ref].path; + } + const arnParts = layer.split(':'); + const layerArn = arnParts.slice(0, -1).join(':'); + const layerVersion = Number(arnParts.slice(-1)[0]); + this.serverless.cli.consoleLog(`Downloading layer ${layer}...`); + return this.provider.request( + 'Lambda', 'getLayerVersion', { LayerName: layerArn, VersionNumber: layerVersion }) + .then(layerInfo => download(layerInfo.Content.Location)) + .then(jsZip.loadAsync) + .then(zip => Promise.all( + Object.keys(zip.files) + .map(filename => zip.files[filename].async('nodebuffer').then(fileData => { + if (filename.endsWith('/')) { + return Promise.resolve(); + } + mkdirp.sync( + path.dirname(`.serverless/invokeLocal/layers/${arnParts[6]}/${filename}`)); + return fs.writeFileAsync( + `.serverless/invokeLocal/layers/${arnParts[6]}/${filename}`, fileData, { + mode: zip.files[filename].unixPermissions, + }); + })))) + .then(() => `.serverless/invokeLocal/layers/${arnParts[6]}`); + })); const imageName = 'sls-docker'; diff --git a/lib/plugins/aws/package/compile/functions/index.js b/lib/plugins/aws/package/compile/functions/index.js index 6eee5d75e295..410480c743e4 100644 --- a/lib/plugins/aws/package/compile/functions/index.js +++ b/lib/plugins/aws/package/compile/functions/index.js @@ -361,10 +361,9 @@ class AwsCompileFunctions { if (functionObject.layers && _.isArray(functionObject.layers)) { newFunction.Properties.Layers = functionObject.layers; - /* TODO - is a DependsOn needed? - newLayer.DependsOn = [NEW LAYER??] - .concat(newLayer.DependsOn || []); - */ + } else if (this.serverless.service.provider.layers && _.isArray( + this.serverless.service.provider.layers)) { + newFunction.Properties.Layers = this.serverless.service.provider.layers; } const functionLogicalId = this.provider.naming From 9e5512e66e21d9a6477d067a0e96aa79f0d186ca Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Mon, 25 Feb 2019 09:21:51 -0500 Subject: [PATCH 05/18] use path.join, right log func, and pre-download base image if needed --- lib/plugins/aws/invokeLocal/index.js | 69 +++++++++++++++++++--------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index 6a75407df6b2..3ac4014d1e65 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -128,6 +128,28 @@ class AwsInvokeLocal { return BbPromise.resolve(); } + checkDockerImage() { + const runtime = this.options.functionObj.runtime + || this.serverless.service.provider.runtime + || 'nodejs4.3'; + + return new BbPromise((resolve, reject) => { + const docker = spawn('docker', ['pull', `lambci/lambda:${runtime}`]); + let stdout; + docker.stdout.on('data', (buf) => { stdout = buf.toString()}); + docker.on('close', error => (error ? reject(error) : resolve(stdout))); + }).then(stdout => Boolean(stdout.trim())); + } + + pullDockerImage() { + this.serverless.cli.log('Downloading base docker image...'); + + return new BbPromise((resolve, reject) => { + const docker = spawn('docker', ['pull', `lambci/lambda:${runtime}`]); + docker.on('close', error => (error ? reject(error) : resolve())); + }); + + } buildDockerImage() { const runtime = this.options.functionObj.runtime || this.serverless.service.provider.runtime @@ -146,7 +168,7 @@ class AwsInvokeLocal { const arnParts = layer.split(':'); const layerArn = arnParts.slice(0, -1).join(':'); const layerVersion = Number(arnParts.slice(-1)[0]); - this.serverless.cli.consoleLog(`Downloading layer ${layer}...`); + this.serverless.cli.log(`Downloading layer ${layer}...`); return this.provider.request( 'Lambda', 'getLayerVersion', { LayerName: layerArn, VersionNumber: layerVersion }) .then(layerInfo => download(layerInfo.Content.Location)) @@ -154,17 +176,17 @@ class AwsInvokeLocal { .then(zip => Promise.all( Object.keys(zip.files) .map(filename => zip.files[filename].async('nodebuffer').then(fileData => { - if (filename.endsWith('/')) { + if (filename.endsWith(path.sep)) { return Promise.resolve(); } - mkdirp.sync( - path.dirname(`.serverless/invokeLocal/layers/${arnParts[6]}/${filename}`)); - return fs.writeFileAsync( - `.serverless/invokeLocal/layers/${arnParts[6]}/${filename}`, fileData, { + mkdirp.sync(path.join( + '.serverless', 'invokeLocal', 'layers', 'arnParts[6]', 'filename')); + return fs.writeFileAsync(path.join( + '.serverless', 'invokeLocal', 'layers', arnParts[6], filename), fileData, { mode: zip.files[filename].unixPermissions, }); })))) - .then(() => `.serverless/invokeLocal/layers/${arnParts[6]}`); + .then(() => path.join('.serverless', 'invokeLocal', 'layers', arnParts[6])); })); const imageName = 'sls-docker'; @@ -173,13 +195,13 @@ class AwsInvokeLocal { let dockerfile = `FROM lambci/lambda:${runtime}`; for (const layerPath of layerPaths) { dockerfile += `\nADD --chown=sbx_user1051:495 ${layerPath} /opt`; - // fs.copySync(layerPath, `.serverless/invokeLocal/${layerPath}`); } - mkdirp.sync(path.dirname('.serverless/invokeLocal')); - fs.writeFileSync('.serverless/invokeLocal/Dockerfile', dockerfile); - this.serverless.cli.consoleLog('Building docker image...'); + mkdirp.sync(path.join('.serverless', 'invokeLocal')); + const dockerfilePath = path.join('.serverless', 'invokeLocal', 'Dockerfile'); + fs.writeFileSync(dockerfilePath, dockerfile); + this.serverless.cli.log('Building docker image...'); const docker = spawn('docker', ['build', '-t', imageName, - `${this.serverless.config.servicePath}`, '-f', '.serverless/invokeLocal/Dockerfile']); + `${this.serverless.config.servicePath}`, '-f', dockerfilePath]); docker.on('close', error => (error ? reject(error) : resolve(imageName))); })); } @@ -187,16 +209,19 @@ class AwsInvokeLocal { invokeLocal() { const handler = this.options.functionObj.handler; - return this.buildDockerImage().then(imageName => new BbPromise((resolve, reject) => { - const dockerArgs = [ - 'run', '--rm', '-v', `${this.serverless.config.servicePath}:/var/task`, imageName, - handler, JSON.stringify(this.options.data), - ]; - const docker = spawn('docker', dockerArgs); - docker.stdout.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); - docker.stderr.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); - docker.on('close', error => (error ? reject(error) : resolve(imageName))); - })); + return this.checkDockerImage() + .then(exists => (exists ? Promise.resolve(): this.pullDockerImage())) + .then(() => this.buildDockerImage()) + .then(imageName => new BbPromise((resolve, reject) => { + const dockerArgs = [ + 'run', '--rm', '-v', `${this.serverless.config.servicePath}:/var/task`, imageName, + handler, JSON.stringify(this.options.data), + ]; + const docker = spawn('docker', dockerArgs); + docker.stdout.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); + docker.stderr.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); + docker.on('close', error => (error ? reject(error) : resolve(imageName))); + })); } invokeLocalPython(runtime, handlerPath, handlerName, event, context) { From 47d08005de5fe8cb4832ded77c20dab4456a8f4a Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Mon, 25 Feb 2019 09:54:10 -0500 Subject: [PATCH 06/18] use 's builtin unzipping --- lib/plugins/aws/invokeLocal/index.js | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index 3ac4014d1e65..3318a4295c89 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -11,7 +11,6 @@ const stdin = require('get-stdin'); const spawn = require('child_process').spawn; const inspect = require('util').inspect; const download = require('download'); -const jsZip = require('jszip'); const mkdirp = require('mkdirp'); class AwsInvokeLocal { @@ -171,21 +170,10 @@ class AwsInvokeLocal { this.serverless.cli.log(`Downloading layer ${layer}...`); return this.provider.request( 'Lambda', 'getLayerVersion', { LayerName: layerArn, VersionNumber: layerVersion }) - .then(layerInfo => download(layerInfo.Content.Location)) - .then(jsZip.loadAsync) - .then(zip => Promise.all( - Object.keys(zip.files) - .map(filename => zip.files[filename].async('nodebuffer').then(fileData => { - if (filename.endsWith(path.sep)) { - return Promise.resolve(); - } - mkdirp.sync(path.join( - '.serverless', 'invokeLocal', 'layers', 'arnParts[6]', 'filename')); - return fs.writeFileAsync(path.join( - '.serverless', 'invokeLocal', 'layers', arnParts[6], filename), fileData, { - mode: zip.files[filename].unixPermissions, - }); - })))) + .then(layerInfo => download( + layerInfo.Content.Location, + path.join('.serverless', 'invokeLocal', 'layers', arnParts[6]), + { extract: true })) .then(() => path.join('.serverless', 'invokeLocal', 'layers', arnParts[6])); })); From dbd6b0a19ca01649b9d22dfbfc62962ca1a44361 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Mon, 25 Feb 2019 11:51:31 -0500 Subject: [PATCH 07/18] rework a bit --- lib/plugins/aws/invokeLocal/index.js | 48 ++++++++++++++++++---------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index 3318a4295c89..7e133555274e 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -133,32 +133,33 @@ class AwsInvokeLocal { || 'nodejs4.3'; return new BbPromise((resolve, reject) => { - const docker = spawn('docker', ['pull', `lambci/lambda:${runtime}`]); - let stdout; - docker.stdout.on('data', (buf) => { stdout = buf.toString()}); - docker.on('close', error => (error ? reject(error) : resolve(stdout))); - }).then(stdout => Boolean(stdout.trim())); + const docker = spawn('docker', ['images', '-q', `lambci/lambda:${runtime}`]); + let stdout = ''; + docker.stdout.on('data', (buf) => { stdout += buf.toString(); }); + docker.on('close', error => (error ? reject(error) : resolve(Boolean(stdout.trim())))); + }); } pullDockerImage() { + const runtime = this.options.functionObj.runtime + || this.serverless.service.provider.runtime + || 'nodejs4.3'; + this.serverless.cli.log('Downloading base docker image...'); return new BbPromise((resolve, reject) => { const docker = spawn('docker', ['pull', `lambci/lambda:${runtime}`]); docker.on('close', error => (error ? reject(error) : resolve())); }); - } - buildDockerImage() { - const runtime = this.options.functionObj.runtime - || this.serverless.service.provider.runtime - || 'nodejs4.3'; + getLayerPaths() { const layers = _.mapKeys( this.serverless.service.layers, (value, key) => this.provider.naming.getLambdaLayerLogicalId(key) ); - const layerPathsPromise = Promise.all( + + return Promise.all( (this.options.functionObj.layers || this.serverless.service.provider.layers || []) .map(layer => { if (layer.Ref) { @@ -167,19 +168,31 @@ class AwsInvokeLocal { const arnParts = layer.split(':'); const layerArn = arnParts.slice(0, -1).join(':'); const layerVersion = Number(arnParts.slice(-1)[0]); + const layerContentsPath = path.join( + '.serverless', 'invokeLocal', 'layers', arnParts[6], arnParts[7]); + if (fs.existsSync(layerContentsPath)) { + return layerContentsPath; + } this.serverless.cli.log(`Downloading layer ${layer}...`); return this.provider.request( 'Lambda', 'getLayerVersion', { LayerName: layerArn, VersionNumber: layerVersion }) .then(layerInfo => download( layerInfo.Content.Location, - path.join('.serverless', 'invokeLocal', 'layers', arnParts[6]), + layerContentsPath, { extract: true })) - .then(() => path.join('.serverless', 'invokeLocal', 'layers', arnParts[6])); + .then(() => layerContentsPath); })); + } + + buildDockerImage(layerPaths) { + const runtime = this.options.functionObj.runtime + || this.serverless.service.provider.runtime + || 'nodejs4.3'; + const imageName = 'sls-docker'; - return layerPathsPromise.then(layerPaths => new BbPromise((resolve, reject) => { + return new BbPromise((resolve, reject) => { let dockerfile = `FROM lambci/lambda:${runtime}`; for (const layerPath of layerPaths) { dockerfile += `\nADD --chown=sbx_user1051:495 ${layerPath} /opt`; @@ -191,15 +204,16 @@ class AwsInvokeLocal { const docker = spawn('docker', ['build', '-t', imageName, `${this.serverless.config.servicePath}`, '-f', dockerfilePath]); docker.on('close', error => (error ? reject(error) : resolve(imageName))); - })); + }); } invokeLocal() { const handler = this.options.functionObj.handler; return this.checkDockerImage() - .then(exists => (exists ? Promise.resolve(): this.pullDockerImage())) - .then(() => this.buildDockerImage()) + .then(exists => (exists ? Promise.resolve() : this.pullDockerImage())) + .then(() => this.getLayerPaths()) + .then((layerPaths) => this.buildDockerImage(layerPaths)) .then(imageName => new BbPromise((resolve, reject) => { const dockerArgs = [ 'run', '--rm', '-v', `${this.serverless.config.servicePath}:/var/task`, imageName, From 6ceef7071857cb8618dcaea407a4316240fd0b54 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Mon, 25 Feb 2019 14:15:48 -0500 Subject: [PATCH 08/18] move docker to own func to retain old functionality --- lib/plugins/aws/invokeLocal/index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index 7e133555274e..9f81be73b232 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -208,6 +208,10 @@ class AwsInvokeLocal { } invokeLocal() { + return this.invokeLocalDocker(); + } + + invokeLocalDocker() { const handler = this.options.functionObj.handler; return this.checkDockerImage() From df1860dc463160f27e40691c08b2b38fcb1bb264 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Mon, 25 Feb 2019 14:52:15 -0500 Subject: [PATCH 09/18] restore original functionality! docker as fallback/flag --- lib/plugins/aws/invokeLocal/index.js | 64 ++++++++++++++++++++++++++-- lib/plugins/invoke/invoke.js | 2 + 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index 9f81be73b232..361019b7c7bf 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -127,6 +127,66 @@ class AwsInvokeLocal { return BbPromise.resolve(); } + invokeLocal() { + const runtime = this.options.functionObj.runtime + || this.serverless.service.provider.runtime + || 'nodejs4.3'; + const handler = this.options.functionObj.handler; + + if (this.options.docker) { + return this.invokeLocalDocker(); + } + + if (runtime.startsWith('nodejs')) { + const handlerPath = handler.split('.')[0]; + const handlerName = handler.split('.')[1]; + return this.invokeLocalNodeJs( + handlerPath, + handlerName, + this.options.data, + this.options.context); + } + + if (_.includes(['python2.7', 'python3.6', 'python3.7'], runtime)) { + const handlerComponents = handler.split(/\./); + const handlerPath = handlerComponents.slice(0, -1).join('.'); + const handlerName = handlerComponents.pop(); + return this.invokeLocalPython( + process.platform === 'win32' ? 'python.exe' : runtime, + handlerPath, + handlerName, + this.options.data, + this.options.context); + } + + if (runtime === 'java8') { + const className = handler.split('::')[0]; + const handlerName = handler.split('::')[1] || 'handleRequest'; + return this.invokeLocalJava( + 'java', + className, + handlerName, + this.serverless.service.package.artifact, + this.options.data, + this.options.context); + } + + if (runtime === 'ruby2.5') { + const handlerComponents = handler.split(/\./); + const handlerPath = handlerComponents[0]; + const handlerName = handlerComponents.slice(1).join('.'); + return this.invokeLocalRuby( + process.platform === 'win32' ? 'ruby.exe' : 'ruby', + handlerPath, + handlerName, + this.options.data, + this.options.context); + } + + return this.invokeLocalDocker(); + } + + checkDockerImage() { const runtime = this.options.functionObj.runtime || this.serverless.service.provider.runtime @@ -207,10 +267,6 @@ class AwsInvokeLocal { }); } - invokeLocal() { - return this.invokeLocalDocker(); - } - invokeLocalDocker() { const handler = this.options.functionObj.handler; diff --git a/lib/plugins/invoke/invoke.js b/lib/plugins/invoke/invoke.js index b01212b82949..d07db02163c1 100644 --- a/lib/plugins/invoke/invoke.js +++ b/lib/plugins/invoke/invoke.js @@ -87,7 +87,9 @@ class Invoke { usage: 'Override environment variables. e.g. --env VAR1=val1 --env VAR2=val2', shortcut: 'e', }, + docker: { usage: 'Flag to turn on docker use for node/python/ruby/java' }, }, + }, }, }, From 50d2c737fa31b8da07d655c221555ba24e9875f4 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Mon, 25 Feb 2019 15:42:42 -0500 Subject: [PATCH 10/18] comment out failing test for non node/java/python/ruby runtimes --- lib/plugins/aws/invokeLocal/index.test.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/plugins/aws/invokeLocal/index.test.js b/lib/plugins/aws/invokeLocal/index.test.js index c8bba83f4bfe..fda6fdfb8fac 100644 --- a/lib/plugins/aws/invokeLocal/index.test.js +++ b/lib/plugins/aws/invokeLocal/index.test.js @@ -468,11 +468,13 @@ describe('AwsInvokeLocal', () => { }); }); + /* it('throw error when using runtime other than Node.js, Python, Java or Ruby', () => { awsInvokeLocal.options.functionObj.runtime = 'invalid-runtime'; expect(() => awsInvokeLocal.invokeLocal()).to.throw(Error); delete awsInvokeLocal.options.functionObj.runtime; }); + */ }); describe('#invokeLocalNodeJs', () => { From 1e0d7509a101520cca6b0ce6ae51a240bba78c0c Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Mon, 25 Feb 2019 15:53:40 -0500 Subject: [PATCH 11/18] add mkdirp as an explicit deplendency --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 8113de4c4c5e..098a6d884683 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,7 @@ "jwt-decode": "^2.2.0", "lodash": "^4.13.1", "minimist": "^1.2.0", + "mkdirp": "^0.5.1", "moment": "^2.13.0", "nanomatch": "^1.2.13", "node-fetch": "^1.6.0", From 60c4151baf78a3f909814addd7f225f5a5a71901 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Mon, 25 Feb 2019 16:43:42 -0500 Subject: [PATCH 12/18] global layer cache --- lib/plugins/aws/invokeLocal/index.js | 29 ++++++++++++++++++++-------- package-lock.json | 5 +++++ package.json | 1 + 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index 361019b7c7bf..0e57fb5dc946 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -4,6 +4,7 @@ const BbPromise = require('bluebird'); const _ = require('lodash'); const os = require('os'); const fs = BbPromise.promisifyAll(require('fs')); +const fse = require('fs-extra'); const path = require('path'); const validate = require('../lib/validate'); const chalk = require('chalk'); @@ -12,6 +13,10 @@ const spawn = require('child_process').spawn; const inspect = require('util').inspect; const download = require('download'); const mkdirp = require('mkdirp'); +const cachedir = require('cachedir'); + +const cachePath = path.join(cachedir('serverless'), 'invokeLocal'); + class AwsInvokeLocal { constructor(serverless, options) { @@ -229,17 +234,25 @@ class AwsInvokeLocal { const layerArn = arnParts.slice(0, -1).join(':'); const layerVersion = Number(arnParts.slice(-1)[0]); const layerContentsPath = path.join( - '.serverless', 'invokeLocal', 'layers', arnParts[6], arnParts[7]); + '.serverless', 'layers', arnParts[6], arnParts[7]); + const layerContentsCachePath = path.join( + cachePath, 'layers', arnParts[6], arnParts[7]); if (fs.existsSync(layerContentsPath)) { return layerContentsPath; } - this.serverless.cli.log(`Downloading layer ${layer}...`); - return this.provider.request( - 'Lambda', 'getLayerVersion', { LayerName: layerArn, VersionNumber: layerVersion }) - .then(layerInfo => download( - layerInfo.Content.Location, - layerContentsPath, - { extract: true })) + let downloadPromise = Promise.resolve() + if (!fs.existsSync(layerContentsCachePath)) { + this.serverless.cli.log(`Downloading layer ${layer}...`); + mkdirp.sync(path.join(layerContentsCachePath)); + downloadPromise = this.provider.request( + 'Lambda', 'getLayerVersion', { LayerName: layerArn, VersionNumber: layerVersion }) + .then(layerInfo => download( + layerInfo.Content.Location, + layerContentsPath, + { extract: true })) + } + return downloadPromise + .then(() => fse.copySync(layerContentsCachePath, layerContentsPath)) .then(() => layerContentsPath); })); } diff --git a/package-lock.json b/package-lock.json index 0c506ee2003f..c769a3879dc8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1166,6 +1166,11 @@ "unset-value": "^1.0.0" } }, + "cachedir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.1.0.tgz", + "integrity": "sha512-xGBpPqoBvn3unBW7oxgb8aJn42K0m9m1/wyjmazah10Fq7bROGG3kRAE6OIyr3U3PIJUqGuebhCEdMk9OKJG0A==" + }, "caller-id": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/caller-id/-/caller-id-0.1.0.tgz", diff --git a/package.json b/package.json index 098a6d884683..f66bf30efa84 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,7 @@ "async": "^1.5.2", "aws-sdk": "^2.373.0", "bluebird": "^3.5.0", + "cachedir": "^2.1.0", "chalk": "^2.0.0", "ci-info": "^1.1.1", "download": "^5.0.2", From ee3fef217d6c00c1d714ef1d1da692746d82726b Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Mon, 25 Feb 2019 17:29:03 -0500 Subject: [PATCH 13/18] support for package artifact based services! --- lib/plugins/aws/invokeLocal/index.js | 47 +++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index 0e57fb5dc946..aa88f93993e5 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -14,6 +14,7 @@ const inspect = require('util').inspect; const download = require('download'); const mkdirp = require('mkdirp'); const cachedir = require('cachedir'); +const jszip = require('jszip'); const cachePath = path.join(cachedir('serverless'), 'invokeLocal'); @@ -240,7 +241,7 @@ class AwsInvokeLocal { if (fs.existsSync(layerContentsPath)) { return layerContentsPath; } - let downloadPromise = Promise.resolve() + let downloadPromise = Promise.resolve(); if (!fs.existsSync(layerContentsCachePath)) { this.serverless.cli.log(`Downloading layer ${layer}...`); mkdirp.sync(path.join(layerContentsCachePath)); @@ -249,7 +250,7 @@ class AwsInvokeLocal { .then(layerInfo => download( layerInfo.Content.Location, layerContentsPath, - { extract: true })) + { extract: true })); } return downloadPromise .then(() => fse.copySync(layerContentsCachePath, layerContentsPath)) @@ -280,16 +281,46 @@ class AwsInvokeLocal { }); } + extractArtifact() { + const artifact = _.get(this.options.functionObj, 'package.artifact', _.get( + this.serverless.service, 'package.artifact' + )); + if (!artifact) { + return this.serverless.config.servicePath; + } + return fs.readFileAsync(artifact) + .then(jszip.loadAsync) + .then(zip => Promise.all( + Object.keys(zip.files) + .map(filename => zip.files[filename].async('nodebuffer').then(fileData => { + if (filename.endsWith(path.sep)) { + return Promise.resolve(); + } + mkdirp.sync(path.join( + '.serverless', 'invokeLocal', 'artifact')); + return fs.writeFileAsync(path.join( + '.serverless', 'invokeLocal', 'artifact', filename), fileData, { + mode: zip.files[filename].unixPermissions, + }); + })))) + .then(() => path.join( + this.serverless.config.servicePath, '.serverless', 'invokeLocal', 'artifact')); + } + + invokeLocalDocker() { const handler = this.options.functionObj.handler; - return this.checkDockerImage() - .then(exists => (exists ? Promise.resolve() : this.pullDockerImage())) - .then(() => this.getLayerPaths()) - .then((layerPaths) => this.buildDockerImage(layerPaths)) - .then(imageName => new BbPromise((resolve, reject) => { + return Promise.all([ + this.checkDockerImage().then(exists => (exists ? {} : this.pullDockerImage())), + this.getLayerPaths().then(layerPaths => this.buildDockerImage(layerPaths)), + this.extractArtifact(), + ]) + .then((results) => new BbPromise((resolve, reject) => { + const imageName = results[1]; + const artifactPath = results[2]; const dockerArgs = [ - 'run', '--rm', '-v', `${this.serverless.config.servicePath}:/var/task`, imageName, + 'run', '--rm', '-v', `${artifactPath}:/var/task`, imageName, handler, JSON.stringify(this.options.data), ]; const docker = spawn('docker', dockerArgs); From accd9316fe41fb5b1d0e513a6f9768c64c17b582 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Tue, 5 Mar 2019 16:46:59 -0500 Subject: [PATCH 14/18] test covvg! --- lib/plugins/aws/invokeLocal/index.test.js | 108 ++++++++++++++++++++-- 1 file changed, 102 insertions(+), 6 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.test.js b/lib/plugins/aws/invokeLocal/index.test.js index fda6fdfb8fac..c390ad113c9c 100644 --- a/lib/plugins/aws/invokeLocal/index.test.js +++ b/lib/plugins/aws/invokeLocal/index.test.js @@ -32,6 +32,7 @@ describe('AwsInvokeLocal', () => { function: 'first', }; serverless = new Serverless(); + serverless.config.servicePath = 'servicePath'; serverless.cli = new CLI(serverless); provider = new AwsProvider(serverless, options); serverless.setProvider('aws', provider); @@ -334,6 +335,7 @@ describe('AwsInvokeLocal', () => { let invokeLocalPythonStub; let invokeLocalJavaStub; let invokeLocalRubyStub; + let invokeLocalDockerStub; beforeEach(() => { invokeLocalNodeJsStub = @@ -344,6 +346,8 @@ describe('AwsInvokeLocal', () => { sinon.stub(awsInvokeLocal, 'invokeLocalJava').resolves(); invokeLocalRubyStub = sinon.stub(awsInvokeLocal, 'invokeLocalRuby').resolves(); + invokeLocalDockerStub = + sinon.stub(awsInvokeLocal, 'invokeLocalDocker').resolves(); awsInvokeLocal.serverless.service.service = 'new-service'; awsInvokeLocal.provider.options.stage = 'dev'; @@ -468,13 +472,24 @@ describe('AwsInvokeLocal', () => { }); }); - /* - it('throw error when using runtime other than Node.js, Python, Java or Ruby', () => { - awsInvokeLocal.options.functionObj.runtime = 'invalid-runtime'; - expect(() => awsInvokeLocal.invokeLocal()).to.throw(Error); - delete awsInvokeLocal.options.functionObj.runtime; + it('should call invokeLocalDocker if using runtime provided', () => { + awsInvokeLocal.options.functionObj.runtime = 'provided'; + awsInvokeLocal.options.functionObj.handler = 'handler.foobar'; + return awsInvokeLocal.invokeLocal().then(() => { + expect(invokeLocalDockerStub.calledOnce).to.be.equal(true); + expect(invokeLocalDockerStub.calledWithExactly()).to.be.equal(true); + }); + }); + + it('should call invokeLocalDocker if using --docker option with nodejs8.10', () => { + awsInvokeLocal.options.functionObj.runtime = 'nodejs8.10'; + awsInvokeLocal.options.functionObj.handler = 'handler.foobar'; + awsInvokeLocal.options.docker = true; + return awsInvokeLocal.invokeLocal().then(() => { + expect(invokeLocalDockerStub.calledOnce).to.be.equal(true); + expect(invokeLocalDockerStub.calledWithExactly()).to.be.equal(true); + }); }); - */ }); describe('#invokeLocalNodeJs', () => { @@ -1099,4 +1114,85 @@ describe('AwsInvokeLocal', () => { }); }); }); + + describe('#invokeLocalDocker()', () => { + let awsInvokeLocalMocked; + let spawnStub; + beforeEach(() => { + awsInvokeLocal.provider.options.stage = 'dev'; + awsInvokeLocal.options = { + function: 'first', + functionObj: { + handler: 'handler.hello', + name: 'hello', + timeout: 4, + }, + data: {}, + }; + + spawnStub = sinon.stub().returns({ + stderr: new EventEmitter().on('data', () => {}), + stdout: new EventEmitter().on('data', () => {}), + stdin: { + write: () => {}, + end: () => {}, + }, + on: (key, callback) => callback(), + }); + mockRequire('child_process', { spawn: spawnStub }); + + // Remove Node.js internal "require cache" contents and re-require ./index.js + delete require.cache[require.resolve('./index')]; + delete require.cache[require.resolve('child_process')]; + + const AwsInvokeLocalMocked = require('./index'); // eslint-disable-line global-require + + serverless.setProvider('aws', new AwsProvider(serverless, options)); + awsInvokeLocalMocked = new AwsInvokeLocalMocked(serverless, options); + + awsInvokeLocalMocked.options = { + stage: 'dev', + function: 'first', + functionObj: { + handler: 'handler.hello', + name: 'hello', + timeout: 4, + runtime: 'nodejs8.10', + }, + data: {}, + }; + }); + + + afterEach(() => { + delete require.cache[require.resolve('./index')]; + delete require.cache[require.resolve('child_process')]; + }); + + it('watwat', () => + awsInvokeLocalMocked.invokeLocalDocker().then(() => { + expect(spawnStub.getCall(0).args).to.deep.equal(['docker', + ['images', '-q', 'lambci/lambda:nodejs8.10']]); + expect(spawnStub.getCall(1).args).to.deep.equal(['docker', [ + 'build', + '-t', + 'sls-docker', + 'servicePath', + '-f', + '.serverless/invokeLocal/Dockerfile', + ]]); + expect(spawnStub.getCall(2).args).to.deep.equal(['docker', + ['pull', 'lambci/lambda:nodejs8.10']]); + expect(spawnStub.getCall(3).args).to.deep.equal(['docker', [ + 'run', + '--rm', + '-v', + 'servicePath:/var/task', + 'sls-docker', + 'handler.hello', + '{}', + ]]); + }) + ); + }); }); From af92c09335688c6560e9b89e148b89a76f75eb85 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Wed, 6 Mar 2019 09:27:42 -0500 Subject: [PATCH 15/18] review feedback --- lib/plugins/aws/invokeLocal/index.js | 18 +++++++++--------- lib/plugins/aws/invokeLocal/index.test.js | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index aa88f93993e5..4177f638f645 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -211,7 +211,7 @@ class AwsInvokeLocal { || this.serverless.service.provider.runtime || 'nodejs4.3'; - this.serverless.cli.log('Downloading base docker image...'); + this.serverless.cli.log('Downloading base Docker image...'); return new BbPromise((resolve, reject) => { const docker = spawn('docker', ['pull', `lambci/lambda:${runtime}`]); @@ -225,7 +225,7 @@ class AwsInvokeLocal { (value, key) => this.provider.naming.getLambdaLayerLogicalId(key) ); - return Promise.all( + return BbPromise.all( (this.options.functionObj.layers || this.serverless.service.provider.layers || []) .map(layer => { if (layer.Ref) { @@ -241,7 +241,7 @@ class AwsInvokeLocal { if (fs.existsSync(layerContentsPath)) { return layerContentsPath; } - let downloadPromise = Promise.resolve(); + let downloadPromise = BbPromise.resolve(); if (!fs.existsSync(layerContentsCachePath)) { this.serverless.cli.log(`Downloading layer ${layer}...`); mkdirp.sync(path.join(layerContentsCachePath)); @@ -274,7 +274,7 @@ class AwsInvokeLocal { mkdirp.sync(path.join('.serverless', 'invokeLocal')); const dockerfilePath = path.join('.serverless', 'invokeLocal', 'Dockerfile'); fs.writeFileSync(dockerfilePath, dockerfile); - this.serverless.cli.log('Building docker image...'); + this.serverless.cli.log('Building Docker image...'); const docker = spawn('docker', ['build', '-t', imageName, `${this.serverless.config.servicePath}`, '-f', dockerfilePath]); docker.on('close', error => (error ? reject(error) : resolve(imageName))); @@ -290,11 +290,11 @@ class AwsInvokeLocal { } return fs.readFileAsync(artifact) .then(jszip.loadAsync) - .then(zip => Promise.all( + .then(zip => BbPromise.all( Object.keys(zip.files) .map(filename => zip.files[filename].async('nodebuffer').then(fileData => { if (filename.endsWith(path.sep)) { - return Promise.resolve(); + return BbPromise.resolve(); } mkdirp.sync(path.join( '.serverless', 'invokeLocal', 'artifact')); @@ -311,7 +311,7 @@ class AwsInvokeLocal { invokeLocalDocker() { const handler = this.options.functionObj.handler; - return Promise.all([ + return BbPromise.all([ this.checkDockerImage().then(exists => (exists ? {} : this.pullDockerImage())), this.getLayerPaths().then(layerPaths => this.buildDockerImage(layerPaths)), this.extractArtifact(), @@ -509,7 +509,7 @@ class AwsInvokeLocal { this.serverless.cli.consoleLog(JSON.stringify(result, null, 4)); } - return new Promise((resolve) => { + return new BbPromise((resolve) => { const callback = (err, result) => { if (!hasResponded) { hasResponded = true; @@ -557,7 +557,7 @@ class AwsInvokeLocal { const maybeThennable = lambda(event, context, callback); if (!_.isUndefined(maybeThennable)) { - return Promise.resolve(maybeThennable) + return BbPromise.resolve(maybeThennable) .then( callback.bind(this, null), callback.bind(this) diff --git a/lib/plugins/aws/invokeLocal/index.test.js b/lib/plugins/aws/invokeLocal/index.test.js index c390ad113c9c..6165549c9676 100644 --- a/lib/plugins/aws/invokeLocal/index.test.js +++ b/lib/plugins/aws/invokeLocal/index.test.js @@ -1169,11 +1169,13 @@ describe('AwsInvokeLocal', () => { delete require.cache[require.resolve('child_process')]; }); - it('watwat', () => + it('calls docker', () => awsInvokeLocalMocked.invokeLocalDocker().then(() => { expect(spawnStub.getCall(0).args).to.deep.equal(['docker', ['images', '-q', 'lambci/lambda:nodejs8.10']]); - expect(spawnStub.getCall(1).args).to.deep.equal(['docker', [ + expect(spawnStub.getCall(1).args).to.deep.equal(['docker', + ['pull', 'lambci/lambda:nodejs8.10']]); + expect(spawnStub.getCall(2).args).to.deep.equal(['docker', [ 'build', '-t', 'sls-docker', @@ -1181,8 +1183,6 @@ describe('AwsInvokeLocal', () => { '-f', '.serverless/invokeLocal/Dockerfile', ]]); - expect(spawnStub.getCall(2).args).to.deep.equal(['docker', - ['pull', 'lambci/lambda:nodejs8.10']]); expect(spawnStub.getCall(3).args).to.deep.equal(['docker', [ 'run', '--rm', From 8c0d2825fd598e9d1f019cf7de10ee09786bd050 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Mon, 11 Mar 2019 14:48:07 -0400 Subject: [PATCH 16/18] document docker support! --- docs/providers/aws/cli-reference/invoke-local.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/providers/aws/cli-reference/invoke-local.md b/docs/providers/aws/cli-reference/invoke-local.md index 120aa51a4662..6597cdf246f4 100644 --- a/docs/providers/aws/cli-reference/invoke-local.md +++ b/docs/providers/aws/cli-reference/invoke-local.md @@ -29,6 +29,8 @@ serverless invoke local --function functionName - `--contextPath` or `-x`, The path to a json file holding input context to be passed to the invoked function. This path is relative to the root directory of the service. - `--context` or `-c`, String data to be passed as a context to your function. Same like with `--data`, context included in `--contextPath` will overwrite the context you passed with `--context` flag. * `--env` or `-e` String representing an environment variable to set when invoking your function, in the form `=`. Can be repeated for more than one environment variable. +* `--docker` Enable docker support for NodeJS/Python/Ruby/Java. Enabled by default for other + runtimes. ## Environment @@ -107,7 +109,11 @@ serverless invoke local -f functionName -e VAR1=value1 -e VAR2=value2 ### Limitations -Currently, `invoke local` only supports the NodeJs, Python, Java, & Ruby runtimes. +Use of the `--docker` flag and runtimes other than NodeJs, Python, Java, & Ruby depend on having +[Docker](https://www.docker.com/) installed. On MacOS & Windows, install +[Docker Desktop](https://www.docker.com/products/docker-desktop); On Linux install +[Docker engine](https://www.docker.com/products/docker-engine) and ensure your user is in the +`docker` group so that you can invoke docker without `sudo`. **Note:** In order to get correct output when using Java runtime, your Response class must implement `toString()` method. From 6cf35631390e6ccea0033c61e602556a32965d1e Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Wed, 13 Mar 2019 09:27:34 -0400 Subject: [PATCH 17/18] jszip as main dep, DRY getting runtime, use on exit instead of on close --- lib/plugins/aws/invokeLocal/index.js | 30 +++++++++++++--------------- package-lock.json | 21 ++++++------------- package.json | 2 +- 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index 4177f638f645..fde036ea52ae 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -36,6 +36,12 @@ class AwsInvokeLocal { }; } + getRuntime() { + return this.options.functionObj.runtime + || this.serverless.service.provider.runtime + || 'nodejs4.3'; + } + validateFile(filePath, key) { const absolutePath = path.isAbsolute(filePath) ? filePath : @@ -134,9 +140,7 @@ class AwsInvokeLocal { } invokeLocal() { - const runtime = this.options.functionObj.runtime - || this.serverless.service.provider.runtime - || 'nodejs4.3'; + const runtime = this.getRuntime(); const handler = this.options.functionObj.handler; if (this.options.docker) { @@ -194,28 +198,24 @@ class AwsInvokeLocal { checkDockerImage() { - const runtime = this.options.functionObj.runtime - || this.serverless.service.provider.runtime - || 'nodejs4.3'; + const runtime = this.getRuntime(); return new BbPromise((resolve, reject) => { const docker = spawn('docker', ['images', '-q', `lambci/lambda:${runtime}`]); let stdout = ''; docker.stdout.on('data', (buf) => { stdout += buf.toString(); }); - docker.on('close', error => (error ? reject(error) : resolve(Boolean(stdout.trim())))); + docker.on('exit', error => (error ? reject(error) : resolve(Boolean(stdout.trim())))); }); } pullDockerImage() { - const runtime = this.options.functionObj.runtime - || this.serverless.service.provider.runtime - || 'nodejs4.3'; + const runtime = this.getRuntime(); this.serverless.cli.log('Downloading base Docker image...'); return new BbPromise((resolve, reject) => { const docker = spawn('docker', ['pull', `lambci/lambda:${runtime}`]); - docker.on('close', error => (error ? reject(error) : resolve())); + docker.on('exit', error => (error ? reject(error) : resolve())); }); } @@ -259,9 +259,7 @@ class AwsInvokeLocal { } buildDockerImage(layerPaths) { - const runtime = this.options.functionObj.runtime - || this.serverless.service.provider.runtime - || 'nodejs4.3'; + const runtime = this.getRuntime(); const imageName = 'sls-docker'; @@ -277,7 +275,7 @@ class AwsInvokeLocal { this.serverless.cli.log('Building Docker image...'); const docker = spawn('docker', ['build', '-t', imageName, `${this.serverless.config.servicePath}`, '-f', dockerfilePath]); - docker.on('close', error => (error ? reject(error) : resolve(imageName))); + docker.on('exit', error => (error ? reject(error) : resolve(imageName))); }); } @@ -326,7 +324,7 @@ class AwsInvokeLocal { const docker = spawn('docker', dockerArgs); docker.stdout.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); docker.stderr.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); - docker.on('close', error => (error ? reject(error) : resolve(imageName))); + docker.on('exit', error => (error ? reject(error) : resolve(imageName))); })); } diff --git a/package-lock.json b/package-lock.json index c769a3879dc8..f4de13c7e443 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3740,8 +3740,7 @@ "immediate": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=", - "dev": true + "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" }, "import-lazy": { "version": "2.1.0", @@ -4989,7 +4988,6 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.1.5.tgz", "integrity": "sha512-5W8NUaFRFRqTOL7ZDDrx5qWHJyBXy6velVudIzQUSoqAAYqzSh2Z7/m0Rf1QbmQJccegD0r+YZxBjzqoBiEeJQ==", - "dev": true, "requires": { "core-js": "~2.3.0", "es6-promise": "~3.0.2", @@ -5001,26 +4999,22 @@ "core-js": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.3.0.tgz", - "integrity": "sha1-+rg/uwstjchfpjbEudNMdUIMbWU=", - "dev": true + "integrity": "sha1-+rg/uwstjchfpjbEudNMdUIMbWU=" }, "es6-promise": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.0.2.tgz", - "integrity": "sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y=", - "dev": true + "integrity": "sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y=" }, "process-nextick-args": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", - "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=", - "dev": true + "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" }, "readable-stream": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=", - "dev": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.1", @@ -5033,8 +5027,7 @@ "string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", - "dev": true + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" } } }, @@ -5128,7 +5121,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz", "integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=", - "dev": true, "requires": { "immediate": "~3.0.5" } @@ -6089,8 +6081,7 @@ "pako": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.8.tgz", - "integrity": "sha512-6i0HVbUfcKaTv+EG8ZTr75az7GFXcLYk9UyLEg7Notv/Ma+z/UG3TCoz6GiNeOrn1E/e63I0X/Hpw18jHOTUnA==", - "dev": true + "integrity": "sha512-6i0HVbUfcKaTv+EG8ZTr75az7GFXcLYk9UyLEg7Notv/Ma+z/UG3TCoz6GiNeOrn1E/e63I0X/Hpw18jHOTUnA==" }, "parse-github-url": { "version": "1.0.2", diff --git a/package.json b/package.json index f66bf30efa84..68d51735beeb 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,6 @@ "eslint-plugin-react": "^6.1.1", "istanbul": "^0.4.4", "jest-cli": "^23.1.0", - "jszip": "^3.1.2", "markdown-link": "^0.1.1", "markdown-magic": "^0.1.19", "markdown-table": "^1.1.1", @@ -91,6 +90,7 @@ "sinon-chai": "^2.9.0" }, "dependencies": { + "jszip": "^3.1.2", "archiver": "^1.1.0", "async": "^1.5.2", "aws-sdk": "^2.373.0", From 2dbc267eac7906f6145c3faf2a20066ad62b2c92 Mon Sep 17 00:00:00 2001 From: Philipp Muens Date: Thu, 14 Mar 2019 13:05:24 +0100 Subject: [PATCH 18/18] Add check if Docker daemon is running --- lib/plugins/aws/invokeLocal/index.js | 17 ++++++++++++++--- lib/plugins/aws/invokeLocal/index.test.js | 10 ++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index fde036ea52ae..26140da9a6fa 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -18,7 +18,6 @@ const jszip = require('jszip'); const cachePath = path.join(cachedir('serverless'), 'invokeLocal'); - class AwsInvokeLocal { constructor(serverless, options) { this.serverless = serverless; @@ -196,6 +195,17 @@ class AwsInvokeLocal { return this.invokeLocalDocker(); } + checkDockerDaemonStatus() { + return new BbPromise((resolve, reject) => { + const docker = spawn('docker', ['version']); + docker.on('exit', error => { + if (error) { + reject('Please start the Docker daemon to use the invoke local Docker integration.'); + } + resolve(); + }); + }); + } checkDockerImage() { const runtime = this.getRuntime(); @@ -310,13 +320,14 @@ class AwsInvokeLocal { const handler = this.options.functionObj.handler; return BbPromise.all([ + this.checkDockerDaemonStatus(), this.checkDockerImage().then(exists => (exists ? {} : this.pullDockerImage())), this.getLayerPaths().then(layerPaths => this.buildDockerImage(layerPaths)), this.extractArtifact(), ]) .then((results) => new BbPromise((resolve, reject) => { - const imageName = results[1]; - const artifactPath = results[2]; + const imageName = results[2]; + const artifactPath = results[3]; const dockerArgs = [ 'run', '--rm', '-v', `${artifactPath}:/var/task`, imageName, handler, JSON.stringify(this.options.data), diff --git a/lib/plugins/aws/invokeLocal/index.test.js b/lib/plugins/aws/invokeLocal/index.test.js index 6165549c9676..c88c8ec316f9 100644 --- a/lib/plugins/aws/invokeLocal/index.test.js +++ b/lib/plugins/aws/invokeLocal/index.test.js @@ -1118,6 +1118,7 @@ describe('AwsInvokeLocal', () => { describe('#invokeLocalDocker()', () => { let awsInvokeLocalMocked; let spawnStub; + beforeEach(() => { awsInvokeLocal.provider.options.stage = 'dev'; awsInvokeLocal.options = { @@ -1171,11 +1172,12 @@ describe('AwsInvokeLocal', () => { it('calls docker', () => awsInvokeLocalMocked.invokeLocalDocker().then(() => { - expect(spawnStub.getCall(0).args).to.deep.equal(['docker', - ['images', '-q', 'lambci/lambda:nodejs8.10']]); + expect(spawnStub.getCall(0).args).to.deep.equal(['docker', ['version']]); expect(spawnStub.getCall(1).args).to.deep.equal(['docker', + ['images', '-q', 'lambci/lambda:nodejs8.10']]); + expect(spawnStub.getCall(2).args).to.deep.equal(['docker', ['pull', 'lambci/lambda:nodejs8.10']]); - expect(spawnStub.getCall(2).args).to.deep.equal(['docker', [ + expect(spawnStub.getCall(3).args).to.deep.equal(['docker', [ 'build', '-t', 'sls-docker', @@ -1183,7 +1185,7 @@ describe('AwsInvokeLocal', () => { '-f', '.serverless/invokeLocal/Dockerfile', ]]); - expect(spawnStub.getCall(3).args).to.deep.equal(['docker', [ + expect(spawnStub.getCall(4).args).to.deep.equal(['docker', [ 'run', '--rm', '-v',