New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle scoped npm packages in ${file()} variables #5312

Merged
merged 2 commits into from Jan 7, 2019
File filter...
Filter file types
Jump to file or symbol
Failed to load files and symbols.
+51 −34
Diff settings

Always

Just for now

@@ -18,7 +18,7 @@ They are especially useful when providing secrets for your service to use and wh


## Syntax ## Syntax


To use variables, you will need to reference values enclosed in `${}` brackets. To use variables, you will need to reference values enclosed in `${}` brackets.


```yml ```yml
# serverless.yml file # serverless.yml file
@@ -27,11 +27,11 @@ yamlKeyXYZ: ${variableSource} # see list of current variable sources below
otherYamlKey: ${variableSource, defaultValue} otherYamlKey: ${variableSource, defaultValue}
``` ```


You can define your own variable syntax (regex) if it conflicts with CloudFormation's syntax. You can define your own variable syntax (regex) if it conflicts with CloudFormation's syntax.


**Note:** You can only use variables in `serverless.yml` property **values**, not property keys. So you can't use variables to generate dynamic logical IDs in the custom resources section for example. **Note:** You can only use variables in `serverless.yml` property **values**, not property keys. So you can't use variables to generate dynamic logical IDs in the custom resources section for example.


## Current variable sources: ## Current variable sources:


- [environment variables](https://serverless.com/framework/docs/providers/aws/guide/variables#referencing-environment-variables) - [environment variables](https://serverless.com/framework/docs/providers/aws/guide/variables#referencing-environment-variables)
- [CLI options](https://serverless.com/framework/docs/providers/aws/guide/variables#referencing-cli-options) - [CLI options](https://serverless.com/framework/docs/providers/aws/guide/variables#referencing-cli-options)
@@ -57,7 +57,7 @@ provider:
MY_SECRET: ${file(./config.${self:provider.stage}.json):CREDS} MY_SECRET: ${file(./config.${self:provider.stage}.json):CREDS}
``` ```


If `sls deploy --stage qa` is run, the option `stage=qa` is used inside the `${file(./config.${self:provider.stage}.json):CREDS}` variable and it will resolve the `config.qa.json` file and use the `CREDS` key defined. If `sls deploy --stage qa` is run, the option `stage=qa` is used inside the `${file(./config.${self:provider.stage}.json):CREDS}` variable and it will resolve the `config.qa.json` file and use the `CREDS` key defined.


**How that works:** **How that works:**


@@ -427,8 +427,8 @@ service: new-service
provider: provider:
name: aws name: aws
runtime: nodejs6.10 runtime: nodejs6.10
variableSyntax: "\\${{([ ~:a-zA-Z0-9._\\'\",\\-\\/\\(\\)]+?)}}" # notice the double quotes for yaml to ignore the escape characters! variableSyntax: "\\${{([ ~:a-zA-Z0-9._@\\'\",\\-\\/\\(\\)]+?)}}" # notice the double quotes for yaml to ignore the escape characters!
# variableSyntax: "\\${((?!AWS)[ ~:a-zA-Z0-9._'\",\\-\\/\\(\\)]+?)}" # Use this for allowing CloudFormation Pseudo-Parameters in your serverless.yml -- e.g. ${AWS::Region}. All other Serverless variables work as usual. # variableSyntax: "\\${((?!AWS)[ ~:a-zA-Z0-9._@'\",\\-\\/\\(\\)]+?)}" # Use this for allowing CloudFormation Pseudo-Parameters in your serverless.yml -- e.g. ${AWS::Region}. All other Serverless variables work as usual.
custom: custom:
myStage: ${{opt:stage}} myStage: ${{opt:stage}}
Copy path View file
@@ -19,7 +19,7 @@ class Service {
this.provider = { this.provider = {
stage: 'dev', stage: 'dev',
region: 'us-east-1', region: 'us-east-1',
variableSyntax: '\\${([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}', variableSyntax: '\\${([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}',
}; };
this.custom = {}; this.custom = {};
this.plugins = []; this.plugins = [];
Copy path View file
@@ -31,7 +31,7 @@ describe('Service', () => {
expect(serviceInstance.provider).to.deep.equal({ expect(serviceInstance.provider).to.deep.equal({
stage: 'dev', stage: 'dev',
region: 'us-east-1', region: 'us-east-1',
variableSyntax: '\\${([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}', variableSyntax: '\\${([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}',
}); });
expect(serviceInstance.custom).to.deep.equal({}); expect(serviceInstance.custom).to.deep.equal({});
expect(serviceInstance.plugins).to.deep.equal([]); expect(serviceInstance.plugins).to.deep.equal([]);
@@ -131,7 +131,7 @@ describe('Service', () => {
name: 'aws', name: 'aws',
stage: 'dev', stage: 'dev',
region: 'us-east-1', region: 'us-east-1',
variableSyntax: '\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}', variableSyntax: '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}',
}, },
plugins: ['testPlugin'], plugins: ['testPlugin'],
functions: { functions: {
@@ -163,7 +163,7 @@ describe('Service', () => {
expect(serviceInstance.service).to.be.equal('new-service'); expect(serviceInstance.service).to.be.equal('new-service');
expect(serviceInstance.provider.name).to.deep.equal('aws'); expect(serviceInstance.provider.name).to.deep.equal('aws');
expect(serviceInstance.provider.variableSyntax).to.equal( expect(serviceInstance.provider.variableSyntax).to.equal(
'\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}' '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}'
); );
expect(serviceInstance.plugins).to.deep.equal(['testPlugin']); expect(serviceInstance.plugins).to.deep.equal(['testPlugin']);
expect(serviceInstance.resources.aws).to.deep.equal({ resourcesProp: 'value' }); expect(serviceInstance.resources.aws).to.deep.equal({ resourcesProp: 'value' });
@@ -186,7 +186,7 @@ describe('Service', () => {
name: 'aws', name: 'aws',
stage: 'dev', stage: 'dev',
region: 'us-east-1', region: 'us-east-1',
variableSyntax: '\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}', variableSyntax: '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}',
}, },
plugins: ['testPlugin'], plugins: ['testPlugin'],
functions: { functions: {
@@ -217,7 +217,7 @@ describe('Service', () => {
expect(serviceInstance.service).to.be.equal('new-service'); expect(serviceInstance.service).to.be.equal('new-service');
expect(serviceInstance.provider.name).to.deep.equal('aws'); expect(serviceInstance.provider.name).to.deep.equal('aws');
expect(serviceInstance.provider.variableSyntax).to.equal( expect(serviceInstance.provider.variableSyntax).to.equal(
'\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}' '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}'
); );
expect(serviceInstance.plugins).to.deep.equal(['testPlugin']); expect(serviceInstance.plugins).to.deep.equal(['testPlugin']);
expect(serviceInstance.resources.aws).to.deep.equal({ resourcesProp: 'value' }); expect(serviceInstance.resources.aws).to.deep.equal({ resourcesProp: 'value' });
@@ -240,7 +240,7 @@ describe('Service', () => {
name: 'aws', name: 'aws',
stage: 'dev', stage: 'dev',
region: 'us-east-1', region: 'us-east-1',
variableSyntax: '\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}', variableSyntax: '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}',
}, },
plugins: ['testPlugin'], plugins: ['testPlugin'],
functions: { functions: {
@@ -272,7 +272,7 @@ describe('Service', () => {
expect(serviceInstance.service).to.be.equal('new-service'); expect(serviceInstance.service).to.be.equal('new-service');
expect(serviceInstance.provider.name).to.deep.equal('aws'); expect(serviceInstance.provider.name).to.deep.equal('aws');
expect(serviceInstance.provider.variableSyntax).to.equal( expect(serviceInstance.provider.variableSyntax).to.equal(
'\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}' '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}'
); );
expect(serviceInstance.plugins).to.deep.equal(['testPlugin']); expect(serviceInstance.plugins).to.deep.equal(['testPlugin']);
expect(serviceInstance.resources.aws).to.deep.equal({ resourcesProp: 'value' }); expect(serviceInstance.resources.aws).to.deep.equal({ resourcesProp: 'value' });
@@ -295,7 +295,7 @@ describe('Service', () => {
name: 'aws', name: 'aws',
stage: 'dev', stage: 'dev',
region: 'us-east-1', region: 'us-east-1',
variableSyntax: '\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}', variableSyntax: '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}',
}, },
plugins: ['testPlugin'], plugins: ['testPlugin'],
functions: { functions: {
@@ -327,7 +327,7 @@ describe('Service', () => {
expect(serviceInstance.service).to.be.equal('new-service'); expect(serviceInstance.service).to.be.equal('new-service');
expect(serviceInstance.provider.name).to.deep.equal('aws'); expect(serviceInstance.provider.name).to.deep.equal('aws');
expect(serviceInstance.provider.variableSyntax).to.equal( expect(serviceInstance.provider.variableSyntax).to.equal(
'\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}' '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}'
); );
expect(serviceInstance.plugins).to.deep.equal(['testPlugin']); expect(serviceInstance.plugins).to.deep.equal(['testPlugin']);
expect(serviceInstance.resources.aws).to.deep.equal({ resourcesProp: 'value' }); expect(serviceInstance.resources.aws).to.deep.equal({ resourcesProp: 'value' });
@@ -363,7 +363,7 @@ describe('Service', () => {
name: 'aws', name: 'aws',
stage: 'dev', stage: 'dev',
region: 'us-east-1', region: 'us-east-1',
variableSyntax: '\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}', variableSyntax: '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}',
}, },
plugins: ['testPlugin'], plugins: ['testPlugin'],
functions: { functions: {
@@ -792,7 +792,7 @@ describe('Service', () => {
name: 'aws', name: 'aws',
stage: 'dev', stage: 'dev',
region: 'us-east-1', region: 'us-east-1',
variableSyntax: '\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}', variableSyntax: '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}',
}, },
plugins: ['testPlugin'], plugins: ['testPlugin'],
functions: { functions: {
Copy path View file
@@ -252,7 +252,7 @@ class Utils {
} }


let hasCustomVariableSyntaxDefined = false; let hasCustomVariableSyntaxDefined = false;
const defaultVariableSyntax = '\\${([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}'; const defaultVariableSyntax = '\\${([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}';


// check if the variableSyntax in the provider section is defined // check if the variableSyntax in the provider section is defined
if (provider && provider.variableSyntax if (provider && provider.variableSyntax
Copy path View file
@@ -35,7 +35,7 @@ const PromiseTracker = require('./PromiseTracker');
* adornments that the framework's components make to the originally loaded service. As a result, * adornments that the framework's components make to the originally loaded service. As a result,
* it is important to reset this object for each use. * it is important to reset this object for each use.
* 3. Note that despite some AWS code herein that this class is used in all plugins. Obviously * 3. Note that despite some AWS code herein that this class is used in all plugins. Obviously
* users avoid the AWS-specific variable types when targetting other clouds. * users avoid the AWS-specific variable types when targeting other clouds.
*/ */
class Variables { class Variables {
constructor(serverless) { constructor(serverless) {
@@ -46,7 +46,7 @@ class Variables {
this.deep = []; this.deep = [];
this.deepRefSyntax = RegExp(/(\${)?deep:\d+(\.[^}]+)*()}?/); this.deepRefSyntax = RegExp(/(\${)?deep:\d+(\.[^}]+)*()}?/);
this.overwriteSyntax = RegExp(/\s*(?:,\s*)+/g); this.overwriteSyntax = RegExp(/\s*(?:,\s*)+/g);
this.fileRefSyntax = RegExp(/^file\((~?[a-zA-Z0-9._\-/]+?)\)/g); this.fileRefSyntax = RegExp(/^file\(([^?%*:|"<>]+?)\)/g);
this.envRefSyntax = RegExp(/^env:/g); this.envRefSyntax = RegExp(/^env:/g);
this.optRefSyntax = RegExp(/^opt:/g); this.optRefSyntax = RegExp(/^opt:/g);
this.selfRefSyntax = RegExp(/^self:/g); this.selfRefSyntax = RegExp(/^self:/g);
Copy path View file
@@ -48,7 +48,7 @@ describe('Variables', () => {
describe('#loadVariableSyntax()', () => { describe('#loadVariableSyntax()', () => {
it('should set variableSyntax', () => { it('should set variableSyntax', () => {
// eslint-disable-next-line no-template-curly-in-string // eslint-disable-next-line no-template-curly-in-string
serverless.service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}'; serverless.service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}';
serverless.variables.loadVariableSyntax(); serverless.variables.loadVariableSyntax();
expect(serverless.variables.variableSyntax).to.be.a('RegExp'); expect(serverless.variables.variableSyntax).to.be.a('RegExp');
}); });
@@ -295,7 +295,7 @@ describe('Variables', () => {
beforeEach(() => { beforeEach(() => {
service = makeDefault(); service = makeDefault();
// eslint-disable-next-line no-template-curly-in-string // eslint-disable-next-line no-template-curly-in-string
service.provider.variableSyntax = '\\${([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}'; // default service.provider.variableSyntax = '\\${([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}'; // default
serverless.variables.service = service; serverless.variables.service = service;
serverless.variables.loadVariableSyntax(); serverless.variables.loadVariableSyntax();
delete service.provider.variableSyntax; delete service.provider.variableSyntax;
@@ -635,7 +635,7 @@ describe('Variables', () => {
.should.become(expected); .should.become(expected);
}); });
it('should handle deep variables regardless of custom variableSyntax', () => { it('should handle deep variables regardless of custom variableSyntax', () => {
service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._\\\'",\\-\\/\\(\\)]+?)}}'; service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._@\\\'",\\-\\/\\(\\)]+?)}}';
serverless.variables.loadVariableSyntax(); serverless.variables.loadVariableSyntax();
delete service.provider.variableSyntax; delete service.provider.variableSyntax;
service.custom = { service.custom = {
@@ -652,7 +652,7 @@ describe('Variables', () => {
.should.become(expected); .should.become(expected);
}); });
it('should handle deep variables regardless of recursion into custom variableSyntax', () => { it('should handle deep variables regardless of recursion into custom variableSyntax', () => {
service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._\\\'",\\-\\/\\(\\)]+?)}}'; service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._@\\\'",\\-\\/\\(\\)]+?)}}';
serverless.variables.loadVariableSyntax(); serverless.variables.loadVariableSyntax();
delete service.provider.variableSyntax; delete service.provider.variableSyntax;
service.custom = { service.custom = {
@@ -673,7 +673,7 @@ describe('Variables', () => {
.should.become(expected); .should.become(expected);
}); });
it('should handle deep variables in complex recursions of custom variableSyntax', () => { it('should handle deep variables in complex recursions of custom variableSyntax', () => {
service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._\\\'",\\-\\/\\(\\)]+?)}}'; service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._@\\\'",\\-\\/\\(\\)]+?)}}';
serverless.variables.loadVariableSyntax(); serverless.variables.loadVariableSyntax();
delete service.provider.variableSyntax; delete service.provider.variableSyntax;
service.custom = { service.custom = {
@@ -704,7 +704,7 @@ describe('Variables', () => {
fse.removeSync(tmpDirPath); fse.removeSync(tmpDirPath);
}); });
const makeTempFile = (fileName, fileContent) => { const makeTempFile = (fileName, fileContent) => {
fse.writeFileSync(path.join(tmpDirPath, fileName), fileContent); fse.outputFileSync(path.join(tmpDirPath, fileName), fileContent);
}; };
const asyncFileName = 'async.load.js'; const asyncFileName = 'async.load.js';
const asyncContent = `'use strict'; const asyncContent = `'use strict';
@@ -792,6 +792,23 @@ module.exports = {
return serverless.variables.populateObject(service.custom) return serverless.variables.populateObject(service.custom)
.should.become(expected); .should.become(expected);
}); });
it('should populate variables from filesnames including \'@\', e.g scoped npm packages',
() => {
const fileName = `./node_modules/@scoped-org/${asyncFileName}`;
makeTempFile(
fileName,
asyncContent
);
service.custom = {
val0: `\${file(${fileName}):str}`,
};
const expected = {
val0: 'my-async-value-1',
};
return serverless.variables
.populateObject(service.custom)
.should.become(expected);
});
const selfFileName = 'self.yml'; const selfFileName = 'self.yml';
const selfContent = `foo: baz const selfContent = `foo: baz
bar: \${self:custom.self.foo} bar: \${self:custom.self.foo}
@@ -1025,7 +1042,7 @@ module.exports = {


describe('#overwrite()', () => { describe('#overwrite()', () => {
beforeEach(() => { beforeEach(() => {
serverless.service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}'; serverless.service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}';
serverless.variables.loadVariableSyntax(); serverless.variables.loadVariableSyntax();
delete serverless.service.provider.variableSyntax; delete serverless.service.provider.variableSyntax;
}); });
@@ -1243,7 +1260,7 @@ module.exports = {


describe('#getValueFromSelf()', () => { describe('#getValueFromSelf()', () => {
beforeEach(() => { beforeEach(() => {
serverless.service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}}'; serverless.service.provider.variableSyntax = '\\${{([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}}';
serverless.variables.loadVariableSyntax(); serverless.variables.loadVariableSyntax();
delete serverless.service.provider.variableSyntax; delete serverless.service.provider.variableSyntax;
}); });
Copy path View file
@@ -54,7 +54,7 @@ class Print {
service.provider = _.merge({ service.provider = _.merge({
stage: 'dev', stage: 'dev',
region: 'us-east-1', region: 'us-east-1',
variableSyntax: '\\${([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}', variableSyntax: '\\${([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}',
}, service.provider); }, service.provider);
} }
strip(svc) { strip(svc) {
@@ -32,7 +32,7 @@ describe('Print', () => {
}); });


afterEach(() => { afterEach(() => {
serverless.service.provider.variableSyntax = '\\${([ ~:a-zA-Z0-9._\'",\\-\\/\\(\\)]+?)}'; serverless.service.provider.variableSyntax = '\\${([ ~:a-zA-Z0-9._@\'",\\-\\/\\(\\)]+?)}';
}); });


it('should print standard config', () => { it('should print standard config', () => {
@@ -256,10 +256,10 @@ describe('Print', () => {
provider: { provider: {
name: 'aws', name: 'aws',
stage: '${{opt:stage}}', stage: '${{opt:stage}}',
variableSyntax: "\\${{([ ~:a-zA-Z0-9._\\'\",\\-\\/\\(\\)]+?)}}", variableSyntax: "\\${{([ ~:a-zA-Z0-9._@\\'\",\\-\\/\\(\\)]+?)}}",
}, },
}; };
serverless.service.provider.variableSyntax = "\\${{([ ~:a-zA-Z0-9._\\'\",\\-\\/\\(\\)]+?)}}"; serverless.service.provider.variableSyntax = "\\${{([ ~:a-zA-Z0-9._@\\'\",\\-\\/\\(\\)]+?)}}";
getServerlessConfigFileStub.resolves(conf); getServerlessConfigFileStub.resolves(conf);


serverless.processedInput = { serverless.processedInput = {
@@ -272,7 +272,7 @@ describe('Print', () => {
provider: { provider: {
name: 'aws', name: 'aws',
stage: 'dev', stage: 'dev',
variableSyntax: "\\${{([ ~:a-zA-Z0-9._\\'\",\\-\\/\\(\\)]+?)}}", variableSyntax: "\\${{([ ~:a-zA-Z0-9._@\\'\",\\-\\/\\(\\)]+?)}}",
}, },
}; };


ProTip! Use n and p to navigate between commits in a pull request.