Add parameter guessing (#97)

* indexTest.d.ts -> lf * Allow parameter guessing to be controlled * changelog
martysweet · Nov 29, 2017 · bd1fa4b · bd1fa4b
1 parent 6361132
commit bd1fa4b
Show file tree

Hide file tree

Showing 9 changed files with 265 additions and 126 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@ Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 - Merge PR #96, fix invalid List<ParameterValue> validation
+- Merge PR #97, allow control over whether parameters are guessed or not.
 
 ## [1.3.4] - 2017-11-19
 ### Changed

diff --git a/README.md b/README.md
@@ -29,6 +29,8 @@ If you get `/usr/bin/env: 'node': No such file or directory` ensure your system
 
 `cfn-lint validate my_template.yaml --parameters key="my value",key2=value2,key3=3`
 
+`cfn-lint validate my_template.yaml --parameters key="my value",key2=value2 --no-guess-parameters`
+
 `cfn-lint validate my_template.yaml --pseudo AWS::StackName="My-Stack"`
 
 `cfn-lint validate my_template.yaml --parameters key="my value" --pseudo AWS::Region=ap-northeast-1,AWS::AccountId=000000000000`
@@ -57,6 +59,21 @@ Documentation: http://docs.aws.amazon.com/search/doc-search.html?searchPath=docu
 Template invalid!
 ```
 
+### Flags
+`--parameters <param values>`: Provide a list of comma-separated key=value pairs of parameters to use when validating your template. If a parameter is not specified here, `cfn-lint` will guess a mock value based on the Parameter's Type and AllowedValues. e.g.
+  - `--parameters InstanceType=t2.micro,Memory=512`
+
+`--pseudo <psuedo param values>`: Provide a list of comma-separated key=value pairs of CloudFormation pseudo-parameters to use when validating your template. e.g.
+  - `--pseudo AWS::Region=ap-southeast-2`
+
+`--guess-parameters`: Guess any parameters if they don't have any Default value in the template. Parameters will be guessed/mocked based on their `AllowedValues` or `Type`. This is the default behaviour; it's only included as an option for explicitness.
+
+`--no-guess-parameters`: Disable the guessing of parameters if they don't have a Default. If you don't provide them on the CLI in this situation, a critical error will be raised instead of the parameter value being mocked.
+
+`--only-guess-parameters <param names>`: Only guess the provided parameters, and disable the guessing of all others without Defaults. A critical error will be raised for missing parameters, as above. e.g.
+ - `--only-guess-parameters InstanceType,Memory`
+
+
 ### What can cfn-lint do?
 * Read JSON + YAML (Including YAML short form)
 * Detect invalid property names
@@ -128,11 +145,13 @@ interface ValidationOptions {
   pseudoParameters?: {
     'AWS::Region': 'ap-southeast-2',
     // ...
-  }
+  },
+  guessParameters?: string[] | undefined // default undefined
 }
 ```
 `parameters` get passed into the template's Parameters before validation, and `pseudoParameters` are used to override AWS' pseudo-parameters, like `AWS::Region`, `AWS::AccountId`, etc.
 
+If `guessParameters` is set to a list of parameter names, a critical error will be raised if any Parameter with no Default is not specified in the `parameters` or `guessParameters` options. An empty list can be used to enforce that all parameters must be specified in `parameters`. Leaving as `undefined` preserves the default loose behaviour, where parameters are guessed as needed without causing an error.
 
 ```ts
 interface ErrorRecord {

diff --git a/src/api.ts b/src/api.ts
@@ -1,15 +1,18 @@
 import validator = require('./validator');
 
-export interface ValidationOptions {
+export interface APIValidationOptions {
   parameters: {[parameterName: string]: any}
   pseudoParameters: {[pseudoParameterName: string]: any}
 }
 
+export type ValidationOptions = APIValidationOptions & validator.ValidateOptions;
+
 export type ValidationResult = validator.ErrorObject;
 
 const defaultOptions: ValidationOptions = {
   parameters: {},
-  pseudoParameters: {}
+  pseudoParameters: {},
+  guessParameters: undefined
 }
 
 /**
@@ -19,7 +22,7 @@ const defaultOptions: ValidationOptions = {
  */
 export function validateFile(fileName: string, options?: Partial<ValidationOptions>): ValidationResult {
   setupValidator(options);
-  return validator.validateFile(fileName);
+  return validator.validateFile(fileName, options);
 }
 
 /**
@@ -30,7 +33,7 @@ export function validateFile(fileName: string, options?: Partial<ValidationOptio
  */
 export function validateJsonObject(objectToValidate: any, options?: Partial<ValidationOptions>): ValidationResult {
   setupValidator(options);
-  return validator.validateJsonObject(objectToValidate);
+  return validator.validateJsonObject(objectToValidate, options);
 }
 
 function setupValidator(passedOptions?: Partial<ValidationOptions>) {

diff --git a/src/index.ts b/src/index.ts
@@ -19,6 +19,12 @@ program
     .arguments('<cmd> <file>')
     .option('-p, --parameters <items>', 'List of params', list)
     .option('-p, --pseudo <items>', 'List of pseudo overrides', list)
+    // https://github.com/tj/commander.js/issues/108
+    // might get fixed by https://github.com/tj/commander.js/issues/691
+    // as a workaround, we can actually leave this out. It defaults to true and the unparsed parameter will be ignored.
+//  .option('--guess-parameters', 'Guess any parameters that are not explicitely passed in and have no Default. This is the default behaviour.')
+    .option('-G, --no-guess-parameters', 'Fail validation if a parameter with no Default is not passed')
+    .option('-g, --only-guess-parameters <items>', 'Guess the provided parameters, and fail validation if a parameter with no Default is passed', list)
     .action(function (arg1, arg2) {
         firstArg = arg1;
         secondArg = arg2;
@@ -55,7 +61,20 @@ if(firstArg == "validate"){
         }
     }
 
-    let result = validator.validateFile(secondArg);
+    let guessParameters: string[] | undefined;
+    if (program.guessParameters === false) {
+        guessParameters = [];
+    } else if (program.onlyGuessParameters) {
+        guessParameters = program.onlyGuessParameters;
+    } else {
+        guessParameters = undefined;
+    }
+
+    const options = {
+        guessParameters
+    };
+
+    let result = validator.validateFile(secondArg, options);
     // Show the errors
     console.log((result['errors']['info'].length + " infos").grey);
     for(let info of result['errors']['info']){

diff --git a/src/test/indexTest.ts b/src/test/indexTest.ts
@@ -1,77 +1,105 @@
-import chai = require('chai');
-const expect = chai.expect;
-const assert = chai.assert;
-
-import childProcess = require('child_process');
-const exec = childProcess.exec;
-
-describe('index', () => {
-
-
-    describe('CLI', () => {
-
-
-        it('no parameters', (done) => {
-            exec('node lib/index.js', function(error, stdout, stderr) {
-                expect(stderr).to.contain('no command given!');
-                done();
-            });
-        }).timeout(5000);;
-
-        it('missing file parameter', (done) => {
-            exec('node lib/index.js validate', function(error, stdout, stderr) {
-                expect(stderr).to.contain('missing required argument');
-                done();
-            });
-        }).timeout(5000);;
-
-
-        it('validate simple yaml', (done) => {
-
-            exec('node lib/index.js validate testData/valid/yaml/issue-28-custom-resource.yaml', function(error, stdout, stderr) {
-                console.log(stderr);
-                console.log(stdout);
-                expect(stdout).to.contain('0 crit');
-                done();
-            });
-        }).timeout(5000);
-
-        it('validate parameter flag', (done) => {
-
-            exec('node lib/index.js validate testData/valid/json/2.json --parameters InstanceType="t1.micro"', function(error, stdout, stderr) {
-                expect(stdout).to.contain('0 crit');
-                done();
-            });
-        }).timeout(5000);
-
-        it('validate pseudo flag', (done) => {
-
-            exec('node lib/index.js validate testData/valid/yaml/pseudo-parameters.yaml ' +
-                '--pseudo AWS::Region=us-east-1,AWS::AccountId=000000000000', function(error, stdout, stderr) {
-                expect(stdout).to.contain('0 crit');
-                done();
-            });
-        }).timeout(5000);
-
-
-        it('validate pseudo + parameter flag', (done) => {
-
-            exec('node lib/index.js validate testData/valid/yaml/pseudo-w-parameter.yaml ' +
-                '--parameters MyInput=abcd --pseudo AWS::Region=us-east-1', function(error, stdout, stderr) {
-                expect(stdout).to.contain('0 crit');
-                done();
-            });
-        }).timeout(5000);
-
-        it('invalid pseudo flag throws 2 critical error', (done) => {
-
-            exec('node lib/index.js validate testData/valid/yaml/pseudo-parameters.yaml ' +
-                '--pseudo AWS::Region=us-east-1,Something=000000000000', function(error, stdout, stderr) {
-                expect(stdout).to.contain('2 crit');
-                done();
-            });
-        }).timeout(5000);
-
-    });
-
-});
+import chai = require('chai');
+const expect = chai.expect;
+const assert = chai.assert;
+
+import childProcess = require('child_process');
+const exec = childProcess.exec;
+
+describe('index', () => {
+
+
+    describe('CLI', () => {
+
+
+        it('no parameters', (done) => {
+            exec('node lib/index.js', function(error, stdout, stderr) {
+                expect(stderr).to.contain('no command given!');
+                done();
+            });
+        }).timeout(5000);;
+
+        it('missing file parameter', (done) => {
+            exec('node lib/index.js validate', function(error, stdout, stderr) {
+                expect(stderr).to.contain('missing required argument');
+                done();
+            });
+        }).timeout(5000);;
+
+
+        it('validate simple yaml', (done) => {
+
+            exec('node lib/index.js validate testData/valid/yaml/issue-28-custom-resource.yaml', function(error, stdout, stderr) {
+                expect(stdout).to.contain('0 crit');
+                done();
+            });
+        }).timeout(5000);
+
+        it('validate parameter flag', (done) => {
+
+            exec('node lib/index.js validate testData/valid/json/2.json --parameters InstanceType="t1.micro"', function(error, stdout, stderr) {
+                expect(stdout).to.contain('0 crit');
+                done();
+            });
+        }).timeout(5000);
+
+        it('validate pseudo flag', (done) => {
+
+            exec('node lib/index.js validate testData/valid/yaml/pseudo-parameters.yaml ' +
+                '--pseudo AWS::Region=us-east-1,AWS::AccountId=000000000000', function(error, stdout, stderr) {
+                expect(stdout).to.contain('0 crit');
+                done();
+            });
+        }).timeout(5000);
+
+
+        it('validate pseudo + parameter flag', (done) => {
+
+            exec('node lib/index.js validate testData/valid/yaml/pseudo-w-parameter.yaml ' +
+                '--parameters MyInput=abcd --pseudo AWS::Region=us-east-1', function(error, stdout, stderr) {
+                expect(stdout).to.contain('0 crit');
+                done();
+            });
+        }).timeout(5000);
+
+        it('invalid pseudo flag throws 2 critical error', (done) => {
+
+            exec('node lib/index.js validate testData/valid/yaml/pseudo-parameters.yaml ' +
+                '--pseudo AWS::Region=us-east-1,Something=000000000000', function(error, stdout, stderr) {
+                expect(stdout).to.contain('2 crit');
+                done();
+            });
+        }).timeout(5000);
+
+
+        it('guess-parameters should explicitely opt in to parameter mocking', (done) => {
+            exec('node lib/index.js validate testData/valid/yaml/no-guess-parameters.yaml --guess-parameters', function(error, stdout, stderr) {
+                expect(stdout).to.contain('0 crit');
+                done();
+            });
+        }).timeout(5000);
+
+        it('no-guess-parameters throws errors if we leave out parameters', (done) => {
+            exec('node lib/index.js validate testData/valid/yaml/no-guess-parameters.yaml --no-guess-parameters', function(error, stdout, stderr) {
+                expect(stdout).to.contain('2 crit');
+                expect(stdout).to.contain('Value for parameter was not provided');
+                done();
+            });
+        }).timeout(5000);
+
+        it('only-guess-parameters should allow opting in to parameter mocking', (done) => {
+            exec('node lib/index.js validate testData/valid/yaml/no-guess-parameters.yaml --only-guess-parameters Param1', function(error, stdout, stderr) {
+                expect(stdout).to.contain('1 crit');
+                expect(stdout).to.contain('Value for parameter was not provided');
+                done();
+            });
+        }).timeout(5000);
+
+        it('only-guess-parameters should allow opting in to parameter mocking with multiple params', (done) => {
+            exec('node lib/index.js validate testData/valid/yaml/no-guess-parameters.yaml --only-guess-parameters Param1,Param2', function(error, stdout, stderr) {
+                expect(stdout).to.contain('0 crit');
+                done();
+            });
+        }).timeout(5000);
+    });
+
+});
diff --git a/src/test/validatorTest.ts b/src/test/validatorTest.ts
@@ -523,6 +523,22 @@ describe('validator', () => {
             expect(result['errors']['crit']).to.have.lengthOf(1);
             expect(result['errors']['crit'][0]).to.has.property('message', 'Parameter value \'\' for Env is not within the parameters AllowedValues');
         });
+
+        it('missing parameters should cause an error when guessParameters is set', () => {
+            const input = 'testData/valid/yaml/parameters.yaml';
+            let result = validator.validateFile(input, {guessParameters: []});
+            expect(result).to.have.deep.property('templateValid', false);
+            expect(result['errors']['crit']).to.have.lengthOf(1);
+            expect(result['errors']['crit'][0]).to.has.property('message', 'Value for parameter was not provided');
+        });
+
+        it('parameters in guessParameters should be permitted to be guessed', () => {
+            const input = 'testData/valid/yaml/parameters.yaml';
+            let result = validator.validateFile(input, {guessParameters: ['Env']});
+            expect(result).to.have.deep.property('templateValid', true);
+            expect(result['errors']['crit']).to.have.lengthOf(0);
+            expect(result['errors']['info']).to.have.lengthOf(0);
+        })
     });
 
     describe('pseudo-parmeters', () => {