Merge pull request #11 from joanvila/response_schema_validation

Response schema validation

README.md

@@ -42,7 +42,11 @@ In order to say if a response code is valid or not, `raml-validate` uses two pri
 
 However, if the response codes are not present in the API definition, the only accepted code will be the 200 one. Again, you can change the default accepted response codes in the file `config.js`.
 
-## Uri parameters
+## Checking the response schema
+
+By default, `raml-validate` doesn't check the response schema. If you want to check it, you can use the option `--validate-response`. This option will check that the json returned from the API matches the one defined in the raml. However, only the first level of the response will be taken into account. In the next versions, there will be an option to check all the schema recursively.
+
+## Sending uri parameters
 
 In order to allow `raml-validate` to check the API with the appropriate uri parameters you should define them with examples:
 
@@ -59,7 +63,7 @@ In order to allow `raml-validate` to check the API with the appropriate uri para
 
 If a parameter doesn't have an example, a non deterministic value will be used instead.
 
-## Query parameters
+## Sending query parameters
 
 Query parameters will be sent to the API if they are defined with an example value:
 
@@ -79,7 +83,7 @@ Query parameters will be sent to the API if they are defined with an example val
 
 In this case, the tested endpoint would be `baseUri/url/with/params?today=2016-11-23&tomorrow=2016-11-24`. Note that required field doesn't affect.
 
-## Post data in the request body
+## Sending post data in the request body
 
 In some cases, it is necessary to send data in the body of the request. Specially in `POST` methods. `raml-validate` supports it according to the RAML 1.0 syntax:
 

bin/raml-validate.js

@@ -10,15 +10,37 @@ const yargs = require('yargs');
 const Validator = require('../lib/validator');
 
 const argv = yargs
-    .usage('Usage:\n  raml-validate.js </path/to/raml> [target]' +
-    '\n\nExample:\n  ' + 'raml-validate definition.raml --target http://localhost:8080')
+    .usage('Usage:\n  raml-validate.js </path/to/raml> [options]')
+
+    .help('h')
+    .alias('h', 'help')
+
+    .option('t', {
+        alias: 'target',
+        demand: false,
+        describe: 'The endpoint to test',
+        type: 'string'
+    })
+
+    .option('r', {
+        alias: 'validate-response',
+        demand: false,
+        describe: 'Validate the response schema',
+        type: 'boolean'
+    })
+
+    .example('raml-validate definition.raml', '--target http://localhost:8080')
+    .example('raml-validate definition.raml', '--validate-response')
+
     .check(argv => {
         if (argv._.length < 1) {
             throw new Error('raml-validate.js: Must specify path to RAML file');
         }
         return true;
     })
-    .epilog("Website:\n  " + 'https://github.com/joanvila/raml-js-validator')
+
+    .epilog('More info and docs:\n' +
+    '  ' + 'https://github.com/joanvila/raml-js-validator')
     .argv;
 
 const fileName = path.resolve(process.cwd(), argv._[0]);
@@ -34,7 +56,11 @@ try {
     throw err.message;
 }
 
-console.log('RAML parsing success. Querying api now...');
+console.log('RAML parsing success. Querying the API now...');
 
-const validator = new Validator(api, argv.target);
+const validator = new Validator(
+    api,
+    argv.target,
+    argv['validate-response']
+);
 validator.validate();

examples/definition.raml

@@ -9,15 +9,17 @@ baseUri: /
             200:
                 body:
                     application/json:
-                        type: string
+                        type: object
+                        properties:
+                            message: string
 
 /tasks:
     get:
         responses:
             200:
                 body:
                     application/json:
-                        type: array
+                        type: object
 
 /task/{taskid}:
     description: get task by id

examples/routes/mainEndpoint.js

@@ -4,14 +4,18 @@ const express = require('express');
 const router = express.Router();
 
 router.get('/', (req, res, next) => {
-    res.status(200).json('Hello world!');
+    res.status(200).json({
+        message: 'Hello world!'
+    });
 });
 
 router.get('/tasks', (req, res, next) => {
-    res.status(200).json([
-        'Do something',
-        'Another thing'
-    ]);
+    res.status(200).json({
+        tasks: [
+            'Do something',
+            'Another thing'
+        ]
+    });
 });
 
 router.get('/task/:taskid', (req, res, next) => {

lib/endpoint-checker.js

@@ -4,28 +4,57 @@ const Q = require('q');
 const request = require('request');
 const config = require('../config');
 
+const ResponseSchemaChecker = require('./response-schema-checker');
+
 const EndpointChecker = function() {
 
-    this.check = (endpointToTest, method, acceptedResponseCodes, postData) => {
+    this.responseSchemaChecker = new ResponseSchemaChecker();
+
+    this.check = (endpointToTest, method, expectedResponses, postData, validateResponse) => {
         let d = Q.defer();
 
-        if (!acceptedResponseCodes.length) {
-            acceptedResponseCodes = config.defaultAcceptedResponseCodes;
-        }
+        let expectedResponseCodes = getExpectedResponseCodes(expectedResponses);
 
         request[method](endpointToTest, {form: postData}, (error, response, body) => {
-            if (error || !acceptedResponseCodes.includes(response.statusCode)) {
+            if (error) {
+                d.reject(new Error('Error: ' + error.syscall + ' ' + error.code +
+                    ' ' + error.address + ':' + error.port));
+            } else if (!expectedResponseCodes.includes(response.statusCode)) {
                 d.reject(new Error('[ERROR][' + method + '] ' + endpointToTest +
                     '\n    Request body: ' + JSON.stringify(postData) +
                     '\n    Response: ' + response.statusCode + ' ' + response.body));
             } else {
-                d.resolve('Resource OK');
+                if (validateResponse) {
+                    const validationResult = this.responseSchemaChecker.check(expectedResponses, response);
+                    if (validationResult.valid) {
+                        d.resolve('Resource OK');
+                    } else {
+                        d.reject(new Error('[ERROR][' + method + '] ' + endpointToTest +
+                        '\n    Response schema error: ' + validationResult.validationErrorReason));
+                    }
+                } else {
+                    d.resolve('Resource OK');
+                }
             }
         });
 
         return d.promise;
     };
 
+    function getExpectedResponseCodes(expectedResponses) {
+        let expectedResponseCodes = [];
+
+        expectedResponses.forEach((response) => {
+            expectedResponseCodes.push(parseInt(response.code().value(), 10));
+        });
+
+        if (!expectedResponseCodes.length) {
+            expectedResponseCodes = config.defaultAcceptedResponseCodes;
+        }
+
+        return expectedResponseCodes;
+    }
+
 };
 
 module.exports = EndpointChecker;

lib/response-schema-checker.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const _ = require('lodash/core');
+
+const ResponseSchemaChecker = function() {
+
+    const noResponseValidation = {valid: true, validationErrorReason: 'no body'};
+
+    this.check = (expectedResponses, response) => {
+        if (_.isEmpty(expectedResponses)) return noResponseValidation;
+
+        const expectedBody = getBodyFromCode(expectedResponses, response.statusCode);
+        if (!expectedBody) return noResponseValidation;
+
+        // If a body is defined we need to check it's schema
+        const responseBody = JSON.parse(response.body);
+        return checkSchema(expectedBody[0], responseBody);
+    };
+
+    function getBodyFromCode(expectedResponses, responseCode) {
+        let expectedBody = null;
+
+        for (var i in expectedResponses) {
+            const expectedCodeValue = parseInt(expectedResponses[i].code().value(), 10);
+            if(expectedCodeValue === responseCode) {
+                expectedBody = expectedResponses[i].body();
+                if (_.isEmpty(expectedBody)) expectedBody = null;
+                break;
+            }
+        }
+
+        return expectedBody;
+    }
+
+    function checkSchema(expectedBody, responseBody) {
+        let validationResult = {};
+
+        if (expectedBody.type()[0] === typeof(responseBody)) {
+            validationResult.valid = true;
+            validationResult.validationErrorReason = '';
+        } else {
+            validationResult.valid = false;
+            validationResult.validationErrorReason = expectedBody.type()[0] + ' !== ' + typeof(responseBody);
+        }
+
+        return validationResult;
+    }
+
+};
+
+module.exports = ResponseSchemaChecker;

lib/validator.js

@@ -6,10 +6,11 @@ const EndpointBuilder = require('./endpoint-builder');
 const EndpointChecker = require('./endpoint-checker');
 const PostDataGenerator = require('./post-data-generator');
 
-const Validator = function(api, target) {
+const Validator = function(api, target, validateResponse) {
 
     this.api = api;
     this.target = target;
+    this.validateResponse = validateResponse;
 
     this.endpointBuilder = new EndpointBuilder();
     this.postDataGenerator = new PostDataGenerator();
@@ -38,12 +39,15 @@ const Validator = function(api, target) {
 
                 endpointToTest = this.endpointBuilder.addQueryParams(endpointToTest, method.queryParameters());
 
-                let acceptedResponseCodes = [];
-                method.responses().forEach((response) => {
-                    acceptedResponseCodes.push(parseInt(response.code().value(), 10));
-                });
+                const expectedResponses = method.responses();
 
-                this.endpointChecker.check(endpointToTest, methodName, acceptedResponseCodes, postData).then(() => {
+                this.endpointChecker.check(
+                    endpointToTest,
+                    methodName,
+                    expectedResponses,
+                    postData,
+                    validateResponse
+                ).then(() => {
                     console.log('[OK][' + methodName + '] ' + endpointToTest);
                     if (!_.isEmpty(postData)) console.log('    With body: ' + JSON.stringify(postData));
                 }).catch(err => {

test/test-endpoint-checker.js

@@ -17,6 +17,28 @@ describe('endpoint-checker', () => {
 
     const endpointChecker = new EndpointChecker();
 
+    const expectedResponses200Mock = [{
+        code: function() {
+            return {
+                value: function() {return 200;}
+            };
+        }
+    }];
+
+    const expectedResponses200500Mock = [{
+        code: function() {
+            return {
+                value: function() {return 200;}
+            };
+        }
+    }, {
+        code: function() {
+            return {
+                value: function() {return 500;}
+            };
+        }
+    }];
+
     describe('check() good endpoint', () => {
 
         before(() => {
@@ -31,12 +53,12 @@ describe('endpoint-checker', () => {
 
 
         it('should accept promise on 200 code', () => {
-            return expect(endpointChecker.check('http://localhost:80', 'get', [200], {})).to.eventually
+            return expect(endpointChecker.check('http://localhost:80', 'get', expectedResponses200Mock, {}, false)).to.eventually
               .equal('Resource OK');
         });
 
         it('should check default response code when no one specified', () => {
-            return expect(endpointChecker.check('http://localhost:80', 'get', [], {})).to.eventually
+            return expect(endpointChecker.check('http://localhost:80', 'get', [], {}, false)).to.eventually
               .equal('Resource OK');
         });
 
@@ -55,13 +77,13 @@ describe('endpoint-checker', () => {
         });
 
         it('should reject promise with error', () => {
-            return expect(endpointChecker.check('http://localhost:80', 'get', [200], {})).to.eventually
+            return expect(endpointChecker.check('http://localhost:80', 'get', expectedResponses200Mock, {}, false)).to.eventually
               .be.rejectedWith('[ERROR][get] http://localhost:80')
               .and.be.an.instanceOf(Error);
         });
 
         it('should accept if the expected code is 500', () => {
-            return expect(endpointChecker.check('http://localhost:80', 'get', [200, 500], {})).to.eventually
+            return expect(endpointChecker.check('http://localhost:80', 'get', expectedResponses200500Mock, {}, false)).to.eventually
               .equal('Resource OK');
         });
 
@@ -82,12 +104,12 @@ describe('endpoint-checker', () => {
         });
 
         it('should return a 200 when sending a post', () => {
-            return expect(endpointChecker.check('http://localhost:80', 'post', [200], {})).to.eventually
+            return expect(endpointChecker.check('http://localhost:80', 'post', expectedResponses200Mock, {}, false)).to.eventually
               .equal('Resource OK');
         });
 
         it('should send the post data when it is not empty', () => {
-            endpointChecker.check('http://localhost:80', 'post', [200], postDataMock);
+            endpointChecker.check('http://localhost:80', 'post', expectedResponses200Mock, postDataMock, false);
 
             request.post.should.have.been.calledWith(
                 'http://localhost:80', {form: postDataMock});