From 4ede633e0c73db917af450e3fefd86b76969d417 Mon Sep 17 00:00:00 2001 From: Martin Buhr Date: Thu, 28 Sep 2017 16:31:45 -0700 Subject: [PATCH] Adds JSON Schema validation to the gateway Fixes #1163 Based on https://github.com/TykTechnologies/tyk/pull/1180 Adds a new JSON Validation middleware that can be configured as follows: ``` "version_data": { "not_versioned": true, "versions": { "default": { "name": "default", "use_extended_paths": true, "extended_paths": { "validate_json": [{ "method": "POST", "path": "me", "validate_with": "BASE64 ENCODED SCHEMA" }] } } } }, ``` The schema must be a draft v4 JSON Schema spec. The gateway will attempt to validate the inbound request against it, if fields are failing the validation process, a detailed error response is provided for the user to fix their payload. This will require a new Dashboard UI to handle input. make Base64 Schema readable in tests removing base64 as not necessary using make for known length slice --- api_definition.go | 26 +++ api_loader.go | 2 + apidef/api_definitions.go | 9 + mw_validate_json.go | 86 ++++++++ mw_validate_json_test.go | 208 ++++++++++++++++++ .../github.com/xeipuuv/gojsonschema/schema.go | 5 - vendor/vendor.json | 6 +- 7 files changed, 334 insertions(+), 8 deletions(-) create mode 100644 mw_validate_json.go create mode 100644 mw_validate_json_test.go diff --git a/api_definition.go b/api_definition.go index 6aaaadea6b7..39a10ab66a4 100644 --- a/api_definition.go +++ b/api_definition.go @@ -52,6 +52,7 @@ const ( MethodTransformed RequestTracked RequestNotTracked + ValidateJSONRequest ) // RequestStatus is a custom type to avoid collisions @@ -80,6 +81,7 @@ const ( StatusRequestSizeControlled RequestStatus = "Request Size Limited" StatusRequesTracked RequestStatus = "Request Tracked" StatusRequestNotTracked RequestStatus = "Request Not Tracked" + StatusValidateJSON RequestStatus = "Validate JSON" ) // URLSpec represents a flattened specification for URLs, used to check if a proxy URL @@ -101,6 +103,7 @@ type URLSpec struct { MethodTransform apidef.MethodTransformMeta TrackEndpoint apidef.TrackEndpointMeta DoNotTrackEndpoint apidef.TrackEndpointMeta + ValidatePathMeta apidef.ValidatePathMeta } type TransformSpec struct { @@ -701,6 +704,20 @@ func (a APIDefinitionLoader) compileTrackedEndpointPathspathSpec(paths []apidef. return urlSpec } +func (a APIDefinitionLoader) compileValidateJSONPathspathSpec(paths []apidef.ValidatePathMeta, stat URLStatus) []URLSpec { + urlSpec := make([]URLSpec, len(paths)) + + for i, stringSpec := range paths { + newSpec := URLSpec{} + a.generateRegex(stringSpec.Path, &newSpec, stat) + // Extend with method actions + newSpec.ValidatePathMeta = stringSpec + urlSpec[i] = newSpec + } + + return urlSpec +} + func (a APIDefinitionLoader) compileUnTrackedEndpointPathspathSpec(paths []apidef.TrackEndpointMeta, stat URLStatus) []URLSpec { urlSpec := []URLSpec{} @@ -734,6 +751,7 @@ func (a APIDefinitionLoader) getExtendedPathSpecs(apiVersionDef apidef.VersionIn methodTransforms := a.compileMethodTransformSpec(apiVersionDef.ExtendedPaths.MethodTransforms, MethodTransformed) trackedPaths := a.compileTrackedEndpointPathspathSpec(apiVersionDef.ExtendedPaths.TrackEndpoints, RequestTracked) unTrackedPaths := a.compileUnTrackedEndpointPathspathSpec(apiVersionDef.ExtendedPaths.DoNotTrackEndpoints, RequestNotTracked) + validateJSON := a.compileValidateJSONPathspathSpec(apiVersionDef.ExtendedPaths.ValidateJSON, ValidateJSONRequest) combinedPath := []URLSpec{} combinedPath = append(combinedPath, ignoredPaths...) @@ -752,6 +770,7 @@ func (a APIDefinitionLoader) getExtendedPathSpecs(apiVersionDef apidef.VersionIn combinedPath = append(combinedPath, methodTransforms...) combinedPath = append(combinedPath, trackedPaths...) combinedPath = append(combinedPath, unTrackedPaths...) + combinedPath = append(combinedPath, validateJSON...) return combinedPath, len(whiteListPaths) > 0 } @@ -797,6 +816,9 @@ func (a *APISpec) getURLStatus(stat URLStatus) RequestStatus { return StatusRequesTracked case RequestNotTracked: return StatusRequestNotTracked + case ValidateJSONRequest: + return StatusValidateJSON + default: log.Error("URL Status was not one of Ignored, Blacklist or WhiteList! Blocking.") return EndPointNotAllowed @@ -930,6 +952,10 @@ func (a *APISpec) CheckSpecMatchesStatus(r *http.Request, rxPaths []URLSpec, mod if r.Method == v.DoNotTrackEndpoint.Method { return true, &v.DoNotTrackEndpoint } + case ValidateJSONRequest: + if r.Method == v.ValidatePathMeta.Method { + return true, &v.ValidatePathMeta + } } } return false, nil diff --git a/api_loader.go b/api_loader.go index 081fd7bc87c..f1aba6564d1 100644 --- a/api_loader.go +++ b/api_loader.go @@ -304,6 +304,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int, mwAppendEnabled(&chainArray, &CertificateCheckMW{BaseMiddleware: baseMid}) mwAppendEnabled(&chainArray, &OrganizationMonitor{BaseMiddleware: baseMid}) mwAppendEnabled(&chainArray, &RateLimitForAPI{BaseMiddleware: baseMid}) + mwAppendEnabled(&chainArray, &ValidateJSON{BaseMiddleware: baseMid}) mwAppendEnabled(&chainArray, &MiddlewareContextVars{BaseMiddleware: baseMid}) mwAppendEnabled(&chainArray, &VersionCheck{BaseMiddleware: baseMid}) mwAppendEnabled(&chainArray, &RequestSizeLimitMiddleware{baseMid}) @@ -441,6 +442,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int, mwAppendEnabled(&chainArray, &RateLimitForAPI{BaseMiddleware: baseMid}) mwAppendEnabled(&chainArray, &RateLimitAndQuotaCheck{baseMid}) mwAppendEnabled(&chainArray, &GranularAccessMiddleware{baseMid}) + mwAppendEnabled(&chainArray, &ValidateJSON{BaseMiddleware: baseMid}) mwAppendEnabled(&chainArray, &TransformMiddleware{baseMid}) mwAppendEnabled(&chainArray, &TransformHeaders{BaseMiddleware: baseMid}) mwAppendEnabled(&chainArray, &URLRewriteMiddleware{BaseMiddleware: baseMid}) diff --git a/apidef/api_definitions.go b/apidef/api_definitions.go index a4a2d775476..2695b6785da 100644 --- a/apidef/api_definitions.go +++ b/apidef/api_definitions.go @@ -165,6 +165,14 @@ type MethodTransformMeta struct { ToMethod string `bson:"to_method" json:"to_method"` } +type ValidatePathMeta struct { + Path string `bson:"path" json:"path"` + Method string `bson:"method" json:"method"` + ValidateWith map[string]interface{} `json:"validate_with"` + // Allows override of default 422 Unprocessible Entity response code for validation errors. + ValidationErrorResponseCode int `bson:"validation_error_response_code" json:"validation_error_response_code"` +} + type ExtendedPathsSet struct { Ignored []EndPointMeta `bson:"ignored" json:"ignored,omitempty"` WhiteList []EndPointMeta `bson:"white_list" json:"white_list,omitempty"` @@ -182,6 +190,7 @@ type ExtendedPathsSet struct { MethodTransforms []MethodTransformMeta `bson:"method_transforms" json:"method_transforms,omitempty"` TrackEndpoints []TrackEndpointMeta `bson:"track_endpoints" json:"track_endpoints,omitempty"` DoNotTrackEndpoints []TrackEndpointMeta `bson:"do_not_track_endpoints" json:"do_not_track_endpoints,omitempty"` + ValidateJSON []ValidatePathMeta `bson:"validate_json" json:"validate_json,omitempty"` } type VersionInfo struct { diff --git a/mw_validate_json.go b/mw_validate_json.go new file mode 100644 index 00000000000..84b0085e728 --- /dev/null +++ b/mw_validate_json.go @@ -0,0 +1,86 @@ +package main + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + + "github.com/xeipuuv/gojsonschema" + + "github.com/TykTechnologies/tyk/apidef" +) + +type ValidateJSON struct { + BaseMiddleware + schemaLoader gojsonschema.JSONLoader +} + +func (k *ValidateJSON) Name() string { + return "ValidateJSON" +} + +func (k *ValidateJSON) EnabledForSpec() bool { + for _, v := range k.Spec.VersionData.Versions { + if len(v.ExtendedPaths.ValidateJSON) > 0 { + return true + } + } + + return false +} + +// ProcessRequest will run any checks on the request on the way through the system, return an error to have the chain fail +func (k *ValidateJSON) ProcessRequest(w http.ResponseWriter, r *http.Request, _ interface{}) (error, int) { + + _, versionPaths, _, _ := k.Spec.Version(r) + found, meta := k.Spec.CheckSpecMatchesStatus(r, versionPaths, ValidateJSONRequest) + if !found { + return nil, http.StatusOK + } + + vPathMeta := meta.(*apidef.ValidatePathMeta) + fmt.Printf("%T: %+v\n", vPathMeta.ValidateWith, vPathMeta.ValidateWith) + if vPathMeta.ValidateWith == nil { + return errors.New("no schemas to validate against"), http.StatusInternalServerError + } + + rCopy := copyRequest(r) + bodyBytes, err := ioutil.ReadAll(rCopy.Body) + if err != nil { + return err, http.StatusInternalServerError + } + defer rCopy.Body.Close() + + schema := vPathMeta.ValidateWith + + result, err := k.validate(bodyBytes, schema) + if err != nil { + return err, http.StatusInternalServerError + } + + if !result.Valid() { + errStr := "payload validation failed" + for _, desc := range result.Errors() { + errStr = fmt.Sprintf("%s, %s", errStr, desc) + } + + if vPathMeta.ValidationErrorResponseCode == 0 { + vPathMeta.ValidationErrorResponseCode = http.StatusUnprocessableEntity + } + + return errors.New(errStr), vPathMeta.ValidationErrorResponseCode + } + + return nil, http.StatusOK +} + +func (k *ValidateJSON) validate(input []byte, schema map[string]interface{}) (*gojsonschema.Result, error) { + inputLoader := gojsonschema.NewBytesLoader(input) + + if k.schemaLoader == nil { + k.schemaLoader = gojsonschema.NewGoLoader(schema) + } + + return gojsonschema.Validate(k.schemaLoader, inputLoader) +} diff --git a/mw_validate_json_test.go b/mw_validate_json_test.go new file mode 100644 index 00000000000..052da17eeb5 --- /dev/null +++ b/mw_validate_json_test.go @@ -0,0 +1,208 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/justinas/alice" + + "github.com/TykTechnologies/tyk/user" +) + +type res struct { + Error string + Code int +} + +type fixture struct { + in string + out res + name string +} + +var schema = `{ + "title": "Person", + "type": "object", + "properties": { + "firstName": { + "type": "string" + }, + "lastName": { + "type": "string" + }, + "age": { + "description": "Age in years", + "type": "integer", + "minimum": 0 + } + }, + "required": ["firstName", "lastName"] +}` + +var fixtures = []fixture{ + { + in: `{"age":23, "firstName": "Harry"}`, + out: res{Error: `lastName: lastName is required`, Code: http.StatusUnprocessableEntity}, + name: "missing field", + }, + { + in: `{"age":23}`, + out: res{Error: `firstName: firstName is required, lastName: lastName is required`, Code: http.StatusUnprocessableEntity}, + name: "missing two fields", + }, + { + in: `{}`, + out: res{Error: `firstName: firstName is required, lastName: lastName is required`, Code: http.StatusUnprocessableEntity}, + name: "empty object", + }, + { + in: `[]`, + out: res{Error: `(root): Invalid type. Expected: object, given: array`, Code: http.StatusUnprocessableEntity}, + name: "array", + }, + { + in: `{"age":"23", "firstName": "Harry", "lastName": "Potter"}`, + out: res{Error: `age: Invalid type. Expected: integer, given: string`, Code: http.StatusUnprocessableEntity}, + name: "wrong type", + }, + { + in: `{"age":23, "firstName": "Harry", "lastName": "Potter"}`, + out: res{Error: `null`, Code: http.StatusOK}, + name: "valid", + }, +} + +func getJsonPathGatewaySetup() string { + + validateJSONPathGatewaySetup := `{ + "api_id": "jsontest", + "definition": { + "location": "header", + "key": "version" + }, + "auth": {"auth_header_name": "authorization"}, + "version_data": { + "not_versioned": true, + "versions": { + "default": { + "name": "default", + "use_extended_paths": true, + "extended_paths": { + "validate_json": [{ + "method": "POST", + "path": "me", + "validate_with": REPLACE_SCHEMA + }] + } + } + } + }, + "proxy": { + "listen_path": "/validate/", + "target_url": "` + testHttpAny + `" + } +}` + validateJSONPathGatewaySetup = strings.Replace(validateJSONPathGatewaySetup, "REPLACE_SCHEMA", schema, 1) + + return validateJSONPathGatewaySetup +} + +func TestValidateJSON_validate(t *testing.T) { + + for _, f := range fixtures { + + t.Run(f.name, func(st *testing.T) { + vj := ValidateJSON{} + + dat := map[string]interface{}{} + + if err := json.Unmarshal([]byte(schema), &dat); err != nil { + t.Fatal(err) + } + + res, err := vj.validate([]byte(f.in), dat) + if err != nil { + t.Fatal(err) + } + + if !res.Valid() && f.out.Code != http.StatusUnprocessableEntity { + st.Fatal("Expected invalid") + } + + if res.Valid() && f.out.Code != http.StatusOK { + t.Log(res.Errors()) + st.Fatal("expected valid", res.Valid(), f.out.Code) + } + }) + } +} + +func TestValidateJSON_ProcessRequest(t *testing.T) { + + for _, f := range fixtures { + + t.Run(f.name, func(st *testing.T) { + + spec := createSpecTest(st, getJsonPathGatewaySetup()) + recorder := httptest.NewRecorder() + req := testReq(t, "POST", "/validate/me", f.in) + + session := createJSONVersionedSession() + spec.SessionManager.UpdateSession("986968696869688869696999", session, 60) + req.Header.Set("Authorization", "986968696869688869696999") + + chain := getJSONValidChain(spec) + chain.ServeHTTP(recorder, req) + + if recorder.Code != f.out.Code { + st.Errorf("failed: %v, code: %v (body: %v)", req.URL.String(), recorder.Code, recorder.Body) + } + + if f.out.Code == http.StatusUnprocessableEntity { + recorderBody := recorder.Body.String() + if !strings.Contains(recorderBody, f.out.Error) { + st.Errorf("Incorrect error msg:\nwant: %v\ngot: %v", f.out.Error, recorderBody) + } + } + }) + } +} + +func createJSONVersionedSession() *user.SessionState { + session := new(user.SessionState) + session.Rate = 10000 + session.Allowance = session.Rate + session.LastCheck = time.Now().Unix() + session.Per = 60 + session.Expires = -1 + session.QuotaRenewalRate = 300 // 5 minutes + session.QuotaRenews = time.Now().Unix() + session.QuotaRemaining = 10 + session.QuotaMax = -1 + session.AccessRights = map[string]user.AccessDefinition{"jsontest": {APIName: "Tyk Test API", APIID: "jsontest", Versions: []string{"default"}}} + return session +} + +func getJSONValidChain(spec *APISpec) http.Handler { + remote, _ := url.Parse(spec.Proxy.TargetURL) + proxy := TykNewSingleHostReverseProxy(remote, spec) + proxyHandler := ProxyHandler(proxy, spec) + baseMid := BaseMiddleware{spec, proxy} + chain := alice.New(mwList( + &IPWhiteListMiddleware{baseMid}, + &MiddlewareContextVars{BaseMiddleware: baseMid}, + &AuthKey{baseMid}, + &VersionCheck{BaseMiddleware: baseMid}, + &KeyExpired{baseMid}, + &AccessRightsCheck{baseMid}, + &RateLimitAndQuotaCheck{baseMid}, + &ValidateJSON{BaseMiddleware: baseMid}, + &TransformHeaders{baseMid}, + )...).Then(proxyHandler) + return chain +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/schema.go b/vendor/github.com/xeipuuv/gojsonschema/schema.go index cc6cdbc0e34..2cac71e9b86 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/schema.go +++ b/vendor/github.com/xeipuuv/gojsonschema/schema.go @@ -586,11 +586,6 @@ func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) formatString, ok := m[KEY_FORMAT].(string) if ok && FormatCheckers.Has(formatString) { currentSchema.format = formatString - } else { - return errors.New(formatErrorDescription( - Locale.MustBeValidFormat(), - ErrorDetails{"key": KEY_FORMAT, "given": m[KEY_FORMAT]}, - )) } } diff --git a/vendor/vendor.json b/vendor/vendor.json index a6534d67c1a..cf6f0d97888 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -511,10 +511,10 @@ "revisionTime": "2015-08-08T06:50:54Z" }, { - "checksumSHA1": "7f3isFdJvljMc0lgQbl+zaaVeBw=", + "checksumSHA1": "wrBUTFFexM4Kz27RheBx34WhNwk=", "path": "github.com/xeipuuv/gojsonschema", - "revision": "3f523f4c14b6e925da10475eb0447c2f28614aac", - "revisionTime": "2017-09-14T03:15:16Z" + "revision": "212d8a0df7acfab8bdd190a7a69f0ab7376edcc8", + "revisionTime": "2017-10-25T06:06:43Z" }, { "checksumSHA1": "vE43s37+4CJ2CDU6TlOUOYE0K9c=",