Skip to content

Commit

Permalink
Merge branch 'master' into wait-for-redis-before-proxying
Browse files Browse the repository at this point in the history
  • Loading branch information
@buger
buger committed Feb 20, 2018
2 parents 5b0179d + 94a4397 commit e4664b5
Show file tree
Hide file tree
Showing 55 changed files with 1,618 additions and 5,110 deletions.
25 changes: 17 additions & 8 deletions api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,14 @@ func checkAndApplyTrialPeriod(keyName, apiId string, newSession *user.SessionSta
}
}

func applyPoliciesAndSave(keyName string, session *user.SessionState, spec *APISpec) error {
// use basic middleware to apply policies to key/session (it also saves it)
mw := BaseMiddleware{
Spec: spec,
}
return mw.ApplyPolicies(keyName, session)
}

func doAddOrUpdate(keyName string, newSession *user.SessionState, dontReset bool) error {
newSession.LastUpdated = strconv.Itoa(int(time.Now().Unix()))

Expand Down Expand Up @@ -137,8 +145,8 @@ func doAddOrUpdate(keyName string, newSession *user.SessionState, dontReset bool
newSession.QuotaRenews = time.Now().Unix() + newSession.QuotaRenewalRate
}

err := apiSpec.SessionManager.UpdateSession(keyName, newSession, newSession.Lifetime(apiSpec.SessionLifetime))
if err != nil {
// apply polices (if any) and save key
if err := applyPoliciesAndSave(keyName, newSession, apiSpec); err != nil {
return err
}
}
Expand All @@ -158,8 +166,9 @@ func doAddOrUpdate(keyName string, newSession *user.SessionState, dontReset bool
newSession.QuotaRenews = time.Now().Unix() + newSession.QuotaRenewalRate
}
checkAndApplyTrialPeriod(keyName, spec.APIID, newSession)
err := spec.SessionManager.UpdateSession(keyName, newSession, newSession.Lifetime(spec.SessionLifetime))
if err != nil {

// apply polices (if any) and save key
if err := applyPoliciesAndSave(keyName, newSession, spec); err != nil {
return err
}
}
Expand Down Expand Up @@ -901,8 +910,8 @@ func createKeyHandler(w http.ResponseWriter, r *http.Request) {
apiSpec.SessionManager.ResetQuota(newKey, newSession)
newSession.QuotaRenews = time.Now().Unix() + newSession.QuotaRenewalRate
}
err := apiSpec.SessionManager.UpdateSession(newKey, newSession, newSession.Lifetime(apiSpec.SessionLifetime))
if err != nil {
// apply polices (if any) and save key
if err := applyPoliciesAndSave(newKey, newSession, apiSpec); err != nil {
doJSONWrite(w, 500, apiError("Failed to create key - "+err.Error()))
return
}
Expand Down Expand Up @@ -941,8 +950,8 @@ func createKeyHandler(w http.ResponseWriter, r *http.Request) {
spec.SessionManager.ResetQuota(newKey, newSession)
newSession.QuotaRenews = time.Now().Unix() + newSession.QuotaRenewalRate
}
err := spec.SessionManager.UpdateSession(newKey, newSession, newSession.Lifetime(spec.SessionLifetime))
if err != nil {
// apply polices (if any) and save key
if err := applyPoliciesAndSave(newKey, newSession, spec); err != nil {
doJSONWrite(w, 500, apiError("Failed to create key - "+err.Error()))
return
}
Expand Down
42 changes: 41 additions & 1 deletion api_definition.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ const (
MethodTransformed
RequestTracked
RequestNotTracked
ValidateJSONRequest
)

// RequestStatus is a custom type to avoid collisions
Expand Down Expand Up @@ -80,6 +81,7 @@ const (
StatusRequestSizeControlled RequestStatus = "Request Size Limited"
StatusRequesTracked RequestStatus = "Request Tracked"
StatusRequestNotTracked RequestStatus = "Request Not Tracked"
StatusValidateJSON RequestStatus = "Validate JSON"
)

// URLSpec represents a flattened specification for URLs, used to check if a proxy URL
Expand All @@ -101,6 +103,7 @@ type URLSpec struct {
MethodTransform apidef.MethodTransformMeta
TrackEndpoint apidef.TrackEndpointMeta
DoNotTrackEndpoint apidef.TrackEndpointMeta
ValidatePathMeta apidef.ValidatePathMeta
}

type TransformSpec struct {
Expand Down Expand Up @@ -138,6 +141,7 @@ type APISpec struct {
HasRun bool
ServiceRefreshInProgress bool
HTTPTransport http.RoundTripper
HTTPTransportCreated time.Time
}

// APIDefinitionLoader will load an Api definition from a storage
Expand Down Expand Up @@ -298,8 +302,14 @@ func (a APIDefinitionLoader) FromDashboardService(endpoint, secret string) []*AP

// FromCloud will connect and download ApiDefintions from a Mongo DB instance.
func (a APIDefinitionLoader) FromRPC(orgId string) []*APISpec {
if rpcEmergencyMode {
return LoadDefinitionsFromRPCBackup()
}

store := RPCStorageHandler{UserKey: config.Global.SlaveOptions.APIKey, Address: config.Global.SlaveOptions.ConnectionString}
store.Connect()
if !store.Connect() {
return nil
}

// enable segments
var tags []string
Expand Down Expand Up @@ -693,6 +703,13 @@ func (a APIDefinitionLoader) compileTrackedEndpointPathspathSpec(paths []apidef.
for _, stringSpec := range paths {
newSpec := URLSpec{}
a.generateRegex(stringSpec.Path, &newSpec, stat)

// set Path if it wasn't set
if stringSpec.Path == "" {
// even if it is empty (and regex matches everything) some middlewares expect to be value here
stringSpec.Path = "/"
}

// Extend with method actions
newSpec.TrackEndpoint = stringSpec
urlSpec = append(urlSpec, newSpec)
Expand All @@ -701,6 +718,20 @@ func (a APIDefinitionLoader) compileTrackedEndpointPathspathSpec(paths []apidef.
return urlSpec
}

func (a APIDefinitionLoader) compileValidateJSONPathspathSpec(paths []apidef.ValidatePathMeta, stat URLStatus) []URLSpec {
urlSpec := make([]URLSpec, len(paths))

for i, stringSpec := range paths {
newSpec := URLSpec{}
a.generateRegex(stringSpec.Path, &newSpec, stat)
// Extend with method actions
newSpec.ValidatePathMeta = stringSpec
urlSpec[i] = newSpec
}

return urlSpec
}

func (a APIDefinitionLoader) compileUnTrackedEndpointPathspathSpec(paths []apidef.TrackEndpointMeta, stat URLStatus) []URLSpec {
urlSpec := []URLSpec{}

Expand Down Expand Up @@ -734,6 +765,7 @@ func (a APIDefinitionLoader) getExtendedPathSpecs(apiVersionDef apidef.VersionIn
methodTransforms := a.compileMethodTransformSpec(apiVersionDef.ExtendedPaths.MethodTransforms, MethodTransformed)
trackedPaths := a.compileTrackedEndpointPathspathSpec(apiVersionDef.ExtendedPaths.TrackEndpoints, RequestTracked)
unTrackedPaths := a.compileUnTrackedEndpointPathspathSpec(apiVersionDef.ExtendedPaths.DoNotTrackEndpoints, RequestNotTracked)
validateJSON := a.compileValidateJSONPathspathSpec(apiVersionDef.ExtendedPaths.ValidateJSON, ValidateJSONRequest)

combinedPath := []URLSpec{}
combinedPath = append(combinedPath, ignoredPaths...)
Expand All @@ -752,6 +784,7 @@ func (a APIDefinitionLoader) getExtendedPathSpecs(apiVersionDef apidef.VersionIn
combinedPath = append(combinedPath, methodTransforms...)
combinedPath = append(combinedPath, trackedPaths...)
combinedPath = append(combinedPath, unTrackedPaths...)
combinedPath = append(combinedPath, validateJSON...)

return combinedPath, len(whiteListPaths) > 0
}
Expand Down Expand Up @@ -797,6 +830,9 @@ func (a *APISpec) getURLStatus(stat URLStatus) RequestStatus {
return StatusRequesTracked
case RequestNotTracked:
return StatusRequestNotTracked
case ValidateJSONRequest:
return StatusValidateJSON

default:
log.Error("URL Status was not one of Ignored, Blacklist or WhiteList! Blocking.")
return EndPointNotAllowed
Expand Down Expand Up @@ -930,6 +966,10 @@ func (a *APISpec) CheckSpecMatchesStatus(r *http.Request, rxPaths []URLSpec, mod
if r.Method == v.DoNotTrackEndpoint.Method {
return true, &v.DoNotTrackEndpoint
}
case ValidateJSONRequest:
if r.Method == v.ValidatePathMeta.Method {
return true, &v.ValidatePathMeta
}
}
}
return false, nil
Expand Down
75 changes: 0 additions & 75 deletions api_definition_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"time"

"github.com/garyburd/redigo/redis"
"github.com/lonelycode/gorpc"

"github.com/TykTechnologies/tyk/apidef"
"github.com/TykTechnologies/tyk/config"
Expand Down Expand Up @@ -191,80 +190,6 @@ func TestIgnored(t *testing.T) {
})
}

func startRPCMock(dispatcher *gorpc.Dispatcher) *gorpc.Server {
configMu.Lock()
defer configMu.Unlock()

config.Global.SlaveOptions.UseRPC = true
config.Global.SlaveOptions.RPCKey = "test_org"
config.Global.SlaveOptions.APIKey = "test"

server := gorpc.NewTCPServer("127.0.0.1:0", dispatcher.NewHandlerFunc())
list := &customListener{}
server.Listener = list
server.LogError = gorpc.NilErrorLogger

if err := server.Start(); err != nil {
panic(err)
}
config.Global.SlaveOptions.ConnectionString = list.L.Addr().String()

return server
}

func stopRPCMock(server *gorpc.Server) {
config.Global.SlaveOptions.ConnectionString = ""
config.Global.SlaveOptions.RPCKey = ""
config.Global.SlaveOptions.APIKey = ""
config.Global.SlaveOptions.UseRPC = false

server.Listener.Close()
server.Stop()

RPCCLientSingleton.Stop()
RPCClientIsConnected = false
RPCCLientSingleton = nil
RPCFuncClientSingleton = nil
}

func TestSyncAPISpecsRPCFailure(t *testing.T) {
// Mock RPC
dispatcher := gorpc.NewDispatcher()
dispatcher.AddFunc("GetApiDefinitions", func(clientAddr string, dr *DefRequest) (string, error) {
return "malformed json", nil
})
dispatcher.AddFunc("Login", func(clientAddr, userKey string) bool {
return true
})

rpc := startRPCMock(dispatcher)
defer stopRPCMock(rpc)

count := syncAPISpecs()
if count != 0 {
t.Error("Should return empty value for malformed rpc response", apiSpecs)
}
}

func TestSyncAPISpecsRPCSuccess(t *testing.T) {
// Mock RPC
dispatcher := gorpc.NewDispatcher()
dispatcher.AddFunc("GetApiDefinitions", func(clientAddr string, dr *DefRequest) (string, error) {
return "[{}]", nil
})
dispatcher.AddFunc("Login", func(clientAddr, userKey string) bool {
return true
})

rpc := startRPCMock(dispatcher)
defer stopRPCMock(rpc)

count := syncAPISpecs()
if count != 1 {
t.Error("Should return array with one spec", apiSpecs)
}
}

func TestSyncAPISpecsDashboardSuccess(t *testing.T) {
// Mock Dashboard
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
Expand Down
4 changes: 2 additions & 2 deletions api_loader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
mwAppendEnabled(&chainArray, &CertificateCheckMW{BaseMiddleware: baseMid})
mwAppendEnabled(&chainArray, &OrganizationMonitor{BaseMiddleware: baseMid})
mwAppendEnabled(&chainArray, &RateLimitForAPI{BaseMiddleware: baseMid})
mwAppendEnabled(&chainArray, &ValidateJSON{BaseMiddleware: baseMid})
mwAppendEnabled(&chainArray, &MiddlewareContextVars{BaseMiddleware: baseMid})
mwAppendEnabled(&chainArray, &VersionCheck{BaseMiddleware: baseMid})
mwAppendEnabled(&chainArray, &RequestSizeLimitMiddleware{baseMid})
Expand Down Expand Up @@ -441,6 +442,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
mwAppendEnabled(&chainArray, &RateLimitForAPI{BaseMiddleware: baseMid})
mwAppendEnabled(&chainArray, &RateLimitAndQuotaCheck{baseMid})
mwAppendEnabled(&chainArray, &GranularAccessMiddleware{baseMid})
mwAppendEnabled(&chainArray, &ValidateJSON{BaseMiddleware: baseMid})
mwAppendEnabled(&chainArray, &TransformMiddleware{baseMid})
mwAppendEnabled(&chainArray, &TransformHeaders{BaseMiddleware: baseMid})
mwAppendEnabled(&chainArray, &URLRewriteMiddleware{BaseMiddleware: baseMid})
Expand Down Expand Up @@ -665,8 +667,6 @@ func loadApps(specs []*APISpec, muxer *mux.Router) {
}).Info("Initialised API Definitions")

if config.Global.SlaveOptions.UseRPC {
//log.Warning("TODO: PUT THE KEEPALIVE WATCHER BACK")
startRPCKeepaliveWatcher(rpcAuthStore)
startRPCKeepaliveWatcher(rpcOrgStore)
}
}
60 changes: 60 additions & 0 deletions api_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,12 +135,39 @@ func TestKeyHandler(t *testing.T) {
masterKey := createStandardSession()
masterKeyJSON, _ := json.Marshal(masterKey)

// with access
withAccess := createStandardSession()
withAccess.AccessRights = map[string]user.AccessDefinition{"test": {
APIID: "test", Versions: []string{"v1"},
}}
withAccessJSON, _ := json.Marshal(withAccess)

// with policy
policiesMu.Lock()
policiesByID["abc_policy"] = user.Policy{
Active: true,
QuotaMax: 1234567890,
}
policiesMu.Unlock()
withPolicy := createStandardSession()
withPolicy.AccessRights = map[string]user.AccessDefinition{"test": {
APIID: "test", Versions: []string{"v1"},
}}
withPolicy.ApplyPolicies = []string{
"abc_policy",
}
withPolicyJSON, _ := json.Marshal(withPolicy)

// with invalid policy
withBadPolicy := createStandardSession()
withBadPolicy.AccessRights = map[string]user.AccessDefinition{"test": {
APIID: "test", Versions: []string{"v1"},
}}
withBadPolicy.ApplyPolicies = []string{
"xyz_policy",
}
withBadPolicyJSON, _ := json.Marshal(withBadPolicy)

knownKey := createSession()

t.Run("Create key", func(t *testing.T) {
Expand All @@ -151,6 +178,39 @@ func TestKeyHandler(t *testing.T) {
}...)
})

t.Run("Create key with policy", func(t *testing.T) {
ts.Run(t, []test.TestCase{
{
Method: "POST",
Path: "/tyk/keys/create",
Data: string(withPolicyJSON),
AdminAuth: true,
Code: 200,
},
{
Method: "POST",
Path: "/tyk/keys/create",
Data: string(withBadPolicyJSON),
AdminAuth: true,
Code: 500,
},
{
Method: "POST",
Path: "/tyk/keys/my_key_id",
Data: string(withPolicyJSON),
AdminAuth: true,
Code: 200,
},
{
Method: "GET",
Path: "/tyk/keys/my_key_id" + "?api_id=test",
AdminAuth: true,
Code: 200,
BodyMatch: `"quota_max":1234567890`,
},
}...)
})

t.Run("Get key", func(t *testing.T) {
ts.Run(t, []test.TestCase{
{Method: "GET", Path: "/tyk/keys/unknown", AdminAuth: true, Code: 404},
Expand Down
11 changes: 11 additions & 0 deletions apidef/api_definitions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,16 @@ type MethodTransformMeta struct {
ToMethod string `bson:"to_method" json:"to_method"`
}

type ValidatePathMeta struct {
Path string `bson:"path" json:"path"`
Method string `bson:"method" json:"method"`
Schema map[string]interface{} `bson:"schema" json:"schema"`
// TODO: Implement multi schema support
SchemaVersion string `bson:"schema_version" json:"schema_version"`
// Allows override of default 422 Unprocessible Entity response code for validation errors.
ErrorResponseCode int `bson:"error_response_code" json:"error_response_code"`
}

type ExtendedPathsSet struct {
Ignored []EndPointMeta `bson:"ignored" json:"ignored,omitempty"`
WhiteList []EndPointMeta `bson:"white_list" json:"white_list,omitempty"`
Expand All @@ -182,6 +192,7 @@ type ExtendedPathsSet struct {
MethodTransforms []MethodTransformMeta `bson:"method_transforms" json:"method_transforms,omitempty"`
TrackEndpoints []TrackEndpointMeta `bson:"track_endpoints" json:"track_endpoints,omitempty"`
DoNotTrackEndpoints []TrackEndpointMeta `bson:"do_not_track_endpoints" json:"do_not_track_endpoints,omitempty"`
ValidateJSON []ValidatePathMeta `bson:"validate_json" json:"validate_json,omitempty"`
}

type VersionInfo struct {
Expand Down
Loading

0 comments on commit e4664b5

Please sign in to comment.