Merge latest master

TykTechnologies · Oct 3, 2019 · ae8bba9 · ae8bba9
2 parents 6cf02c7 + b91aff9
commit ae8bba9
Show file tree

Hide file tree

Showing 159 changed files with 13,650 additions and 6,763 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -7,9 +7,9 @@ assignees:
 
 ---
 
-**Branch/Envrionment/Version**
+**Branch/Environment/Version**
 - Branch/Version: [e.g. Master/Release/Stable/Feature branch]
-- Environemnt: [e.g. On-prem/Hybrid/MDCB]
+- Environment: [e.g. On-prem/Hybrid/MDCB]
 
 **Describe the bug**
 A clear and concise description of what the bug is.

diff --git a/.travis.yml b/.travis.yml
@@ -14,6 +14,7 @@ addons:
     packages:
     - python3.5
     - python3-pip
+    - python3.5-dev
     - libluajit-5.1-dev
 
 matrix:

diff --git a/Makefile b/Makefile
@@ -0,0 +1,80 @@
+SHELL := /bin/bash
+
+GOCMD=go
+GOTEST=$(GOCMD) test
+GOCLEAN=$(GOCMD) clean
+GOBUILD=$(GOCMD) build
+GOINSTALL=$(GOCMD) install
+
+BINARY_NAME=tyk
+BINARY_LINUX=tyk
+TAGS=coprocess grpc goplugin
+CONF=tyk.conf
+
+TEST_REGEX=.
+TEST_COUNT=1
+
+BENCH_REGEX=.
+BENCH_RUN=NONE
+
+.PHONY: test
+test:
+	$(GOTEST) -run=$(TEST_REGEX) -count=$(TEST_COUNT) ./...
+
+.PHONY: bench
+bench:
+	$(GOTEST) -run=$(BENCH_RUN) -bench=$(BENCH_REGEX) ./...
+
+.PHONY: clean
+clean:
+	$(GOCLEAN)
+	rm -f $(BINARY_NAME)
+
+.PHONY: dev
+dev:
+	$(GOBUILD) -tags "$(TAGS)" -o $(BINARY_NAME) -v .
+	./$(BINARY_NAME) --conf $(CONF)
+
+.PHONY: build
+build:
+	$(GOBUILD) -tags "$(TAGS)" -o $(BINARY_NAME) -v .
+
+.PHONY: build-linux
+build-linux:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -tags "$(TAGS)" -o $(BINARY_LINUX) -v .
+
+.PHONY: install
+install:
+	$(GOINSTALL) -tags "$(TAGS)"
+
+.PHONY: db-start
+db-start: redis-start mongo-start
+
+.PHONY: db-stop
+db-stop: redis-stop mongo-stop
+
+# Docker start redis
+.PHONY: redis-start
+redis-start:
+	docker run -itd --rm --name redis -p 127.0.0.1:6379:6379 redis:4.0-alpine redis-server --appendonly yes
+
+.PHONY: redis-stop
+redis-stop:
+	docker stop redis
+
+.PHONY: redis-cli
+redis-cli:
+	docker exec -it redis redis-cli
+
+# Docker start mongo
+.PHONY: mongo-start
+mongo-start:
+	docker run -itd --rm --name mongo -p 127.0.0.1:27017:27017 mongo:3.4-jessie
+
+.PHONY: mongo-stop
+mongo-stop:
+	docker stop mongo
+
+.PHONY: mongo-shell
+mongo-shell:
+	docker exec -it mongo mongo
diff --git a/apidef/api_definitions.go b/apidef/api_definitions.go
@@ -226,7 +226,7 @@ type ExtendedPathsSet struct {
 	TrackEndpoints          []TrackEndpointMeta   `bson:"track_endpoints" json:"track_endpoints,omitempty"`
 	DoNotTrackEndpoints     []TrackEndpointMeta   `bson:"do_not_track_endpoints" json:"do_not_track_endpoints,omitempty"`
 	ValidateJSON            []ValidatePathMeta    `bson:"validate_json" json:"validate_json,omitempty"`
-	Internal                []InternalMeta        `bson:"internal" json:"internal"`
+	Internal                []InternalMeta        `bson:"internal" json:"internal,omitempty"`
 }
 
 type VersionInfo struct {
@@ -306,10 +306,19 @@ type ResponseProcessor struct {
 }
 
 type HostCheckObject struct {
-	CheckURL string            `bson:"url" json:"url"`
-	Method   string            `bson:"method" json:"method"`
-	Headers  map[string]string `bson:"headers" json:"headers"`
-	Body     string            `bson:"body" json:"body"`
+	CheckURL            string            `bson:"url" json:"url"`
+	Protocol            string            `bson:"protocol" json:"protocol"`
+	Timeout             time.Duration     `bson:"timeout" json:"timeout"`
+	EnableProxyProtocol bool              `bson:"enable_proxy_protocol" json:"enable_proxy_protocol"`
+	Commands            []CheckCommand    `bson:"commands" json:"commands"`
+	Method              string            `bson:"method" json:"method"`
+	Headers             map[string]string `bson:"headers" json:"headers"`
+	Body                string            `bson:"body" json:"body"`
+}
+
+type CheckCommand struct {
+	Name    string `bson:"name" json:"name"`
+	Message string `bson:"message" json:"message"`
 }
 
 type ServiceDiscoveryConfiguration struct {
@@ -339,16 +348,19 @@ type OpenIDOptions struct {
 //
 // swagger:model
 type APIDefinition struct {
-	Id               bson.ObjectId `bson:"_id,omitempty" json:"id,omitempty"`
-	Name             string        `bson:"name" json:"name"`
-	Slug             string        `bson:"slug" json:"slug"`
-	APIID            string        `bson:"api_id" json:"api_id"`
-	OrgID            string        `bson:"org_id" json:"org_id"`
-	UseKeylessAccess bool          `bson:"use_keyless" json:"use_keyless"`
-	UseOauth2        bool          `bson:"use_oauth2" json:"use_oauth2"`
-	UseOpenID        bool          `bson:"use_openid" json:"use_openid"`
-	OpenIDOptions    OpenIDOptions `bson:"openid_options" json:"openid_options"`
-	Oauth2Meta       struct {
+	Id                  bson.ObjectId `bson:"_id,omitempty" json:"id,omitempty"`
+	Name                string        `bson:"name" json:"name"`
+	Slug                string        `bson:"slug" json:"slug"`
+	ListenPort          int           `bson:"listen_port" json:"listen_port"`
+	Protocol            string        `bson:"protocol" json:"protocol"`
+	EnableProxyProtocol bool          `bson:"enable_proxy_protocol" json:"enable_proxy_protocol"`
+	APIID               string        `bson:"api_id" json:"api_id"`
+	OrgID               string        `bson:"org_id" json:"org_id"`
+	UseKeylessAccess    bool          `bson:"use_keyless" json:"use_keyless"`
+	UseOauth2           bool          `bson:"use_oauth2" json:"use_oauth2"`
+	UseOpenID           bool          `bson:"use_openid" json:"use_openid"`
+	OpenIDOptions       OpenIDOptions `bson:"openid_options" json:"openid_options"`
+	Oauth2Meta          struct {
 		AllowedAccessTypes     []osin.AccessRequestType    `bson:"allowed_access_types" json:"allowed_access_types"`
 		AllowedAuthorizeTypes  []osin.AuthorizeRequestType `bson:"allowed_authorize_types" json:"allowed_authorize_types"`
 		AuthorizeLoginRedirect string                      `bson:"auth_login_redirect" json:"auth_login_redirect"`

diff --git a/apidef/rpc.go b/apidef/rpc.go
@@ -0,0 +1,30 @@
+package apidef
+
+type InboundData struct {
+	KeyName      string
+	Value        string
+	SessionState string
+	Timeout      int64
+	Per          int64
+	Expire       int64
+}
+
+type DefRequest struct {
+	OrgId string
+	Tags  []string
+}
+
+type GroupLoginRequest struct {
+	UserKey string
+	GroupID string
+}
+
+type GroupKeySpaceRequest struct {
+	OrgID   string
+	GroupID string
+}
+
+type KeysValuesPair struct {
+	Keys   []string
+	Values []string
+}
diff --git a/apidef/schema.go b/apidef/schema.go
@@ -279,6 +279,15 @@ const Schema = `{
         "domain": {
             "type": "string"
         },
+        "listen_port": {
+            "type": "number"
+        },
+        "protocol": {
+            "type": "string"
+        },
+        "enable_proxy_protocol": {
+            "type": "boolean"
+        },
         "certificates": {
             "type": ["array", "null"]
         },

diff --git a/bin/ci-test.sh b/bin/ci-test.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-TEST_TIMEOUT=3m
+TEST_TIMEOUT=5m
 
 # print a command and execute it
 show() {
@@ -33,10 +33,12 @@ if [[ ${LATEST_GO} ]]; then
     race="-race"
 fi
 
-PKGS="$(go list -tags "coprocess python grpc" ./...)"
+PKGS="$(go list ./...)"
 
 go get -t
 
+export PKG_PATH=$GOPATH/src/github.com/TykTechnologies/tyk
+
 # build Go-plugin used in tests
 go build ${race} -o ./test/goplugins/goplugins.so -buildmode=plugin ./test/goplugins || fatal "building Go-plugin failed"
 
@@ -45,13 +47,7 @@ for pkg in $PKGS; do
 
     # TODO: Remove skipRace variable after solving race conditions in tests.
     skipRace=false
-    if [[ ${pkg} == *"coprocess/grpc" ]]; then
-        tags="-tags 'coprocess grpc'"
-        skipRace=true
-    elif [[ ${pkg} == *"coprocess/python" ]]; then
-        tags="-tags 'coprocess python'"
-    elif [[ ${pkg} == *"coprocess" ]]; then
-        tags="-tags 'coprocess'"
+    if [[ ${pkg} == *"grpc" ]]; then
         skipRace=true
     elif [[ ${pkg} == *"goplugin" ]]; then
         tags="-tags 'goplugin'"

diff --git a/bin/dist_build.sh b/bin/dist_build.sh
@@ -58,14 +58,9 @@ do
 done
 
 echo "Building Tyk binaries"
-gox -osarch="linux/arm64 linux/amd64 linux/386" -tags 'coprocess grpc'
-
-echo "Building Tyk CP binaries"
-export CPBINNAME_LUA=tyk_linux_amd64_lua
-export CPBINNAME_PYTHON=tyk_linux_amd64_python
-
-gox -osarch="linux/amd64" -tags 'coprocess python' -output '{{.Dir}}_{{.OS}}_{{.Arch}}_python'
-gox -osarch="linux/amd64" -tags 'coprocess lua' -output '{{.Dir}}_{{.OS}}_{{.Arch}}_lua'
+gox -osarch="linux/amd64 linux/386" -cgo
+# Build arm64 without CGO (no Python plugins), an improved cross-compilation toolkit is needed for that
+gox -osarch="linux/arm64"
 
 TEMPLATEDIR=${ARCHTGZDIRS[i386]}
 echo "Prepping TGZ Dirs"
@@ -99,8 +94,6 @@ do
     mv tyk_linux_${arch/i386/386} $archDir/$SOURCEBIN
     cp $cliTmpDir/tyk-cli_linux_${arch/i386/386} $archDir/utils/$CLIBIN
 done
-mv $CPBINNAME_LUA ${ARCHTGZDIRS[amd64]}/$SOURCEBIN-lua
-mv $CPBINNAME_PYTHON ${ARCHTGZDIRS[amd64]}/$SOURCEBIN-python
 
 echo "Compressing"
 for arch in ${!ARCHTGZDIRS[@]}

diff --git a/bin/dist_push.sh b/bin/dist_push.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 : ${ORGDIR:="/src/github.com/TykTechnologies"}
 : ${SOURCEBINPATH:="${ORGDIR}/tyk"}
-: ${DEBVERS:="ubuntu/precise ubuntu/trusty ubuntu/xenial debian/jessie"}
+: ${DEBVERS:="ubuntu/trusty ubuntu/xenial ubuntu/bionic debian/jessie debian/stretch debian/buster"}
 : ${RPMVERS:="el/6 el7"}
 : ${PKGNAME:="tyk-gateway"}
 

diff --git a/cli/linter/schema.json b/cli/linter/schema.json
@@ -58,6 +58,32 @@
           "type": "string"
         }
       }
+    },
+    "PortWhiteList": {
+      "type": [
+        "object"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "ranges": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "from": {
+              "type": "integer"
+            },
+            "to": {
+              "type": "integer"
+            }
+          }
+        },
+        "ports": {
+          "type": "array",
+          "items": {
+            "type": "integer"
+          }
+        }
+      }
     }
   },
   "properties": {
@@ -228,6 +254,9 @@
         },
         "python_path_prefix": {
           "type": "string"
+        },
+        "python_version": {
+          "type": "string"
         }
       }
     },
@@ -677,6 +706,30 @@
         }
       }
     },
+    "disable_ports_whitelist": {
+      "type": "boolean"
+    },
+    "ports_whitelist": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "http": {
+          "$ref": "#/definitions/PortWhiteList"
+        },
+        "https": {
+          "$ref": "#/definitions/PortWhiteList"
+        },
+        "tcp": {
+          "$ref": "#/definitions/PortWhiteList"
+        },
+        "tls": {
+          "$ref": "#/definitions/PortWhiteList"
+        }
+      }
+    },
     "proxy_default_timeout": {
       "type": "integer"
     },