refactor: Rules are loaded into a radix trie instead of a list (#1358)

Co-authored-by: Dimitrij Drus <dadrus@gmx.de> Co-authored-by: David van der Spek <davidspek@users.noreply.github.com> --- perf: O(log(n)) time complexity for lookup of rules feat: Support for free and single (named) wildcards for request path matching and access of the captured values from the pipeline feat: Support for backtracking while matching rules feat: Multiple rules can be defined for the same path, e.g. to have separate rules for read and write requests feat: Glob expressions are context aware and use `.` for host related expressions and `/` for path related ones as separators refactor!: Rule matching configuration API redesigned refactor!: Default rule rejects requests with encoded slashes in the path of the URL with `400 Bad Request`
dadrus · Apr 30, 2024 · f2f6867 · f2f6867
1 parent 64938b4
commit f2f6867
Show file tree

Hide file tree

Showing 147 changed files with 4,721 additions and 3,425 deletions.
diff --git a/DockerHub-README.md b/DockerHub-README.md
@@ -107,9 +107,6 @@ mechanisms:
     type: jwt
 
 default_rule:
-  methods:
-  - GET
-  - POST
   execute:
   - authenticator: anonymous_authenticator
   - authorizer: deny_all_requests
@@ -124,11 +121,11 @@ providers:
 Create a rule file (`rule.yaml`) with the following contents:
 
 ```yaml
-version: "1alpha3"
+version: "1alpha4"
 rules:
   - id: test-rule
     match: 
-      url: http://<**>/<**>
+      path: /**
     forward_to:
       host: upstream
     execute:

diff --git a/charts/heimdall/Chart.yaml b/charts/heimdall/Chart.yaml
@@ -17,7 +17,7 @@
 apiVersion: v2
 name: heimdall
 description: A cloud native Identity Aware Proxy and Access Control Decision Service
-version: 0.13.1
+version: 0.14.0
 appVersion: latest
 kubeVersion: ^1.19.0
 type: application
@@ -43,5 +43,4 @@ keywords:
   - iap
   - auth-proxy
   - identity-aware-proxy
-  - decision-api
   - auth-filter
diff --git a/charts/heimdall/crds/ruleset.yaml b/charts/heimdall/crds/ruleset.yaml
@@ -27,7 +27,7 @@ spec:
     singular: ruleset
     listKind: RuleSetList
   versions:
-    - name: v1alpha3
+    - name: v1alpha4
       served: true
       storage: true
       schema:
@@ -75,20 +75,66 @@ spec:
                         description: How to match the rule
                         type: object
                         required:
-                          - url
+                          - path
                         properties:
-                          url:
-                            description: The url to match
+                          path:
+                            description: The path to match
                             type: string
-                            maxLength: 512
-                          strategy:
-                            description: Strategy to match the url. Can either be regex or glob.
-                            type: string
-                            maxLength: 5
-                            default: glob
-                            enum:
-                              - regex
-                              - glob
+                            maxLength: 256
+                          backtracking_enabled:
+                            description: Wither this rule allows backtracking. Defaults to the value inherited from the default rule
+                            type: boolean
+                          with:
+                            description: Additional constraints during request matching
+                            type: object
+                            properties:
+                              methods:
+                                description: The HTTP methods to match
+                                type: array
+                                minItems: 1
+                                items:
+                                  type: string
+                                  maxLength: 16
+                                  enum:
+                                    - "CONNECT"
+                                    - "!CONNECT"
+                                    - "DELETE"
+                                    - "!DELETE"
+                                    - "GET"
+                                    - "!GET"
+                                    - "HEAD"
+                                    - "!HEAD"
+                                    - "OPTIONS"
+                                    - "!OPTIONS"
+                                    - "PATCH"
+                                    - "!PATCH"
+                                    - "POST"
+                                    - "!POST"
+                                    - "PUT"
+                                    - "!PUT"
+                                    - "TRACE"
+                                    - "!TRACE"
+                                    - "ALL"
+                              scheme:
+                                description: The HTTP scheme, which should be matched. If not set, http and https are matched
+                                type: string
+                                maxLength: 5
+                              host_glob:
+                                description: Glob expression to match the host if required. If not set, all hosts are matched. Mutually exclusive with 'host_regex'.
+                                type: string
+                                maxLength: 512
+                              host_regex:
+                                description: Regular expression to match the host if required. If not set, all hosts are matched. Mutually exclusive with 'host_glob'.
+                                type: string
+                                maxLength: 512
+                              path_glob:
+                                description: Additional glob expression the matched path should be matched against. Mutual exclusive with 'regex'.
+                                type: string
+                                maxLength: 256
+                              path_regex:
+                                description: Additional regular expression the matched path should be matched against. Mutual exclusive with 'glob'
+                                type: string
+                                maxLength: 256
                       forward_to:
                         description: Where to forward the request to. Required only if heimdall is used in proxy operation mode.
                         type: object
@@ -125,33 +171,6 @@ spec:
                                 items:
                                   type: string
                                   maxLength: 128
-                      methods:
-                        description: The allowed HTTP methods
-                        type: array
-                        minItems: 1
-                        items:
-                          type: string
-                          maxLength: 16
-                          enum:
-                            - "CONNECT"
-                            - "!CONNECT"
-                            - "DELETE"
-                            - "!DELETE"
-                            - "GET"
-                            - "!GET"
-                            - "HEAD"
-                            - "!HEAD"
-                            - "OPTIONS"
-                            - "!OPTIONS"
-                            - "PATCH"
-                            - "!PATCH"
-                            - "POST"
-                            - "!POST"
-                            - "PUT"
-                            - "!PUT"
-                            - "TRACE"
-                            - "!TRACE"
-                            - "ALL"
                       execute:
                         description: The pipeline mechanisms to execute
                         type: array

diff --git a/charts/heimdall/templates/demo/configmap.yaml b/charts/heimdall/templates/demo/configmap.yaml
@@ -74,9 +74,6 @@ data:
         type: noop
 
     default_rule:
-      methods:
-        - GET
-        - POST
       execute:
         - authenticator: anonymous_authenticator
         - authorizer: deny_all_requests

diff --git a/charts/heimdall/templates/demo/test-rule.yaml b/charts/heimdall/templates/demo/test-rule.yaml
@@ -15,7 +15,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 {{- if .Values.demo.enabled }}
-apiVersion: heimdall.dadrus.github.com/v1alpha3
+apiVersion: heimdall.dadrus.github.com/v1alpha4
 kind: RuleSet
 metadata:
   name: {{ include "heimdall.demo.fullname" . }}-test-rule
@@ -26,7 +26,7 @@ spec:
   rules:
     - id: public-access
       match:
-        url: http://<**>/pub/<**>
+        path: /pub/**
       forward_to:
         host: {{ include "heimdall.demo.fullname" . }}.heimdall-demo.svc.cluster.local:8080
       execute:
@@ -35,7 +35,7 @@ spec:
         - finalizer: noop_finalizer
     - id: anonymous-access
       match:
-        url: http://<**>/anon/<**>
+        path: /anon/**
       forward_to:
         host: {{ include "heimdall.demo.fullname" . }}.heimdall-demo.svc.cluster.local:8080
       execute:

diff --git a/charts/heimdall/templates/heimdall/admissioncontroller.yaml b/charts/heimdall/templates/heimdall/admissioncontroller.yaml
@@ -23,7 +23,7 @@ webhooks:
     {{- end }}
     rules:
       - apiGroups:   ["heimdall.dadrus.github.com"]
-        apiVersions: ["v1alpha3"]
+        apiVersions: ["v1alpha4"]
         operations:  ["CREATE", "UPDATE"]
         resources:   ["rulesets"]
         scope:       "Namespaced"

diff --git a/cmd/validate/ruleset.go b/cmd/validate/ruleset.go
@@ -24,10 +24,11 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/dadrus/heimdall/internal/config"
+	"github.com/dadrus/heimdall/internal/heimdall"
 	"github.com/dadrus/heimdall/internal/rules"
-	"github.com/dadrus/heimdall/internal/rules/event"
 	"github.com/dadrus/heimdall/internal/rules/mechanisms"
 	"github.com/dadrus/heimdall/internal/rules/provider/filesystem"
+	"github.com/dadrus/heimdall/internal/rules/rule"
 )
 
 // NewValidateRulesCommand represents the "validate rules" command.
@@ -55,8 +56,6 @@ func NewValidateRulesCommand() *cobra.Command {
 }
 
 func validateRuleSet(cmd *cobra.Command, args []string) error {
-	const queueSize = 50
-
 	envPrefix, _ := cmd.Flags().GetString("env-config-prefix")
 	logger := zerolog.Nop()
 
@@ -90,14 +89,17 @@ func validateRuleSet(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	queue := make(event.RuleSetChangedEventQueue, queueSize)
-
-	defer close(queue)
-
-	provider, err := filesystem.NewProvider(conf, rules.NewRuleSetProcessor(queue, rFactory, logger), logger)
+	provider, err := filesystem.NewProvider(conf, rules.NewRuleSetProcessor(&noopRepository{}, rFactory), logger)
 	if err != nil {
 		return err
 	}
 
 	return provider.Start(context.Background())
 }
+
+type noopRepository struct{}
+
+func (*noopRepository) FindRule(_ heimdall.Context) (rule.Rule, error) { return nil, nil }
+func (*noopRepository) AddRuleSet(_ string, _ []rule.Rule) error       { return nil }
+func (*noopRepository) UpdateRuleSet(_ string, _ []rule.Rule) error    { return nil }
+func (*noopRepository) DeleteRuleSet(_ string) error                   { return nil }
diff --git a/cmd/validate/ruleset_test.go b/cmd/validate/ruleset_test.go
@@ -103,7 +103,7 @@ func TestRunValidateRulesCommand(t *testing.T) {
 			proxyMode: true,
 			confFile:  "test_data/config.yaml",
 			rulesFile: "test_data/invalid-ruleset-for-proxy-usage.yaml",
-			expError:  "no forward_to",
+			expError:  "requires forward_to",
 		},
 		{
 			uc:        "everything is valid for proxy mode usage",

diff --git a/cmd/validate/test_data/config.yaml b/cmd/validate/test_data/config.yaml
@@ -171,9 +171,7 @@ mechanisms:
         to: http://127.0.0.1:4433/self-service/login/browser?return_to={{ .Request.URL | urlenc }}
 
 default_rule:
-  methods:
-    - GET
-    - POST
+  backtracking_enabled: false
   execute:
     - authenticator: anonymous_authenticator
     - finalizer: jwt
@@ -189,8 +187,8 @@ providers:
     watch_interval: 5m
     endpoints:
       - url: http://foo.bar/rules.yaml
-        rule_path_match_prefix: /foo
-        enable_http_cache: true
+        http_cache:
+          enabled: true
       - url: http://bar.foo/rules.yaml
         headers:
           bla: bla
@@ -209,10 +207,8 @@ providers:
     buckets:
       - url: gs://my-bucket
         prefix: service1
-        rule_path_match_prefix: /service1
       - url: gs://my-bucket
         prefix: service2
-        rule_path_match_prefix: /service2
       - url: s3://my-bucket/my-rule-set
 
   kubernetes:

diff --git a/cmd/validate/test_data/invalid-ruleset-for-proxy-usage.yaml b/cmd/validate/test_data/invalid-ruleset-for-proxy-usage.yaml
@@ -1,13 +1,13 @@
-version: "1alpha3"
+version: "1alpha4"
 name: test-rule-set
 rules:
 - id: rule:foo
   match:
-    url: http://foo.bar/<**>
-    strategy: glob
-#  methods: # reuses default
-#    - GET
-#    - POST
+    path: /**
+    with:
+      scheme: http
+      host_glob: foo.bar
+      methods: [ GET, POST ]
   execute:
     - authenticator: unauthorized_authenticator
     - authenticator: jwt_authenticator1

diff --git a/cmd/validate/test_data/valid-ruleset.yaml b/cmd/validate/test_data/valid-ruleset.yaml
@@ -1,19 +1,22 @@
-version: "1alpha3"
+version: "1alpha4"
 name: test-rule-set
 rules:
 - id: rule:foo
   match:
-    url: http://foo.bar/<**>
-    strategy: glob
+    path: /**
+    backtracking_enabled: true
+    with:
+      scheme: http
+      host_glob: foo.bar
+      methods:
+        - POST
+        - PUT
   forward_to:
     host: bar.foo
     rewrite:
       strip_path_prefix: /foo
       add_path_prefix: /baz
       strip_query_parameters: [boo]
-#  methods: # reuses default
-#    - GET
-#    - POST
   execute:
     - authenticator: unauthorized_authenticator
     - authenticator: jwt_authenticator1

diff --git a/docs/content/_index.adoc b/docs/content/_index.adoc
@@ -15,15 +15,18 @@ Use declarative techniques you are already familiar with
 
 [source, yaml]
 ----
-apiVersion: heimdall.dadrus.github.com/v1alpha3
+apiVersion: heimdall.dadrus.github.com/v1alpha4
 kind: RuleSet
 metadata:
   name: My awesome service
 spec:
   rules:
     - id: my_api_rule
       match:
-        url: http://127.0.0.1:9090/api/<**>
+        path: /api/**
+        with:
+          scheme: http
+          host_glob: 127.0.0.1:9090
       execute:
         - authenticator: keycloak
         - authorizer: opa

diff --git a/docs/content/docs/concepts/operating_modes.adoc b/docs/content/docs/concepts/operating_modes.adoc
@@ -76,9 +76,12 @@ And there is a rule, which allows anonymous requests and sets a header with subj
 ----
 id: rule:my-service:anonymous-api-access
 match:
-  url: http://my-backend-service/my-service/api
-methods:
-  - GET
+  path: /my-service/api
+  with:
+    scheme: http
+    host_glob: my-backend-service
+    methods:
+      - GET
 execute:
   - authenticator: anonymous-authn
   - finalizer: id-header
@@ -144,11 +147,12 @@ And there is a rule, which allows anonymous requests and sets a header with subj
 ----
 id: rule:my-service:anonymous-api-access
 match:
-  url: <**>/my-service/api
+  path: /my-service/api
+  with:
+    methods:
+      - GET
 forward_to:
   host: my-backend-service:8888
-methods:
-  - GET
 execute:
   - authenticator: anonymous-authn
   - finalizer: id-header