diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d27b86dec..03ed931a63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,10 @@ Adding a new version? You'll need three changes: a corresponding environment variable `CONTROLLER_KONG_ADMIN_URL` (which can specify multiple values separated by a comma). [#3268](https://github.com/Kong/kubernetes-ingress-controller/pull/3268) +- Added a new `dbless-konnect` configuration variant to the manifests. It can + be used to deploy a DB-less variant of KIC that will also synchronise its + data-plane configuration with Konnect cloud. + [#3448](https://github.com/Kong/kubernetes-ingress-controller/pull/3448) ### Fixed diff --git a/Makefile b/Makefile index 62f9be7d00..ee2cd3ef34 100644 --- a/Makefile +++ b/Makefile @@ -494,6 +494,18 @@ debug.skaffold: skaffold TAG=$(TAG)-debug REPO_INFO=$(REPO_INFO) COMMIT=$(COMMIT) \ $(SKAFFOLD) debug --port-forward=pods --profile=debug $(SKAFFOLD_FLAGS) +# This will port-forward 40000 from KIC's debugger to localhost. Connect to that +# port with debugger/IDE of your choice. +# +# To make it work with Konnect, you must provide following files under ./config/variants/konnect/debug: +# * `konnect.env` with CONTROLLER_KONNECT_RUNTIME_GROUP env variable set +# to the UUID of a Runtime Group you have created in Konnect. +# * `tls.crt` and `tls.key` with TLS client cerificate and its key (generated by Konnect). +.PHONY: debug.konnect.skaffold +debug.konnect.skaffold: skaffold + TAG=$(TAG)-debug REPO_INFO=$(REPO_INFO) COMMIT=$(COMMIT) \ + $(SKAFFOLD) debug --port-forward=pods --profile=debug-konnect $(SKAFFOLD_FLAGS) + # This will port-forward 40000 from KIC's debugger to localhost. Connect to that # port with debugger/IDE of your choice .PHONY: debug.skaffold.sync diff --git a/config/variants/konnect/base/kustomization.yaml b/config/variants/konnect/base/kustomization.yaml new file mode 100644 index 0000000000..2a0401e110 --- /dev/null +++ b/config/variants/konnect/base/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../../base + +patchesStrategicMerge: + - manager.yaml diff --git a/config/variants/konnect/base/manager.yaml b/config/variants/konnect/base/manager.yaml new file mode 100644 index 0000000000..283f2eab2d --- /dev/null +++ b/config/variants/konnect/base/manager.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ingress-kong + namespace: kong +spec: + template: + spec: + containers: + - name: ingress-controller + envFrom: + - configMapRef: + # konnect-config ConfigMap is expected to specify: + # * CONTROLLER_KONNECT_RUNTIME_GROUP (required) + # * CONTROLLER_KONNECT_ADDRESS (optional) + name: konnect-config + env: + - name: CONTROLLER_KONNECT_SYNC_ENABLED + value: "true" + - name: CONTROLLER_KONNECT_TLS_CLIENT_CERT + valueFrom: + secretKeyRef: + name: konnect-client-tls + key: tls.crt + - name: CONTROLLER_KONNECT_TLS_CLIENT_KEY + valueFrom: + secretKeyRef: + name: konnect-client-tls + key: tls.key diff --git a/config/variants/konnect/debug/.gitignore b/config/variants/konnect/debug/.gitignore new file mode 100644 index 0000000000..b734fc7845 --- /dev/null +++ b/config/variants/konnect/debug/.gitignore @@ -0,0 +1,3 @@ +tls.crt +tls.key +konnect.env diff --git a/config/variants/konnect/debug/kustomization.yaml b/config/variants/konnect/debug/kustomization.yaml new file mode 100644 index 0000000000..eea603eb83 --- /dev/null +++ b/config/variants/konnect/debug/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../base + +patchesStrategicMerge: + - manager_debug.yaml + +generatorOptions: + disableNameSuffixHash: true + +secretGenerator: + - name: konnect-client-tls + namespace: kong + type: tls + files: + - tls.crt + - tls.key + +configMapGenerator: + - name: konnect-config + namespace: kong + envs: [konnect.env] diff --git a/config/variants/konnect/debug/manager_debug.yaml b/config/variants/konnect/debug/manager_debug.yaml new file mode 100644 index 0000000000..59a6c00701 --- /dev/null +++ b/config/variants/konnect/debug/manager_debug.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ingress-kong + namespace: kong +spec: + template: + spec: + containers: + - name: ingress-controller + command: + - /go/bin/dlv + - --continue + - --accept-multiclient + - --listen=:40000 + - --check-go-version=false + - --headless=true + - --api-version=2 + - --log=true + - --log-output=debugger,debuglineerr,gdbwire + - exec + - /manager-debug + - -- + args: + - --feature-gates=GatewayAlpha=true + - --anonymous-reports=false + env: + - name: CONTROLLER_LOG_LEVEL + value: debug + - name: CONTROLLER_KONNECT_ADDRESS + value: https://us.kic.api.konghq.tech + image: kic-placeholder:placeholder diff --git a/deploy/single/all-in-one-dbless-konnect.yaml b/deploy/single/all-in-one-dbless-konnect.yaml new file mode 100644 index 0000000000..9cfddd85c5 --- /dev/null +++ b/deploy/single/all-in-one-dbless-konnect.yaml @@ -0,0 +1,1784 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kong +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: ingressclassparameterses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + kind: IngressClassParameters + listKind: IngressClassParametersList + plural: ingressclassparameterses + singular: ingressclassparameters + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: IngressClassParameters is the Schema for the IngressClassParameters + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec is the IngressClassParameters specification. + properties: + enableLegacyRegexDetection: + default: false + description: EnableLegacyRegexDetection automatically detects if ImplementationSpecific + Ingress paths are regular expression paths using the legacy 2.x + heuristic. The controller adds the "~" prefix to those paths if + the Kong version is 3.0 or higher. + type: boolean + serviceUpstream: + default: false + description: Offload load-balancing to kube-proxy or sidecar. + type: boolean + type: object + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: kongclusterplugins.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongClusterPlugin + listKind: KongClusterPluginList + plural: kongclusterplugins + shortNames: + - kcp + singular: kongclusterplugin + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Name of the plugin + jsonPath: .plugin + name: Plugin-Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Indicates if the plugin is disabled + jsonPath: .disabled + name: Disabled + priority: 1 + type: boolean + - description: Configuration of the plugin + jsonPath: .config + name: Config + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: KongClusterPlugin is the Schema for the kongclusterplugins API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + config: + description: Config contains the plugin configuration. It's a list of + keys and values required to configure the plugin. Please read the documentation + of the plugin being configured to set values in here. For any plugin + in Kong, anything that goes in the `config` JSON key in the Admin API + request, goes into this property. Only one of `config` or `configFrom` + may be used in a KongClusterPlugin, not both at once. + type: object + x-kubernetes-preserve-unknown-fields: true + configFrom: + description: ConfigFrom references a secret containing the plugin configuration. + This should be used when the plugin configuration contains sensitive + information, such as AWS credentials in the Lambda plugin or the client + secret in the OIDC plugin. Only one of `config` or `configFrom` may + be used in a KongClusterPlugin, not both at once. + properties: + secretKeyRef: + description: Specifies a name, a namespace, and a key of a secret + to refer to. + properties: + key: + description: The key containing the value. + type: string + name: + description: The secret containing the key. + type: string + namespace: + description: The namespace containing the secret. + type: string + required: + - key + - name + - namespace + type: object + type: object + consumerRef: + description: ConsumerRef is a reference to a particular consumer. + type: string + disabled: + description: Disabled set if the plugin is disabled or not. + type: boolean + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + ordering: + description: 'Ordering overrides the normal plugin execution order. It''s + only available on Kong Enterprise. `` is a request processing + phase (for example, `access` or `body_filter`) and `` is the + name of the plugin that will run before or after the KongPlugin. For + example, a KongPlugin with `plugin: rate-limiting` and `before.access: + ["key-auth"]` will create a rate limiting plugin that limits requests + _before_ they are authenticated.' + properties: + after: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + before: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + type: object + plugin: + description: PluginName is the name of the plugin to which to apply the + config. + type: string + protocols: + description: Protocols configures plugin to run on requests received on + specific protocols. + items: + description: KongProtocol is a valid Kong protocol. This alias is necessary + to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + type: array + run_on: + description: RunOn configures the plugin to run on the first or the second + or both nodes in case of a service mesh deployment. + enum: + - first + - second + - all + type: string + required: + - plugin + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: kongconsumers.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongConsumer + listKind: KongConsumerList + plural: kongconsumers + shortNames: + - kc + singular: kongconsumer + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Username of a Kong Consumer + jsonPath: .username + name: Username + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: KongConsumer is the Schema for the kongconsumers API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + credentials: + description: Credentials are references to secrets containing a credential + to be provisioned in Kong. + items: + type: string + type: array + custom_id: + description: CustomID is a Kong cluster-unique existing ID for the consumer + - useful for mapping Kong with users in your existing database. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + username: + description: Username is a Kong cluster-unique username of the consumer. + type: string + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: kongingresses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongIngress + listKind: KongIngressList + plural: kongingresses + shortNames: + - ki + singular: kongingress + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: KongIngress is the Schema for the kongingresses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + proxy: + description: Proxy defines additional connection options for the routes + to be configured in the Kong Gateway, e.g. `connection_timeout`, `retries`, + etc. + properties: + connect_timeout: + description: "The timeout in milliseconds for\testablishing a connection + to the upstream server. Deprecated: use Service's \"konghq.com/connect-timeout\" + annotation instead." + minimum: 0 + type: integer + path: + description: '(optional) The path to be used in requests to the upstream + server. Deprecated: use Service''s "konghq.com/path" annotation + instead.' + pattern: ^/.*$ + type: string + protocol: + description: 'The protocol used to communicate with the upstream. + Deprecated: use Service''s "konghq.com/protocol" annotation instead.' + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + read_timeout: + description: 'The timeout in milliseconds between two successive read + operations for transmitting a request to the upstream server. Deprecated: + use Service''s "konghq.com/read-timeout" annotation instead.' + minimum: 0 + type: integer + retries: + description: 'The number of retries to execute upon failure to proxy. + Deprecated: use Service''s "konghq.com/retries" annotation instead.' + minimum: 0 + type: integer + write_timeout: + description: 'The timeout in milliseconds between two successive write + operations for transmitting a request to the upstream server. Deprecated: + use Service''s "konghq.com/write-timeout" annotation instead.' + minimum: 0 + type: integer + type: object + route: + description: Route define rules to match client requests. Each Route is + associated with a Service, and a Service may have multiple Routes associated + to it. + properties: + headers: + additionalProperties: + items: + type: string + type: array + description: 'Headers contains one or more lists of values indexed + by header name that will cause this Route to match if present in + the request. The Host header cannot be used with this attribute. + Deprecated: use Ingress'' "konghq.com/headers" annotation instead.' + type: object + https_redirect_status_code: + description: 'HTTPSRedirectStatusCode is the status code Kong responds + with when all properties of a Route match except the protocol. Deprecated: + use Ingress'' "ingress.kubernetes.io/force-ssl-redirect" or "konghq.com/https-redirect-status-code" + annotations instead.' + type: integer + methods: + description: 'Methods is a list of HTTP methods that match this Route. + Deprecated: use Ingress'' "konghq.com/methods" annotation instead.' + items: + type: string + type: array + path_handling: + description: 'PathHandling controls how the Service path, Route path + and requested path are combined when sending a request to the upstream. + Deprecated: use Ingress'' "konghq.com/path-handling" annotation + instead.' + enum: + - v0 + - v1 + type: string + preserve_host: + description: 'PreserveHost sets When matching a Route via one of the + hosts domain names, use the request Host header in the upstream + request headers. If set to false, the upstream Host header will + be that of the Service’s host. Deprecated: use Ingress'' "konghq.com/preserve-host" + annotation instead.' + type: boolean + protocols: + description: 'Protocols is an array of the protocols this Route should + allow. Deprecated: use Ingress'' "konghq.com/protocols" annotation + instead.' + items: + description: KongProtocol is a valid Kong protocol. This alias is + necessary to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + type: array + regex_priority: + description: 'RegexPriority is a number used to choose which route + resolves a given request when several routes match it using regexes + simultaneously. Deprecated: use Ingress'' "konghq.com/regex-priority" + annotation instead.' + type: integer + request_buffering: + description: 'RequestBuffering sets whether to enable request body + buffering or not. Deprecated: use Ingress'' "konghq.com/request-buffering" + annotation instead.' + type: boolean + response_buffering: + description: 'ResponseBuffering sets whether to enable response body + buffering or not. Deprecated: use Ingress'' "konghq.com/response-buffering" + annotation instead.' + type: boolean + snis: + description: 'SNIs is a list of SNIs that match this Route when using + stream routing. Deprecated: use Ingress'' "konghq.com/snis" annotation + instead.' + items: + type: string + type: array + strip_path: + description: 'StripPath sets When matching a Route via one of the + paths strip the matching prefix from the upstream request URL. Deprecated: + use Ingress'' "konghq.com/strip-path" annotation instead.' + type: boolean + type: object + upstream: + description: Upstream represents a virtual hostname and can be used to + loadbalance incoming requests over multiple targets (e.g. Kubernetes + `Services` can be a target, OR `Endpoints` can be targets). + properties: + algorithm: + description: Algorithm is the load balancing algorithm to use. + enum: + - round-robin + - consistent-hashing + - least-connections + type: string + hash_fallback: + description: 'HashFallback defines What to use as hashing input if + the primary hash_on does not return a hash. Accepted values are: + "none", "consumer", "ip", "header", "cookie".' + type: string + hash_fallback_header: + description: HashFallbackHeader is the header name to take the value + from as hash input. Only required when "hash_fallback" is set to + "header". + type: string + hash_fallback_query_arg: + description: HashFallbackQueryArg is the "hash_fallback" version of + HashOnQueryArg. + type: string + hash_fallback_uri_capture: + description: HashFallbackURICapture is the "hash_fallback" version + of HashOnURICapture. + type: string + hash_on: + description: 'HashOn defines what to use as hashing input. Accepted + values are: "none", "consumer", "ip", "header", "cookie", "path", + "query_arg", "uri_capture".' + type: string + hash_on_cookie: + description: The cookie name to take the value from as hash input. + Only required when "hash_on" or "hash_fallback" is set to "cookie". + type: string + hash_on_cookie_path: + description: The cookie path to set in the response headers. Only + required when "hash_on" or "hash_fallback" is set to "cookie". + type: string + hash_on_header: + description: HashOnHeader defines the header name to take the value + from as hash input. Only required when "hash_on" is set to "header". + type: string + hash_on_query_arg: + description: HashOnQueryArg is the query string parameter whose value + is the hash input when "hash_on" is set to "query_arg". + type: string + hash_on_uri_capture: + description: HashOnURICapture is the name of the capture group whose + value is the hash input when "hash_on" is set to "uri_capture". + type: string + healthchecks: + description: Healthchecks defines the health check configurations + in Kong. + properties: + active: + description: ActiveHealthcheck configures active health check + probing. + properties: + concurrency: + minimum: 1 + type: integer + healthy: + description: Healthy configures thresholds and HTTP status + codes to mark targets healthy for an upstream. + properties: + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + successes: + minimum: 0 + type: integer + type: object + http_path: + pattern: ^/.*$ + type: string + https_sni: + type: string + https_verify_certificate: + type: boolean + timeout: + minimum: 0 + type: integer + type: + type: string + unhealthy: + description: Unhealthy configures thresholds and HTTP status + codes to mark targets unhealthy. + properties: + http_failures: + minimum: 0 + type: integer + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + tcp_failures: + minimum: 0 + type: integer + timeouts: + minimum: 0 + type: integer + type: object + type: object + passive: + description: PassiveHealthcheck configures passive checks around + passive health checks. + properties: + healthy: + description: Healthy configures thresholds and HTTP status + codes to mark targets healthy for an upstream. + properties: + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + successes: + minimum: 0 + type: integer + type: object + type: + type: string + unhealthy: + description: Unhealthy configures thresholds and HTTP status + codes to mark targets unhealthy. + properties: + http_failures: + minimum: 0 + type: integer + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + tcp_failures: + minimum: 0 + type: integer + timeouts: + minimum: 0 + type: integer + type: object + type: object + threshold: + type: number + type: object + host_header: + description: HostHeader is The hostname to be used as Host header + when proxying requests through Kong. + type: string + slots: + description: Slots is the number of slots in the load balancer algorithm. + minimum: 10 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: kongplugins.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: KongPlugin + listKind: KongPluginList + plural: kongplugins + shortNames: + - kp + singular: kongplugin + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Name of the plugin + jsonPath: .plugin + name: Plugin-Type + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Indicates if the plugin is disabled + jsonPath: .disabled + name: Disabled + priority: 1 + type: boolean + - description: Configuration of the plugin + jsonPath: .config + name: Config + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: KongPlugin is the Schema for the kongplugins API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + config: + description: Config contains the plugin configuration. It's a list of + keys and values required to configure the plugin. Please read the documentation + of the plugin being configured to set values in here. For any plugin + in Kong, anything that goes in the `config` JSON key in the Admin API + request, goes into this property. Only one of `config` or `configFrom` + may be used in a KongPlugin, not both at once. + type: object + x-kubernetes-preserve-unknown-fields: true + configFrom: + description: ConfigFrom references a secret containing the plugin configuration. + This should be used when the plugin configuration contains sensitive + information, such as AWS credentials in the Lambda plugin or the client + secret in the OIDC plugin. Only one of `config` or `configFrom` may + be used in a KongPlugin, not both at once. + properties: + secretKeyRef: + description: Specifies a name and a key of a secret to refer to. The + namespace is implicitly set to the one of referring object. + properties: + key: + description: The key containing the value. + type: string + name: + description: The secret containing the key. + type: string + required: + - key + - name + type: object + type: object + consumerRef: + description: ConsumerRef is a reference to a particular consumer. + type: string + disabled: + description: Disabled set if the plugin is disabled or not. + type: boolean + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + ordering: + description: 'Ordering overrides the normal plugin execution order. It''s + only available on Kong Enterprise. `` is a request processing + phase (for example, `access` or `body_filter`) and `` is the + name of the plugin that will run before or after the KongPlugin. For + example, a KongPlugin with `plugin: rate-limiting` and `before.access: + ["key-auth"]` will create a rate limiting plugin that limits requests + _before_ they are authenticated.' + properties: + after: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + before: + additionalProperties: + items: + type: string + type: array + description: PluginOrderingPhase indicates which plugins in a phase + should affect the target plugin's order + type: object + type: object + plugin: + description: PluginName is the name of the plugin to which to apply the + config. + type: string + protocols: + description: Protocols configures plugin to run on requests received on + specific protocols. + items: + description: KongProtocol is a valid Kong protocol. This alias is necessary + to deal with https://github.com/kubernetes-sigs/controller-tools/issues/342 + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + - udp + type: string + type: array + run_on: + description: RunOn configures the plugin to run on the first or the second + or both nodes in case of a service mesh deployment. + enum: + - first + - second + - all + type: string + required: + - plugin + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: tcpingresses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: TCPIngress + listKind: TCPIngressList + plural: tcpingresses + singular: tcpingress + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Address of the load balancer + jsonPath: .status.loadBalancer.ingress[*].ip + name: Address + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: TCPIngress is the Schema for the tcpingresses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec is the TCPIngress specification. + properties: + rules: + description: A list of rules used to configure the Ingress. + items: + description: IngressRule represents a rule to apply against incoming + requests. Matching is performed based on an (optional) SNI and + port. + properties: + backend: + description: Backend defines the referenced service endpoint + to which the traffic will be forwarded to. + properties: + serviceName: + description: Specifies the name of the referenced service. + minLength: 1 + type: string + servicePort: + description: Specifies the port of the referenced service. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - serviceName + - servicePort + type: object + host: + description: Host is the fully qualified domain name of a network + host, as defined by RFC 3986. If a Host is not specified, + then port-based TCP routing is performed. Kong doesn't care + about the content of the TCP stream in this case. If a Host + is specified, the protocol must be TLS over TCP. A plain-text + TCP request cannot be routed based on Host. It can only be + routed based on Port. + type: string + port: + description: Port is the port on which to accept TCP or TLS + over TCP sessions and route. It is a required field. If a + Host is not specified, the requested are routed based only + on Port. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - backend + - port + type: object + type: array + tls: + description: TLS configuration. This is similar to the `tls` section + in the Ingress resource in networking.v1beta1 group. The mapping + of SNIs to TLS cert-key pair defined here will be used for HTTP + Ingress rules as well. Once can define the mapping in this resource + or the original Ingress resource, both have the same effect. + items: + description: IngressTLS describes the transport layer security. + properties: + hosts: + description: Hosts are a list of hosts included in the TLS certificate. + The values in this list must match the name/s used in the + tlsSecret. Defaults to the wildcard host setting for the loadbalancer + controller fulfilling this Ingress, if left unspecified. + items: + type: string + type: array + secretName: + description: SecretName is the name of the secret used to terminate + SSL traffic. + type: string + type: object + type: array + type: object + status: + description: TCPIngressStatus defines the observed state of TCPIngress. + properties: + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer. + properties: + ingress: + description: Ingress is a list containing ingress points for the + load-balancer. Traffic intended for the service should be sent + to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a + load-balancer ingress point: traffic intended for the service + should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points + that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points + that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports + If used, every port defined in the service should have + an entry in it + items: + properties: + error: + description: 'Error is to record the problem with + the service port The format of the error shall comply + with the following rules: - built-in error values + shall be specified in this file and those shall + use CamelCase names - cloud provider specific error + values must have names that comply with the format + foo.example.com/CamelCase. --- The regex it matches + is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service + port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service + port of which status is recorded here The supported + values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: udpingresses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + categories: + - kong-ingress-controller + kind: UDPIngress + listKind: UDPIngressList + plural: udpingresses + singular: udpingress + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Address of the load balancer + jsonPath: .status.loadBalancer.ingress[*].ip + name: Address + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: UDPIngress is the Schema for the udpingresses API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec is the UDPIngress specification. + properties: + rules: + description: A list of rules used to configure the Ingress. + items: + description: UDPIngressRule represents a rule to apply against incoming + requests wherein no Host matching is available for request routing, + only the port is used to match requests. + properties: + backend: + description: Backend defines the Kubernetes service which accepts + traffic from the listening Port defined above. + properties: + serviceName: + description: Specifies the name of the referenced service. + minLength: 1 + type: string + servicePort: + description: Specifies the port of the referenced service. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - serviceName + - servicePort + type: object + port: + description: Port indicates the port for the Kong proxy to accept + incoming traffic on, which will then be routed to the service + Backend. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - backend + - port + type: object + type: array + type: object + status: + description: UDPIngressStatus defines the observed state of UDPIngress. + properties: + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer. + properties: + ingress: + description: Ingress is a list containing ingress points for the + load-balancer. Traffic intended for the service should be sent + to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a + load-balancer ingress point: traffic intended for the service + should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points + that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points + that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports + If used, every port defined in the service should have + an entry in it + items: + properties: + error: + description: 'Error is to record the problem with + the service port The format of the error shall comply + with the following rules: - built-in error values + shall be specified in this file and those shall + use CamelCase names - cloud provider specific error + values must have names that comply with the format + foo.example.com/CamelCase. --- The regex it matches + is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service + port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service + port of which status is recorded here The supported + values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kong-leader-election + namespace: kong +rules: +- apiGroups: + - "" + - coordination.k8s.io + resources: + - configmaps + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: kong-ingress +rules: +- apiGroups: + - "" + resources: + - endpoints + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - endpoints/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - secrets/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - ingressclassparameterses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongclusterplugins + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongclusterplugins/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongconsumers + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongconsumers/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongingresses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongplugins + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - kongplugins/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - tcpingresses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - tcpingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.konghq.com + resources: + - udpingresses + verbs: + - get + - list + - watch +- apiGroups: + - configuration.konghq.com + resources: + - udpingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: kong-ingress-gateway +rules: +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways + verbs: + - get + - list + - update + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - referencegrants + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - referencegrants/status + verbs: + - get +- apiGroups: + - gateway.networking.k8s.io + resources: + - tcproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tcproutes/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - tlsroutes/status + verbs: + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - udproutes + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - udproutes/status + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: kong-ingress-knative +rules: +- apiGroups: + - networking.internal.knative.dev + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.internal.knative.dev + resources: + - ingresses/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kong-leader-election + namespace: kong +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kong-leader-election +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-gateway +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress-knative +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-knative +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: v1 +kind: Secret +metadata: + annotations: + kubernetes.io/service-account.name: kong-serviceaccount + name: kong-serviceaccount-token + namespace: kong +type: kubernetes.io/service-account-token +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp + service.beta.kubernetes.io/aws-load-balancer-type: nlb + name: kong-proxy + namespace: kong +spec: + ports: + - name: proxy + port: 80 + protocol: TCP + targetPort: 8000 + - name: proxy-ssl + port: 443 + protocol: TCP + targetPort: 8443 + selector: + app: ingress-kong + type: LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + name: kong-validation-webhook + namespace: kong +spec: + ports: + - name: webhook + port: 443 + protocol: TCP + targetPort: 8080 + selector: + app: ingress-kong +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ingress-kong + name: ingress-kong + namespace: kong +spec: + replicas: 1 + selector: + matchLabels: + app: ingress-kong + template: + metadata: + annotations: + kuma.io/gateway: enabled + kuma.io/service-account-token-volume: kong-serviceaccount-token + traffic.sidecar.istio.io/includeInboundPorts: "" + labels: + app: ingress-kong + spec: + automountServiceAccountToken: false + containers: + - env: + - name: CONTROLLER_KONNECT_SYNC_ENABLED + value: "true" + - name: CONTROLLER_KONNECT_TLS_CLIENT_CERT + valueFrom: + secretKeyRef: + key: tls.crt + name: konnect-client-tls + - name: CONTROLLER_KONNECT_TLS_CLIENT_KEY + valueFrom: + secretKeyRef: + key: tls.key + name: konnect-client-tls + - name: CONTROLLER_KONG_ADMIN_URL + value: https://127.0.0.1:8444 + - name: CONTROLLER_KONG_ADMIN_TLS_SKIP_VERIFY + value: "true" + - name: CONTROLLER_PUBLISH_SERVICE + value: kong/kong-proxy + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + envFrom: + - configMapRef: + name: konnect-config + image: kong/kubernetes-ingress-controller:2.8.1 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: ingress-controller + ports: + - containerPort: 8080 + name: webhook + protocol: TCP + - containerPort: 10255 + name: cmetrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: 10254 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: kong-serviceaccount-token + readOnly: true + - env: + - name: KONG_PROXY_LISTEN + value: 0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport + backlog=16384 + - name: KONG_PORT_MAPS + value: 80:8000, 443:8443 + - name: KONG_ADMIN_LISTEN + value: 127.0.0.1:8444 http2 ssl reuseport backlog=16384 + - name: KONG_STATUS_LISTEN + value: 0.0.0.0:8100 + - name: KONG_DATABASE + value: "off" + - name: KONG_NGINX_WORKER_PROCESSES + value: "2" + - name: KONG_KIC + value: "on" + - name: KONG_ADMIN_ACCESS_LOG + value: /dev/stdout + - name: KONG_ADMIN_ERROR_LOG + value: /dev/stderr + - name: KONG_PROXY_ERROR_LOG + value: /dev/stderr + - name: KONG_ROUTER_FLAVOR + value: traditional + image: kong:3.1 + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - kong quit + livenessProbe: + failureThreshold: 3 + httpGet: + path: /status + port: 8100 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: proxy + ports: + - containerPort: 8000 + name: proxy + protocol: TCP + - containerPort: 8443 + name: proxy-ssl + protocol: TCP + - containerPort: 8100 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /status + port: 8100 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + serviceAccountName: kong-serviceaccount + volumes: + - name: kong-serviceaccount-token + secret: + items: + - key: token + path: token + - key: ca.crt + path: ca.crt + - key: namespace + path: namespace + secretName: kong-serviceaccount-token +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: kong +spec: + controller: ingress-controllers.konghq.com/kong diff --git a/scripts/build-single-manifests.sh b/scripts/build-single-manifests.sh index 6539654bec..0109057ba3 100755 --- a/scripts/build-single-manifests.sh +++ b/scripts/build-single-manifests.sh @@ -12,3 +12,4 @@ ${REPO_ROOT}/bin/kustomize build config/base > deploy/single/all-in-one-dbless.y ${REPO_ROOT}/bin/kustomize build config/variants/postgres > deploy/single/all-in-one-postgres.yaml ${REPO_ROOT}/bin/kustomize build config/variants/enterprise > deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml ${REPO_ROOT}/bin/kustomize build config/variants/enterprise-postgres > deploy/single/all-in-one-postgres-enterprise.yaml +${REPO_ROOT}/bin/kustomize build config/variants/konnect/base > deploy/single/all-in-one-dbless-konnect.yaml diff --git a/skaffold.yaml b/skaffold.yaml index b998d5d3c6..4f80f07b1d 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -57,3 +57,18 @@ profiles: TAG: ${{ .TAG }} COMMIT: ${{ .COMMIT }} REPO_INFO: ${{ .REPO_INFO }} +- name: debug-konnect + manifests: + kustomize: + paths: + - config/variants/konnect/debug + build: + artifacts: + - image: kic-placeholder + docker: + dockerfile: Dockerfile.debug + target: debug + buildArgs: + TAG: ${{ .TAG }} + COMMIT: ${{ .COMMIT }} + REPO_INFO: ${{ .REPO_INFO }}