From a2d6251c182e3587fc6a9dbb32b92876d8bdb021 Mon Sep 17 00:00:00 2001 From: ffforest Date: Fri, 21 Jul 2023 14:08:50 +0800 Subject: [PATCH] doc: Add design documentation for Istio support in kusion --- docs/istio.md | 466 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 466 insertions(+) create mode 100644 docs/istio.md diff --git a/docs/istio.md b/docs/istio.md new file mode 100644 index 00000000..a1101b49 --- /dev/null +++ b/docs/istio.md @@ -0,0 +1,466 @@ +# Istio Integration + +## Purpose +This document captures the intended design for KusionStack's AppConfiguration model to support Istio-related configuration when describing an application. + +## Background +### AppConfiguration +AppConfiguration model provides an interface to describe all attributes that are tied to an application. + +More details on AppConfiguration model can be found in the [AppConfiguration design doc](). + +### Istio +Istio is an open source service mesh[1] that layers transparently onto existing distributed applications. Istio’s powerful features provide a uniform and more efficient way to secure, connect, and monitor services. Istio is the path to load balancing, service-to-service authentication, and monitoring – with few or no service code changes. + +## Glossary +### Kusion concepts +Frontend model: The model that is exposed to the end user. The goal of the frontend model is to provide a user-friendly interface to define the application. The frontend model omits configurations that are repetitive or derivable, and only exposes essential attributes that users might be interested in. +An example of a frontend model is [here](https://github.com/KusionStack/konfig/blob/main/base/pkg/kusion_models/kube/frontend/server.k#L13-L143). + +An example of using this frontend model: +``` +wordpress: frontend.Server { + image = "wordpress:4.8-apache" + database = storage.DataBase { + dataBaseType = "aws_rds" + dataBaseAttr = storage.DBAttr { + databaseEngine = "MySQL" + databaseEngineVersion = "5.7" + databaseAccountName = "root" + databaseAccountPassword = option("db_password") + internetAccess = True + } + } + mainContainer: { + env += [ + { + name = "ENV_ADD_EXAMPLE" + value = "wordpress-example" + } + ] + } +} +``` + +Backend model: The backend model represents the opinionated logic to render and convert the user-defined application properties to the actual resources (both Kubernetes and cloud infrastructure resources) that need to be applied. + +An example of a backend model (The full example can be found [here](https://github.com/KusionStack/konfig/blob/main/base/pkg/kusion_models/kube/backend/server_backend.k#L26-L152)): +``` +schema ServerBackend[inputConfig: server.Server]: + mixin [ + ... + ] + _applicationLabel: {str: str} = { + "app.kubernetes.io/component": workloadName + } + workloadAttributes: {str:} = { + metadata = utils.MetadataBuilder(config) | { + ... + } + spec = { + ... + } + } + _workloadInstance?: appsv1.Deployment | appsv1.StatefulSet + ... + kubernetes: resource.ResourceMapping = { + if _workloadInstance: + "${typeof(_workloadInstance)}" = [_workloadInstance] + if _headlessServiceInstance: + "${typeof(_headlessServiceInstance)}" = [_headlessServiceInstance] + } +``` + +Backbone model: The backbone model does not contain any abstraction or logic. It is usually generated with kcl-openapi tool from an OpenAPI spec or a Kubernetes CRD. The attributes in the backbone model are mapped one-to-one to the underlying resources (e.g. Kubernetes objects). + +An example of a backbone model: +``` +""" +This is the secret module in kusion_kubernetes.api.core.v1 package. +This file was generated by the KCL auto-gen tool. DO NOT EDIT. +Editing this file might prove futile when you re-run the KCL auto-gen generate command. +""" +import base.pkg.kusion_kubernetes.apimachinery.apis + +schema Secret: + """ Secret holds secret data of a certain type. The total bytes of the values in the Data field must be less than MaxSecretSize bytes. + """ + + apiVersion: "v1" = "v1" + data?: {str:str} + immutable?: bool + kind: "Secret" = "Secret" + stringData?: {str:str} + $type?: str + metadata?: apis.ObjectMeta +``` + +### Istio Concepts + +#### Traffic Management +##### VirtualService +A VirtualService defines a set of traffic routing rules that are applied when a host is addressed. Each routing rule defines matching criteria for traffic of a specific protocol. If the traffic is matched, then it is sent to a named destination service (or subset/version of it) defined in the registry[2]. + +An example of a VirtualService where traffic is sent to v1 version by default, with an exception to requests with `Jason` in the `end-user` field in the http header, which would be routed to v2 of the `reviews` service in the mesh: +``` +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: reviews + namespace: default +spec: + hosts: + - reviews + http: + - match: + - headers: + end-user: + exact: jason + route: + - destination: + host: reviews + subset: v2 + - route: + - destination: + host: reviews + subset: v1 +``` + +##### DestinationRule +DestinationRule defines policies that apply to traffic intended for a service after routing has occurred. These rules specify configuration for load balancing, connection pool size from the sidecar, and outlier detection settings to detect and evict unhealthy hosts from the load balancing pool[3]. + +For example, a simple load balancing policy, along with the definition (match by labels) of `v1` and `v2` of the `reviews` service: +``` +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: reviews + namespace: default +spec: + host: reviews + subsets: + - labels: + version: v1 + name: v1 + - labels: + version: v2 + name: v2 + trafficPolicy: + loadBalancer: + simple: LEAST_REQUEST +``` + +##### ServiceEntry +To be completed soon. + +##### Gateway +To be completed soon. + +##### EnvoyFilter +To be completed soon. + +##### ProxyConfig +To be completed soon. + +##### Sidecar +To be completed soon. + +##### AuthorizationPolicy +To be completed soon. + +##### PeerAuthentication +To be completed soon. + +##### RequestAuthentication +To be completed soon. + +##### Telemetry +To be completed soon. + +## Constraints +KusionStack is application-centric by design and aims to provide a homogeneous experience in application delivery for all kinds of applications. + +As of today, the design in this document only captures the elements Istio supports that can be tied to an application. For example, the steps to install and configure Istio control plane is not within consideration of this document. + +Kusion's support for Istio are focused on abstracting, rendering and managing the Istio-related CRs. + +As of today, the context in this document does not cover a federated usage of Istio control plane across multiple Kubernetes clusters. + +### Istio automated sidecar injection +In order to take advantage of all of Istio’s features, pods in the mesh must be running an Istio sidecar proxy. + +The recommended way to include the sidecar container in an application deployment is to use the automatic sidecar injection with a mutating webhook admission controller. + +More details on sidecar injection can be found [here](https://istio.io/latest/docs/setup/additional-setup/sidecar-injection/). + +## Design + +### Pre-requisites +- Istio control plane is installed in the target Kubernetes cluster +- Sidecar injection is configured to ensure all Istio sidecars are created properly + +### Goals +- Users are able to declare Istio-related attributes in AppConfiguration frontend model +- The backend model is able to convert the user-defined attributes to Istio-managed CRs while rendering the Kubernetes manifests +- The Istio-related CRs are managed in the exact same workflow as the rest of the Kubernetes resources managed by Kusion + +### List of supported Istio-managed CRs +- Routing + - DestinationRule + - VirtualService + - ServiceEntry + - Gateway(?) +- Authentication + - PeerAuthentication + - RequestAuthentication +- Authorization + - AuthorizationPolicy +- Configuration + - EnvoyFilter + - ProxyConfig + - Sidecar +- Observability + - Telemetry + +### Generating backbone models from Istio CRDs + +Kusion backbone models can be automatically generated from Istio CRDs using the kcl-openapi tool[4]. + +Istio CRDs are located [here](https://github.com/istio/istio/blob/master/manifests/charts/base/crds/crd-all.gen.yaml). + +The kcl-openapi tool is located [here](https://github.com/kcl-lang/kcl-openapi) + +The backbone models can be generated using the following command: +``` +kcl-openapi generate model --crd -f [crd YAML] +``` + +Generated backbone models: +``` +~/playground/kcl-to-openapi$ tree models/ +models/ +├── extensions_istio_io_v1alpha1_wasm_plugin.k +├── kusion_kubernetes +│   └── apimachinery +│   └── apis +│   ├── managed_fields_entry.k +│   ├── object_meta.k +│   └── owner_reference.k +├── networking_istio_io_v1alpha3_destination_rule.k +├── networking_istio_io_v1alpha3_envoy_filter.k +├── networking_istio_io_v1alpha3_gateway.k +├── networking_istio_io_v1alpha3_service_entry.k +├── networking_istio_io_v1alpha3_sidecar.k +├── networking_istio_io_v1alpha3_virtual_service.k +├── networking_istio_io_v1alpha3_workload_entry.k +├── networking_istio_io_v1beta1_proxy_config.k +├── security_istio_io_v1_authorization_policy.k +├── security_istio_io_v1_request_authentication.k +├── security_istio_io_v1beta1_peer_authentication.k +└── telemetry_istio_io_v1alpha1_telemetry.k +``` + +### Frontend models + +Frontend models determine the list of Istio-related attributes that are exposed to the end user. + +The full list of supported attributes will be completed over time. +Below is a sample for `DestinationRule`(not finalized), one of the most used Istio-managed resources for traffic management. + +#### DestinationRule + +##### Attributes +- `host` - (string, Required) The service address this destination rule applies to. This can be any service from Istio's service registry, including the ones defined using `ServiceEntry`. Short names can be used (e.g. `reviews` instead of `reviews.default.svc.cluster.local`). +- `subsets` - ([]subset, Optional) A list of named sets that represent subsets of the applications. Usually used to represent different versions of a service. See `subset` object below. +- `namespaces` - ([]string, Optional) A list of the namespaces that have visibility to, and can resolve this destination rule. The value `"."` represents the current application namespace. Default to all namespaces. +- `workloadSelector` - (map[string]string, Optional) The labels used to select a specific set of workloads in the current namespace. Default to all workloads. +- `trafficPolicy` - (trafficPolicy, Optional) Traffic policies to apply. See `trafficPolicy` object defined below. + +- A `subset` object supports the following attributes: + - `name` - (string, Required) The name of the subset. + - `labels` - (map[string]string, Optional) The labels that are used to filter over the endpoints of a service and define the subset. + - `trafficPolicy` - (Optional) See `trafficPolicy` object as defined below. This will override the `trafficPolicy` defined at `DestinationRule` level. + +- A `trafficPolicy` object supports the following attributes: + - `loadBalancerSettings` - (Optional) A `loadBalancerSettings` block as defined below. + - `tlsSettings` - (Optional) A `tlsSettings` block as defined below. + - `ejectionSettings` - (Optional) An `ejectionSettings` block as defined below. This defines the circuit breaker behavior for the host. + - `connectionPoolSettings` - (Optional) A `connectionPoolSettings` block as defined below. + +- A `loadBalancerSettings` object supports the following attributes: + - `strategy` - (string, Optional) The kind of load balancing strategy to enforce. Possible values are `ROUND_ROBIN`,`LEAST_REQUEST`,`LEAST_CONN`,`PASSTHROUGH`,`RANDOM`. + +- A `tlsSettings` object supports the following attributes: + - `mode` - (string, Optional) How TLS will be enforced when connecting to this host. Possible values are `DISABLE`,`SIMPLE`,`MUTUAL`,`ISTIO_MUTUAL`. + - `clientCertificate` - (string, Optional). The path to the file holding the client TLS certificate. Required when `mode` is set to `MUTUAL`. + - `clientPrivateKey` - (string, Optional). The path to the file holding the client private key. Required when `mode` is set to `MUTUAL`. + - `caCertificates` - (string, Optional). The path to the file holding the client TLS certificate. Only applicable when `mode` is set to `MUTUAL`. + +- An `ejectionSettings` object supports the following attributes: + - `consecutiveErrors` - (int, Optional) The number of consecutive errors before a host is ejected from the connection pool. Default to 5. + - `interval` - (protobuf.Duration, Optional) The time interval between the ejection sweep occurs. Format: 1h/1m/1s/1ms. Default to 10s. + - `minimumEjectionTime` - (protobuf.Duration, Optional) The minimum time a host will remain ejected. Format: 1h/1m/1s/1ms. Default to 30s. + +- A `connectionPoolSettings` object supports the following attributes: + - `tcpSettings` - (Optional) See `tcpSettings` object defined below. + - `httpSettings` - (Optional) See `httpSettings` object defined below. + +- A `tcpSettings` object supports the following attributes: + - `maxTcpConnections` - (int, Optional) Max number of TCP connections to a destination host. Default is 2^32-1. + - `connectionTimeout` - (protobuf.Duration, Optional) TCP connection timeout. Format: 1h/1m/1s/1ms. Default to 10s. + +- An `httpSettings` object supports the following attributes: + - `maxPendingHttpConnections` - (int, Optional) Max number of HTTP that will be queued while waiting for a ready connection to a destination host. Default is 1024. + - `maxActiveHttpConnections` - (int, Optional) Max number of active requests to a destination. Default 1024. + - `maxHttpRequestsPerConnection` - (int, Optional) Maximum number of requests per connection to a host. Default to unlimited. + +#### VirtualService +To be completed soon. + +#### ServiceEntry +To be completed soon. + +#### Gateway +To be completed soon. + +#### EnvoyFilter +To be completed soon. + +#### ProxyConfig +To be completed soon. + +#### Sidecar +To be completed soon. + +#### AuthorizationPolicy +To be completed soon. + +#### PeerAuthentication +To be completed soon. + +#### RequestAuthentication +To be completed soon. + +#### Telemetry +To be completed soon. + +### Backend models +Backend models contain the render logic for Istio-related configurations. The backend models for Istio-related configurations is responsible for transcribing the values defined in the frontend model into the corresponding fields in the backbone model. For those attributes that aren't directly exposed in the frontend model for the sake of simplicity, the backend model should decide between populating them with default values that represents an opinionated platform implementation, or using the Istio-provided defaults (by omission) as the fallback. + +A sample snippet of the backend rendering logic in KCL: +``` + import base.pkg.kusion_models.kube.frontend.istio.destination_rule as dr + import base.pkg.kusion_istio.v1alpha3 as istiov1alpha3 + import base.pkg.kusion_models.kube.metadata + ... + ... + destinationRuleName = metadata.__META_APP_NAME + "_destination_rule" + app: utils.ApplicationBuilder = utils.ApplicationBuilder {} + _applicationLabel: {str: str} = { + "app.kubernetes.io/component": metadata.__META_APP_NAME + } + _applicationAnnotation: {str: str} = { + "traffic.sidecar.istio.io/testAnnotation1" = "test1" + "traffic.sidecar.istio.io/testAnnotation2" = "test2" + } + + kubernetes: resource.ResourceMapping = { + DestinationRule = [ + istiov1alpha3.DestinationRule { + metadata = utils.MetadataBuilder(istioConfig) | { + name = destinationRuleName + namespace = metadata.__META_APP_NAME + labels: app.labels | _applicationLabel | istioConfig?.labels + annotations: _applicationAnnotation | istioConfig?.annotations + } + spec.host = host + spec.subsets = subsets + spec.exportTo = exportTo + spec.trafficPolicy = istiov1alpha3.NetworkingIstioIoV1alpha3DestinationRuleSpecTrafficPolicy { + loadBalancer = istiov1alpha3.NetworkingIstioIoV1alpha3DestinationRuleSpecTrafficPolicyLoadBalancer { + simple = trafficPolicy.loadBalancerSettings.simpleLB + } + ... + } + ... + } + ] + } +``` + +## Examples +An example of an Istio DestinationRule configuration that splits the `sample-service` service into 3 subsets, identified with the `version` label: +``` +destination_rule: istio.DestinationRule{ + host = "sample-service" + subsets = [ + { + labels = {"version" = "v1"} + name = "v1" + }, + { + labels = {"version" = "v2"} + name = "v2" + }, + { + labels = {"version" = "v3"} + name = "v3" + } + ] + namespaces = [ + "default", + "my-namespace" + ] + trafficPolicy = istio.TrafficPolicy{ + loadBalancerSettings = { + strategy = "ROUND_ROBIN" + ... + } + } +} +``` + +Generated DestinationRule spec: +``` +- id: networking.istio.io/v1alpha3:DestinationRule:wordpress-example:wordpress-example_destination_rule + type: Kubernetes + attributes: + apiVersion: networking.istio.io/v1alpha3 + kind: DestinationRule + metadata: + annotations: + traffic.sidecar.istio.io/testAnnotation1: test1 + traffic.sidecar.istio.io/testAnnotation2: test2 + labels: + app.kubernetes.io/component: wordpress-example + app.kubernetes.io/env: dev + app.kubernetes.io/instance: wordpress-example-dev + app.kubernetes.io/name: wordpress-example + cluster.x-k8s.io/cluster-name: default + name: wordpress-example_destination_rule + namespace: wordpress-example + spec: + exportTo: + - default + - my-namespace + host: sample-service + subsets: + - labels: + version: v1 + name: v1 + - labels: + version: v2 + name: v2 + - labels: + version: v3 + name: v3 + trafficPolicy: + loadBalancer: + simple: ROUND_ROBIN +``` + +## References +1. Istio: https://istio.io/latest/about/service-mesh/ +2. Istio VirtualService: https://istio.io/latest/docs/reference/config/networking/virtual-service/ +3. Istio DestinationRule: https://istio.io/latest/docs/reference/config/networking/destination-rule/ +4. Generating backbone models: https://github.com/kcl-lang/kcl-openapi \ No newline at end of file