diff --git a/keps/prod-readiness/sig-storage/1979.yaml b/keps/prod-readiness/sig-storage/1979.yaml new file mode 100644 index 00000000000..2c651330a38 --- /dev/null +++ b/keps/prod-readiness/sig-storage/1979.yaml @@ -0,0 +1,3 @@ +kep-number: 1979 +alpha: + approver: "@deads2k" diff --git a/keps/sig-storage/1979-object-storage-support/README.md b/keps/sig-storage/1979-object-storage-support/README.md index 3b43818020e..13c0b71c939 100644 --- a/keps/sig-storage/1979-object-storage-support/README.md +++ b/keps/sig-storage/1979-object-storage-support/README.md @@ -3,703 +3,901 @@ ## Table of Contents -- [Release Signoff Checklist](#release-signoff-checklist) -- [Summary](#summary) -- [Motivation](#motivation) + - [Release Signoff Checklist](#release-signoff-checklist) + - [Introduction](#introduction) + - [Motivation](#motivation) - [User Stories](#user-stories) - - [Admin](#admin) - - [User](#user) + - [Admin](#admin) + - [DevOps](#devops) + - [Object Storage Provider (OSP)](#object-storage-provider-osp) - [Goals](#goals) - - [Non-Goals](#non-goals) -- [Vocabulary](#vocabulary) -- [Proposal](#proposal) - - [APIs](#apis) - - [Storage APIs](#storage-apis) - - [BucketRequest](#bucketrequest) - - [Bucket](#bucket) - - [BucketClass](#bucketclass) - - [Access APIs](#access-apis) - - [BucketAccessRequest](#bucketaccessrequest) - - [BucketAccess](#bucketaccess) - - [BucketAccessClass](#bucketaccessclass) - - [App Pod](#app-pod) - - [Topology](#topology) -- [Object Relationships](#object-relationships) -- [Workflows](#workflows) - - [Create Bucket](#create-bucket) - - [Sharing COSI Created Buckets](#sharing-cosi-created-buckets) - - [Delete Bucket](#delete-bucket) - - [Grant Bucket Access](#grant-bucket-access) - - [Revoke Bucket Access](#revoke-bucket-access) - - [Delete BucketAccess](#delete-bucketaccess) - - [Setting Access Permissions](#setting-access-permissions) - - [Dynamic Provisioning](#dynamic-provisioning) - - [Static Provisioning](#static-provisioning) -- [gRPC Definitions](#grpc-definitions) - - [ProvisionerGetInfo](#provisionergetinfo) - - [ProvisonerCreateBucket](#provisonercreatebucket) - - [ProvisonerDeleteBucket](#provisonerdeletebucket) - - [ProvisionerGrantBucketAccess](#provisionergrantbucketaccess) - - [ProvisionerRevokeBucketAccess](#provisionerrevokebucketaccess) + - [Functionality](#functionality) + - [System Properties](#system-properties) + - [Non-Goals](#non-goals) + - [COSI architecture](#cosi-architecture) + - [COSI API](#cosi-api) + - [Bucket Creation](#bucket-creation) + - [Generating Access Credentials for Buckets](#generating-access-credentials-for-buckets) + - [Attaching Bucket Information to Pods](#attaching-bucket-information-to-pods) + - [Sharing Buckets](#sharing-buckets) + - [Accessing existing Buckets](#accessing-existing-buckets) + - [Bucket deletion](#bucket-deletion) +- [Usability](#usability) + - [Self Service](#self-service) + - [Mutating Buckets](#mutating-buckets) +- [Object Lifecycle](#object-lifecycle) +- [COSI API Reference](#cosi-api-reference) + - [Bucket](#bucket) + - [BucketClaim](#bucketclaim) + - [BucketClass](#bucketclass) + - [BucketAccess](#bucketaccess) + - [BucketAccessClass](#bucketaccessclass) + - [BucketInfo](#bucketinfo) +- [COSI Driver](#cosi-driver) + - [COSI Driver gRPC API](#cosi-driver-grpc-api) + - [DriverGetInfo](#drivergetinfo) + - [DriverCreateBucket](#drivercreatebucket) + - [DriverGrantBucketAccess](#drivergrantbucketaccess) + - [DriverDeleteBucket](#driverdeletebucket) + - [DriverRevokeBucketAccess](#driverrevokebucketaccess) - [Test Plan](#test-plan) - [Graduation Criteria](#graduation-criteria) - [Alpha](#alpha) + - [Alpha -> Beta](#alpha---beta) + - [Beta -> GA](#beta---ga) - [Alternatives Considered](#alternatives-considered) - [Add Bucket Instance Name to BucketAccessClass (brownfield)](#add-bucket-instance-name-to-bucketaccessclass-brownfield) - [Motivation](#motivation-1) - [Problems](#problems) + - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy) + - [Version Skew Strategy](#version-skew-strategy) + - [Production Readiness Review Questionnaire](#production-readiness-review-questionnaire) + - [Feature Enablement and Rollback](#feature-enablement-and-rollback) + - [Rollout, Upgrade and Rollback Planning](#rollout-upgrade-and-rollback-planning) + - [Monitoring Requirements](#monitoring-requirements) + - [Dependencies](#dependencies) + - [Scalability](#scalability) + - [Infrastructure Needed (Optional)](#infrastructure-needed-optional) -# Release Signoff Checklist +## Release Signoff Checklist -- [X] (R) Enhancement issue in release milestone, which links to KEP dir in [kubernetes/enhancements] (not the initial KEP PR) +- [X] (R) Enhancement issue in release milestone, which links to KEP dir in [kubernetes/enhancements][57] (not the initial KEP PR) - [ ] (R) KEP approvers have approved the KEP status as `implementable` - [ ] (R) Design details are appropriately documented -- [ ] (R) Test plan is in place, giving consideration to SIG Architecture and SIG Testing input -- [ ] (R) Graduation criteria is in place -- [ ] (R) Production readiness review completed +- [X] (R) Test plan is in place, giving consideration to SIG Architecture and SIG Testing input +- [X] (R) Graduation criteria is in place +- [X] (R) Production readiness review completed - [ ] Production readiness review approved - [ ] "Implementation History" section is up-to-date for milestone -- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io] +- [ ] User-facing documentation has been created in [kubernetes/website][58], for publication to [kubernetes.io][59] - [ ] Supporting documentation e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes -[kubernetes.io]: https://kubernetes.io/ -[kubernetes/enhancements]: https://git.k8s.io/enhancements -[kubernetes/kubernetes]: https://git.k8s.io/kubernetes -[kubernetes/website]: https://git.k8s.io/website +## Introduction +This proposal introduces the *Container Object Storage Interface* (COSI), a standard for provisioning and consuming object storage in Kubernetes. -# Summary -This proposal introduces the *Container Object Storage Interface* (COSI), a system composed of Custom Resources (CRs), APIs for bucket provisioning and access, a controller automation architecture, a gRPC specification, and "COSI node adapter" that interfaces with the kubelet on each node. These components combine to support a standard object storage representation in Kubernetes. +## Motivation -This KEP describes the above components, defines our goals and target use-cases, and sets the scope of the proposal by defining higher level objectives. The vocabulary section defines terminology. Relationships between the APIs are provided to illustrate the interconnections between object storage APIs, users' workloads, and object store service instances. Lastly, the document describes the proposed API specs, the create and delete workflows, and the gRPC spec. +File and block storage are treated as first class citizens in the Kubernetes ecosystem via CSI. Workloads using CSI volumes enjoy the benefits of portability across vendors and across Kubernetes clusters without the need to change application manifests. **An equivalent standard does not exist for Object storage.** -# Motivation - -File and block are first class citizens within the Kubernetes ecosystem. Object, though very different under the hood, is a popular means of storing data, especially against very large data sources. As such, we feel it is in the interest of the community to integrate object storage into Kubernetes, supported by the SIG-Storage community. In doing so, we can provide Kubernetes cluster users and administrators a normalized and familiar means of managing object storage. -When a workload (app pod, deployment, configs) is moved to another cluster, as long as the new cluster runs a COSI controller that supports the same object protocol, then the workload can be moved to a new cluster without requiring any changes in the user manifests. +Object storage has been rising in popularity in the recent years as an alternative form of storage to filesystems and block devices. Object storage paradigm promotes disaggregation of compute and storage. This is done by making data available over the network, rather than locally. Disaggregated architectures allow compute workloads to be stateless, which consequently makes them easier to manage, scale and automate. ## User Stories -### Admin +We define 3 kinds of stakeholders: + ++ **Admins** - Members with escalated privileges and authority to manage site wide/cluster wide policies to control access, assign storage quotas and other concerns related to hardware or application resources. ++ **Application Developer/Operator (User)** - Members with privileges to run workloads, request storage for them, and manage resources inside a specific namespace. ++ **Object Storage Provider (OSP)** - Vendors offering Object Storage capabilities + +#### Admin + ++ Establish cluster/site wide policies for data redundancy, durability and other data related parameters ++ Establish cluster/site wide policies for access control over buckets ++ Enable/restrict users who can create, delete and reuse buckets -- As a cluster administrator, I can control access to new and existing buckets when accessed from the cluster, regardless of the backing object store. +#### DevOps -### User ++ Request a bucket to be created and/or provisioned for workloads ++ Request access to be created/deleted for a specific bucket ++ Request deletion of a created bucket -- As a developer, I can define my object storage needs in the same manifest as my workload, so that deployments are streamlined and encapsulated within the Kubernetes interface. -- As a developer, I can define a manifest containing my workload and object storage configuration once, so that my app may be ported between clusters as long as the storage provided supports my designated data path protocol. -- As a developer, I want to create a workload controller which is bucket API aware, so that it can dynamically connect workloads to underlying object storage. +### Object Storage Provider (OSP) + ++ Integrate with the official standard for orchestrating Object Storage resources in Kubernetes ## Goals -+ Specify object storage Kubernetes APIs for the purpose of orchestrating/managing object store operations. -+ Implement a Kubernetes controller automation design with support for pluggable provisioners. -+ Support workload portability across clusters. -+ As MVP, be accessible to the largest groups of consumers by supporting the major object storage protocols (S3, Google Cloud Storage, Azure Blob) while being extensible for future protocol additions.. -+ Present similar workflows for both greenfield and brownfield bucket operations. - -## Non-Goals - -+ Defining the _data-plane_ object store protocol to replace or supplement existing vendor protcols/APIs is not within scope. -+ Object store deployment/management is left up to individual vendors. -+ Bucket access lifecycle management is not within the scope of this KEP. ACLs, access policies, and credentialing need to be handled out-of-band. - -# Vocabulary - -+ _backend bucket_ - any bucket provided by the object store system, completely separate from Kubernetes. -+ _brownfield bucket_ - an existing backend bucket. -+ _Bucket (Bucket Instance)_ - a cluster-scoped custom resource referenced by a `BucketRequest` and containing connection information and metadata for a backend bucket. -+ _BucketAccess (BA)_ - a cluster-scoped custom resource for granting bucket access. -+ _BucketAccessClass (BAC)_ - a cluster-scoped custom resource containing fields defining the provisioner and a ConfigMap reference where policy is defined. -+ _BucketAccessRequest (BAR)_ - a user-namespaced custom resource representing a request for access to an existing bucket. -+ _BucketClass (BC)_ - a cluster-scoped custom resource containing fields defining the provisioner and an immutable parameter set for creating new buckets. -+ _BucketRequest (BR)_ - a user-namespaced custom resource representing a request for a new backend bucket, or access to an existing bucket. -+ _COSI_ - Container _Object_ Store Interface, modeled after CSI. -+ _cosi-node-adapter_ - a pod per node which receives Kubelet gRPC _NodeStageVolume_ and _NodeUnstageVolume_ requests, ensures the target bucket has been provisioned, and notifies the kubelet when the pod can be run. -+ _driver_ - a container (usually paired with a sidecar container) that is responsible for communicating with the underlying object store to create, delete, grant access to, and revoke access from buckets. Drivers talk gRPC and need no knowledge of Kubernetes. Drivers are typically written by storage vendors, and should not be given any access outside of their namespace. -+ _driverless_ - a use-case where no driver exists to support the backend object store. Some COSI automation may still be provided, but backend operations such as creating a bucket or minting credentials will not occur. -+ _greenfield bucket_ - a new backend bucket created by COSI. -+ _green-to-brownfield_ - a use-case where COSI creates a new bucket on behalf of a user, and then supports ways for other users in the cluster to share this bucket. -+ _provisioner_ - a generic term meant to describe the combination of a sidecar and driver. "Provisioning" a bucket can mean creating a new bucket or granting access to an existing bucket. -+ _sidecar_ - a Kubernetes-aware container (usually paired with a driver) that is responsible for fullfilling COSI requests with the driver via gRPC calls. The sidecar's access can be restricted to the namespace where it resides, which is expected to be the same namespace as the provisioner. The COSI sidecar is provided by the Kubernetes community. -+ _static provisioning_ - custom resource creation is done by the admin rather than via COSI automation. This may also include _driverless_ but they are independent. - -# Proposal - -## APIs - -### Storage APIs - -#### BucketRequest - -A user facing, namespaced custom resource requesting provisioning of a new bucket, or requesting access to an existing bucket. A `BucketRequest` (BR) lives in the app's namespace. In addition to a `BucketRequest`, a [BucketAccessRequest](#bucketaccessrequest) is necessary in order to grant credentials to access the bucket. BRs are required for both greenfield and brownfield uses. - -There is a 1:1 mapping of a `BucketRequest` and the cluster scoped `Bucket` instance, meaning there is a single `Bucket` instance for every BR. - -```yaml -apiVersion: cosi.io/v1alpha1 -kind: BucketRequest -metadata: - name: - namespace: - labels: - - cosi.io/provisioner: [1] - finalizers: - - cosi.io/finalizer [2] -spec: - protocol: - name: [3] - version: [4] - bucketPrefix: [5] - bucketClassName: [6] - bucketInstanceName: [7] -status: - bucketAvailable: [8] -``` - -1. `labels`: added by COSI. Key’s value should be the provisioner name. Characters that do not adhere to [Kubernetes label conventions](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) will be converted to ‘-’. -1. `finalizers`: added by COSI to defer `BucketRequest` deletion until backend deletion succeeds. -1. `protocol.name`: (required) specifies the desired protocol. One of {“s3”, “gs”, or “azureBlob”}. -1. `protocol.version`: (optional) specifies the desired version of the `protocol`. For "s3", a value of "v2" or "v4" could be used. -1. `bucketPrefix`: (optional for greenfield, ignored for brownfield) prefix prepended to a generated new bucket name, eg. “yosemite-photos-". If `bucketInstanceName` is supplied then `bucketPrefix` is ignored because the request is for access to an existing bucket. -1. `bucketClassName`: (optional for greenfield, ignored for brownfield) name of the `BucketClass` used to provision this request. If omitted for a greenfield bucket request, a default bucket class matching the protocol, if available, is used. If the greenfield bucket class is missing or does not support the requested protocol, an error is logged and the request is retried (with exponential backoff). A `BucketClass` is necessary for greenfield requests since BCs support a list of allowed namespaces. A `BucketClass` is not needed for brownfield requests since the `Bucket` instance, created by the admin, also contains `allowedNamespaces`. -1. `bucketInstanceName`: (required for brownfield, omitted for greenfield) name of the cluster-wide `Bucket` instance. If blank, then COSI assumes this is a greenfield request and will fill in the name during the binding step. If set by the user, then this names the `Bucket` instance created by the admin. -1. `bucketAvailable`: if true the bucket has been provisioned. If false then the bucket has not been provisioned and is unable to be accessed. - -#### Bucket - -A cluster-scoped custom resource representing the abstraction of a single backend bucket. A `Bucket` instance stores enough identifying information so that drivers can accurately target the backend object store (e.g. needed during a deletion process). The relevant bucket class fields are copied to the `Bucket`. This is done so that the `Bucket` instance reflects the `BucketClass` at the time of `Bucket` creation. This is needed to handle cases where the BC is either deleted or re-created. Additionally, data returned by the driver is copied to the `Bucket` by the sidecar. - -For greenfield, COSI creates the `Bucket` based on values in the `BucketRequest` and `BucketClass`. For brownfield, an admin manually creates the `Bucket`, filling in BucketClass fields, such as `allowedNamespaces`. COSI populates fields returned by the provisioner, and binds the `Bucket` to the `BucketAccess`. - -Since there is a 1:1 mapping between a BR and a `Bucket`, when multiple BRs request access to the same backend bucket, each separate `Bucket` points to the same backend bucket. - -```yaml -apiVersion: cosi.io/v1alpha1 -kind: Bucket -metadata: - name: [1] - labels: [2] - cosi.io/provisioner: - cosi.io/bucket-prefix: - finalizers: - - cosi.io/finalizer [3] -spec: - provisioner: [4] - retentionPolicy: [5] - anonymousAccessMode: [6] - bucketClassName: [7] - bucketRequest: [8] - allowedNamespaces: [9] - - name: - protocol: [10] - name: - version: - azureBlob: - containerName: - storageAccount: - s3: - endpoint: - bucketName: - region: - signatureVersion: - gs: - bucketName: - privateKeyName: - projectId: - serviceAccount: - parameters: [11] -status: - bucketAvailable: [12] -``` - -1. `name`: when created by COSI, the `Bucket` name is generated in this format: _"br-"+uuid_. If an admin creates a `Bucket`, as is necessary for brownfield access, they can use any name. The uuid is unique within a cluster. -1. `labels`: added by COSI. The "cosi.io/provisioner" key's value is the provisioner name. The "cosi.io/bucket-prefix" key's value is the `BucketRequest.bucketPrefix` value when specified. Characters that do not adhere to [Kubernetes label conventions](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) will be converted to ‘-’. -1. `finalizers`: added by COSI to defer `Bucket` deletion until backend deletion ops succeed. Implementation may add one finalizer for the BR and one for the BA. -1. `provisioner`: (optional) The provisioner field as defined in the `BucketClass`. If empty then there is no driver/sidecar, thus the admin had to create the `Bucket` and `BucketAccess` instances. -1. `retentionPolicy`: Prescribes the outcome when the `BucketRequest` is deleted. - - _Retain_: (default) the `Bucket` and its data are preserved. The `Bucket` can potentially be reused. - - _Delete_: the bucket and its contents are expected to be deleted by the backend store. -A `Bucket` is not deleted if it is bound to a `BucketRequest`. -1. `anonymousAccessMode`: a string specifying *uncredentialed* access to the Bucket. This is applicable to cases where the bucket objects are intended to be publicly readable and/or writable. One of: - - "private": Default, disallow uncredentialed access to the backend storage. - - "publicReadOnly": Read only, uncredentialed users can call ListBucket and GetObject. - - "publicWriteOnly": Write only, uncredentialed users can only call PutObject. - - "publicReadWrite": Read/Write, same as `ro` with the addition of PutObject being allowed. - > Note: `anonymousAccessMode` is intended for workloads to consume the bucket out of band, i.e. COSI does not provide a automated mechanism for a k8s workload to consume the bucket anonymously -1. `bucketClassName`: (set by COSI for greenfield, and ignored for brownfield) Name of the associated bucket class. -1. `bucketRequest`: (set by COSI for greenfield, and ignored for brownfield) an `objectReference` containing the name, namespace and UID of the associated `BucketRequest`. -1. `allowedNamespaces`: a list of namespaces that are permitted to either create new buckets or to access existing buckets. -1. `protocol`: protocol-specific field the application will use to access the backend storage. - - `name`: supported protocols are: "s3", "gs", and "azureBlob". - - `version`: version being used by the backend storage. - - `azureBlob`: data required to target a provisioned azure container. - - `gs`: data required to target a provisioned GS bucket. - - `s3`: data required to target a provisioned S3 bucket. -1. `parameters`: a copy of the BucketClass parameters. -1. `bucketAvailable`: if true the bucket has been provisioned. If false then the bucket has not been provisioned and is unable to be accessed. - -#### BucketClass - -An immutable, cluster-scoped, custom resource to provide admins control over the handling of bucket provisioning. The `BucketClass` (BC) defines a retention policy, driver specific parameters, and the provisioner name. A list of allowed namespaces can be specified to restrict new bucket creation and access to existing buckets. A default bucket class can be defined for each supported protocol. This allows the bucket class to be omitted from a `BucketRequest`. Relevant `BucketClass` fields are copied to the `Bucket` instance to handle the case of the BC being deleted or re-created. If an object store supports more than one protocol then the admin should create a `BucketClass` per protocol. - -NOTE: the `BucketClass` object is immutable except for the field `isDefaultBucketClass` - -```yaml -apiVersion: cosi.io/v1alpha1 -kind: BucketClass -metadata: - name: -provisioner: [1] -isDefaultBucketClass: [2] -protocol: - name: [3] - version: [4] -anonymousAccessMode: [5] -retentionPolicy: {"Delete", "Retain"} [6] -allowedNamespaces: [7] - - name: -parameters: [8] -``` - -1. `provisioner`: (required) the name of the vendor-specific driver supporting the `protocol`. -1. `isDefaultBucketClass`: (optional) boolean, default is false. If set to true then a `BucketRequest` may omit the `BucketClass` reference. If a greenfield `BucketRequest` omits the `BucketClass` and a default `BucketClass`'s protocol matches the `BucketRequest`'s protocol then the default bucket class is used; otherwise an error is logged. It is not possible for more than one default `BucketClass` of the same protocol to exist due to an admission controller which enforces the default rule. -1. `protocol.name`: (required) specifies the desired protocol. One of {“s3”, “gs”, or “azureBlob”}. -1. `protocol.version`: (optional) specifies the desired version of the `protocol`. For "s3", a value of "v2" or "v4" could be used. -1. `anonymousAccessMode`: (optional) a string specifying *uncredentialed* access to the backend bucket. This is applicable for cases where the backend storage is intended to be publicly readable and/or writable. One of: - - "private": Default, disallow uncredentialed access to the backend storage. - - "publicReadOnly": Read only, uncredentialed users can call ListBucket and GetObject. - - "publicWriteOnly": Write only, uncredentialed users can only call PutObject. - - "publicReadWrite": Read/Write, same as `ro` with the addition of PutObject being allowed. -1. `retentionPolicy`: defines bucket retention for greenfield `BucketRequest`. - - _Retain_: (default) the `Bucket` instance and the backend bucket are preserved. The `Bucket` may potentially be reused. - - _Delete_: the backend bucket and the `Bucket` instance are deleted. -1. `allowedNamespaces`: a list of namespaces that are permitted to either create new buckets or to access existing buckets. This list is static for the life of the `BucketClass`, but the `Bucket` instance's list of allowed namespaces can be mutated by the admin. -1. `parameters`: (optional) a map of string:string key values. Allows admins to set provisioner key-values. - -### Access APIs - -The Access APIs abstract the backend policy system. Access policy and user identities are an integral part of most object stores. As such, a system must be implemented to manage both user/credential creation and the binding of those users to individual buckets via policies. Object stores differ from file and block storage in how they manage users, with cloud providers typically integrating with an IAM platform. This API includes support for cloud platform identity integration with Kubernetes ServiceAccounts. On-prem solutions usually provide their own user management systems, which may look very different from each other and from IAM platforms. We also account for third party authentication solutions that may be integrated with an on-prem service. - -#### BucketAccessRequest - -A user namespaced custom resource representing an object store user and an access policy defining the user’s relation to that storage. A user creates a `BucketAccessRequest` (BAR) in the app's namespace (which is the same namespace as the `BucketRequest`) A 'BucketAccessRequest' optionally specifies a ServiceAccount, and a config map which contains access policy. Specifying a ServiceAccount supports provisioners that implement cloud provider identity integration with their respective Kubernetes offerings. - -```yaml -apiVersion: cosi.io/v1alpha1 -kind: BucketAccessRequest -metadata: - name: - namespace: - labels: - cosi.io/provisioner: [1] - finalizers: - - cosi.io/finalizer [2] -spec: - serviceAccountName: [3] - bucketRequestName: [4] - bucketAccessClassName: [5] - bucketAccessName: [6] -status: - accessGranted: [7] -``` - -1. `labels`: added by COSI. Key’s value should be the provisioner name. Characters that do not adhere to [Kubernetes label conventions](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) will be converted to ‘-’. -1. `finalizers`: added by COSI to defer `BucketAccessRequest` deletion until backend deletion ops succeed. -1. `serviceAccountName`: (optional) the name of a Kubernetes ServiceAccount, in the same namespace, which is expected to be linked to an identity in the cloud provider, such that a pod using the ServiceAccount is authenticated as the linked identity. In this case, COSI applies the policy defined in the `BAC.policyActionsConfigMap` to the identity. If `serviceAccountName` is omitted, a "minted" user is generated by the driver and stored as `BucketAccess.principal`. -1. `bucketRequestName`: (required) the name of the `BucketRequest` associated with this access request. From the bucket request, COSI knows the `Bucket` instance and thus the backend bucket and its properties. -1. `bucketAccessClassName`: (required) name of the `BucketAccessClass` specifying the desired set of policy actions to be set for a user identity or ServiceAccount. -1. `bucketAccessName`: name of the bound cluster-scoped `BucketAccess` instance. Set by COSI. -1. `accessGranted`: if true access has been granted to the bucket. If false then access to the bucket has not been granted. - -#### BucketAccess - -A cluster-scoped administrative custom resource which encapsulates fields from the `BucketAccessRequest` (BAR) and the `BucketAccessClass` (BAC). The purpose of the `BucketAccess` (BA) is to serve as an access communication path between provisioners and the central COSI controller. The COSI controller creates a `BucketAccess` instance for a new `BucketAccessRequest`, and there is a 1:1 mapping between `BucketAccess` and `BucketAccessRequest`. - -```yaml -apiVersion: cosi.io/v1alpha1 -kind: BucketAccess -metadata: - name: [1] - labels: - cosi.io/provisioner: [2] - finalizers: - - cosi.io/finalizer [3] - spec: - provisioner: [4] - bucketInstanceName: [5] - bucketAccessRequest: [6] - serviceAccount: [7] - mintedSecretName: [8] - policyActionsConfigMapData: [9] - principal: [10] - parameters: [11] - status: - accessGranted: [12] -``` - -1. `name`: when created by COSI, the `BucketAccess` name is generated in this format: _"ba-"+uuid_. If an admin creates a `BucketAccess`, as is necessary for driverless access, they can use any name. The uuid is unique within a cluster. -1. `labels`: added COSI. Key’s value should be the provisioner name. Characters that do not adhere to [Kubernetes label conventions](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) will be converted to ‘-’. -1. `finalizers`: added by COSI to defer `BucketAccess` deletion until the pod that uses this access terminates, and the related `BucketAccessRequest` is deleted. -1. `provisioner`: (optional) The provisioner field as defined in the `BucketAccessClass`. If empty then there is no driver/sidecar, thus the admin had to create the `Bucket` and `BucketAccess` instances. -1. `bucketInstanceName`: name of the `Bucket` instance bound to this BA. -1. `bucketAccessRequest`: an `objectReference` containing the name, namespace and UID of the associated `BucketAccessRequest`. -1. `serviceAccount`: an `ObjectReference` containing the name, namespace and UID of the associated `BAR.serviceAccountName`. If empty then integrated Kubernetes -> cloud identity is not being used, in which case, `BucketAccess.principal` contains the user identity, which is minted by the provisioner. -1. `mintedSecretName`: (omitted if serviceAccount is specified) name of the provisioner-generated Secret containing access credentials. This Secret exists in the provisioner’s namespace, is read by the cosi-node-adapter, and written to the secret mount defined in the app pod's csi-driver spec. -1. `policyActionsConfigMapData`: encoded data, known to the driver, and defined by the admin when creating a `BucketAccessClass`. Contains a set of provisioner/platform defined policy actions to a given user identity. Contents of the ConfigMap that the *policyActionsConfigMap* field in the `BucketAccessClass` refers to. A sample value of this field could look like: -``` - {“Effect”:“Allow”,“Action”:“s3:PutObject”,“Resource”:“arn:aws:s3:::profilepics/*“}, - {“Effect”:“Allow”,“Action”:“s3:GetObject”,“Resource”:“arn:aws:s3:::profilepics/*“}, - {“Effect”:“Deny”,“Action”:“s3:*“,”NotResource”:“arn:aws:s3:::profilepics/*“} -``` -1. `principal`: username/access-key for the object storage provider to uniquely identify the user who has access to this bucket. This value is minted by the provisioner (and set in the BA) when the `BucketAccessRequest` omits a ServiceAccount. There are several use cases involving `principal`: - - Not relevant if serviceAccount is supplied. - - For brownfield cases when there is already a storage-side principal that you want to use for access, the admin can supply this, the driver will (possibly) enable access for that principal to the bucket in question and will return credentials. - - For greenfield cases, this could be populated by the driver. Maybe so that the driver can request deletion of this principal during deprovisioning -1. `parameters`: A map of string:string key values. Allows admins to control user and access provisioning by setting provisioner key-values. Copied from `BucketAccessClass`. -1. `accessGranted`: if true access has been granted to the bucket. If false then access to the bucket has not been granted. - -#### BucketAccessClass - -An immutable, cluster-scoped, custom resource providing a way for admins to specify policies that may be used to access buckets. A `BucketAccessClass` (BAC) is required for both greenfield and brownfield use cases. Besides naming the provisioner, a BAC references a config map which is expected to define the access policy for a given provider. It is typical for these config maps to reside in each provisioner's namespace, though this is not required. Unlike BucketClasses, there is no default BAC. - -```yaml -apiVersion: cosi.io/v1alpha1 -kind: BucketAccessClass -metadata: - name: -provisioner: [1] -policyActionsConfigMap: [2] - name: - namespace: -parameters: [3] -``` - -1. `provisioner`: (optional) the name of the driver that `BucketAccess` instances should be managed by. If empty then there is no driver/sidecar, thus the admin had to create the `Bucket` and `BucketAccess` instances. -1. `policyActionsConfigMap`: (required) a reference to a ConfigMap that contains a set of provisioner/platform-defined policy actions for bucket access. It is seen as typical that this config map's namespace is the same as for the provisioner. In othe words, a namespace that has locked down RBAC rules or prevent modification of this config map. -1. `parameters`: (Optional) A map of string:string key values. Allows admins to control user and access provisioning by setting provisioner key-values. Optional reserved keys cosi.io/configMap and cosi.io/secrets are used to reference user created resources with provider specific access policies. ---- - -### App Pod -The application pod utilizes CSI's inline ephemeral volume support to provide the endpoint and secret credentials in an in-memory volume. This approach also, importantly, prevents the pod from launching before the bucket has been provisioned since the kubelet waits to start the pod until it has received the cosi-node-adapter's _NodeStageVolume_ response. - -Here is a sample pod manifest: - -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: my-app-pod - namespace: dev-ns -spec: - serviceAccountName: [1] - containers: - - name: my-app - image: ... - volumeMounts: - - mountPath: /cosi [2] - name: cosi-vol - volumes: - - name: cosi-vol - csi: [3] - driver: cosi.sigs.k8s.io [4] - volumeAttributes: [5] - bucketAccessRequestName: +#### Functionality + ++ Support automated **Bucket Creation** ++ Support automated **Bucket Deletion** ++ Support automated **Access Credential Generation** ++ Support automated **Access Credential Revokation** ++ Support automated **Bucket Provisioning** for workloads (enabling pods to access buckets) ++ Support **Bucket Reuse** (use existing buckets via COSI) ++ Support **Automated Bucket Sharing across namespaces** + +#### System Properties + ++ Support **workload portability** across clusters ++ Achieve the above goals in a **vendor neutral** manner ++ Standardize mechanism for third-party vendors to integrate easily with COSI ++ Allow users (non-admins) to create and utilize buckets (self-service) ++ Establish best-in-class **Access Control** practices for bucket creation and sharing + +### Non-Goals + ++ **Data Plane** API standardization will not be addressed by this project ++ **Bucket Mutation** will not be supported as of now + +## COSI architecture + +Since this is an entirely new feature, it is possible to implement this completely out of tree. The following components are proposed for this architecture: + + - COSI ControllerManager + - COSI Sidecar + - COSI Driver + +1. The COSI ControllerManager is the central controller that validates, authorizes and binds COSI created buckets to BucketClaims. Only one active instance of ControllerManager should be present. +2. The COSI Sidecar is the point of integration between COSI and drivers. All operations that require communication with the OSP is triggered by the Sidecar using gRPC calls to the driver. One active instance of Sidecar should be present **for each driver**. +3. The COSI driver communicates with the OSP to conduct Bucket related operations. + +More information about COSI driver is [here](#cosi-driver) + +## COSI API + +COSI defines 5 new API types + + - [Bucket](#bucket) + - [BucketClaim](#bucketclaim) + - [BucketAccess](#bucketaccess) + - [BucketClass](#bucketclass) + - [BucketAccessClass](#bucketaccessclass) + +Detailed information about these API types are provided inline with user stories. + +Here is a TL;DR version: + + - BucketClaims/Bucket are similar to PVC/PV. + - BucketClaim is used to request generation of new buckets. + - Buckets represent the actual Bucket. + - BucketClass is similar to StorageClass. It is meant for admins to define and control policies for Bucket Creation + - BucketAccess is required before a bucket can be "attached" to a pod. + - BucketAccess both represents the attachment status and holds a pointer to the access credentials secret. + - BucketAccessClass is meant for admins to control authz/authn for users requesting access to buckets. + +## Bucket Creation + +The following stakeholders are involved in the lifecycle of bucket creation: + + - Users - request buckets to be created for their workload + - Admins - setup COSI drivers in the cluster, and specify their configurations + +Here are the steps for creating a Bucket: + +###### 1. Admin creates BucketClass, User requests Bucket to be created using BucketClaim + +The BucketClass represents a set of common properties shared by multiple buckets. It is used to specify the driver for creating Buckets, and also for configuring driver-specific parameters. More information about BucketClass is [here](#bucketclass) + +The BucketClaim is a claim to create a new Bucket. This resource can be used to specify the bucketClassName, which in-turn includes configuration pertinent to bucket creation. More information about BucketClaim is available [here](#bucketclaim) + ``` -1. the service account may be needed depending on cloud IAM integration with kubernetes. -1. the mount path is the directory where the app will read the credentials and endpoint. -1. this is an inline CSI volume. -1. the name of the cosi-node-adapter. -1. information needed by the cosi-node-adapter to verify that the bucket has been provisioned. + BucketClaim - bcl1 BucketClass - bc1 + |------------------------------| |--------------------------------| + | metadata: | | deletionPolicy: delete | + | namespace: ns1 | | driverName: s3.amazonaws.com | + | spec: | | parameters: | + | bucketClassName: bc1 | | key: value | + | protocols: | |--------------------------------| + | - s3 | + |------------------------------| + +``` -The contents of the secret generated by the provisioner is expected to have the following fields in the provider specific format: +###### 2. COSI creates an intermediate Bucket object -- Endpoint for the bucket. The Endpoint should encode the URL for the provider -- Access Key: Only set when serviceAccount is empty -- Secret Key: Only set when serviceAccount is empty +The ControllerManager creates a Bucket object by copying fields over from both the BucketClaim and BucketClass. This step can only proceed if the BucketClass exists. This Bucket object is an intermediate object, with its status condition `BucketReady` to false. This indicates that the Bucket is yet to be created in the OSP. -### Topology +More information about Bucket is [here](#bucket) -![Architecture Diagram](images/arch.png) +``` + Bucket - bcl-$uuid + |--------------------------------------| + | name: bcl-$uuid | + | spec: | + | bucketClassName: bc1 | + | protocols: | + | - s3 | + | parameters: | + | key: value | + | driverName: s3.amazonaws.com | + | status: | + | conditions: | + | - type: BucketReady | + | status: False | + |--------------------------------------| +``` -# Object Relationships +###### 3. COSI calls appropriate driver to create Bucket -The diagram below describes the relationships between various resources in the COSI ecosystem. +The COSI sidecar, which runs alongside each of the drivers, listens for Bucket events. All but the sidecar with the specified driver will ignore the Bucket object. Only the appropriate sidecar will make a gRPC request to create a Bucket in the OSP. -![Object Relationships](images/object-rel.png) +More information about COSI gRPC API is [here](#cosi-grpc-api) -# Workflows -Here we describe the workflows used to automate provisioning of new and existing buckets, and the de-provisioning of these buckets. +``` + COSI Driver (s3.amazonaws.com) + |------------------------------------------| + | grpc DriverCreateBucket({ | + | "name": "bcl-$uuid", | + | "protocols": ["s3"], | + | "parameters": { | + | "key": "value" | + | } | + | }) | + |------------------------------------------| +``` +Once the Bucket is successfully created in the OSP, and a successful status is retrieved from the gRPC request, sidecar sets `BucketReady`condition to `True`. At this point, the Bucket is ready to be utilized by workloads. -When the app pod is started, the kubelet will gRPC call _NodeStageVolume_ and _NodePublishVolume_, which are received by the cosi-node-adapter. The pod waits until the adapter responds to the gRPC requests. The adapter ensures that the target bucket has been provisioned and is ready to be accessed. +## Generating Access Credentials for Buckets -When the app pod terminates, the kubelet gRPC calls _NodeUnstageVolume_ and _NodeUnpublishVolume_, which the cosi-node-adapter also receives. The adapter orchestrates tear-down of the ephemeral volume provisioned for this bucket. +The following stakeholders are involved in the lifecycle of access credential generation: -General prep: -+ admin creates the needed bucket classes and bucket access classes. + - Users - request access to buckets + - Admins - establish cluster wide access policies -Prep for brownfield: -+ admin creates `Bucket` instances representing backend buckets. +Access credentials are represented by BucketAccess objects. The separation of BucketClaim and BucketAccess is a reflection of the usage pattern of Object Storage, where buckets are always accessed over the network, and all access is subject to authentication and authorization i.e. lifecycle of a bucket and its access are not tightly coupled. -## Create Bucket +__Example: for the same bucket, one might need a BucketAccess with a "read-only" policy and another to with a "write" policy__ + -![CreateBucket Workflow](images/create-bucket.png) +Here are the steps for creating a BucketAccess: -This workflow describes the automation supporting creating a new (greenfield) backend bucket. Accessing this bucket is covered in [Sharing COSI Created Buckets](#sharing-cosi-created-buckets). COSI determines that a request is for an existing bucket when `BucketRequest.bucketInstanceName` is specified. +###### 1. Admin creates BucketAccessClass, User creates BucketAccess -Here is the workflow: +The BucketAccessClass represents a set of common properties shared by multiple BucketAccesses. It is used to specify policies for creating access credentials, and also for configuring driver-specific access parameters. More information about BucketAccessClass is [here](#bucketaccessclass) -+ COSI central controller sees a new `BucketRequest` (BR). -+ COSI gets the `BR.BucketClass` (directly or via the matching default) and verifies that the BR's namespace is allowed. -+ If BR.bucketInstanceName is empty, COSI creates the associated `Bucket`, filling in fields from the `BucketClass` and BR. Else, we have a brownfield BR. -+ The sidecar sees the new `Bucket` and gRPC calls the driver to create a backend bucket. -+ COSI updates the BR and `Bucket` instance to reference each other. +The BucketAccess is used to request access to a bucket. It contains fields for choosing the Bucket for which the credentials will be generated, and also includes a bucketAccessClassName field, which in-turn contains configuration for authorizing users to access buckets. More information about BucketAccess is [here](#bucketaccess) -In addition, the cosi-node-adapter sees a new app pod references a BAR: -+ cosi-node-adapter receives _NodeStageVolume_ request from kubelet. -+ adapter add a finalizer to both the BAR.BA and the BAR.BR. -+ the finalizer value is `pod.name` +BucketAccessClass can be used to specify a authorization mechanism. It can be one of + - KEY (__default__) + - IAM -## Sharing COSI Created Buckets -This is the greenfield -> brownfield access use case, where COSI has created the `Bucket` instance and the driver has provisioned a new bucket. Now, we want to share access to this bucket in other namespaces. +The KEY based mechanism is where access and secret keys are generated to be provided to pods. IAM style is where pods are implicitly granted access to buckets by means of a metadata service. IAM style access provides greater control for the infra/cluster administrator to rotate secret tokens, revoke access, change authorizations etc., which makes it more secure. -![ShareBucket Workflow](images/share-bucket.png) +``` + BucketAccess - ba1 BucketAccessClass - bac1 + |---------------------------------------| |----------------------------------| + | metadata: | | driverName: s3.amazonaws.com | + | namespace: ns1 | | parameters: | + | spec: | | key: value | + | bucketAccessClassName: bac1 | | authenticationType: KEY | + | bucketClaimName: bcl1 | |----------------------------------| + | credentialsSecretName: bucketcreds1 | + | protocol: s3 | + | status: | + | conditions: | + | - name: AccessGranted | + | value: False | + |---------------------------------------| +``` -If the bucket sharing is all within the same namespace then each `BucketAccessRequest` (in the namespace) only needs to reference the existing BR. +The `credentialsSecretName` is the name of the secret that COSI will generate containing credentials to access the bucket. The same secret name has to be set in the podSpec as well as the projected secret volumes. -Here is the workflow: +In case of IAM style authentication, along with the `credentialsSecretName`, `serviceAccountName` field must also be specified. This will map the specified serviceaccount to the appropriate service account in the OSP. -+ A `Bucket` instance created by COSI must be cloned by the admin (each `Bucket` instance needs a unique name). -+ The user creates a `BucketRequest` (BR) that specifies the `Bucket` instance name (BR.bucketInstanceName) provided by the admin. -+ The user creates a `BucketAccessRequest` (BAR) - see [grant bucket access](#grant-bucket-access) -- and the BAR references their BR. +``` + BucketAccess - ba1 BucketAccessClass - bac2 + |---------------------------------------| |----------------------------------| + | metadata: | | driverName: s3.amazonaws.com | + | namespace: ns1 | | parameters: | + | spec: | | key: value | + | bucketAccessClassName: bac2 | | authenticationType: IAM | + | bucketClaimName: bcl1 | |----------------------------------| + | credentialsSecretName: bucketcreds1 | + | serviceAccountName: svacc1 | + | protocol: s3 | + | status: | + | conditions: | + | - name: AccessGranted | + | value: False | + |---------------------------------------| +``` -## Delete Bucket +###### 3. COSI calls driver to generate credentials -![DeleteBucket Workflow](images/delete-bucket.png) +All but the sidecar for the specified driver will ignore the BucketClaim object. Only the appropriate sidecar will make a gRPC request to its driver to generate credentials/map service accounts. -This workflow describes the automation designed for deleting a `Bucket` instance and optionally the related backend bucket. Revoking access to this bucket is covered in [Revoke Bucket Access](#revoke-bucket-access). +This step will only proceed if the Bucket already exist. The Bucket's `BucketReady` condition should be true. Until access been granted, the `AccessGranted` condition in BucketAccess will be false. -It is important to understand that when a `BucketRequest` deletion triggers the delete of the associated `Bucket` instance, the `Bucket` instance is deleted regardless of access. This means that even if a `BucketAccess` references a `Bucket`, the `Bucket is still deleted and the BA is orphaned. Additionally, even if multiple `Bucket` instances point to the same backend bucket, if the retention policy is "Delete" then the bucket is deleted and workloads may fail. Potentially, this may only apply to MVP. +``` + COSI Driver (s3.amazonaws.com) + |------------------------------------------| + | grpc DriverGrantBucketAccess({ | + | "name": "ba-$uuid", | + | "bucketID": "br-$uuid", | + | "parameters": { | + | "key": "value" | + | } | + | }) | + |------------------------------------------| +``` -A `BucketAccess` instance is deleted once the app pod terminates. In this case, the associated BAR ismodified to reflect a delete condition, but this resource is not deleted. Deleting the BAR is deferred until the app terminates, in which case the BA is deleted, finalizers removed, and the BAR is garbage collected. The admin can delete the BA directly but this is an anti-pattern and may not be supported in MVP. If a pod is using the BA, direct BA deletion waits until pod is deleted. Once the pod using the BA terminates, the BA can be deleted by deleting the BAR. +Each BucketAccess is meant to map to a unique service account in the OSP. Once the requested privileges have been granted, a secret by the name specified in `credentialsSecretName` in the BucketClaim is created. The secret will reside in the namespace of the BucketClaim. The secret will contain either access keys or service account tokens based on the chosen authentication type. The format of this secret can be found [here](#bucketinfo) -It is up to each provisioner whether or not to physically delete bucket content. But, for idempotency, the expectation is that the backend bucket will at least be made unavailable, pending out-of-band bucket lifecycle policies. +If this call returns successfully, the sidecar sets `AccessGranted` condition to `True` in the BucketAccess. -The delete workflow is described as a synchronous, but it will likely be asynchronous to accommodate potentially long delete times when buckets contain many objects. The steps should still mostly follow what's outlined below. +NOTE: + - The secret will not be created until the credentials are generated/service account mappings are complete. + - Within a namespace, one BucketAccess and secret pair is generally sufficient, but cases which may want to control this more granularly can use multiple. + - The secret will be created with a finalizer that prevents it from being deleted until the associated bucketAccess is deleted. -Here is the workflow: +## Attaching Bucket Information to Pods -+ User deletes their `BucketRequest` (BR) which is deferred until finalizers have been removed. -+ COSI gets the `Bucket` instance from the BR and notes the retention policy. -+ If the `Bucket` instance was created by COSI then COSI deletes the `Bucket`, which sets its deleteTimestamp but does not delete it due to finalizer. -+ If the `Bucket` instance was not created by COSI then the `Bucket` is not deleted, but finalizers are removed so that the admin can manually delete it. -+ COSI unbinds the BR from the `Bucket`. -+ Sidecar sees the `Bucket` is unbound, and if the retention policy is "Delete", the gRPC calls the provisoner's _Delete_ interface. Upon successful completion, the `Bucket` is updated to Deleted. -+ If the retention policy is "Retain" then the `Bucket` remains "Released" and it can potentially be reused. -+ When COSI sees the `Bucket` is "Deleted", it removes all finalizers and the resources are garbage collected. +The following stakeholders are involved in the lifecycle of attaching bucket information into pods: -## Grant Bucket Access -This workflow describes the automation supporting granting access to an existing backend bucket. + - Users - specify bucket in the pod definition -Here is the workflow: +Once a valid BucketAccess is available (`AccessGranted` is `True`), pods can use them. -+ The admin creates a cluster-scoped `Bucket` representing the backend bucket. -+ The user creates a `BucketRequest` referencing this `Bucket` instance and COSI follows the "Create Bucket" workflow. -+ The user creates a `BucketAccessRequest` (BAR) which references their BR. -+ COSI central controller sees a new `BucketAccessRequest` (BAR) and gets the `BAR.BucketAccessClass` (BAC). -+ COSI creates the cluster-scoped `BucketAccess` (BA) filling in the field from the BAR and BAC. -+ The sidecar sees the new BA and gRPC calls the driver to grant access to the backend bucket. -+ COSI updates the BAR and BA to reference each other. -+ The cosi-node-adapter, which has been waiting for the BAR to be processed, writes access tokens to the app pod's specified mount point. +###### 1. User creates pod with a projected volume pointing to the secret in BucketAccess -## Revoke Bucket Access -This workflow describes the automation supporting revoking access to an existing backend bucket, and the deletion of the cluster-scoped `BucketAccess` instance. +The secret mentioned in the `credentialsSecretName` field of the BucketAccess should be set as a projected volume in the PodSpec. If either the Bucket provisioning is incomplete, or the access generation is incomplete - the pod will not run. It will wait indefinitely for those two conditions to be true. -Here is the workflow: +``` + PodSpec - pod1 + |-------------------------------------------------| + | spec: | + | containers: | + | - volumeMounts: | + | - name: cosi-bucket1 | + | mountPath: /cosi/bucket1 | + | - name: cosi-bucket2 | + | mountPath: /cosi/bucket2 | + | volumes: | + | - name: cosi-bucket1 | + | projected: | + | sources: | + | - secret: | + | name: bucketcreds1 | + | - name: cosi-bucket2 | + | projected: | + | sources: | + | - secret: | + | name: bucketcreds2 | + |-------------------------------------------------| +``` -+ COSI central controller sees the `BucketAccessRequest` (BAR) has been deleted. -+ COSI gets the `BAR.BucketAccess` (BA). -+ COSI deletes the BA, but finalizers prevent it from being garbage collected. -+ The sidecar sees the BA has been deleted and gRPC calls the driver to revoke access to the backend bucket. -+ COSI removes the BAR and BA finalizers and these instances are garbage collected. +If IAM style authentication was specified, then the `serviceAccountName` specified in BucketAccess must be specified as the `serviceAccountName` in the podSpec. -In addition, the cosi-node-adapter sees the app pod is terminating: -+ cosi-node-adapter receives _NodeUnstageVolume_ request from kubelet. -+ adapter removes its finalizer from both the BAR.BA and the BAR.BR. +``` + PodSpec - pod1 + |-------------------------------------------------| + | spec: | + | serviceAccountName: svacc1 | + | containers: | + | - volumeMounts: | + | name: cosi-bucket | + | mountPath: /cosi/bucket1 | + | volumes: | + | - name: cosi-bucket | + | projected: | + | sources: | + | - secret: | + | name: bucketcreds1 | + |-------------------------------------------------| +``` -## Delete BucketAccess -The above workflows are triggered by the user. Now we cover worflows triggerd by the admin. -The most common scenario is likely the case where tokens are compromised and the admin needs to stop their use. In this case the admin may terminate the app pod(s) and delete the `BucketAccess` instances. +The volume `mountPath` will be the directory where bucket credentials and other related information will be served. -Here is the workflow: +NOTE: the contents of the files served in mountPath will be a COSI generated file containing credentials and other information required for accessing the bucket. **This is NOT intended to specify a mountpoint to expose the bucket as a filesystem.** -+ Admin deletes a `BucketAccess` (BA) which is deferred until the finalizer has been removed. -+ The sidecar detects the delete and calls the driver to revoke access to the backend bucket. -+ Admin deletes the app pod. -+ The cosi-node-adapter is notified (_NodeUnpublishVolume_) and removes the pod.name finalizer from the BA, and the BA is garbage collected. -+ The user BAR remains pointing to a BA that no longer exists. It is felt that COSI should not delete user-created instances. +NOTE: the secret containing bucketInfo can be provided to the pod using any other {secret -> pod} provisioning mechanism, including environment variables. In case of environment variables, the secrets will be exposed to other processes in the same host, as environment variables are not inherently secure. -## Setting Access Permissions -### Dynamic Provisioning -Incoming `BucketAccessRequest`s contains a *serviceAccountName* where a cloud provider supports identity integration. The incoming `BucketAccessRequest` represents a user to access the `Bucket` and a corresponding `BucketAccess` will provide the access credentials to the workloads using *serviceAccount* or *mintedSecretName* . -When requesting access to a bucket, workloads will go through the scenarios described here: -+ New User: In this scenario, we do not have user account in the backend storage system as well as no access for this user to the `Bucket`. - + Create user account in the backend storage system. - + add the access privileges for the user to the `Bucket`. - + return the credentials to the workload owning the `BucketAccessRequest`. -+ Existing User and New Bucket: In this scenario, we have the user account created in the backend storage system, but the account is not associated to the `Bucket`. - + add the access privileges to the `Bucket`. - + return the credentials to the workload owning the `BucketAccessRequest`. -+ Existing User and existing Bucket: In this scenario, the user account has access policy defined on the `Bucket`. The existing user privileges in the backend may conflict with the privileges that the user is requesting. - + FAIL, if existing access policy is different from the requested policy. - + if the existing privileges match the requested privileges, return the credentials to the workload owning the `BucketAccessRequest`. -+ A Service Account specified and the cloud platform identity integration maps Kubernetes ServiceAccounts to the account in the backend storage system. No need to create credentials here. - -Upon success, the `BucketAccess` instance is ready and the app workload can access backend storage. +###### 2. The secret containing BucketInfo is mounted in the specified directory -### Static Provisioning -"Driverless" allows the existing workloads to make use of COSI without the need for vendors to create drivers. The following steps show the details of the workflow: -+ Admin creates `Bucket` instance that references an existing backend bucket. -+ User creates a `BucketRequest` (BR) that references this `Bucket`. -+ User creates `BucketAccessRequest` (BAR) that references their BR, a `BucketAccessClass`, and their secret containing access tokens. -+ COSI detects the existence of the BAR and follows the [grant access](#grant-bucket-access) steps. -+ COSI detects the existence of the BAR, BA and executes a validation process. -+ As a validation step, COSI ensures that the `BucketRequest.bucketInstanceName` matches the `BucketAccessRequest.bucketAccessName.bucketInstanceName`. In other words, the two related `Bucket` references match, meaning that the BR and BAR->BA point to the same `Bucket`. +The above volume definition will prompt kubernetes to retrieve the secret and place it in the volumeMount path defined above. The contents of the secret will be of the format shown below: ---- +``` + bucket_info.json + |-----------------------------------------------| + | { | + | apiVersion: "v1alpha1", | + | kind: "BucketInfo", | + | metadata: { | + | name: "bc-$uuid" | + | }, | + | spec: { | + | bucketName: "bc-$uuid", | + | authenticationType: "KEY", | + | endpoint: "https://s3.amazonaws.com", | + | accessKeyID: "AKIAIOSFODNN7EXAMPLE", | + | accessSecretKey: "wJalrXUtnFEMI/K...", | + | region: "us-west-1", | + | protocols: [ | + | "s3" | + | ] | + | } | + | } | + |-----------------------------------------------| -# gRPC Definitions +``` -There is one service defined by COSI - `Provisioner`. This will need to be satisfied by the vendor-provisioner in order to be COSI-compatible +In case IAM style authentication was specified, then workloadIdentityToken will be provided. ``` -service Provisioner { - rpc ProvisionerGetInfo (ProvisionerGetInfoRequest) returns (ProvisionerGetInfoResponse) {} + bucket_info.json + |-------------------------------------------------| + | { | + | apiVersion: "v1alpha1", | + | kind: "BucketInfo", | + | metadata: { | + | name: "bc-$uuid" | + | }, | + | spec: { | + | bucketName: "bc-$uuid", | + | authenticationType: "IAM", | + | workloadIdentityToken: "ZXlKaGJHY2...", | + | endpoint: "https://s3.amazonaws.com", | + | region: "us-west-1", | + | protocols: [ | + | "s3" | + | ] | + | } | + | } | + |-------------------------------------------------| - rpc ProvisionerCreateBucket (ProvisionerCreateBucketRequest) returns (ProvisionerCreateBucketResponse) {} - rpc ProvisionerDeleteBucket (ProvisionerDeleteBucketRequest) returns (ProvisionerDeleteBucketResponse) {} +``` + +Workloads are expected to read the definitions in this file to access a bucket. The `BucketInfo` API will not be a CRD in the cluster, however, it follows the same conventions as the rest of the COSI APIs. More details can be found [here](#bucketinfo) + +## Sharing Buckets + +This section describes the current design for sharing buckets with other namespaces. As of the current milestone (alpha) of COSI, any bucket created in one namespace cannot be accessed in another namespace. i.e. no bucket sharing is possible. In future versions, a namespace level access control will be enforced, and buckets will be constrained to particular namespaces using selectors. Admins will be able to control which namespaces can access which buckets using namespace selectors. + +## Accessing existing Buckets + +The benefits of COSI can also be brought to existing buckets/ones created outside of COSI. This user story explains the steps to import a bucket: + +###### 1. Admin creates a Bucket API object + +When a Bucket object is manually created, and has its `bucketID` set, then COSI assumes that this Bucket has already been created. + +The admin must ensure that this bucket binds only to a specific BucketClaim by specifying the BucketClaim. - rpc ProvisionerGrantBucketAccess (ProvisionerGrantBucketAccessRequest) returns (ProvisionerGrantBucketAccessResponse); - rpc ProvisionerRevokeBucketAccess (ProvisionerRevokeBucketAccessRequest) returns (ProvisionerRevokeBucketAccessResponse); -} ``` + Bucket - br-$uuid + |-------------------------------------------------| + | name: bucketName123 | + | spec: | + | bucketID: bucketname123 | + | bucketClaim: | + | name: bucketClaim123 | + | namespace: ns1 | + | protocols: | + | - s3 | + | parameters: | + | key: value | + | driverName: s3.amazonaws.com | + |-------------------------------------------------| +``` + -## ProvisionerGetInfo +###### 2. User creates BucketClaim referring to the bucket -This call is meant to retrieve the unique provisioner Identity. This identity will have to be set in `BucketRequest.Provisioner` field in order to invoke this specific provisioner. +Only the specified BucketClaim will be able to bind to the Bucket. ``` -message ProvisionerGetInfoRequest { - // Intentionally left blank -} + BucketClaim - bucketClaim123 + |------------------------------------------| + | name: bucketClaim123 | + | spec: | + | existingBucketName: bucketName123 | + | | + |------------------------------------------| -message ProvisionerGetInfoResponse { - // The name MUST follow domain name notation format - // (https://tools.ietf.org/html/rfc1035#section-2.3.1). It SHOULD - // include the plugin's host company name and the plugin name, - // to minimize the possibility of collisions. It MUST be 63 - // characters or less, beginning and ending with an alphanumeric - // character ([a-z0-9A-Z]) with dashes (-), dots (.), and - // alphanumerics between. This field is REQUIRED. - string name = 1; -} ``` -## ProvisonerCreateBucket +###### 3. User creates BucketAccess to generate credentials for that bucket -This call is made to create the bucket in the backend. If a bucket that matches both name and parameters already exists, then OK (success) must be returned. If a bucket by same name, but different parameters is provided, then the appropriate error code `ALREADY_EXISTS` must be returned by the provisioner. The call to *ProvisonerCreateBucket* MUST be idempotent. +Similar to the BucketAccess for COSI created bucket, this BucketAccess should reference the BucketClaim that refers to this bucket. ``` -message ProvisionerCreateBucketRequest { - // This field is REQUIRED - string bucket_name = 1; + BucketAccess - bac2 + |-------------------------------| + | spec: | + | bucketClaim: bucketClaim123 | + | | + |-------------------------------| +``` - map bucket_context = 2; +## Bucket deletion - enum AnonymousBucketAccessMode { - PRIVATE = 0; - PUBLIC_READ_ONLY = 1; - PUBLIC_WRITE_ONLY = 2; - PUBLIC_READ_WRITE = 3; - } - - AnonymousBucketAccessMode anonymous_bucket_access_mode = 3; -} + - A Bucket created by COSI as a result of a BucketClaim can deleted by deleting the BucketClaim + - A Bucket created outside of COSI, once bound, can be deleted by deleting the BucketClaim to which it is bound + - A Bucket created outside of COSI, unless it is bound to a particular BucketClaim, cannot be deleted by users from any particular namespace. Privileged users can however delete the Bucket object at their discretion. + +Once a delete has been issued to a bucket, no new BucketAccesses can be created for it. Buckets having valid BucketAccesses (Buckets in use) will not be deleted until all the BucketAccesses are cleaned up. + +Buckets can be created with one of two deletion policies: + - Retain + - Delete + +When the deletion policy is Retain, then the underlying bucket is not cleaned up when the Bucket object is deleted. When the deletion policy is Delete, then the underlying bucket is cleaned up when the Bucket object is deleted. + +Only when all accessors (BucketAccesses) of the Bucket are deleted, is the Bucket itself cleaned up. There is a finalizer on the Bucket that prevents it from being deleted until all the accessors are done using it. + +When a user deletes a BucketAccess, the corresponding secret/serviceaccount are also deleted. If a pod has that secret mounted when delete is called, then a finalizer on the secret will prevent it from being deleted. Instead, the deletionTimestamp will be set on the secret. In this way, access to a Bucket is preserved until the application pod dies. + +When an admin deletes any of the class objects, it does not affect existing Buckets as fields from the class objects are copied into the Buckets during creation. + +If a Bucket is manually deleted by an admin, then a finalizer on the Bucket prevents it from being deleted until the binding BucketClaim is deleted. + +# Usability + +## Self Service + +Self service is easily possible with the current design as both the BucketRequest and BucketClaim resources are namespace scoped, and users need not have admin privileges to create, modify and delete them. + +The only admin steps are creation of class objects(BucketClass, BucketAccessClass) and Bucket imports. The creation of a class object is no different from requiring a StorageClass for provisioning PVCs. It is a well-understood pattern among Kubernetes users. Importing a Bucket requires special permissions because its lifecycle is not managed by COSI, and special care needs to be taken to prevent clones, accidental deletions and other mishaps (for instance, setting the deletion policy to Delete). + +## Mutating Buckets + +As of the current design of COSI, mutating bucket properties is not supported. However, the current design does not prevent us from supporting it in the future. Mutable properties include encryption, object lifecycle configuration, replication configuration etc. These properties will be supported in future versions along with the capability to mutate them. + +These properties will be specified in the BucketRequest and follow the same pattern of events as Bucket creation. i.e. Bucket API object will be updated to reflect the properties set in BucketRequest, and then a controller will pick-up these changes and call the appropriate APIs to reflect them in the backend. + +# Object Lifecycle + +The following resources are managed by admins + +- Bucket in case of brownfield buckets +- BucketClass +- BucketAccessClass + +The following resources are managed by a user. These are created with a reference to their corresponding class objects: + +- BucketClaim -> BucketClass +- BucketAccess -> Bucket, BucketClaim, BucketAccessClass + +The COSI controller responds by creating the following objects + +- BucketRequest <-> Bucket (1:1) +- BucketClaim -> Bucket -message ProvisionerCreateBucketResponse { - // Intentionally left blank +Notes: + + - There are **NO** cycles in the relationship graph of the above mentioned API objects. + - Mutations are not supported in the API. + +# COSI API Reference + +## Bucket + +Resource to represent a Bucket in OSP. Buckets are cluster-scoped. + +```go +Bucket { + TypeMeta + ObjectMeta + + Spec BucketSpec { + // DriverName is the name of driver associated with this bucket + DriverName string + + // DeletionPolicy is used to specify how COSI should handle deletion of this + // bucket. There are 3 possible values: + // - Retain: Indicates that the bucket should not be deleted from the OSP (default) + // - Delete: Indicates that the bucket should be deleted from the OSP + // once all the workloads accessing this bucket are done + // +optional + DeletionPolicy DeletionPolicy + + // Name of the BucketClass specified in the BucketRequest + BucketClassName string + + // Name of the BucketClaim that resulted in the creation of this Bucket + // In case the Bucket object was created manually, then this should refer + // to the BucketClaim with which this Bucket should be bound + BucketClaim corev1.ObjectReference + + // Protocols are the set of data APIs this bucket is expected to support. + // The possible values for protocol are: + // - S3: Indicates Amazon S3 protocol + // - Azure: Indicates Microsoft Azure BlobStore protocol + // - GCS: Indicates Google Cloud Storage protocol + Protocols []Protocol + + // Parameters is an opaque map for passing in configuration to a driver + // for creating the bucket + // +optional + Parameters map[string]string + + // ExistingBucketID is the unique id of the bucket in the OSP. This field should be + // used to specify a bucket that has been created outside of COSI. + // This field will be empty when the Bucket is dynamically provisioned by COSI. + // +optional + ExistingBucketID string + } + + Status BucketStatus { + // BucketReady is a boolean condition to reflect the successful creation + // of a bucket. + BucketReady bool + + // BucketID is the unique id of the bucket in the OSP. This field will be + // populated by COSI. + // +optional + BucketID string + } } ``` -## ProvisonerDeleteBucket +## BucketClaim -This call is made to delete the bucket in the backend. If the bucket has already been deleted, then no error should be returned. The call to *ProvisonerDeleteBucket* MUST be idempotent. +A claim to create Bucket. BucketClaim is namespace-scoped -``` -message ProvisionerDeleteBucketRequest { - // This field is REQUIRED - string bucket_name = 1; +```go +BucketClaim { + TypeMeta + ObjectMeta + + Spec BucketClaimSpec { + // Name of the BucketClass + BucketClassName string - map bucket_context = 2; -} + // Protocols are the set of data API this bucket is required to support. + // The possible values for protocol are: + // - S3: Indicates Amazon S3 protocol + // - Azure: Indicates Microsoft Azure BlobStore protocol + // - GCS: Indicates Google Cloud Storage protocol + Protocols []Protocol + + // Name of a bucket object that was manually + // created to import a bucket created outside of COSI + // If unspecified, then a new Bucket will be dynamically provisioned + // +optional + ExistingBucketName string + } + + Status BucketClaimStatus { + // BucketReady indicates that the bucket is ready for consumpotion + // by workloads + BucketReady bool + + // BucketName is the name of the provisioned Bucket in response + // to this BucketClaim. It is generated and set by the COSI controller + // before making the creation request to the OSP backend. + // +optional + BucketName string + } +``` +## BucketClass + +Resouce for configuring common properties for multiple Buckets. BucketClass is cluster-scoped. + +```go +BucketClass { + TypeMeta + ObjectMeta + + // DriverName is the name of driver associated with this bucket + DriverName string -message ProvisionerDeleteBucketResponse { - // Intentionally left blank + // DeletionPolicy is used to specify how COSI should handle deletion of this + // bucket. There are 3 possible values: + // - Retain: Indicates that the bucket should not be deleted from the OSP + // - Delete: Indicates that the bucket should be deleted from the OSP + // once all the workloads accessing this bucket are done + DeletionPolicy DeletionPolicy + + // Parameters is an opaque map for passing in configuration to a driver + // for creating the bucket + // +optional + Parameters map[string]string } ``` -## ProvisionerGrantBucketAccess +## BucketAccess -This call grants access to a particular principal. The principal is the account for which this access should be granted. +A resource to access a Bucket. BucketAccess is namespace-scoped -If the principal is set, then it should be used as the username of the created credentials or in some way should be deterministically used to generate a new credential for this request. This principal will be used as the unique identifier for deleting this access by calling ProvisionerRevokeBucketAccess +```go +BucketAccess { + TypeMeta + ObjectMeta -If the `principal` is empty, then a new service account should be created in the backend that satisfies the requested `access_policy`. The username/principal for this service account should be set in the `principal` field of `ProvisionerGrantBucketAccessResponse`. + Spec BucketAccessSpec { + // BucketClaimName is the name of the BucketClaim. + BucketClaimName string -``` -message ProvisionerGrantBucketAccessRequest { - // This field is REQUIRED - string bucket_name = 1; - - map bucket_context = 2; + // Protocol is the name of the Protocol + // that this access credential is supposed to support + // If left empty, it will choose the protocol supported + // by the bucket. If the bucket supports multiple protocols, + // the end protocol is determined by the driver. + // +optional + Protocol Protocol - string principal = 3; + // BucketAccessClassName is the name of the BucketAccessClass + BucketAccessClassName string + + // CredentialsSecretName is the name of the secret that COSI should populate + // with the credentials. If a secret by this name already exists, then it is + // assumed that credentials have already been generated. It is not overridden. + // This secret is deleted when the BucketAccess is delted. + CredentialsSecretName string + + // ServiceAccountName is the name of the serviceAccount that COSI will map + // to the OSP service account when IAM styled authentication is specified + // +optional + ServiceAccountName string + } + + Status BucketAccessStatus { + // AccessGranted indicates the successful grant of privileges to access the bucket + AccessGranted bool - // This field is REQUIRED - string access_policy = 4; + // AccountID is the unique ID for the account in the OSP. It will be populated + // by the COSI sidecar once access has been successfully granted. + // +optional + AccountID string + } +``` + +## BucketAccessClass + +Resource for configuring common properties for multiple BucketClaims. BucketAccessClass is a clustered resource + +```go +BucketAccessClass { + TypeMeta + ObjectMeta + + // DriverName is the name of driver associated with + // this BucketAccess + DriverName string + + // AuthenticationType denotes the style of authentication + // It can be one of + // KEY - access, secret tokens based authentication + // IAM - implicit authentication of pods to the OSP based on service account mappings + AuthenticationType AuthenticationType + + // Parameters is an opaque map for passing in configuration to a driver + // for granting access to a bucket + // +optional + Parameters map[string]string } +``` -message ProvisionerGrantBucketAccessResponse { - // This is the account that is being provided access. This will - // be required later to revoke access. - string principal = 1; +## BucketInfo + +Resource mounted into pods containing information for applications to gain access to buckets. + +```go +BucketInfo { + TypeMeta + ObjectMeta + + Spec BucketInfoSpec { + // BucketName is the name of the Bucket + BucketName string + + // AuthenticationType denotes the style of authentication + // It can be one of + // KEY - access, secret tokens based authentication + // IAM - implicit authentication of pods to the OSP based on service account mappings + AuthenticationType AuthenticationType + + // Endpoint is the URL at which the bucket can be accessed + Endpoint string - string credentials_file_contents = 2; + // Region is the vendor-defined region where the bucket "resides" + Region string - string credentials_file_path = 3; -} + // Protocols are the set of data APIs this bucket is expected to support. + // The possible values for protocol are: + // - S3: Indicates Amazon S3 protocol + // - Azure: Indicates Microsoft Azure BlobStore protocol + // - GCS: Indicates Google Cloud Storage protocol + Protocols []Protocol + } +} ``` -## ProvisionerRevokeBucketAccess +# COSI Driver + +A component that runs alongside COSI Sidecar and satisfies the COSI gRPC API specification. Sidecar and driver work together to orchestrate changes in the OSP. The driver acts as a gRPC server to the COSI Sidecar. Each COSI driver is identified by a unique id. -This call revokes all access to a particular bucket from a principal. +The sidecar uses the unique id to direct requests to the appropriate driver. Multiple instances of drivers with the same id will be added into a group and only one of them will act as the leader at any given time. + +## COSI Driver gRPC API + +#### DriverGetInfo + +This gRPC call responds with the name of the driver. The name is used to identify the appropriate driver for a given BucketRequest or BucketClaim. ``` -message ProvisionerRevokeBucketAccessRequest { - // This field is REQUIRED - string bucket_name = 1; - - map bucket_context = 2; + DriverGetInfo + |------------------------------------------| |---------------------------------------| + | grpc DriverGetInfoRequest{} | ===> | DriverGetInfoResponse{ | + |------------------------------------------| | "name": "s3.amazonaws.com" | + | } | + |---------------------------------------| +``` - // This field is REQUIRED - string principal = 3; -} +#### DriverCreateBucket + +This gRPC call creates a bucket in the OSP, and returns information about the new bucket. This api must be idempotent. The input to this call is the name of the bucket and an opaque parameters field. + +The returned `bucketID` should be a unique identifier for the bucket in the OSP. This value could be the name of the bucket too. This value will be used by COSI to make all subsequent calls related to this bucket. + +``` + DriverCreateBucket + |------------------------------------------| |-----------------------------------------------| + | grpc DriverCreateBucketRequest{ | ===> | DriverCreateBucketResponse{ | + | "name": "br-$uuid", | | "bucketID": "br-$uuid", | + | "parameters": { | | "bucketInfo": { | + | "key": "value" | | "s3": { | + | } | | "bucketName": "br-$uuid", | + | } | | "region": "us-west1", | + | -----------------------------------------| | "endpoint": "s3.amazonaws.com" | + | } | + | } | + | } | + |-----------------------------------------------| +``` + +#### DriverGrantBucketAccess + +This gRPC call creates a set of access credentials for a bucket. This api must be idempotent. The input to this call is the id of the bucket, a set of opaque parameters and name of the account. This `accountName` field is the concatenation of the characters ba (short for BucketAccess) and its UID. It is used as the idempotency key for requests to the drivers regarding a particular BA. + +The returned `accountID` should be a unique identifier for the account in the OSP. This value could be the name of the account too. This value will be included in all subsequent calls to the driver for changes to the BucketAccess. + +``` + DriverGrantBucketAccess + |---------------------------------------------| |-----------------------------------------------| + | grpc DriverGrantBucketAccessRequest{ | ===> | DriverGrantBucketAccessResponse{ | + | "bucketID": "br-$uuid", | | "accountID": "bar-$uuid", | + | "accountName": "bar-$uuid" | | "credentials": { | + | "authenticationType": "KEY" | | "s3": { | + | "parameters": { | | "accessKeyID": "AKIAODNN7EXAMPLE", | + | "key": "value", | | "accessSecretKey": "wJaUtnFEMI/K..." | + | } | | } | + | } | | } | + |---------------------------------------------| | } | + |-----------------------------------------------| +``` + +#### DriverDeleteBucket + +This gRPC call deletes a bucket in the OSP. + +``` + DriverDeleteBucket + |---------------------------------------------| |-----------------------------------------------| + | grpc DriverDeleteBucketRequest{ | ===> | DriverDeleteBucketResponse{} | + | "bucketID": "br-$uuid" | |-----------------------------------------------| + | } | + |---------------------------------------------| +``` + +#### DriverRevokeBucketAccess + +This gRPC call revokes access granted to a particular account. + +``` + DriverDeleteBucket + |---------------------------------------------| |-----------------------------------------------| + | grpc DriverRevokeBucketAccessRequest{ | ===> | DriverRevokeBucketAccessResponse{} | + | "bucketID": "br-$uuid", | |-----------------------------------------------| + | "accountID": "bar-$uuid" | + | } | + |---------------------------------------------| -message ProvisionerRevokeBucketAccessResponse { - // Intentionally left blank -} ``` # Test Plan @@ -714,12 +912,24 @@ message ProvisionerRevokeBucketAccessResponse { # Graduation Criteria ## Alpha - - API is reviewed and accepted -- Implement all COSI components to support Greenfield, Green/Brown Field, Brownfield and Static Driverless provisioning -- Evaluate gaps, update KEP and conduct reviews for all design changes +- Design COSI APIs to support Greenfield, Green/Brown Field, Brownfield and Static Driverless provisioning +- Design COSI APIs to support authentication using access/secret keys, and IAM. +- Evaluate gaps, update KEP and conduct reviews for all design changes - Develop unit test cases to demonstrate that the above mentioned use cases work correctly +## Alpha -\> Beta +- Consider using a typed configuration for Bucket properties (parameter fields in Bucket, BucketClass, BucketAccess, BucketAccessClass) +- Implement all COSI components to support agreed design. +- Design and implement support for sharing buckets across namespaces. +- Design and implement quotas/restrictions for Buckets and BucketAccess. +- Basic unit and e2e tests as outlined in the test plan. +- Metrics for bucket create and delete, and granting and revoking bucket access. +- Metrics in provisioner for bucket create and delete, and granting and revoking bucket access. + +## Beta -\> GA +- Stress tests to iron out possible race conditions in the controllers. +- Users deployed in production and have gone through at least one K8s upgrade. # Alternatives Considered This KEP has had a long journey and many revisions. Here we capture the main alternatives and the reasons why we decided on a different solution. @@ -732,8 +942,398 @@ This KEP has had a long journey and many revisions. Here we capture the main alt 1. If the `Bucket` instance name is in the BAC instead of the BAR then the user is not burdened with knowledge of `Bucket` names, and there is some centralized admin control over brownfield bucket access. ### Problems -1. The greenfield -> brownfield workflow is very awkward with this approach. The user creates a `BucketRequest` (BR) to provision a new bucket which they then want to access. The user creates a BAR pointing to a BAC which must contain the name of this newly created ``Bucket` instance. Since the `Bucket`'s name is non-deterministic the admin cannot create the BAC in advance. Instead, the user must ask the admin to find the new `Bucket` instance and add its name to new (or maybe existing) BAC. +1. The greenfield -\> brownfield workflow is very awkward with this approach. The user creates a `BucketRequest` (BR) to provision a new bucket which they then want to access. The user creates a BAR pointing to a BAC which must contain the name of this newly created \``Bucket` instance. Since the `Bucket`'s name is non-deterministic the admin cannot create the BAC in advance. Instead, the user must ask the admin to find the new `Bucket` instance and add its name to new (or maybe existing) BAC. 1. App portability is still a concern but we believe that deterministic, unique `Bucket` and `BucketAccess` names can be generated and referenced in BRs and BARs. -1. Since, presumably, all or most BACs will be known to users, there is no real "control" offered to the admin with this approach. Instead, adding _allowedNamespaces_ or similar to the BAC may help with this. + +### Upgrade / Downgrade Strategy + +No changes are required on upgrade to maintain previous behaviour. + +### Version Skew Strategy + +COSI is out-of-tree, so version skew strategy is N/A + +## Production Readiness Review Questionnaire + + + +### Feature Enablement and Rollback + + + +###### How can this feature be enabled / disabled in a live cluster? + + + +- [ ] Feature gate (also fill in values in `kep.yaml`) + - Feature gate name: + - Components depending on the feature gate: +- [X] Other + - Describe the mechanism: Create Deployment and DaemonSet resources (along with supporting secrets, configmaps etc.) for the three controllers that COSI requires + - Will enabling / disabling the feature require downtime of the control + plane? No + - Will enabling / disabling the feature require downtime or reprovisioning + of a node? (Do not assume `Dynamic Kubelet Config` feature is enabled). No + +###### Does enabling the feature change any default behavior? + + +No + +###### Can the feature be disabled once it has been enabled (i.e. can we roll back the enablement)? + + + +Yes. Delete the resources created when installing COSI + +###### What happens if we reenable the feature if it was previously rolled back? + +###### Are there any tests for feature enablement/disablement? + + + +N/A since we are only targeting alpha for this Kubernetes release + +### Rollout, Upgrade and Rollback Planning + + + +###### How can a rollout or rollback fail? Can it impact already running workloads? + + + +###### What specific metrics should inform a rollback? + + + +###### Were upgrade and rollback tested? Was the upgrade-\>downgrade-\>upgrade path tested? + + + +###### Is the rollout accompanied by any deprecations and/or removals of features, APIs, fields of API types, flags, etc.? + + + +No + +### Monitoring Requirements + + + +###### How can an operator determine if the feature is in use by workloads? + + + +The operator can query Bucket* objects to find if their workloads are associated with them. + +###### How can someone using this feature know that it is working for their instance? + + + +- [ ] Events + - Event Reason: `Bucket provisioning 'bucket-name' failed` +- [ ] API .status + - Condition `PodReady=False "Error: secrets 'bucket-secret' not found"`) + - Other field: +- [ ] Other (treat as last resort) + - Details: + +###### What are the reasonable SLOs (Service Level Objectives) for the enhancement? + + + +###### What are the SLIs (Service Level Indicators) an operator can use to determine the health of the service? + + + +- [ ] Metrics + - Metric name: + - [Optional] Aggregation method: + - Components exposing the metric: +- [ ] Other (treat as last resort) + - Details: + +###### Are there any missing metrics that would be useful to have to improve observability of this feature? + + + +### Dependencies + + + +###### Does this feature depend on any specific services running in the cluster? + + + +No + +### Scalability + + + +###### Will enabling / using this feature result in any new API calls? + + + +Existing components will not make any new API calls. + +The API load of COSI components will be a factor of the number of buckets being managed and the number of bucket-accessors for these buckets. Essentially O(num-buckets * num-bucket-access). There is no per-node or per-namespace load by COSI. + +###### Will enabling / using this feature result in introducing new API types? + + + +Yes, the following cluster scoped resources + +- Bucket +- BucketClass +- BucketAccessClass + +and the following namespaced scoped resources + +- BucketAccess + +###### Will enabling / using this feature result in any new calls to the cloud provider? + + + +Not by the framework itself. Calls to external systems will be made by vendor drivers for COSI. + +###### Will enabling / using this feature result in increasing size or count of the existing API objects? + + + +No + +###### Will enabling / using this feature result in increasing time taken by any operations covered by existing SLIs/SLOs? + + + +Yes. Containers requesting Buckets will not start until Buckets have been provisioned. This is similar to dynamic volume provisioning + +###### Will enabling / using this feature result in non-negligible increase of resource usage (CPU, RAM, disk, IO, ...) in any components? + + + +Not likely to increase resource consumption in a significant manner + +## Infrastructure Needed (Optional) + + + +We need Linux VMs for e2e testing in CI. + +[1]: #release-signoff-checklist +[2]: #summary +[3]: #motivation +[4]: #user-stories +[5]: #goals +[6]: #non-goals +[7]: #vocabulary +[8]: #proposal +[9]: #apis +[10]: #storage-apis +[11]: #bucketrequest +[12]: #bucket +[13]: #bucketclass +[14]: #access-apis +[15]: #bucketaccessrequest +[16]: #bucketaccess +[17]: #bucketaccessclass +[18]: #app-pod +[19]: #topology +[20]: #object-relationships +[21]: #workflows +[22]: #finalizers +[23]: #create-bucket +[24]: #sharing-cosi-created-buckets +[25]: #delete-bucket +[26]: #grant-bucket-access +[27]: #revoke-bucket-access +[28]: #delete-bucketaccess +[29]: #delete-bucket-1 +[30]: #setting-access-permissions +[31]: #dynamic-provisioning +[32]: #static-provisioning +[33]: #grpc-definitions +[34]: #drivergetinfo +[35]: #drivercreatebucket +[36]: #driverdeletebucket +[37]: #drivergrantbucketaccess +[38]: #driverrevokebucketaccess +[39]: #test-plan +[40]: #graduation-criteria +[41]: #alpha +[42]: #alpha---beta +[43]: #beta---ga +[44]: #alternatives-considered +[45]: #add-bucket-instance-name-to-bucketaccessclass-brownfield +[46]: #motivation-1 +[47]: #problems +[48]: #upgrade--downgrade-strategy +[49]: #version-skew-strategy +[50]: #production-readiness-review-questionnaire +[51]: #feature-enablement-and-rollback +[52]: #rollout-upgrade-and-rollback-planning +[53]: #monitoring-requirements +[54]: #dependencies +[55]: #scalability +[56]: #infrastructure-needed-optional +[57]: https://git.k8s.io/enhancements +[58]: https://git.k8s.io/website +[59]: https://kubernetes.io/ diff --git a/keps/sig-storage/1979-object-storage-support/images/arch.png b/keps/sig-storage/1979-object-storage-support/images/arch.png deleted file mode 100644 index 14901914073..00000000000 Binary files a/keps/sig-storage/1979-object-storage-support/images/arch.png and /dev/null differ diff --git a/keps/sig-storage/1979-object-storage-support/images/create-bucket.png b/keps/sig-storage/1979-object-storage-support/images/create-bucket.png deleted file mode 100644 index 5269a7476e3..00000000000 Binary files a/keps/sig-storage/1979-object-storage-support/images/create-bucket.png and /dev/null differ diff --git a/keps/sig-storage/1979-object-storage-support/images/delete-bucket.png b/keps/sig-storage/1979-object-storage-support/images/delete-bucket.png deleted file mode 100644 index 557bfc57159..00000000000 Binary files a/keps/sig-storage/1979-object-storage-support/images/delete-bucket.png and /dev/null differ diff --git a/keps/sig-storage/1979-object-storage-support/images/object-rel.png b/keps/sig-storage/1979-object-storage-support/images/object-rel.png deleted file mode 100644 index 5658895e3f3..00000000000 Binary files a/keps/sig-storage/1979-object-storage-support/images/object-rel.png and /dev/null differ diff --git a/keps/sig-storage/1979-object-storage-support/images/share-bucket.png b/keps/sig-storage/1979-object-storage-support/images/share-bucket.png deleted file mode 100644 index 74456f83656..00000000000 Binary files a/keps/sig-storage/1979-object-storage-support/images/share-bucket.png and /dev/null differ diff --git a/keps/sig-storage/1979-object-storage-support/kep.yaml b/keps/sig-storage/1979-object-storage-support/kep.yaml index 887dca16cdd..de0bd3c01d7 100644 --- a/keps/sig-storage/1979-object-storage-support/kep.yaml +++ b/keps/sig-storage/1979-object-storage-support/kep.yaml @@ -2,9 +2,7 @@ title: Object Storage Support kep-number: 1979 authors: - "@jeffvance" - - "@copejon" - "@wlan0" - - "@brahmaroutu" owning-sig: "sig-storage" participating-sigs: - sig-storage @@ -13,12 +11,17 @@ reviewers: - "@alarge" - "@erinboyd" - "@guymguym" - - "@rsquared" - - "@krishchow" + - "@thockin" approvers: - "@saad-ali" - "@xing-yang" + - "@msau42" + - "@thockin" editor: TBD creation-date: 2019-11-25 -last-updated: 2020-09-10 -status: provisional +last-updated: 2022-04-06 +status: alpha +stage: alpha +latest-milestone: v1.24 +milestone: + alpha: "v1.24"