From 649b9b569c54e72d152568c161227fa626829d4c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 16 Feb 2023 15:50:14 +0100 Subject: [PATCH 1/9] added initial opm bundle for openshift --- deploy/olm/23.1.0/.gitignore | 4 + deploy/olm/23.1.0/README.md | 1 + deploy/olm/23.1.0/manifests/authclass.yaml | 185 ++++++++++++++++++ deploy/olm/23.1.0/manifests/csv.yaml | 169 ++++++++++++++++ deploy/olm/23.1.0/manifests/s3bucket.yaml | 136 +++++++++++++ deploy/olm/23.1.0/manifests/s3connection.yaml | 117 +++++++++++ deploy/olm/23.1.0/olm_generate.sh | 47 +++++ 7 files changed, 659 insertions(+) create mode 100644 deploy/olm/23.1.0/.gitignore create mode 100644 deploy/olm/23.1.0/README.md create mode 100644 deploy/olm/23.1.0/manifests/authclass.yaml create mode 100644 deploy/olm/23.1.0/manifests/csv.yaml create mode 100644 deploy/olm/23.1.0/manifests/s3bucket.yaml create mode 100644 deploy/olm/23.1.0/manifests/s3connection.yaml create mode 100755 deploy/olm/23.1.0/olm_generate.sh diff --git a/deploy/olm/23.1.0/.gitignore b/deploy/olm/23.1.0/.gitignore new file mode 100644 index 0000000..741604f --- /dev/null +++ b/deploy/olm/23.1.0/.gitignore @@ -0,0 +1,4 @@ +bundle +bundle.Dockerfile +catalog.Dockerfile +catalog diff --git a/deploy/olm/23.1.0/README.md b/deploy/olm/23.1.0/README.md new file mode 100644 index 0000000..f87f5c1 --- /dev/null +++ b/deploy/olm/23.1.0/README.md @@ -0,0 +1 @@ +# TODO \ No newline at end of file diff --git a/deploy/olm/23.1.0/manifests/authclass.yaml b/deploy/olm/23.1.0/manifests/authclass.yaml new file mode 100644 index 0000000..fdd4d9c --- /dev/null +++ b/deploy/olm/23.1.0/manifests/authclass.yaml @@ -0,0 +1,185 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: authenticationclasses.authentication.stackable.tech + annotations: + helm.sh/resource-policy: keep +spec: + group: authentication.stackable.tech + names: + categories: [] + kind: AuthenticationClass + plural: authenticationclasses + shortNames: [] + singular: authenticationclass + scope: Cluster + versions: + - additionalPrinterColumns: [] + name: v1alpha1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for AuthenticationClassSpec via `CustomResource` + properties: + spec: + properties: + provider: + description: Provider used for authentication like LDAP or Kerberos + oneOf: + - required: + - ldap + - required: + - tls + - required: + - static + properties: + ldap: + properties: + bindCredentials: + description: In case you need a special account for searching the LDAP server you can specify it here + nullable: true + properties: + scope: + description: '[Scope](https://docs.stackable.tech/secret-operator/scope.html) of the [SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html)' + nullable: true + properties: + node: + default: false + type: boolean + pod: + default: false + type: boolean + services: + default: [] + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html) containing the LDAP bind credentials' + type: string + required: + - secretClass + type: object + hostname: + description: Hostname of the LDAP server + type: string + ldapFieldNames: + default: + uid: uid + group: memberof + givenName: givenName + surname: sn + email: mail + description: The name of the LDAP object fields + properties: + email: + default: mail + description: The name of the email field + type: string + givenName: + default: givenName + description: The name of the firstname field + type: string + group: + default: memberof + description: The name of the group field + type: string + surname: + default: sn + description: The name of the lastname field + type: string + uid: + default: uid + description: The name of the username field + type: string + type: object + port: + description: Port of the LDAP server. If TLS is used defaults to 636 otherwise to 389 + format: uint16 + minimum: 0.0 + nullable: true + type: integer + searchBase: + default: '' + description: LDAP search base + type: string + searchFilter: + default: '' + description: LDAP query to filter users + type: string + tls: + description: Use a TLS connection. If not specified no TLS will be used + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates + type: object + server: + description: Use TLS and ca certificate to verify the server + properties: + caCert: + description: Ca cert to verify the server + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: Name of the SecretClass which will provide the ca cert. Note that a SecretClass does not need to have a key but can also work with just a ca cert. So if you got provided with a ca cert but don't have access to the key you can still use this method. + type: string + webPki: + description: Use TLS and the ca certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - hostname + type: object + static: + properties: + userCredentialsSecret: + description: Secret providing the usernames and password. The secret must contain an entry for every user, with the key being the username and the value the password in plain text. It must be located in the same namespace as the product using it. + properties: + name: + description: Name of the secret + type: string + required: + - name + type: object + required: + - userCredentialsSecret + type: object + tls: + properties: + clientCertSecretClass: + description: See ``. If `client_cert_secret_class` is not set, the TLS settings may also be used for client authentication. If `client_cert_secret_class` is set, the [SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html) will be used to provision client certificates. + nullable: true + type: string + type: object + type: object + required: + - provider + type: object + required: + - spec + title: AuthenticationClass + type: object + served: true + storage: true + subresources: {} diff --git a/deploy/olm/23.1.0/manifests/csv.yaml b/deploy/olm/23.1.0/manifests/csv.yaml new file mode 100644 index 0000000..c12a4fd --- /dev/null +++ b/deploy/olm/23.1.0/manifests/csv.yaml @@ -0,0 +1,169 @@ +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: commons-operator.v23.1.0 +spec: + annotations: + support: stackable.tech + olm.properties: '[]' + + # The following affect how the package is indexed at OperatorHub.io: + # https://operatorhub.io/?category=Database + # https://sdk.operatorframework.io/docs/advanced-topics/operator-capabilities/operator-capabilities/ + categories: Storage + capabilities: Full Lifecycle + description: Stackable Commons Operator + + + description: Stackable Commons Operator + displayName: Stackable Commons Operator + keywords: + - commons + maintainers: + - email: info@stackable.tech + name: Stackable GmbH + maturity: stable + provider: + name: Stackable GmbH + url: https://stackable.tech + version: 23.1.0 + minKubeVersion: 1.20.0 + + installModes: + - supported: true + type: OwnNamespace + - supported: true + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: false + type: AllNamespaces + + customresourcedefinitions: + owned: + # a list of CRDs that this operator owns + # name is the metadata.name of the CRD (which is of the form .) + - name: s3buckets.s3.stackable.tech + # version is the spec.versions[].name value defined in the CRD + version: v1alpha1 + # kind is the CamelCased singular value defined in spec.names.kind of the CRD. + kind: S3Bucket + # human-friendly display name of the CRD for rendering in graphical consoles (optional) + displayName: S3Bucket + # a short description of the CRDs purpose for rendering in graphical consoles (optional) + description: Represents an S3 bucket + - name: s3connections.s3.stackable.tech + # version is the spec.versions[].name value defined in the CRD + version: v1alpha1 + # kind is the CamelCased singular value defined in spec.names.kind of the CRD. + kind: S3Connection + # human-friendly display name of the CRD for rendering in graphical consoles (optional) + displayName: S3Connection + # a short description of the CRDs purpose for rendering in graphical consoles (optional) + description: Represents an S3Connection + - name: authenticationclasses.authentication.stackable.tech + # version is the spec.versions[].name value defined in the CRD + version: v1alpha1 + # kind is the CamelCased singular value defined in spec.names.kind of the CRD. + kind: AuthenticationClass + # human-friendly display name of the CRD for rendering in graphical consoles (optional) + displayName: AuthenticationClass + # a short description of the CRDs purpose for rendering in graphical consoles (optional) + description: Represents an AuthenticationClass + + + install: + # strategy indicates what type of deployment artifacts are used + strategy: deployment + # spec for the deployment strategy is a list of deployment specs and required permissions - similar to a pod template used in a deployment + spec: + permissions: + - serviceAccountName: commons-operator + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - '*' + # permissions required at the cluster scope + clusterPermissions: + - serviceAccountName: commons-operator + rules: + - apiGroups: + - "" + resources: + - pods + - configmaps + - secrets + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch + - patch + - apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - apiGroups: + - s3.stackable.tech + resources: + - s3buckets + - s3connections + verbs: + - get + - list + - patch + - watch + - apiGroups: + - authentication.stackable.tech + resources: + - authenticationclasses + verbs: + - get + - list + - patch + - watch + - apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create + + deployments: + - name: commons-operator + spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: commons-operator + app.kubernetes.io/instance: commons-operator + template: + metadata: + labels: + app.kubernetes.io/name: commons-operator + app.kubernetes.io/instance: commons-operator + spec: + serviceAccountName: commons-operator + securityContext: {} + containers: + - name: commons-operator + securityContext: {} + image: "docker.stackable.tech/stackable/commons-operator:23.1.0" + imagePullPolicy: IfNotPresent + resources: {} diff --git a/deploy/olm/23.1.0/manifests/s3bucket.yaml b/deploy/olm/23.1.0/manifests/s3bucket.yaml new file mode 100644 index 0000000..5806f6f --- /dev/null +++ b/deploy/olm/23.1.0/manifests/s3bucket.yaml @@ -0,0 +1,136 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: s3buckets.s3.stackable.tech + annotations: + helm.sh/resource-policy: keep +spec: + group: s3.stackable.tech + names: + categories: [] + kind: S3Bucket + plural: s3buckets + shortNames: [] + singular: s3bucket + scope: Namespaced + versions: + - additionalPrinterColumns: [] + name: v1alpha1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for S3BucketSpec via `CustomResource` + properties: + spec: + description: S3 bucket specification containing only the bucket name and an inlined or referenced connection specification. + properties: + bucketName: + nullable: true + type: string + connection: + description: Operators are expected to define fields for this type in order to work with S3 connections. + nullable: true + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: S3 connection definition as CRD. + properties: + accessStyle: + description: Which access style to use. Defaults to virtual hosted-style as most of the data products out there. Have a look at the official documentation on + enum: + - Path + - VirtualHosted + nullable: true + type: string + credentials: + description: If the S3 uses authentication you have to specify you S3 credentials. In the most cases a SecretClass providing `accessKey` and `secretKey` is sufficient. + nullable: true + properties: + scope: + description: '[Scope](https://docs.stackable.tech/secret-operator/scope.html) of the [SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html)' + nullable: true + properties: + node: + default: false + type: boolean + pod: + default: false + type: boolean + services: + default: [] + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html) containing the LDAP bind credentials' + type: string + required: + - secretClass + type: object + host: + description: Hostname of the S3 server without any protocol or port + nullable: true + type: string + port: + description: Port the S3 server listens on. If not specified the products will determine the port to use. + format: uint16 + minimum: 0.0 + nullable: true + type: integer + tls: + description: If you want to use TLS when talking to S3 you can enable TLS encrypted communication with this setting. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates + type: object + server: + description: Use TLS and ca certificate to verify the server + properties: + caCert: + description: Ca cert to verify the server + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: Name of the SecretClass which will provide the ca cert. Note that a SecretClass does not need to have a key but can also work with just a ca cert. So if you got provided with a ca cert but don't have access to the key you can still use this method. + type: string + webPki: + description: Use TLS and the ca certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + type: object + reference: + type: string + type: object + type: object + required: + - spec + title: S3Bucket + type: object + served: true + storage: true + subresources: {} diff --git a/deploy/olm/23.1.0/manifests/s3connection.yaml b/deploy/olm/23.1.0/manifests/s3connection.yaml new file mode 100644 index 0000000..d93144c --- /dev/null +++ b/deploy/olm/23.1.0/manifests/s3connection.yaml @@ -0,0 +1,117 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: s3connections.s3.stackable.tech + annotations: + helm.sh/resource-policy: keep +spec: + group: s3.stackable.tech + names: + categories: [] + kind: S3Connection + plural: s3connections + shortNames: [] + singular: s3connection + scope: Namespaced + versions: + - additionalPrinterColumns: [] + name: v1alpha1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for S3ConnectionSpec via `CustomResource` + properties: + spec: + description: S3 connection definition as CRD. + properties: + accessStyle: + description: Which access style to use. Defaults to virtual hosted-style as most of the data products out there. Have a look at the official documentation on + enum: + - Path + - VirtualHosted + nullable: true + type: string + credentials: + description: If the S3 uses authentication you have to specify you S3 credentials. In the most cases a SecretClass providing `accessKey` and `secretKey` is sufficient. + nullable: true + properties: + scope: + description: '[Scope](https://docs.stackable.tech/secret-operator/scope.html) of the [SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html)' + nullable: true + properties: + node: + default: false + type: boolean + pod: + default: false + type: boolean + services: + default: [] + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html) containing the LDAP bind credentials' + type: string + required: + - secretClass + type: object + host: + description: Hostname of the S3 server without any protocol or port + nullable: true + type: string + port: + description: Port the S3 server listens on. If not specified the products will determine the port to use. + format: uint16 + minimum: 0.0 + nullable: true + type: integer + tls: + description: If you want to use TLS when talking to S3 you can enable TLS encrypted communication with this setting. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates + type: object + server: + description: Use TLS and ca certificate to verify the server + properties: + caCert: + description: Ca cert to verify the server + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: Name of the SecretClass which will provide the ca cert. Note that a SecretClass does not need to have a key but can also work with just a ca cert. So if you got provided with a ca cert but don't have access to the key you can still use this method. + type: string + webPki: + description: Use TLS and the ca certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + type: object + required: + - spec + title: S3Connection + type: object + served: true + storage: true + subresources: {} diff --git a/deploy/olm/23.1.0/olm_generate.sh b/deploy/olm/23.1.0/olm_generate.sh new file mode 100755 index 0000000..93b04f9 --- /dev/null +++ b/deploy/olm/23.1.0/olm_generate.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# what should be the entry point for bundling/deploying - this script? +# - split into a) dockers and yamls b) installation +# - "make opm" (to update csv.yaml etc. and tee everything up) +# How often do we need to create a bundle - every time the manifests change? i.e. every release +# - yes, every new version +# generate-manifests.sh has been removed from templating, can maybe be re-purposed for openshift stuff.. +# manifests: +# - roles.yaml --> same as the ClusterRole definition in helm/zookeeper-operator/templates/roles.yaml +# - zookeeper*.crd.yaml --> same as helm/zookeeper-operator/crds/crds.yaml (less one annotation for helm.sh/resource-policy: keep) +# - csv.yaml --> specific to openshift +# - configmap.yaml --> embeds helm/zookeeper-operator/configs/properties.yaml +# is the templates folder used at all? is it in danger of being removed? can we use for openshift? +# how to parameterize the namespace? +# the operator installs even when the source namespace doesn't match in subscription.yaml +# the catalog-source deploys the operator w/o the subscription? +# Tasks: +# - regenerate charts and then split crds.yaml instead of individual files +# - 23.1.0 --> 23.4.0-rc0 +# - manually prepare a bundle for common and add it to the catalog and see what happens with the subscription +# - does deleting the operator also clean up the crds? (no) +# - split script into: +# prepare operator bundle (operator code: make regenerate-opm) +# one-off for catalog (stackable-utils) +# packaging for all operators (stackable-utils, iterating over all operators) + +set -euo pipefail +set -x + +main() { + VERSION="$1"; + + if [ -d "bundle" ]; then + rm -rf bundle + fi + + opm alpha bundle generate --directory manifests \ + --package commons-operator-package --output-dir bundle \ + --channels stable --default stable + + docker build -t "docker.stackable.tech/sandbox/test/commons-operator-bundle:${VERSION}" -f bundle.Dockerfile . + docker push "docker.stackable.tech/sandbox/test/commons-operator-bundle:${VERSION}" + + opm alpha bundle validate --tag "docker.stackable.tech/sandbox/test/commons-operator-bundle:${VERSION}" --image-builder docker +} + +main "$@" From e5156c3fedaa80db38615444c8ee43d701ff88d6 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 16 Feb 2023 18:11:46 +0100 Subject: [PATCH 2/9] add make regenerate bundle --- Makefile | 16 +++- deploy/olm/23.1.0/README.md | 124 +++++++++++++++++++++++++++++- deploy/olm/23.1.0/olm_generate.sh | 47 ----------- 3 files changed, 138 insertions(+), 49 deletions(-) delete mode 100755 deploy/olm/23.1.0/olm_generate.sh diff --git a/Makefile b/Makefile index c9343c3..fb7d69c 100644 --- a/Makefile +++ b/Makefile @@ -75,4 +75,18 @@ build: regenerate-charts helm-package docker-build publish: build docker-publish helm-publish run-dev: - nix run -f. tilt -- up --port 5431 \ No newline at end of file + nix run -f. tilt -- up --port 5431 + +bundle-clean: + rm -rf "deploy/olm/23.1.0/bundle" + rm -rf "deploy/olm/23.1.0/bundle.Dockerfile" + +build-bundle: + pushd deploy/olm/23.1.0/ && \ + opm alpha bundle generate --directory manifests --package ${OPERATOR_NAME}-package --output-dir bundle --channels stable --default stable && \ + docker build -t "docker.stackable.tech/sandbox/test/${OPERATOR_NAME}-bundle:23.1.0" -f bundle.Dockerfile . && \ + docker push "docker.stackable.tech/sandbox/test/${OPERATOR_NAME}-bundle:23.1.0" && \ + opm alpha bundle validate --tag "docker.stackable.tech/sandbox/test/${OPERATOR_NAME}-bundle:23.1.0" --image-builder docker && \ + popd + +regenerate-bundle: bundle-clean build-bundle \ No newline at end of file diff --git a/deploy/olm/23.1.0/README.md b/deploy/olm/23.1.0/README.md index f87f5c1..eaa2dd2 100644 --- a/deploy/olm/23.1.0/README.md +++ b/deploy/olm/23.1.0/README.md @@ -1 +1,123 @@ -# TODO \ No newline at end of file +# OLM installation files + +The following steps describe how to install a Stackable operator - in this, the operator for Apache Zookeeper - using the [Operator Lifecycle Manager](https://olm.operatorframework.io/) (OLM). + +It specifically installs the version 23.1.0 of the operator. Installing additional versions in the future requires generating new bundle images and updating the catalog as described below. + +## Usage + +Prerequisite is of course a running OpenShift cluster. + +First, install the operator using OLM: + + kubectl apply -f catalog-source.yaml \ + -f operator-group.yaml \ + -f subscription.yaml + +Then, install the operator dependencies with Helm: + + helm install secret-operator stackable/secret-operator + helm install commons-operator stackable/commons-operator + +And finally, create an Apache Zookeeper cluster: + + kubectl create -f examples/simple-zookeeper-cluster.yaml + +NOTE: The `kuttl` tests don't work because they themselves require SCCs which are not available. + +## OLM packaging requirements + +- An [OpenShift](https://developers.redhat.com/products/openshift-local/overview) cluster. +- [opm](https://github.com/operator-framework/operator-registry/) +- docker and kubectl +- `kubeadmin` access + +It was tested with: + + $ crc version + WARN A new version (2.5.1) has been published on https://developers.redhat.com/content-gateway/file/pub/openshift-v4/clients/crc/2.5.1/crc-linux-amd64.tar.xz + CRC version: 2.4.1+b877358 + OpenShift version: 4.10.14 + Podman version: 4.0.2 + + $ oc version + Client Version: 4.10.14 + Server Version: 4.10.14 + Kubernetes Version: v1.23.5+b463d71 + + $ opm version + Version: version.Version{OpmVersion:"v1.23.2", GitCommit:"82505333", BuildDate:"2022-07-04T13:45:39Z", GoOs:"linux", GoArch:"amd64"} + +## Open questions + +- OLM [doesn't support DaemonSet(s)](https://github.com/operator-framework/operator-lifecycle-manager/issues/1022) and we need them for the secret-operator. Currently we can deploy the secret-operator using Helm but this means we cannot configure the [required](https://olm.operatorframework.io/docs/tasks/creating-operator-manifests/#required-apis) apis of the Zookeeper bundle. What are the consequences for publishing and certification ? +- Here we create a catalog for a single operator. We probably want a catalog for all Stackable operators in the future but this will get large very quickly. Figure out how to handle this. Especially figure out what happens with new versions of the same operator. +- OLM cannot create SecurityContextConstraints objects. The Zookeeper cluster (not the operator) cannot run with the default `restricted` SCC. The current solution is to use the `hostmount-anyuid` SCC for the `zookeeper-clusterrole`. Will this pass the certification process ? +- Everything (catalog, subscription, etc) is installed in the `stackable-operators` namespace. Is this a good idea ? +- The Subscription object uses `installPlanApproval: Automatic` which means the operator is updated automatically for every new version. Is this a good idea? + +See the [OLM documentation](https://olm.operatorframework.io/docs/tasks/) for details. + +## Build and publish operator bundle image + +Each catalog can contain several operator packages, and each operator package can contain multiple channels, each with its own bundles of different versions of the operator. + +### Generate operator bundle (this is operator-specific) + + opm alpha bundle generate --directory manifests --package zookeeper-operator-package --output-dir bundle --channels stable --default stable + +### Build bundle image + + docker build -t docker.stackable.tech/stackable/zookeeper-operator-bundle:23.1.0 -f bundle.Dockerfile . + docker push docker.stackable.tech/stackable/zookeeper-operator-bundle:23.1.0 + +### Validate bundle image + + opm alpha bundle validate --tag docker.stackable.tech/stackable/zookeeper-operator-bundle:23.1.0 --image-builder docker + +## Create catalog + + mkdir catalog + opm generate dockerfile catalog + +## Create a package for each operator + + opm init zookeeper-operator-package \ + --default-channel=stable \ + --description=./README.md \ + --output yaml > catalog/zookeeper-operator-package.yaml + + { + echo "---" + echo "schema: olm.channel" + echo "package: zookeeper-operator-package" + echo "name: stable" + echo "entries:" + echo "- name: zookeeper-operator.v23.1.0" + } >> catalog/zookeeper-operator-package.yaml + +NOTE: with the command below we can add the Stackable logo as icon. + + # add for each operator... + opm render docker.stackable.tech/stackable/zookeeper-operator-bundle:23.1.0 --output=yaml >> catalog/zookeeper-operator-package.yaml + + # ...and then validate the entire catalog + opm validate catalog + +The catalog is correct if the command above returns successfully without any message. If the catalog doesn't validate, the operator will not install. Now build a catalog image and push it to the repository: + + docker build . -f catalog.Dockerfile -t docker.stackable.tech/stackable/zookeeper-operator-catalog:latest + docker push docker.stackable.tech/stackable/zookeeper-operator-catalog:latest + +## Install catalog and the operator group + + kubectl apply -f catalog-source.yaml + kubectl apply -f operator-group.yaml + +## List available operators + + kubectl get packagemanifest -n stackable-operators + +## Install operator + + kubectl apply -f subscription.yaml diff --git a/deploy/olm/23.1.0/olm_generate.sh b/deploy/olm/23.1.0/olm_generate.sh deleted file mode 100755 index 93b04f9..0000000 --- a/deploy/olm/23.1.0/olm_generate.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# what should be the entry point for bundling/deploying - this script? -# - split into a) dockers and yamls b) installation -# - "make opm" (to update csv.yaml etc. and tee everything up) -# How often do we need to create a bundle - every time the manifests change? i.e. every release -# - yes, every new version -# generate-manifests.sh has been removed from templating, can maybe be re-purposed for openshift stuff.. -# manifests: -# - roles.yaml --> same as the ClusterRole definition in helm/zookeeper-operator/templates/roles.yaml -# - zookeeper*.crd.yaml --> same as helm/zookeeper-operator/crds/crds.yaml (less one annotation for helm.sh/resource-policy: keep) -# - csv.yaml --> specific to openshift -# - configmap.yaml --> embeds helm/zookeeper-operator/configs/properties.yaml -# is the templates folder used at all? is it in danger of being removed? can we use for openshift? -# how to parameterize the namespace? -# the operator installs even when the source namespace doesn't match in subscription.yaml -# the catalog-source deploys the operator w/o the subscription? -# Tasks: -# - regenerate charts and then split crds.yaml instead of individual files -# - 23.1.0 --> 23.4.0-rc0 -# - manually prepare a bundle for common and add it to the catalog and see what happens with the subscription -# - does deleting the operator also clean up the crds? (no) -# - split script into: -# prepare operator bundle (operator code: make regenerate-opm) -# one-off for catalog (stackable-utils) -# packaging for all operators (stackable-utils, iterating over all operators) - -set -euo pipefail -set -x - -main() { - VERSION="$1"; - - if [ -d "bundle" ]; then - rm -rf bundle - fi - - opm alpha bundle generate --directory manifests \ - --package commons-operator-package --output-dir bundle \ - --channels stable --default stable - - docker build -t "docker.stackable.tech/sandbox/test/commons-operator-bundle:${VERSION}" -f bundle.Dockerfile . - docker push "docker.stackable.tech/sandbox/test/commons-operator-bundle:${VERSION}" - - opm alpha bundle validate --tag "docker.stackable.tech/sandbox/test/commons-operator-bundle:${VERSION}" --image-builder docker -} - -main "$@" From c63500815cdd5c0cda462902284cb4347a329a28 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 16 Feb 2023 18:13:45 +0100 Subject: [PATCH 3/9] changed to stackable repo --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index fb7d69c..426a274 100644 --- a/Makefile +++ b/Makefile @@ -84,9 +84,9 @@ bundle-clean: build-bundle: pushd deploy/olm/23.1.0/ && \ opm alpha bundle generate --directory manifests --package ${OPERATOR_NAME}-package --output-dir bundle --channels stable --default stable && \ - docker build -t "docker.stackable.tech/sandbox/test/${OPERATOR_NAME}-bundle:23.1.0" -f bundle.Dockerfile . && \ - docker push "docker.stackable.tech/sandbox/test/${OPERATOR_NAME}-bundle:23.1.0" && \ - opm alpha bundle validate --tag "docker.stackable.tech/sandbox/test/${OPERATOR_NAME}-bundle:23.1.0" --image-builder docker && \ + docker build -t "docker.stackable.tech/stackable/${OPERATOR_NAME}-bundle:23.1.0" -f bundle.Dockerfile . && \ + docker push "docker.stackable.tech/stackable/${OPERATOR_NAME}-bundle:23.1.0" && \ + opm alpha bundle validate --tag "docker.stackable.tech/stackable/${OPERATOR_NAME}-bundle:23.1.0" --image-builder docker && \ popd regenerate-bundle: bundle-clean build-bundle \ No newline at end of file From 8be3373151e061b706aac09f6e537d3abdcfab9b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 17 Feb 2023 09:56:49 +0100 Subject: [PATCH 4/9] removed readme --- deploy/olm/23.1.0/README.md | 123 ------------------------------------ 1 file changed, 123 deletions(-) delete mode 100644 deploy/olm/23.1.0/README.md diff --git a/deploy/olm/23.1.0/README.md b/deploy/olm/23.1.0/README.md deleted file mode 100644 index eaa2dd2..0000000 --- a/deploy/olm/23.1.0/README.md +++ /dev/null @@ -1,123 +0,0 @@ -# OLM installation files - -The following steps describe how to install a Stackable operator - in this, the operator for Apache Zookeeper - using the [Operator Lifecycle Manager](https://olm.operatorframework.io/) (OLM). - -It specifically installs the version 23.1.0 of the operator. Installing additional versions in the future requires generating new bundle images and updating the catalog as described below. - -## Usage - -Prerequisite is of course a running OpenShift cluster. - -First, install the operator using OLM: - - kubectl apply -f catalog-source.yaml \ - -f operator-group.yaml \ - -f subscription.yaml - -Then, install the operator dependencies with Helm: - - helm install secret-operator stackable/secret-operator - helm install commons-operator stackable/commons-operator - -And finally, create an Apache Zookeeper cluster: - - kubectl create -f examples/simple-zookeeper-cluster.yaml - -NOTE: The `kuttl` tests don't work because they themselves require SCCs which are not available. - -## OLM packaging requirements - -- An [OpenShift](https://developers.redhat.com/products/openshift-local/overview) cluster. -- [opm](https://github.com/operator-framework/operator-registry/) -- docker and kubectl -- `kubeadmin` access - -It was tested with: - - $ crc version - WARN A new version (2.5.1) has been published on https://developers.redhat.com/content-gateway/file/pub/openshift-v4/clients/crc/2.5.1/crc-linux-amd64.tar.xz - CRC version: 2.4.1+b877358 - OpenShift version: 4.10.14 - Podman version: 4.0.2 - - $ oc version - Client Version: 4.10.14 - Server Version: 4.10.14 - Kubernetes Version: v1.23.5+b463d71 - - $ opm version - Version: version.Version{OpmVersion:"v1.23.2", GitCommit:"82505333", BuildDate:"2022-07-04T13:45:39Z", GoOs:"linux", GoArch:"amd64"} - -## Open questions - -- OLM [doesn't support DaemonSet(s)](https://github.com/operator-framework/operator-lifecycle-manager/issues/1022) and we need them for the secret-operator. Currently we can deploy the secret-operator using Helm but this means we cannot configure the [required](https://olm.operatorframework.io/docs/tasks/creating-operator-manifests/#required-apis) apis of the Zookeeper bundle. What are the consequences for publishing and certification ? -- Here we create a catalog for a single operator. We probably want a catalog for all Stackable operators in the future but this will get large very quickly. Figure out how to handle this. Especially figure out what happens with new versions of the same operator. -- OLM cannot create SecurityContextConstraints objects. The Zookeeper cluster (not the operator) cannot run with the default `restricted` SCC. The current solution is to use the `hostmount-anyuid` SCC for the `zookeeper-clusterrole`. Will this pass the certification process ? -- Everything (catalog, subscription, etc) is installed in the `stackable-operators` namespace. Is this a good idea ? -- The Subscription object uses `installPlanApproval: Automatic` which means the operator is updated automatically for every new version. Is this a good idea? - -See the [OLM documentation](https://olm.operatorframework.io/docs/tasks/) for details. - -## Build and publish operator bundle image - -Each catalog can contain several operator packages, and each operator package can contain multiple channels, each with its own bundles of different versions of the operator. - -### Generate operator bundle (this is operator-specific) - - opm alpha bundle generate --directory manifests --package zookeeper-operator-package --output-dir bundle --channels stable --default stable - -### Build bundle image - - docker build -t docker.stackable.tech/stackable/zookeeper-operator-bundle:23.1.0 -f bundle.Dockerfile . - docker push docker.stackable.tech/stackable/zookeeper-operator-bundle:23.1.0 - -### Validate bundle image - - opm alpha bundle validate --tag docker.stackable.tech/stackable/zookeeper-operator-bundle:23.1.0 --image-builder docker - -## Create catalog - - mkdir catalog - opm generate dockerfile catalog - -## Create a package for each operator - - opm init zookeeper-operator-package \ - --default-channel=stable \ - --description=./README.md \ - --output yaml > catalog/zookeeper-operator-package.yaml - - { - echo "---" - echo "schema: olm.channel" - echo "package: zookeeper-operator-package" - echo "name: stable" - echo "entries:" - echo "- name: zookeeper-operator.v23.1.0" - } >> catalog/zookeeper-operator-package.yaml - -NOTE: with the command below we can add the Stackable logo as icon. - - # add for each operator... - opm render docker.stackable.tech/stackable/zookeeper-operator-bundle:23.1.0 --output=yaml >> catalog/zookeeper-operator-package.yaml - - # ...and then validate the entire catalog - opm validate catalog - -The catalog is correct if the command above returns successfully without any message. If the catalog doesn't validate, the operator will not install. Now build a catalog image and push it to the repository: - - docker build . -f catalog.Dockerfile -t docker.stackable.tech/stackable/zookeeper-operator-catalog:latest - docker push docker.stackable.tech/stackable/zookeeper-operator-catalog:latest - -## Install catalog and the operator group - - kubectl apply -f catalog-source.yaml - kubectl apply -f operator-group.yaml - -## List available operators - - kubectl get packagemanifest -n stackable-operators - -## Install operator - - kubectl apply -f subscription.yaml From 4b2cbec7980f1d9a53bf40e881fa50b4c2eeedc3 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 17 Feb 2023 14:03:24 +0100 Subject: [PATCH 5/9] linting fix --- deploy/olm/23.1.0/manifests/csv.yaml | 104 +++++++++++++-------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/deploy/olm/23.1.0/manifests/csv.yaml b/deploy/olm/23.1.0/manifests/csv.yaml index c12a4fd..e2a22ac 100644 --- a/deploy/olm/23.1.0/manifests/csv.yaml +++ b/deploy/olm/23.1.0/manifests/csv.yaml @@ -63,7 +63,7 @@ spec: # a short description of the CRDs purpose for rendering in graphical consoles (optional) description: Represents an S3Connection - name: authenticationclasses.authentication.stackable.tech - # version is the spec.versions[].name value defined in the CRD + # version is the spec.versions[].name value defined in the CRD version: v1alpha1 # kind is the CamelCased singular value defined in spec.names.kind of the CRD. kind: AuthenticationClass @@ -91,57 +91,57 @@ spec: clusterPermissions: - serviceAccountName: commons-operator rules: - - apiGroups: - - "" - resources: - - pods - - configmaps - - secrets - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - apps - resources: - - statefulsets - verbs: - - get - - list - - watch - - patch - - apiGroups: - - events.k8s.io - resources: - - events - verbs: - - create - - apiGroups: - - s3.stackable.tech - resources: - - s3buckets - - s3connections - verbs: - - get - - list - - patch - - watch - - apiGroups: - - authentication.stackable.tech - resources: - - authenticationclasses - verbs: - - get - - list - - patch - - watch - - apiGroups: - - "" - resources: - - pods/eviction - verbs: - - create + - apiGroups: + - "" + resources: + - pods + - configmaps + - secrets + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch + - patch + - apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - apiGroups: + - s3.stackable.tech + resources: + - s3buckets + - s3connections + verbs: + - get + - list + - patch + - watch + - apiGroups: + - authentication.stackable.tech + resources: + - authenticationclasses + verbs: + - get + - list + - patch + - watch + - apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create deployments: - name: commons-operator From df9f51caa29b161374e32c9a8cba9a4da5b26b6f Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 17 Feb 2023 16:21:41 +0100 Subject: [PATCH 6/9] move generate bundle from makefile to script --- Makefile | 16 +--------------- deploy/olm/23.1.0/bundle.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 15 deletions(-) create mode 100755 deploy/olm/23.1.0/bundle.sh diff --git a/Makefile b/Makefile index 426a274..c9343c3 100644 --- a/Makefile +++ b/Makefile @@ -75,18 +75,4 @@ build: regenerate-charts helm-package docker-build publish: build docker-publish helm-publish run-dev: - nix run -f. tilt -- up --port 5431 - -bundle-clean: - rm -rf "deploy/olm/23.1.0/bundle" - rm -rf "deploy/olm/23.1.0/bundle.Dockerfile" - -build-bundle: - pushd deploy/olm/23.1.0/ && \ - opm alpha bundle generate --directory manifests --package ${OPERATOR_NAME}-package --output-dir bundle --channels stable --default stable && \ - docker build -t "docker.stackable.tech/stackable/${OPERATOR_NAME}-bundle:23.1.0" -f bundle.Dockerfile . && \ - docker push "docker.stackable.tech/stackable/${OPERATOR_NAME}-bundle:23.1.0" && \ - opm alpha bundle validate --tag "docker.stackable.tech/stackable/${OPERATOR_NAME}-bundle:23.1.0" --image-builder docker && \ - popd - -regenerate-bundle: bundle-clean build-bundle \ No newline at end of file + nix run -f. tilt -- up --port 5431 \ No newline at end of file diff --git a/deploy/olm/23.1.0/bundle.sh b/deploy/olm/23.1.0/bundle.sh new file mode 100755 index 0000000..5fe2245 --- /dev/null +++ b/deploy/olm/23.1.0/bundle.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -euo pipefail +set -x + +OPERATOR_NAME="commons-operator" +VERSION="23.1.0" + +bundle-clean() { + rm -rf "deploy/olm/${VERSION}/bundle" + rm -rf "deploy/olm/${VERSION}/bundle.Dockerfile" +} + + +build-bundle() { + opm alpha bundle generate --directory manifests --package "${OPERATOR_NAME}-package" --output-dir bundle --channels stable --default stable + docker build -t "docker.stackable.tech/stackable/${OPERATOR_NAME}-bundle:${VERSION}" -f bundle.Dockerfile . + docker push "docker.stackable.tech/stackable/${OPERATOR_NAME}-bundle:${VERSION}" + opm alpha bundle validate --tag "docker.stackable.tech/stackable/${OPERATOR_NAME}-bundle:${VERSION}" --image-builder docker +} + +main() { + pushd deploy/olm/${VERSION} + bundle-clean + build-bundle + popd +} + +main "$@" \ No newline at end of file From 6ed615c1b539af0a0546e4e4301ee559760c049b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 17 Feb 2023 16:29:43 +0100 Subject: [PATCH 7/9] updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c64f58c..a49290e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,11 @@ All notable changes to this project will be documented in this file. - Specified security context settings needed for OpenShift ([#136]). - Revert openshift settings ([#142]) +- Generate OLM bundle ([#149]) [#136]: https://github.com/stackabletech/commons-operator/pull/136 [#142]: https://github.com/stackabletech/commons-operator/pull/142 +[#149]: https://github.com/stackabletech/commons-operator/pull/149 ## [23.1.0] - 2023-01-23 From 498590d578c1c661a91d453658131d4d1afc6f80 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 17 Feb 2023 16:32:40 +0100 Subject: [PATCH 8/9] moved changelog entry to added --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a49290e..f557bb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,15 +4,19 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- Generate OLM bundle ([#149]) + +[#149]: https://github.com/stackabletech/commons-operator/pull/149 + ### Changed - Specified security context settings needed for OpenShift ([#136]). - Revert openshift settings ([#142]) -- Generate OLM bundle ([#149]) [#136]: https://github.com/stackabletech/commons-operator/pull/136 [#142]: https://github.com/stackabletech/commons-operator/pull/142 -[#149]: https://github.com/stackabletech/commons-operator/pull/149 ## [23.1.0] - 2023-01-23 From 021c81076b9c606d932aac11bf8c0f020c62e9c4 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 17 Feb 2023 17:28:37 +0100 Subject: [PATCH 9/9] make script version-agnostic --- deploy/olm/{23.1.0 => }/bundle.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) rename deploy/olm/{23.1.0 => }/bundle.sh (83%) diff --git a/deploy/olm/23.1.0/bundle.sh b/deploy/olm/bundle.sh similarity index 83% rename from deploy/olm/23.1.0/bundle.sh rename to deploy/olm/bundle.sh index 5fe2245..5c996e6 100755 --- a/deploy/olm/23.1.0/bundle.sh +++ b/deploy/olm/bundle.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash +# usage: bundle.sh , called from base folder: +# e.g. ./deploy/olm/bundle.sh 23.1.0 set -euo pipefail set -x OPERATOR_NAME="commons-operator" -VERSION="23.1.0" bundle-clean() { rm -rf "deploy/olm/${VERSION}/bundle" rm -rf "deploy/olm/${VERSION}/bundle.Dockerfile" } - build-bundle() { opm alpha bundle generate --directory manifests --package "${OPERATOR_NAME}-package" --output-dir bundle --channels stable --default stable docker build -t "docker.stackable.tech/stackable/${OPERATOR_NAME}-bundle:${VERSION}" -f bundle.Dockerfile . @@ -20,7 +20,9 @@ build-bundle() { } main() { - pushd deploy/olm/${VERSION} + VERSION="$1"; + + pushd "deploy/olm/${VERSION}" bundle-clean build-bundle popd