Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OLM - ci -test. Create cdi-olm-catalog to be able to deploy cdi via o… #862

Merged
merged 8 commits into from
Jul 15, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ notifications:

install:
- make generate-verify
- make docker
- make manifests
- make olm-verify
- make docker
- make apidocs

before_script:
Expand Down
17 changes: 11 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,15 @@
#See the License for the specific language governing permissions and
#limitations under the License.

.PHONY: build build-controller build-importer build-cloner build-apiserver build-uploadproxy build-uploadserver build-operator build-functest-file-image-init build-functest-registry-image-init build-functest \
docker docker-controller docker-cloner docker-importer docker-apiserver docker-uploadproxy docker-uploadserver docker-operator docker-functest-image-init docker-functest-image-http docker-functest-registry-populate docker-functest-registry docker-functest-registry-init \
cluster-up cluster-down cluster-sync cluster-sync-controller cluster-sync-cloner cluster-sync-importer cluster-sync-apiserver cluster-sync-uploadproxy cluster-sync-uploadserver \
.PHONY: build build-controller build-importer build-cloner build-apiserver build-uploadproxy build-uploadserver build-operator build-functest-file-image-init build-functest-registry-image-init build-functest \
manifests \
olm-verify olm-push \
docker docker-controller docker-cloner docker-importer docker-apiserver docker-uploadproxy docker-uploadserver docker-operator docker-functest-image-init docker-functest-image-http docker-functest-registry-populate docker-functest-registry docker-functest-registry-init docker-olm-catalog \
cluster-up cluster-down cluster-sync cluster-sync-controller cluster-sync-cloner cluster-sync-importer cluster-sync-apiserver cluster-sync-uploadproxy cluster-sync-uploadserver \
test test-functional test-unit test-lint \
publish \
vet \
format \
manifests \
goveralls \
release-description

Expand Down Expand Up @@ -49,7 +49,7 @@ apidocs:
${DO} "./hack/update-codegen.sh && ./hack/gen-swagger-doc/gen-swagger-docs.sh v1alpha1 html"

build:
${DO} "./hack/build/build-go.sh clean && ./hack/build/build-go.sh build ${WHAT} && ./hack/build/build-cdi-func-test-file-host.sh && ./hack/build/build-cdi-func-test-registry-host.sh && DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} VERBOSITY=${VERBOSITY} PULL_POLICY=${PULL_POLICY} QUAY_NAMESPACE=${QUAY_NAMESPACE} QUAY_REPOSITORY=${QUAY_REPOSITORY} CSV_VERSION=${CSV_VERSION} ./hack/build/build-manifests.sh ${WHAT} && ./hack/build/build-copy-artifacts.sh ${WHAT}"
${DO} "DOCKER_REPO=${DOCKER_REPO} DOCKER_TAG=${DOCKER_TAG} VERBOSITY=${VERBOSITY} PULL_POLICY=${PULL_POLICY} QUAY_NAMESPACE=${QUAY_NAMESPACE} QUAY_REPOSITORY=${QUAY_REPOSITORY} CSV_VERSION=${CSV_VERSION} ./hack/build/build-go.sh clean && ./hack/build/build-go.sh build ${WHAT} && ./hack/build/build-cdi-func-test-file-host.sh && ./hack/build/build-cdi-func-test-registry-host.sh && ./hack/build/build-cdi-olm-catalog.sh && ./hack/build/build-copy-artifacts.sh ${WHAT}"

build-controller: WHAT = cmd/cdi-controller
build-controller: build
Expand All @@ -65,6 +65,8 @@ build-cloner: WHAT = cmd/cdi-cloner
build-cloner: build
build-operator: WHAT = cmd/cdi-operator
build-operator: build
#build-cdi-olm-catalog: WHAT = tools/cdi-olm-catalog
#build-cdi-olm-catalog:
build-functest-file-image-init: WHAT = tools/cdi-func-test-file-host-init
build-functest-file-image-init:
build-functest-registry-image-init: WHAT= tools/cdi-func-test-registry-init
Expand Down Expand Up @@ -106,7 +108,10 @@ docker-uploadserver: WHAT = cmd/cdi-uploadserver
docker-uploadserver: docker
docker-operator: WHAT = cmd/cdi-operator
docker-operator: docker
docker-functest-images: docker-functest-image-http docker-functest-image-init docker-functest-registry-init docker-functest-registry-populate docker-functest-registry
docker-olm-catalog: WHAT = tools/cdi-olm-catalog
docker-olm-catalog:
${DO} "./hack/build/build-cdi-olm-catalog.sh && ./hack/build/build-docker.sh build ${WHAT}"
docker-functest-images: docker-functest-image-http docker-functest-image-init docker-functest-registry-init docker-functest-registry-populate docker-functest-registry docker-functest-block-device docker-olm-catalog
docker-functest-image-init: WHAT = tools/cdi-func-test-file-host-init
docker-functest-image-init: docker
docker-functest-image-http: WHAT = tools/cdi-func-test-file-host-http
Expand Down
4 changes: 3 additions & 1 deletion cluster-sync/ephemeral_provider.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#!/usr/bin/env bash
set -e

source cluster-sync/install.sh

function seed_images(){
container=""
container_alias=""
Expand Down Expand Up @@ -30,4 +32,4 @@ function verify() {

function configure_local_storage() {
echo "Local storage already configured ..."
}
}
1 change: 1 addition & 0 deletions cluster-sync/external/install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CDI_INISTALL=${CDI_INSTALL_OPERATOR}
6 changes: 5 additions & 1 deletion cluster-sync/external/provider.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#!/usr/bin/env bash

source cluster-sync/install.sh

function _kubectl(){
kubectl "$@"
}
Expand All @@ -19,4 +21,6 @@ function up() {

function configure_local_storage() {
echo "Local storage not needed for external provider..."
}
}


47 changes: 47 additions & 0 deletions cluster-sync/install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/usr/bin/env bash

set -e
source ./cluster-sync/${KUBEVIRT_PROVIDER}/install.sh

function install_cdi_olm {
#Install CDI via OLM
_kubectl create ns $NAMESPACE
_kubectl apply -f _out/manifests/release/olm/operatorgroup.yaml
_kubectl apply -f _out/manifests/release/olm/k8s/cdi-catalogsource-registry.yaml
_kubectl apply -f _out/manifests/release/olm/k8s/cdi-subscription.yaml
}

function install_cdi_operator {
_kubectl apply -f "./_out/manifests/release/cdi-operator.yaml"
}


function install_cdi {
case "${CDI_INSTALL}" in
"${CDI_INSTALL_OPERATOR}")
install_cdi_operator
;;
"${CDI_INSTALL_OLM}")
install_cdi_olm
;;
esac
}


function wait_cdi_crd_installed {
timeout=$1
crd_defined=0
while [ $crd_defined -eq 0 ] && [ $timeout > 0 ]; do
crd_defined=$(_kubectl get customresourcedefinition| grep cdis.cdi.kubevirt.io | wc -l)
sleep 1
timeout=timeout-1
done

#In case CDI crd is not defined after 120s - throw error
if [ $timeout \< 1 ]; then
echo "ERROR - CDI CRD is not defined after timeout"
exit 1
fi

}

1 change: 1 addition & 0 deletions cluster-sync/k8s-1.13.3/install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CDI_INSTALL=${CDI_INSTALL_OPERATOR}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think you need the install.sh files for the providers, you can put

CDI_INSTALL=${CDI_INSTALL_OPERATOR}

In the provider.sh right before you source cluster-sync/ephemeral_provider.sh

2 changes: 2 additions & 0 deletions cluster-sync/k8s-1.13.3/provider.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,5 @@ re='^-?[0-9]+$'
if ! [[ $num_nodes =~ $re ]] || [[ $num_nodes -lt 1 ]] ; then
num_nodes=1
fi


1 change: 1 addition & 0 deletions cluster-sync/okd-4.1.2/install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CDI_INSTALL=${CDI_INSTALL_OLM}
1 change: 1 addition & 0 deletions cluster-sync/okd-4.1.2/provider.sh
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,4 @@ function configure_local_storage() {
set -e
fi
}

1 change: 1 addition & 0 deletions cluster-sync/os-3.11.0-crio/install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CDI_INSTALL=${CDI_INSTALL_OPERATOR}
3 changes: 3 additions & 0 deletions cluster-sync/os-3.11.0-crio/provider.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,6 @@ re='^-?[0-9]+$'
if ! [[ $num_nodes =~ $re ]] || [[ $num_nodes -lt 1 ]] ; then
num_nodes=1
fi



10 changes: 8 additions & 2 deletions cluster-sync/sync.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,27 @@ if [ "${KUBEVIRT_PROVIDER}" != "external" ]; then
registry=${IMAGE_REGISTRY:-localhost:$(_port registry)}
DOCKER_PREFIX=${registry}
MANIFEST_REGISTRY="registry:5000"
QUAY_NAMESPACE="none"
fi

# Need to set the DOCKER_PREFIX appropriately in the call to `make docker push`, otherwise make will just pass in the default `kubevirt`
QUAY_NAMESPACE=$QUAY_NAMESPACE DOCKER_PREFIX=$MANIFEST_REGISTRY PULL_POLICY=$(getTestPullPolicy) make manifests
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why the seperate QUAY_NAMESPACE? We could just set DOCKER_PREFIX=quay.io/xxxxx

This also brings back my earlier comment that we should probably change DOCKER_PREFIX to REGISTRY_NAME or something more general. This isn't your problem and shouldn't impact your patch but capturing the point here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

QUAY_NAMESPACE is the namespace in quay where CDI OLM bundle is located. It is not necessarily equals to DOCKER_PREFIX

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think Johns point is, that we either use Quay OR Docker but not both at the same time, so having two separate variables is sort of weird, it would make sense to have a 'REGISTRY_NAME' or something like instead of DOCKER_PREFIX and QUAY_NAMESPACE if they both serve the same purpose but for a different registry provider.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think Johns point is, that we either use Quay OR Docker but not both at the same time, so having two separate variables is sort of weird, it would make sense to have a 'REGISTRY_NAME' or something like instead of DOCKER_PREFIX and QUAY_NAMESPACE if they both serve the same purpose but for a different registry provider.

But we do utilize both dockerhub and quay - we push all cdi images to repo in dockerhub and OLM bundle to quay

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, my point was as @awels mentioned why both, I get that we do push to both, but we don't push to both in the same build iteration. So the idea I was getting at was use a single variable and set it appropriately for the build we're doing.

If you're doing parallel buld/push operations then it would make more sense to still use a single image registry variable but make it a list and just do build, then push to 'n' registries from the list. We should probably be doing something like that anyway if we're not already rather than run a seperate build/release process for each registry.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As discussed we do need both DOCKER_PREFIX and QUAY_REPOSITORY variables in the same make manifests iteration. DOCKER_PREFIX for container images pushed to docker and QUAY related variables for olm manifests and bundle

DOCKER_PREFIX=$DOCKER_PREFIX make docker push
DOCKER_PREFIX=$MANIFEST_REGISTRY PULL_POLICY=$(getTestPullPolicy) make manifests

seed_images

configure_local_storage

# Install CDI
_kubectl apply -f "./_out/manifests/release/cdi-operator.yaml"
install_cdi

#wait cdi crd is installed with timeout
wait_cdi_crd_installed $CDI_INSTALL_TIMEOUT

_kubectl apply -f "./_out/manifests/release/cdi-cr.yaml"
_kubectl wait cdis.cdi.kubevirt.io/cdi --for=condition=running --timeout=120s


# Start functional test HTTP server.
# We skip the functional test additions for external provider for now, as they're specific
if [ "${KUBEVIRT_PROVIDER}" != "external" ]; then
Expand Down
5 changes: 5 additions & 0 deletions cluster-up/hack/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ if [ -z "$KUBEVIRTCI_CONFIG_PATH" ]; then
)"
fi

CDI_INSTALL_OPERATOR="install-operator"
CDI_INSTALL_OLM="install-olm"
CDI_INSTALL=${CDI_INSTALL:-${CDI_INSTALL_OPERATOR}}
CDI_INSTALL_TIMEOUT=${CDI_INSTALL_TIMEOUT:-120}

KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.13.3}
KUBEVIRT_NUM_NODES=${KUBEVIRT_NUM_NODES:-1}
KUBEVIRT_MEMORY_SIZE=${KUBEVIRT_MEMORY_SIZE:-5120M}
Expand Down
132 changes: 129 additions & 3 deletions doc/cdi-operator-olm.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# OLM (Operator Lifecycle Management) CDI (Containerized Data Importer) intergartion
# OLM (Operator Lifecycle Management) CDI (Containerized Data Importer) integration

## Table of Contents
* [OLM Overview](#overview)
Expand All @@ -17,6 +17,7 @@ https://github.com/kubevirt/kubevirt/blob/master/docs/devel/olm-integration.md
| ------------- |:-------------|:--------------|
| _OperatorSource_ | Is used to define the external datastore we are using to store operator bundles |https://github.com/operator-framework/operator-marketplace/blob/master/README.md|
| _CatalogSourceConfig_ | Is used to enable an operator present in the _OperatorSource_ to your cluster. Behind the scenes, it will configure an OLM CatalogSource so that the operator can then be managed by OLM. | https://github.com/operator-framework/operator-marketplace/blob/master/README.md|
| _operator-registry_ | Operator Registry runs in a Kubernetes or OpenShift cluster to provide operator catalog data to Operator Lifecycle Manager. | https://github.com/operator-framework/operator-registry|
| _Subscription_ | Monitors CatalogSource for updates | https://github.com/operator-framework/operator-lifecycle-manager/tree/274df58592c2ffd1d8ea56156c73c7746f57efc0#discovery-catalogs-and-automated-upgrades |
| _OperatorGroup_ | An OperatorGroup is an OLM resource that provides rudimentary multitenant configuration to OLM installed operators. | https://github.com/operator-framework/operator-lifecycle-manager/blob/master/Documentation/design/operatorgroups.md|

Expand Down Expand Up @@ -114,7 +115,7 @@ cluster/kubectl.sh apply -f _out/manifests/release/cdi-cr.yaml
Now CDI deployment should finish its deployment successfully

#### k8s cluster
- Install CDI operatorsource manifest that specifies the location of CDI OLM bundle in quay
- Install CDI operatorsource manifest that specifies the location of CDI OLM bundle in quay.
**Vocabulary**: _OperatorSource_ is used to define the external datastore we are using to store operator bundles
```bash
kubectl apply -f _out/manifests/release/olm/k8s/cdi-operatorsource.yaml
Expand All @@ -127,7 +128,7 @@ kubectl create ns cdi
```bash
kubectl apply -f _out/manifests/release/olm/operatorgroup.yaml
```
- Install CatalogSourceConfig resource
- Install CatalogSourceConfig resource.
**Vocabulary**: _CatalogSourceConfig_ is used to enable an operator present in the _OperatorSource_ to your cluster. Behind the scenes, it will configure an OLM CatalogSource so that the operator can then be managed by OLM.
```bash
kubectl create --save-config -f _out/manifests/release/olm/k8s/cdi-catalogsource.yaml
Expand All @@ -152,6 +153,129 @@ cluster/kubectl.sh apply -f _out/manifests/release/cdi-cr.yaml
```
Now the operator should finish its deployment successfully

<a name="operator-registry"></a>
### CDI Installation via OLM with operator-registry
It is possible to deploy operator via OLM without marketplace operator. Marketplace operator is required in order to fetch OLM bundle from the specified quay repo. Operator framework provides a way to create _CatalogSource_ with manifests without hosting them in quay. This functionlaity is introduced in operator-registry https://github.com/operator-framework/operator-registry

In order to deploy operator-registry a _CatalogSource_ manifest has to reference a container image that is based on _quay.io/openshift/origin-operator-registry_ and has operator OLM manifests under /registry directory.

#####Example of Dockerfile
```
> cat Dockerfile
FROM quay.io/openshift/origin-operator-registry

COPY olm-catalog /registry

# Initialize the database
RUN initializer --manifests /registry --output bundles.db

# There are multiple binaries in the origin-operator-registry
# We want the registry-server
ENTRYPOINT ["registry-server"]
CMD ["--database", "bundles.db"]

```
#####Example of CatalogSource
```
apiVersion: operators.coreos.com/v1alpha1
kind: CatalogSource
metadata:
name: cdi-operatorhub
namespace: cdi
spec:
sourceType: grpc
image: docker.io/kubevirt/cdi-olm-catalog:latest
displayName: KubeVirt CDI
publisher: Red Hat

```

Once such _CatalogSource_ is deployed, it provides operartor's OLM manifests via grpc interface and can be consumed by OLM subscription mechanism.

#### OKD4.x cluster
- Generate CDI OLM manifests
- Create operator-registry container image
```
CSV_VERSION=<version> DOCKER_REPO=<repo> DOCKER_TAG=<tag> make docker-olm-catalog
```
- Push operator-registry container image to dockerhub
```
docker push DOCKER_REPO/cdi-olm-catalog:DOCKER_TAG
```
- Create CDI namespace
```bash
kubectl create ns cdi
```
- Configure namespace to be allowed to create operators there
```bash
kubectl apply -f _out/manifests/release/olm/operatorgroup.yaml
```
- Install catalogsourceconfig that refers to the created operator-registry container image
```bash
kubectl apply -f _out/manifests/release/olm/os/cdi-catalogsource-registry.yaml
```
- Install subscription that will point from which channel the app is downloaded
```bash
kubectl apply -f _out/manifests/release/olm/os/cdi-subscription.yaml
```
- Verify CDI installation plan was created
```bash
kubectl get operatorsource,catalogsourceconfig,catalogsource,subscription,installplan -n cdi
NAME PACKAGE SOURCE CHANNEL
subscription.operators.coreos.com/cdi cdi cdi beta

NAME CSV SOURCE APPROVAL APPROVED
installplan.operators.coreos.com/install-995l9 cdioperator.0.0.0 Automatic true

```
- Now cdi-operator starts running but in to install CDI we need to deploy cdi cr
```bash
cluster/kubectl.sh apply -f _out/manifests/release/cdi-cr.yaml
```
Now CDI deployment should finish its deployment successfully

#### k8s cluster
- Generate CDI OLM manifests
- Create operator-registry container image
```
CSV_VERSION=<version> DOCKER_REPO=<repo> DOCKER_TAG=<tag> make docker-olm-catalog
```
- Push operator-registry container image to dockerhub
```
docker push DOCKER_REPO/cdi-olm-catalog:DOCKER_TAG
```
- Create CDI namespace
```bash
kubectl create ns cdi
```
- Configure namespace to be allowed to create operators there
```bash
kubectl apply -f _out/manifests/release/olm/operatorgroup.yaml
```
- Install _CatalogSource_ that refers to the created operator-registry container image
```bash
kubectl apply -f _out/manifests/release/olm/k8s/cdi-catalogsource-registry.yaml
```
- Install subscription that will point from which channel the app is downloaded
```bash
kubectl apply -f _out/manifests/release/olm/k8s/cdi-subscription.yaml
```
- Verify CDI installation plan was created
```bash
kubectl get operatorsource,catalogsourceconfig,catalogsource,subscription,installplan -n cdi
NAME PACKAGE SOURCE CHANNEL
subscription.operators.coreos.com/cdi cdi cdi beta

NAME CSV SOURCE APPROVAL APPROVED
installplan.operators.coreos.com/install-995l9 cdioperator.0.0.0 Automatic true

```
- Now cdi-operator starts running but in order for it to succeed we need to deploy cdi cr
```bash
cluster/kubectl.sh apply -f _out/manifests/release/cdi-cr.yaml
```
Now the operator should finish its deployment successfully

<a name="update"></a>
### CDI OLM update
OLM mechanism supports operator update via subscription mechanism. Once subscription manifest is installed on cluster, it monitors the catalog source. CatalogSource in its turn monitors the location in quay and when new OLM bundle appears, OLM can trigger update of the operator.
Expand All @@ -172,6 +296,8 @@ DOCKER_REPO=<repo> DOCKER_TAG=<docker tag> PULL_POLICY=<pull policy> VERBOSITY=<
```bash
QUAY_NAMESPACE=<quay namespace> QUAY_REPOSITORY=<quay repo> QUAY_USERNAME=<quay username> QUAY_PASSWORD=<quay password> CSV_VERSION=<csv version > make olm-push
```


<a name="okdui"></a>
### OKD UI
- Grant cluster-admin permissions to kube-system:default
Expand Down