Merge pull request #862 from annastopel/cdi-okd-ci

OLM - ci -test. Create cdi-olm-catalog to be able to deploy cdi via o…

.travis.yml

@@ -24,9 +24,9 @@ notifications:
 
 install:
   - make generate-verify
-  - make docker
   - make manifests
   - make olm-verify
+  - make docker
   - make apidocs
 
 before_script:

Makefile

@@ -12,15 +12,15 @@
 #See the License for the specific language governing permissions and
 #limitations under the License.
 
-.PHONY: build build-controller build-importer build-cloner build-apiserver build-uploadproxy build-uploadserver build-operator build-functest-file-image-init build-functest-registry-image-init  build-functest \
-		docker docker-controller docker-cloner docker-importer docker-apiserver docker-uploadproxy docker-uploadserver docker-operator docker-functest-image-init docker-functest-image-http docker-functest-registry-populate docker-functest-registry docker-functest-registry-init \
-		cluster-up cluster-down cluster-sync cluster-sync-controller cluster-sync-cloner cluster-sync-importer cluster-sync-apiserver cluster-sync-uploadproxy cluster-sync-uploadserver \
+.PHONY: build build-controller build-importer build-cloner build-apiserver build-uploadproxy build-uploadserver build-operator build-functest-file-image-init build-functest-registry-image-init build-functest \
+	    manifests \
 		olm-verify olm-push \
+    	docker docker-controller docker-cloner docker-importer docker-apiserver docker-uploadproxy docker-uploadserver docker-operator docker-functest-image-init docker-functest-image-http docker-functest-registry-populate docker-functest-registry docker-functest-registry-init docker-olm-catalog \
+		cluster-up cluster-down cluster-sync cluster-sync-controller cluster-sync-cloner cluster-sync-importer cluster-sync-apiserver cluster-sync-uploadproxy cluster-sync-uploadserver \
 		test test-functional test-unit test-lint \
 		publish \
 		vet \
 		format \
-		manifests \
 		goveralls \
 		release-description
 
@@ -49,7 +49,7 @@ apidocs:
 	${DO} "./hack/update-codegen.sh && ./hack/gen-swagger-doc/gen-swagger-docs.sh v1alpha1 html"
 
 build:
-	${DO} "./hack/build/build-go.sh clean && ./hack/build/build-go.sh build ${WHAT} && ./hack/build/build-cdi-func-test-file-host.sh && ./hack/build/build-cdi-func-test-registry-host.sh && DOCKER_PREFIX=${DOCKER_PREFIX} DOCKER_TAG=${DOCKER_TAG} VERBOSITY=${VERBOSITY} PULL_POLICY=${PULL_POLICY} QUAY_NAMESPACE=${QUAY_NAMESPACE} QUAY_REPOSITORY=${QUAY_REPOSITORY} CSV_VERSION=${CSV_VERSION} ./hack/build/build-manifests.sh ${WHAT} && ./hack/build/build-copy-artifacts.sh ${WHAT}"
+	${DO} "DOCKER_REPO=${DOCKER_REPO} DOCKER_TAG=${DOCKER_TAG} VERBOSITY=${VERBOSITY} PULL_POLICY=${PULL_POLICY} QUAY_NAMESPACE=${QUAY_NAMESPACE} QUAY_REPOSITORY=${QUAY_REPOSITORY} CSV_VERSION=${CSV_VERSION} ./hack/build/build-go.sh clean && ./hack/build/build-go.sh build ${WHAT} && ./hack/build/build-cdi-func-test-file-host.sh && ./hack/build/build-cdi-func-test-registry-host.sh && ./hack/build/build-cdi-olm-catalog.sh && ./hack/build/build-copy-artifacts.sh ${WHAT}"
 
 build-controller: WHAT = cmd/cdi-controller
 build-controller: build
@@ -65,6 +65,8 @@ build-cloner: WHAT = cmd/cdi-cloner
 build-cloner: build
 build-operator: WHAT = cmd/cdi-operator
 build-operator: build
+#build-cdi-olm-catalog: WHAT = tools/cdi-olm-catalog
+#build-cdi-olm-catalog:  
 build-functest-file-image-init: WHAT = tools/cdi-func-test-file-host-init
 build-functest-file-image-init:
 build-functest-registry-image-init: WHAT= tools/cdi-func-test-registry-init
@@ -106,7 +108,10 @@ docker-uploadserver: WHAT = cmd/cdi-uploadserver
 docker-uploadserver: docker
 docker-operator: WHAT = cmd/cdi-operator
 docker-operator: docker
-docker-functest-images: docker-functest-image-http docker-functest-image-init docker-functest-registry-init docker-functest-registry-populate docker-functest-registry
+docker-olm-catalog: WHAT = tools/cdi-olm-catalog
+docker-olm-catalog:
+	${DO} "./hack/build/build-cdi-olm-catalog.sh && ./hack/build/build-docker.sh build ${WHAT}" 
+docker-functest-images: docker-functest-image-http docker-functest-image-init docker-functest-registry-init docker-functest-registry-populate docker-functest-registry docker-functest-block-device docker-olm-catalog
 docker-functest-image-init: WHAT = tools/cdi-func-test-file-host-init
 docker-functest-image-init: docker
 docker-functest-image-http: WHAT = tools/cdi-func-test-file-host-http

cluster-sync/ephemeral_provider.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -e
 
+source cluster-sync/install.sh
+
 function seed_images(){
   container=""
   container_alias=""
@@ -30,4 +32,4 @@ function verify() {
 
 function configure_local_storage() {
   echo "Local storage already configured ..."
-}
+}

cluster-sync/external/install.sh

@@ -0,0 +1 @@
+CDI_INISTALL=${CDI_INSTALL_OPERATOR}

cluster-sync/external/provider.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+source cluster-sync/install.sh
+
 function _kubectl(){
   kubectl "$@"
 }
@@ -19,4 +21,6 @@ function up() {
 
 function configure_local_storage() {
   echo "Local storage not needed for external provider..."
-}
+}
+
+

cluster-sync/install.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -e
+source ./cluster-sync/${KUBEVIRT_PROVIDER}/install.sh
+
+function install_cdi_olm {
+  #Install CDI via OLM
+  _kubectl create ns $NAMESPACE
+  _kubectl apply -f _out/manifests/release/olm/operatorgroup.yaml
+  _kubectl apply -f _out/manifests/release/olm/k8s/cdi-catalogsource-registry.yaml
+  _kubectl apply -f _out/manifests/release/olm/k8s/cdi-subscription.yaml
+}
+
+function install_cdi_operator {
+  _kubectl apply -f "./_out/manifests/release/cdi-operator.yaml" 
+}
+
+
+function install_cdi {
+    case "${CDI_INSTALL}" in
+    "${CDI_INSTALL_OPERATOR}")
+        install_cdi_operator
+        ;;
+    "${CDI_INSTALL_OLM}")
+        install_cdi_olm
+        ;;
+    esac
+}
+
+
+function wait_cdi_crd_installed {
+  timeout=$1
+  crd_defined=0
+  while [ $crd_defined -eq 0 ] && [ $timeout > 0 ]; do
+      crd_defined=$(_kubectl get customresourcedefinition| grep cdis.cdi.kubevirt.io | wc -l)
+      sleep 1
+      timeout=timeout-1
+  done
+
+  #In case CDI crd is not defined after 120s - throw error
+  if [ $timeout \< 1 ]; then
+     echo "ERROR - CDI CRD is not defined after timeout"
+     exit 1
+  fi  
+
+}
+

cluster-sync/k8s-1.13.3/install.sh

@@ -0,0 +1 @@
+CDI_INSTALL=${CDI_INSTALL_OPERATOR}

cluster-sync/k8s-1.13.3/provider.sh

@@ -9,3 +9,5 @@ re='^-?[0-9]+$'
 if ! [[ $num_nodes =~ $re ]] || [[ $num_nodes -lt 1 ]] ; then
     num_nodes=1
 fi
+
+

cluster-sync/okd-4.1.2/install.sh

@@ -0,0 +1 @@
+CDI_INSTALL=${CDI_INSTALL_OLM}

cluster-sync/okd-4.1.2/provider.sh

@@ -47,3 +47,4 @@ function configure_local_storage() {
     set -e
   fi
 }
+

cluster-sync/os-3.11.0-crio/install.sh

@@ -0,0 +1 @@
+CDI_INSTALL=${CDI_INSTALL_OPERATOR}

cluster-sync/os-3.11.0-crio/provider.sh

@@ -9,3 +9,6 @@ re='^-?[0-9]+$'
 if ! [[ $num_nodes =~ $re ]] || [[ $num_nodes -lt 1 ]] ; then
     num_nodes=1
 fi
+
+
+

cluster-sync/sync.sh

@@ -28,21 +28,27 @@ if [ "${KUBEVIRT_PROVIDER}" != "external" ]; then
   registry=${IMAGE_REGISTRY:-localhost:$(_port registry)}
   DOCKER_PREFIX=${registry}
   MANIFEST_REGISTRY="registry:5000"
+  QUAY_NAMESPACE="none"
 fi
 
 # Need to set the DOCKER_PREFIX appropriately in the call to `make docker push`, otherwise make will just pass in the default `kubevirt`
+QUAY_NAMESPACE=$QUAY_NAMESPACE DOCKER_PREFIX=$MANIFEST_REGISTRY PULL_POLICY=$(getTestPullPolicy) make manifests
 DOCKER_PREFIX=$DOCKER_PREFIX make docker push
-DOCKER_PREFIX=$MANIFEST_REGISTRY PULL_POLICY=$(getTestPullPolicy) make manifests
 
 seed_images
 
 configure_local_storage
 
 # Install CDI
-_kubectl apply -f "./_out/manifests/release/cdi-operator.yaml" 
+install_cdi
+
+#wait cdi crd is installed with timeout
+wait_cdi_crd_installed $CDI_INSTALL_TIMEOUT
+
 _kubectl apply -f "./_out/manifests/release/cdi-cr.yaml"
 _kubectl wait cdis.cdi.kubevirt.io/cdi --for=condition=running --timeout=120s
 
+
 # Start functional test HTTP server.
 # We skip the functional test additions for external provider for now, as they're specific
 if [ "${KUBEVIRT_PROVIDER}" != "external" ]; then

cluster-up/hack/common.sh

@@ -15,6 +15,11 @@ if [ -z "$KUBEVIRTCI_CONFIG_PATH" ]; then
     )"
 fi
 
+CDI_INSTALL_OPERATOR="install-operator"
+CDI_INSTALL_OLM="install-olm"
+CDI_INSTALL=${CDI_INSTALL:-${CDI_INSTALL_OPERATOR}}
+CDI_INSTALL_TIMEOUT=${CDI_INSTALL_TIMEOUT:-120}
+
 KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.13.3}
 KUBEVIRT_NUM_NODES=${KUBEVIRT_NUM_NODES:-1}
 KUBEVIRT_MEMORY_SIZE=${KUBEVIRT_MEMORY_SIZE:-5120M}

doc/cdi-operator-olm.md

@@ -1,4 +1,4 @@
-# OLM (Operator Lifecycle Management) CDI (Containerized Data Importer) intergartion
+# OLM (Operator Lifecycle Management) CDI (Containerized Data Importer) integration
 
 ## Table of Contents 
 * [OLM Overview](#overview)
@@ -17,6 +17,7 @@ https://github.com/kubevirt/kubevirt/blob/master/docs/devel/olm-integration.md
 | ------------- |:-------------|:--------------|
 | _OperatorSource_     | Is used to define the external datastore we are using to store operator bundles |https://github.com/operator-framework/operator-marketplace/blob/master/README.md|
 | _CatalogSourceConfig_      | Is used to enable an operator present in the _OperatorSource_ to your cluster. Behind the scenes, it will configure an OLM CatalogSource so that the operator can then be managed by OLM.      | https://github.com/operator-framework/operator-marketplace/blob/master/README.md|
+| _operator-registry_ | Operator Registry runs in a Kubernetes or OpenShift cluster to provide operator catalog data to Operator Lifecycle Manager. | https://github.com/operator-framework/operator-registry|
 | _Subscription_ |  Monitors CatalogSource  for updates    | https://github.com/operator-framework/operator-lifecycle-manager/tree/274df58592c2ffd1d8ea56156c73c7746f57efc0#discovery-catalogs-and-automated-upgrades |
 | _OperatorGroup_ |  An OperatorGroup is an OLM resource that provides rudimentary multitenant configuration to OLM installed operators.     | https://github.com/operator-framework/operator-lifecycle-manager/blob/master/Documentation/design/operatorgroups.md|
 
@@ -114,7 +115,7 @@ cluster/kubectl.sh apply -f  _out/manifests/release/cdi-cr.yaml
 Now CDI deployment should finish its deployment successfully
 
 #### k8s cluster
-- Install CDI operatorsource manifest that specifies the location of CDI OLM bundle in quay
+- Install CDI operatorsource manifest that specifies the location of CDI OLM bundle in quay.
 **Vocabulary**: _OperatorSource_ is used to define the external datastore we are using to store operator bundles
 ```bash
 kubectl apply -f _out/manifests/release/olm/k8s/cdi-operatorsource.yaml
@@ -127,7 +128,7 @@ kubectl create ns cdi
 ```bash
 kubectl apply -f _out/manifests/release/olm/operatorgroup.yaml
 ```
-- Install CatalogSourceConfig resource
+- Install CatalogSourceConfig resource.
 **Vocabulary**: _CatalogSourceConfig_ is used to enable an operator present in the _OperatorSource_ to your cluster. Behind the scenes, it will configure an OLM CatalogSource so that the operator can then be managed by OLM.
 ```bash
 kubectl create --save-config -f _out/manifests/release/olm/k8s/cdi-catalogsource.yaml
@@ -152,6 +153,129 @@ cluster/kubectl.sh apply -f  _out/manifests/release/cdi-cr.yaml
 ```
 Now the operator should finish its deployment successfully
 
+<a name="operator-registry"></a>
+### CDI Installation via OLM with operator-registry
+It is possible to deploy operator via OLM without marketplace operator. Marketplace  operator is required in order to fetch OLM bundle from the specified quay repo. Operator framework provides a way to create _CatalogSource_ with manifests without hosting them in quay. This functionlaity is introduced in operator-registry https://github.com/operator-framework/operator-registry
+
+In order to deploy operator-registry a _CatalogSource_ manifest has to reference a container image that is based on _quay.io/openshift/origin-operator-registry_ and has operator OLM manifests under /registry directory.
+
+#####Example of Dockerfile 
+```
+> cat Dockerfile
+FROM quay.io/openshift/origin-operator-registry
+
+COPY olm-catalog /registry
+
+# Initialize the database
+RUN initializer --manifests /registry --output bundles.db
+
+# There are multiple binaries in the origin-operator-registry
+# We want the registry-server
+ENTRYPOINT ["registry-server"]
+CMD ["--database", "bundles.db"]
+
+```
+#####Example of CatalogSource
+```
+apiVersion: operators.coreos.com/v1alpha1
+kind: CatalogSource
+metadata:
+  name: cdi-operatorhub
+  namespace: cdi
+spec:
+  sourceType: grpc
+  image: docker.io/kubevirt/cdi-olm-catalog:latest
+  displayName: KubeVirt CDI
+  publisher: Red Hat
+
+```
+
+Once such _CatalogSource_ is deployed, it provides operartor's OLM manifests via grpc interface and can be consumed by OLM subscription mechanism.
+
+#### OKD4.x cluster
+- Generate CDI OLM manifests
+- Create operator-registry container image 
+```
+CSV_VERSION=<version> DOCKER_REPO=<repo> DOCKER_TAG=<tag> make docker-olm-catalog
+```
+- Push operator-registry container image to dockerhub
+```
+docker push DOCKER_REPO/cdi-olm-catalog:DOCKER_TAG
+```
+- Create CDI namespace
+```bash
+kubectl create ns cdi 
+```
+- Configure namespace to be allowed to create operators there
+```bash
+kubectl apply -f _out/manifests/release/olm/operatorgroup.yaml
+```
+- Install catalogsourceconfig that refers to the created operator-registry container image
+```bash
+kubectl apply -f _out/manifests/release/olm/os/cdi-catalogsource-registry.yaml
+```
+- Install subscription that will point from which channel the app is downloaded
+```bash
+kubectl apply -f  _out/manifests/release/olm/os/cdi-subscription.yaml
+```
+- Verify CDI installation plan was created
+```bash
+kubectl get operatorsource,catalogsourceconfig,catalogsource,subscription,installplan -n cdi
+NAME                                    PACKAGE   SOURCE   CHANNEL
+subscription.operators.coreos.com/cdi   cdi       cdi      beta
+
+NAME                                             CSV                 SOURCE   APPROVAL    APPROVED
+installplan.operators.coreos.com/install-995l9   cdioperator.0.0.0            Automatic   true
+
+```
+- Now cdi-operator starts running but in to install CDI we need to deploy cdi cr
+```bash
+cluster/kubectl.sh apply -f  _out/manifests/release/cdi-cr.yaml
+```
+Now CDI deployment should finish its deployment successfully
+
+#### k8s cluster
+- Generate CDI OLM manifests
+- Create operator-registry container image 
+```
+CSV_VERSION=<version> DOCKER_REPO=<repo> DOCKER_TAG=<tag> make docker-olm-catalog
+```
+- Push operator-registry container image to dockerhub
+```
+docker push DOCKER_REPO/cdi-olm-catalog:DOCKER_TAG
+```
+- Create CDI namespace
+```bash
+kubectl create ns cdi 
+```
+- Configure namespace to be allowed to create operators there
+```bash
+kubectl apply -f _out/manifests/release/olm/operatorgroup.yaml
+```
+- Install _CatalogSource_ that refers to the created operator-registry container image
+```bash
+kubectl apply -f _out/manifests/release/olm/k8s/cdi-catalogsource-registry.yaml
+```
+- Install subscription that will point from which channel the app is downloaded
+```bash
+kubectl apply -f  _out/manifests/release/olm/k8s/cdi-subscription.yaml
+```
+- Verify CDI installation plan was created
+```bash
+kubectl get operatorsource,catalogsourceconfig,catalogsource,subscription,installplan -n cdi
+NAME                                    PACKAGE   SOURCE   CHANNEL
+subscription.operators.coreos.com/cdi   cdi       cdi      beta
+
+NAME                                             CSV                 SOURCE   APPROVAL    APPROVED
+installplan.operators.coreos.com/install-995l9   cdioperator.0.0.0            Automatic   true
+
+```
+- Now cdi-operator starts running but in order for it to succeed we need to deploy cdi cr
+```bash
+cluster/kubectl.sh apply -f  _out/manifests/release/cdi-cr.yaml
+```
+Now the operator should finish its deployment successfully
+
 <a name="update"></a>
 ### CDI OLM update
 OLM mechanism supports operator update via subscription mechanism. Once subscription manifest is installed on cluster, it monitors the catalog source. CatalogSource in its turn monitors the location in quay and when new OLM bundle appears, OLM can trigger update of the operator.
@@ -172,6 +296,8 @@ DOCKER_REPO=<repo> DOCKER_TAG=<docker tag> PULL_POLICY=<pull policy> VERBOSITY=<
 ```bash
 QUAY_NAMESPACE=<quay namespace> QUAY_REPOSITORY=<quay repo> QUAY_USERNAME=<quay username> QUAY_PASSWORD=<quay password> CSV_VERSION=<csv version > make olm-push
 ```
+
+
 <a name="okdui"></a>
 ### OKD UI
 - Grant cluster-admin permissions to kube-system:default