[olm] Store previous versions of olm manifests

Makefile

@@ -14,6 +14,7 @@ GIT_VERSION = $(shell git describe --dirty --tags --always)
 GIT_COMMIT = $(shell git rev-parse HEAD)
 K8S_VERSION = v1.18.2
 GOLANGCI_LINT_VER = "1.30.0"
+OLM_VERSION = "0.15.1"
 REPO = github.com/operator-framework/operator-sdk
 PKGS = $(shell go list ./... | grep -v /vendor/)
 TEST_PKGS = $(shell go list ./... | grep -v -E 'github.com/operator-framework/operator-sdk/test/')
@@ -111,6 +112,9 @@ gen-cli-doc: ## Generate CLI documentation
 gen-changelog: ## Generate CHANGELOG.md and migration guide updates
 	./hack/generate/changelog/gen-changelog.sh
 
+bindata:
+	./hack/generate/olm_bindata.sh $(OLM_VERSION)
+
 ##############################
 # Release                    #
 ##############################

changelog/fragments/store-olm-bindata.yaml

@@ -0,0 +1,8 @@
+entries:
+  - description: >
+      Added feature to the command `operator-sdk olm install` that allows installing OLM version 0.15.1 without 
+      fetching the manifest from Github.
+      
+    kind: "addition"
+
+    breaking: false

go.sum

@@ -257,6 +257,7 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-bindata/go-bindata/v3 v3.1.3 h1:F0nVttLC3ws0ojc7p60veTurcOm//D4QBODNM7EGrCI=
 github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I=
 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -502,6 +503,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0 h1:reN85Pxc5larApoH1keMBiu2GWtPqXQ1nc9gx+jOU+E=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=

hack/check-error-log-msg-format.sh

@@ -6,7 +6,7 @@ set -o pipefail
 source "hack/lib/test_lib.sh"
 
 echo "Checking format of error and log messages..."
-allfiles=$(listFiles)
+allfiles=$(listFiles|grep -v ./internal/bindata/...)
 log_case_output=$(grep -PRn '(Error\((.*[Ee]rr|nil), |^(?!.*(fmt|errors)).+\.Error(f)?\(|Fatal(f)?\(|Info(f)?\(|Warn(f)?\()"[[:lower:]]' $allfiles | sort -u)
 if [ -n "${log_case_output}" ]; then
   echo -e "Log messages do not begin with upper case:\n${log_case_output}"

hack/check-license.sh

@@ -7,7 +7,7 @@ set -o pipefail
 source "hack/lib/test_lib.sh"
 
 echo "Checking for license header..."
-allfiles=$(listFiles)
+allfiles=$(listFiles|grep -v ./internal/bindata/...)
 licRes=""
 for file in $allfiles; do
   if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED|Licensed)" ; then

hack/generate/olm_bindata.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+function get_olm_manifests() {
+    mkdir olm-manifests
+    echo "downloading olm manifests for version ${1}"
+    wget -O olm-manifests/olm.yaml "https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${1}/olm.yaml"
+    wget -O olm-manifests/crds.yaml "https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${1}/crds.yaml"
+}
+
+function remove_olm_manifests {
+    rm -rf olm-manifests
+}
+
+# check for files starting with the name "olm-bindata" inside internal/olm folder
+function delete_old_olmbindata {
+    echo "Deleting previous versions of olm-bindata files if they exist"
+    find internal/bindata/olm -maxdepth 1 -type f -name manifests-* -exec rm {} \;
+}
+
+# TODO:
+# 1. Modify this to accept multiple versions and download bindata.
+# 2. Discuss on the number of olm versions of will be supported.
+FILE=internal/bindata/olm/"manifests-"$1.go
+if [ -f "$FILE" ]; then
+    delete_old_olmbindata
+    get_olm_manifests $1
+
+    go get -u github.com/go-bindata/go-bindata/...
+    $(go env GOPATH)/bin/go-bindata -o manifests-$1.go -pkg olm olm-manifests/
+    mv manifests-$1.go internal/bindata/olm
+
+    remove_olm_manifests
+fi
+
+go mod tidy

hack/tests/subcommand-olm-install.sh

@@ -37,4 +37,4 @@ test_version() {
 }
 
 test_version "latest"
-test_version "0.10.1"
+test_version "0.15.1" # Check installation of OLM for locally stored version of binaries

internal/bindata/olm/versions.go

@@ -0,0 +1,25 @@
+// Copyright 2020 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package olm
+
+var availableVersions = map[string]struct{}{
+	"0.15.1": struct{}{},
+}
+
+// HasVersion returns whether version maps to released OLM manifests as bindata.
+func HasVersion(version string) bool {
+	_, ok := availableVersions[version]
+	return ok
+}

internal/olm/installer/client.go

@@ -18,6 +18,7 @@
 package installer
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -27,6 +28,7 @@ import (
 	"time"
 
 	olmapiv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
+	olmmanifests "github.com/operator-framework/operator-sdk/internal/bindata/olm"
 	log "github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -43,6 +45,10 @@ const (
 	olmOperatorName     = "olm-operator"
 	catalogOperatorName = "catalog-operator"
 	packageServerName   = "packageserver"
+
+	// These paths are keys to look up internal OLM bindata.
+	olmManifestBindataPath = "olm-manifests/olm.yaml"
+	crdManifestBindataPath = "olm-manifests/crds.yaml"
 )
 
 type Client struct {
@@ -167,15 +173,34 @@ func (c Client) GetStatus(ctx context.Context, namespace, version string) (*olmr
 
 func (c Client) getResources(ctx context.Context, version string) ([]unstructured.Unstructured, error) {
 	log.Infof("Fetching CRDs for version %q", version)
-	crdResources, err := c.getCRDs(ctx, version)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch CRDs: %v", err)
-	}
 
-	log.Infof("Fetching resources for version %q", version)
-	olmResources, err := c.getOLM(ctx, version)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch resources: %v", err)
+	var crdResources, olmResources []unstructured.Unstructured
+	var err error
+
+	// If the manifests for the requested version are saved as bindata in SDK, use
+	// them instead of fetching them from
+	if olmmanifests.HasVersion(version) {
+		log.Infof("Using locally stored resource manifests")
+		crdResources, err = getPackagedManifests(crdManifestBindataPath)
+		if err != nil {
+			return nil, err
+		}
+
+		olmResources, err = getPackagedManifests(olmManifestBindataPath)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		log.Infof("Fetching resources for version %q", version)
+		crdResources, err = c.getCRDs(ctx, version)
+		if err != nil {
+			return nil, fmt.Errorf("failed to fetch CRDs: %v", err)
+		}
+
+		olmResources, err = c.getOLM(ctx, version)
+		if err != nil {
+			return nil, fmt.Errorf("failed to fetch resources: %v", err)
+		}
 	}
 
 	resources := append(crdResources, olmResources...)
@@ -200,6 +225,20 @@ func (c Client) getOLM(ctx context.Context, version string) ([]unstructured.Unst
 	return decodeResources(resp.Body)
 }
 
+func getPackagedManifests(manifestPath string) ([]unstructured.Unstructured, error) {
+	data, err := olmmanifests.Asset(manifestPath)
+	if err != nil {
+		return nil, fmt.Errorf("error retrieving bindata asset: %v", err)
+	}
+
+	reader := bytes.NewReader(data)
+	resources, err := decodeResources(reader)
+	if err != nil {
+		return nil, err
+	}
+	return resources, nil
+}
+
 func (c Client) crdsURL(version string) string {
 	return fmt.Sprintf("%s/crds.yaml", c.getBaseDownloadURL(version))
 }

test/internal/utils.go

@@ -40,7 +40,14 @@ func NewTestContext(env ...string) (tc TestContext, err error) {
 
 // InstallOLM runs 'operator-sdk olm install' and returns any errors emitted by that command.
 func (tc TestContext) InstallOLM() error {
-	cmd := exec.Command(tc.BinaryName, "olm", "install", "--timeout", "4m")
+	err := tc.InstallOLMVersion("latest")
+	return err
+}
+
+// InstallOLM runs 'operator-sdk olm install' for specific version
+// and returns any errors emitted by that command.
+func (tc TestContext) InstallOLMVersion(version string) error {
+	cmd := exec.Command(tc.BinaryName, "olm", "install", "--version", version, "--timeout", "4m")
 	_, err := tc.Run(cmd)
 	return err
 }

website/content/en/docs/contribution-guidelines/releasing.md

@@ -78,7 +78,7 @@ Each minor release has a corresponding release branch of the form `vX.Y.x`, wher
 release version numbers and the `x` is literal. This branch accepts bug fixes according to our [backport policy][backports].
 
 After the minor release is made, this branch must be fast-forwarded to that release's tag and a post-release PR made
-against this branch. See the [release process](#7-create-a-pr-for-post-release-version-updates) for more details.
+against this branch. See the [release process](#8-create-a-pr-for-post-release-version-updates) for more details.
 
 #### Cherry-picking
 
@@ -112,7 +112,7 @@ Unlock `master` or release branch after the release has completed (after step 6
 ### Releasing
 
 The GitHub [`Releases` tab][release-page] in the operator-sdk repo is where all SDK releases live.
-To create a GitHub release see the [releasing binaries section](#8-releasing-binaries-signatures-and-release-notes).
+To create a GitHub release see the [releasing binaries section](#9-releasing-binaries-signatures-and-release-notes).
 
 #### Release notes
 
@@ -202,13 +202,23 @@ For major and minor releases, `master` should be locked between steps 3 and 6 so
 or have a pre-release version, ex. `v1.2.0+git`. Otherwise commits might be built into a release that shouldn't be.
 For patch releases, ensure all required bugs are [cherry-picked](#cherry-picking), then the release branch `v1.3.x` should be locked down.
 
-### 1. Netlify configuration
+### 1. Update OLM bindata
+
+This step is to be preferably performed only during major releases. Update the `OLM_VERSION` variable in Makefile to the latest successful release of OLM. Run `make bindata` so that `internal/bindata/olm` is updated. Also, update the `availableVersions` map in `internal/bindata/olm/versions.go` to contain the version of OLM which you have specified in the Makefile.
+
+Submit a PR with the changes and merge it with master.
+
+**Important:**
+- Update OLM bindata just before starting the release so that we have the latest OLM version.
+- Verify that the release of OLM version which you specify in Makefile is successful.
+
+### 2. Netlify configuration
 
 **Important:** ensure a release branch-to-subdomain mapping exists in the SDK's Netlify configuration _prior to creating a release_,
 ex. `v1.3.x` to `https://v1-3-x.sdk.operatorframework.io`. You can ping SDK [approvers][doc-owners] to ensure a
 [release branch](#release-branches) is created prior to the release and that this mapping is created.
 
-### 2. Create release branch for Netlify
+### 3. Create release branch for Netlify
 
 The release branch must be created before the release occurs to appease the Netlify website configuration demons.
 You can do so by running the following before proceeding with the release, assuming the upstream SDK is the `upstream` remote repo:
@@ -220,7 +230,7 @@ $ git checkout -b v1.3.x
 $ git push -u upstream v1.3.x
 ```
 
-### 3. Create a PR for release version, CHANGELOG.md, and migration guide updates
+### 4. Create a PR for release version, CHANGELOG.md, and migration guide updates
 
 Once all PR's needed for a release have been merged, branch from `master`:
 
@@ -262,11 +272,11 @@ and add the following lines under `[[params.versions]]` for `master`:
     url = "https://v1-3-x.sdk.operatorframework.io"
   ```
 
-### 4. Lock down proper branch
+### 5. Lock down proper branch
 Create and merge a new PR for `release-v1.3.0`. Once this PR is merged, lock down the master or release branch
 to prevent further commits between this and step 7. See [this section](#locking-down-branches) for steps to do so.
 
-### 5. Create a release tag, binaries, and signatures
+### 6. Create a release tag, binaries, and signatures
 
 The top-level `release.sh` script will take care of verifying versions in files described in step 3, and tagging and verifying the tag, as well as building binaries and generating signatures by calling `make release`.
 
@@ -304,7 +314,7 @@ Once this tag passes CI, go to step 6. For more info on tagging, see the [releas
 
 **Note:** If CI fails for some reason, you will have to revert the tagged commit, re-commit, and make a new PR.
 
-### 6. Fast-forward the `latest` and release branches
+### 7. Fast-forward the `latest` and release branches
 
 The `latest` branch points to the latest release tag to keep the main website subdomain up-to-date.
 Run the following commands to do so:
@@ -323,7 +333,7 @@ $ git reset --hard tags/v1.3.0
 $ git push -f upstream v1.3.x
 ```
 
-### 7. Create a PR for post-release version updates
+### 8. Create a PR for post-release version updates
 
 Check out a new branch from `master` or release branch and commit the following changes:
 
@@ -333,7 +343,7 @@ Check out a new branch from `master` or release branch and commit the following
 
 Create a new PR for this branch targeting the `master` or release branch.
 
-### 8. Releasing binaries, signatures, and release notes
+### 9. Releasing binaries, signatures, and release notes
 
 The final step is to upload binaries, their signature files, and release notes from `CHANGELOG.md` for `v1.3.0`.
 To create a GitHub release:
@@ -346,7 +356,7 @@ To create a GitHub release:
 
 **Note:** if this is a pre-release, make sure to check the `This is a pre-release` box under the file attachment frame. If you are not sure what this means, ask another maintainer.
 
-### 9. Unlock proper branch
+### 10. Unlock proper branch
 Unlock the `master` or release branch after the Github release is complete.
 See [this section](#locking-down-branches) for steps to do so.