diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7100f699..f673d7bb 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -7,7 +7,6 @@ on: pull_request: env: - # github.repository as / IMAGE_BASE: ${{ github.repository }} jobs: @@ -32,23 +31,25 @@ jobs: name: Run tests run: | make test - # - - # name: Log into ghcr.io - # if: ${{ github.event_name != 'pull_request' }} - # uses: docker/login-action@v1 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - # - - # name: Push images to ghcr.io - # if: ${{ github.event_name != 'pull_request' }} - # run: | - # images=(${{ env.IMAGES }}) - # for image in ${images[*]} - # do - # echo "::group::Push ods-$image to ghcr.io" - # docker tag localhost:5000/ods/ods-$image:latest ghcr.io/${{ env.IMAGE_BASE }}/ods-$image:latest - # docker push ghcr.io/${{ env.IMAGE_BASE }}/ods-$image:latest - # echo "::endgroup::" - # done + - + name: Log into ghcr.io + if: ${{ github.event_name != 'pull_request' }} + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Push images to ghcr.io + if: ${{ github.event_name != 'pull_request' }} + env: + IMAGES: finish pipeline-manager start + run: | + images=(${{ env.IMAGES }}) + for image in ${images[*]} + do + echo "::group::Push $image to ghcr.io" + docker tag localhost:5000/ods-pipeline/$image:latest ghcr.io/${{ env.IMAGE_BASE }}/$image:latest + docker push ghcr.io/${{ env.IMAGE_BASE }}/$image:latest + echo "::endgroup::" + done diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0da1ffa2..1a137ef6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,13 +1,11 @@ -name: Release images +name: Release images and chart on: release: types: - released env: - # Use docker.io for Docker Hub if empty REGISTRY: ghcr.io - # github.repository as / IMAGE_BASE: ${{ github.repository }} jobs: @@ -23,7 +21,7 @@ jobs: outputs: imageTag: ${{ steps.createImageTag.outputs.imageTag }} - build: + release-images: name: Build and release images needs: setup runs-on: ubuntu-latest @@ -37,8 +35,8 @@ jobs: id-token: write steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 # Workaround: https://github.com/docker/build-push-action/issues/461 - name: Setup Docker buildx @@ -63,3 +61,23 @@ jobs: file: build/package/Dockerfile.${{ matrix.image }} push: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_BASE }}/ods-${{ matrix.image }}:${{ needs.setup.outputs.imageTag }} + + release-chart: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.5.0 + with: + charts_dir: deploy + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 117c049a..1fee0d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,9 @@ listed in the changelog. - Move pipeline tasks to separate repositories. This is a huge change with many implications. Instead of providing build, package and deploy taks as part of the `ods-pipeline` repository, the tasks are no provided by separate repositories, such as `ods-pipeline-go`, `ods-pipeline-sonar`, `ods-pipeline-image`, `ods-pipeline-helm` and so on. The only tasks that are provided by `ods-pipeline` are the start and finish tasks automatically injected into each pipeline. This change allows to have a different lifecycle for each task (or set of tasks). It also benefits maintenance greatly: running the tests for this repository is much faster now (around 10 minutes compared to 35+ minutes earlier). This repository facilitates task creation, maintenance and testing by providing a few Go packages that can be used by task repositories such as `ods-pipeline-helm`. For more information, see [#722](https://github.com/opendevstack/ods-pipeline/pull/722). -- Build tasks streamlining and avoidance of file copies (#678 fixed by [#710](https://github.com/opendevstack/ods-pipeline/pull/710)). This is an incompatible change. Build tasks were adjusted to (mostly) no longer copy build files in a dedicated location. Instead one should adjust the Dockerfile (or other downstream tasks) to directly consume the build outputs from their natural locations. In addition build task skipping now supports parameter `build-extra-inputs`. The package-image task `dockerfile` and `docker-dir` parameters have been changed to assume that the docker context and file are at the repository root. See the PR for further information and the issue for more context. +- Publish Helm chart and adjust install script to install from there. Instead of requiring users to create a local repository, using `git subtree`` to include the `deploy`` folder, and installing from the local chart, the install script now uses the published chart. This greatly simplifies the installation and upgrade procedure. See [#730](https://github.com/opendevstack/ods-pipeline/pull/730). + +- Build tasks streamlining and avoidance of file copies (#678 fixed by [#710](https://github.com/opendevstack/ods-pipeline/pull/710)). This is an incompatible change. Build tasks were adjusted to (mostly) no longer copy build files in a dedicated location. Instead one should adjust the Dockerfile (or other downstream tasks) to directly consume the build outputs from their natural locations. In addition build task skipping now supports parameter `build-extra-inputs`. The package-image task `dockerfile` and `docker-dir` parameters have been changed to assume that the docker context and file are at the repository root. See the PR for further information and the issue for more context. Note that these changes affect the extracted tasks, as described above. ## [0.13.2] - 2023-07-18 diff --git a/Makefile b/Makefile index 22b2af66..271cb729 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ lint-go: ## Run golangci-lint. .PHONY: lint-go lint-shell: ## Run shellcheck. - shellcheck scripts/*.sh build/package/scripts/* deploy/*.sh + shellcheck scripts/*.sh build/images/scripts/* deploy/*.sh .PHONY: lint-shell docs: ## Render documentation for tasks. diff --git a/deploy/basic-auth-secret.yaml.tmpl b/deploy/basic-auth-secret.yaml.tmpl deleted file mode 100644 index 9f2b9240..00000000 --- a/deploy/basic-auth-secret.yaml.tmpl +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: '{{name}}' - labels: - app.kubernetes.io/name: ods-pipeline -stringData: - password: '{{password}}' - username: '{{username}}' -type: kubernetes.io/basic-auth diff --git a/deploy/chart/Chart.yaml b/deploy/chart/Chart.yaml index e8197bba..b8bb298d 100644 --- a/deploy/chart/Chart.yaml +++ b/deploy/chart/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.13.2 +version: 0.14.0-preview.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.13.2" +appVersion: "0.14.0-preview.1" diff --git a/deploy/install.sh b/deploy/install.sh index 7137c993..dc70121d 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -1,35 +1,55 @@ #!/usr/bin/env bash set -ue -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -VERBOSE="false" -DRY_RUN="false" -DIFF="true" -NAMESPACE="" -RELEASE_NAME="ods-pipeline" -SERVICEACCOUNT="pipeline" -VALUES_FILE="values.yaml" -CHART_DIR="./chart" +verbose=false +dry_run=false +use_local_chart=false +diff="true" +namespace="" +release_name="ods-pipeline" +serviceaccount="pipeline" +values_file="values.yaml" +chart_dir="./chart" +chart_version="0.14.0-preview.1" # Secrets -AUTH_SEPARATOR=":" -BITBUCKET_AUTH="" -BITBUCKET_WEBHOOK_SECRET="" -NEXUS_AUTH="" -PRIVATE_CERT="" +auth_separator=":" +bitbucket_auth="" +bitbucket_webhook_secret="" +nexus_auth="" +private_cert="" +# Templates +basicAuthSecretTemplate="apiVersion: v1 +kind: Secret +metadata: + name: '{{name}}' + labels: + app.kubernetes.io/name: ods-pipeline +stringData: + password: '{{password}}' + username: '{{username}}' +type: kubernetes.io/basic-auth" +opaqueSecretTemplate="apiVersion: v1 +kind: Secret +metadata: + name: '{{name}}' + labels: + app.kubernetes.io/name: ods-pipeline +stringData: + secret: '{{password}}' +type: Opaque" # Check prerequisites. -KUBECTL_BIN="" +kubectl_bin="" if command -v oc &> /dev/null; then - KUBECTL_BIN="oc" + kubectl_bin="oc" elif command -v kubectl &> /dev/null; then - KUBECTL_BIN="kubectl" + kubectl_bin="kubectl" else echo "ERROR: Neither oc nor kubectl in \$PATH"; exit 1 fi -HELM_BIN="" +helm_bin="" if command -v helm &> /dev/null; then - HELM_BIN="helm" + helm_bin="helm" else echo "ERROR: helm is not in \$PATH"; exit 1 fi @@ -39,15 +59,17 @@ function usage { printf "\t-h|--help\t\t\tPrints this usage information.\n" printf "\t-v|--verbose\t\t\tTurn on verbose output.\n" printf "\t-n|--namespace\t\t\tK8s namespace to target.\n" - printf "\t-f|--values\t\t\tValues file to supply to Helm (defaults to '%s'). Multiple files can be specified comma-separated.\n" "$VALUES_FILE" - printf "\t-s|--serviceaccount\t\tServiceaccount to use (defaults to '%s').\n" "$SERVICEACCOUNT" + printf "\t--local-chart\t\t\tUse local chart instead of remote, versioned chart.\n" + printf "\t-f|--values\t\t\tValues file to supply to Helm (defaults to '%s'). Multiple files can be specified comma-separated.\n" "$values_file" + printf "\t-s|--serviceaccount\t\tServiceaccount to use (defaults to '%s').\n" "$serviceaccount" printf "\t--no-diff\t\t\tDo not run Helm diff before running Helm upgrade.\n" printf "\t--dry-run\t\t\tDo not apply any changes, instead just print what the script would do.\n" - printf "\t--auth-separator\t\tCharacter to use as a separator for basic auth flags (defaults to '%s')\n" "$AUTH_SEPARATOR" + printf "\t--auth-separator\t\tCharacter to use as a separator for basic auth flags (defaults to '%s')\n" "$auth_separator" printf "\t--bitbucket-auth\t\tAccess token of a Bitbucket user (if not given, script will prompt for this).\n" printf "\t--bitbucket-webhook-secret\tSecret to protect webhook endpoint with (if not given, script will generate this).\n" - printf "\t--nexus-auth\t\t\tUsername and password (separated by '%s') of a Nexus user (if not given, script will prompt for this).\n" "$AUTH_SEPARATOR" + printf "\t--nexus-auth\t\t\tUsername and password (separated by '%s') of a Nexus user (if not given, script will prompt for this).\n" "$auth_separator" printf "\t--private-cert\t\t\tHost from which to download private certificate (if not given, script will skip this).\n" + printf "\t--chart-version\t\t\tOverwrite chart version (defaults to '%s').\n" "$chart_version" printf "\nExample:\n\n" printf "\t%s \ \ \n\t\t--namespace foo \ \ @@ -61,52 +83,55 @@ while [ "$#" -gt 0 ]; do -h|--help) shift; usage; exit 0;; - -v|--verbose) VERBOSE="true";; + -v|--verbose) verbose=true;; - -n|--namespace) NAMESPACE="$2"; shift;; - -n=*|--namespace=*) NAMESPACE="${1#*=}";; + --no-diff) diff=false;; - -f|--values) VALUES_FILE="$2"; shift;; - -f=*|--values=*) VALUES_FILE="${1#*=}";; + --dry-run) dry_run=true;; - -s|--serviceaccount) SERVICEACCOUNT="$2"; shift;; - -s=*|--serviceaccount=*) SERVICEACCOUNT="${1#*=}";; + --local-chart) use_local_chart=true;; - --no-diff) DIFF="false";; + -n|--namespace) namespace="$2"; shift;; + -n=*|--namespace=*) namespace="${1#*=}";; - --dry-run) DRY_RUN="true";; + -f|--values) values_file="$2"; shift;; + -f=*|--values=*) values_file="${1#*=}";; - --auth-separator) AUTH_SEPARATOR="$2"; shift;; - --auth-separator=*) AUTH_SEPARATOR="${1#*=}";; + -s|--serviceaccount) serviceaccount="$2"; shift;; + -s=*|--serviceaccount=*) serviceaccount="${1#*=}";; - --bitbucket-auth) BITBUCKET_AUTH="$2"; shift;; - --bitbucket-auth=*) BITBUCKET_AUTH="${1#*=}";; + --auth-separator) auth_separator="$2"; shift;; + --auth-separator=*) auth_separator="${1#*=}";; - --bitbucket-webhook-secret) BITBUCKET_WEBHOOK_SECRET="$2"; shift;; - --bitbucket-webhook-secret=*) BITBUCKET_WEBHOOK_SECRET="${1#*=}";; + --bitbucket-auth) bitbucket_auth="$2"; shift;; + --bitbucket-auth=*) bitbucket_auth="${1#*=}";; - --nexus-auth) NEXUS_AUTH="$2"; shift;; - --nexus-auth=*) NEXUS_AUTH="${1#*=}";; + --bitbucket-webhook-secret) bitbucket_webhook_secret="$2"; shift;; + --bitbucket-webhook-secret=*) bitbucket_webhook_secret="${1#*=}";; - --private-cert) PRIVATE_CERT="$2"; shift;; - --private-cert=*) PRIVATE_CERT="${1#*=}";; + --nexus-auth) nexus_auth="$2"; shift;; + --nexus-auth=*) nexus_auth="${1#*=}";; + + --private-cert) private_cert="$2"; shift;; + --private-cert=*) private_cert="${1#*=}";; + + --chart-version) chart_version="$2"; shift;; + --chart-version=*) chart_version="${1#*=}";; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done -cd "${SCRIPT_DIR}" - -VALUES_FILES=$(echo "$VALUES_FILE" | tr "," "\n") -VALUES_ARGS=() -for valueFile in ${VALUES_FILES}; do - VALUES_ARGS+=(--values="${valueFile}") +values_fileS=$(echo "$values_file" | tr "," "\n") +values_args=() +for valueFile in ${values_fileS}; do + values_args+=(--values="${valueFile}") done -if [ "${VERBOSE}" == "true" ]; then +if [ "${verbose}" = true ]; then set -x fi -if [ -z "${NAMESPACE}" ]; then +if [ -z "${namespace}" ]; then echo "--namespace is required" exit 1 fi @@ -120,7 +145,7 @@ kubectlApplySecret () { # To avoid forward slashes messing up sed, escape forward slashes first. # See https://tldp.org/LDP/abs/html/string-manipulation.html. # shellcheck disable=SC2002 - cat "${secretTemplate}" | sed "s/{{name}}/${secretName}/" | sed "s/{{username}}/${username//\//\\/}/" | sed "s/{{password}}/${password//\//\\/}/" | "${KUBECTL_BIN}" -n "${NAMESPACE}" apply -f - + echo "${secretTemplate}" | sed "s/{{name}}/${secretName}/" | sed "s/{{username}}/${username//\//\\/}/" | sed "s/{{password}}/${password//\//\\/}/" | "${kubectl_bin}" -n "${namespace}" apply -f - } installSecret () { @@ -131,14 +156,14 @@ installSecret () { local passwordPrompt="$5" # Split flag value on first occurence of auth separator. - local authUser="${flagValue%%"${AUTH_SEPARATOR}"*}" - local authPassword="${flagValue#*"${AUTH_SEPARATOR}"}" + local authUser="${flagValue%%"${auth_separator}"*}" + local authPassword="${flagValue#*"${auth_separator}"}" # If the secret exists and the flag is present, update the secret. - if "${KUBECTL_BIN}" -n "${NAMESPACE}" get "secret/${secretName}" &> /dev/null; then + if "${kubectl_bin}" -n "${namespace}" get "secret/${secretName}" &> /dev/null; then # In case the secret was previously managed by Helm, we want to instruct Helm # to keep the resource during helm upgrade. - "${KUBECTL_BIN}" -n "${NAMESPACE}" annotate --overwrite secret "${secretName}" "helm.sh/resource-policy=keep" + "${kubectl_bin}" -n "${namespace}" annotate --overwrite secret "${secretName}" "helm.sh/resource-policy=keep" if [ -n "${flagValue}" ]; then echo "Updating secret ${secretName} ..." kubectlApplySecret "${secretName}" "${secretTemplate}" "${authUser}" "${authPassword}" @@ -181,104 +206,112 @@ installTLSSecret () { openssl s_client -showcerts -connect "${privateCert}" "${certFile}" fi - if "${KUBECTL_BIN}" -n "${NAMESPACE}" get "secret/${secretName}" &> /dev/null; then + if "${kubectl_bin}" -n "${namespace}" get "secret/${secretName}" &> /dev/null; then echo "Re-creating secret ${secretName} ..." - "${KUBECTL_BIN}" -n "${NAMESPACE}" delete secret "${secretName}" + "${kubectl_bin}" -n "${namespace}" delete secret "${secretName}" else echo "Creating secret ${secretName} ..." fi - "${KUBECTL_BIN}" -n "${NAMESPACE}" create secret generic "${secretName}" \ + "${kubectl_bin}" -n "${namespace}" create secret generic "${secretName}" \ --from-file=tls.crt="${certFile}" rm private-cert.pem.tmp &>/dev/null || true fi } # Manage serviceaccount ... -if "${KUBECTL_BIN}" -n "${NAMESPACE}" get serviceaccount/"${SERVICEACCOUNT}" &> /dev/null; then +if "${kubectl_bin}" -n "${namespace}" get serviceaccount/"${serviceaccount}" &> /dev/null; then echo "Serviceaccount exists already ..." else echo "Creating serviceaccount ..." - if [ "${DRY_RUN}" == "true" ]; then + if [ "${dry_run}" = true ]; then echo "(skipping in dry-run)" else - "${KUBECTL_BIN}" -n "${NAMESPACE}" create serviceaccount "${SERVICEACCOUNT}" + "${kubectl_bin}" -n "${namespace}" create serviceaccount "${serviceaccount}" - "${KUBECTL_BIN}" -n "${NAMESPACE}" \ - create rolebinding "${SERVICEACCOUNT}-edit" \ + "${kubectl_bin}" -n "${namespace}" \ + create rolebinding "${serviceaccount}-edit" \ --clusterrole edit \ - --serviceaccount "${NAMESPACE}:${SERVICEACCOUNT}" + --serviceaccount "${namespace}:${serviceaccount}" fi fi echo "Installing secrets ..." -if [ "${DRY_RUN}" == "true" ]; then +if [ "${dry_run}" = true ]; then echo "(skipping in dry-run)" else # Bitbucket username is not required as PAT alone is enough. installSecret "ods-bitbucket-auth" \ - "basic-auth-secret.yaml.tmpl" \ - "${BITBUCKET_AUTH}" \ + "${basicAuthSecretTemplate}" \ + "${bitbucket_auth}" \ "Please enter the username of Bitbucket user with write permission." \ "Please enter a personal access token of this Bitbucket user (input will be hidden):" # Webhook secret is a special case, as we do not want the user to set it. # No prompts -> password will be auto-generated if not given. installSecret "ods-bitbucket-webhook" \ - "opaque-secret.yaml.tmpl" \ - "${BITBUCKET_WEBHOOK_SECRET}" \ + "${opaqueSecretTemplate}" \ + "${bitbucket_webhook_secret}" \ "" "" installSecret "ods-nexus-auth" \ - "basic-auth-secret.yaml.tmpl" \ - "${NEXUS_AUTH}" \ + "${basicAuthSecretTemplate}" \ + "${nexus_auth}" \ "Please enter the username of a Nexus user with write permission:" \ "Please enter the password of this Nexus user (input will be hidden):" - installTLSSecret "ods-private-cert" "${PRIVATE_CERT}" + installTLSSecret "ods-private-cert" "${private_cert}" +fi + +chart_location="" +if [ "${use_local_chart}" = true ]; then + chart_name="ods-pipeline" + chart_location="https://github.com/opendevstack/ods-pipeline/releases/download/${chart_name}-${chart_version}/${chart_name}-${chart_version}.tgz" +else + chart_location="${chart_dir}" fi -echo "Installing Helm release ${RELEASE_NAME} ..." -if [ "${DIFF}" == "true" ]; then - if "${HELM_BIN}" -n "${NAMESPACE}" \ +echo "Installing Helm release ${release_name} from ${chart_location} ..." +if [ "${diff}" = true ]; then + if "${helm_bin}" -n "${namespace}" \ diff upgrade --install --detailed-exitcode --three-way-merge --normalize-manifests \ - "${VALUES_ARGS[@]}" \ - ${RELEASE_NAME} ${CHART_DIR}; then + "${values_args[@]}" \ + "${release_name}" "${chart_location}"; then echo "Helm release already up-to-date." else - if [ "${DRY_RUN}" == "true" ]; then + if [ "${dry_run}" = true ]; then echo "(skipping in dry-run)" else - "${HELM_BIN}" -n "${NAMESPACE}" \ + "${helm_bin}" -n "${namespace}" \ upgrade --install \ - "${VALUES_ARGS[@]}" \ - ${RELEASE_NAME} ${CHART_DIR} + "${values_args[@]}" \ + "${release_name}" "${chart_location}" fi fi else - if [ "${DRY_RUN}" == "true" ]; then + if [ "${dry_run}" = true ]; then echo "(skipping in dry-run)" else - "${HELM_BIN}" -n "${NAMESPACE}" \ + "${helm_bin}" -n "${namespace}" \ upgrade --install \ - "${VALUES_ARGS[@]}" \ - ${RELEASE_NAME} ${CHART_DIR} + "${values_args[@]}" \ + "${release_name}" "${chart_location}" fi fi echo "Adding Tekton annotation to ods-bitbucket-auth secret ..." -if [ "${DRY_RUN}" == "true" ]; then +if [ "${dry_run}" = true ]; then echo "(skipping in dry-run)" else - bitbucketUrl=$("${KUBECTL_BIN}" -n "${NAMESPACE}" get cm/ods-bitbucket -ojsonpath='{.data.url}') - "${KUBECTL_BIN}" -n "${NAMESPACE}" annotate --overwrite secret ods-bitbucket-auth "tekton.dev/git-0=${bitbucketUrl}" + bitbucketUrl=$("${kubectl_bin}" -n "${namespace}" get cm/ods-bitbucket -ojsonpath='{.data.url}') + "${kubectl_bin}" -n "${namespace}" annotate --overwrite secret ods-bitbucket-auth "tekton.dev/git-0=${bitbucketUrl}" fi -echo "Adding ods-bitbucket-auth secret to ${SERVICEACCOUNT} serviceaccount ..." -if [ "${DRY_RUN}" == "true" ]; then +echo "Adding ods-bitbucket-auth secret to ${serviceaccount} serviceaccount ..." +if [ "${dry_run}" = true ]; then echo "(skipping in dry-run)" else - "${KUBECTL_BIN}" -n "${NAMESPACE}" \ - patch sa "${SERVICEACCOUNT}" \ + "${kubectl_bin}" -n "${namespace}" \ + patch sa "${serviceaccount}" \ --type json \ -p '[{"op": "add", "path": "/secrets", "value":[{"name": "ods-bitbucket-auth"}]}]' fi diff --git a/deploy/opaque-secret.yaml.tmpl b/deploy/opaque-secret.yaml.tmpl deleted file mode 100644 index 554b26fd..00000000 --- a/deploy/opaque-secret.yaml.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: '{{name}}' - labels: - app.kubernetes.io/name: ods-pipeline -stringData: - secret: '{{password}}' -type: Opaque diff --git a/docs/installation.adoc b/docs/installation.adoc index 4d9ab43a..959f1b1e 100644 --- a/docs/installation.adoc +++ b/docs/installation.adoc @@ -5,144 +5,89 @@ This guide will show how to install ODS Pipeline in an existing ODS project. It An ODS Pipeline installation consists of the following resources: -* `Task` resources -* `ConfigMap` and `Secret` resources, e.g. holding credentials of centrally installed tools such as Nexus and SonarQube * A pipeline manager, which is creating pipeline runs in response to Bitbucket webhook requests +* A start and finish task which will get injected into every pipeline run +* `ConfigMap` and `Secret` resources, e.g. holding credentials of centrally installed tools such as Nexus and Bitbucket + == Prerequisites You'll need: -* A namespace in an OpenShift cluster (such as `foo-cd` from an existing ODS project) and a project in Bitbucket (such as `FOO`). +* A namespace in an OpenShift/Kubernetes cluster (such as `foo-cd` from an existing ODS project) and a project in Bitbucket (such as `FOO`). * `git`, link:https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html[`oc`] (or link:https://kubernetes.io/docs/reference/kubectl/[`kubectl`]) and link:https://helm.sh[`helm`] installed locally. The plugin link:https://github.com/databus23/helm-diff[`helm-diff`] is optional but recommended. == Installation Instructions -The installation procedure consists of two steps: - -1. Creating a Git repository defining the configuration of the ODS Pipeline installation -2. Using Helm to install from this Git repository +ODS Pipeline is packaged as a Helm chart. The installation procedure consists of three quick steps: -=== Creating a Git repository +1. Configuring the chart values +2. Running the install script (which will deploy the Helm chart) +3. Exposing a route to the pipeline manager -Create a new repository in Bitbucket, e.g. `foo-cd`. The name can be anything, but since the repository will define the K8s resources of namespace `foo-cd` in code, it makes sense to mirror the namespace name. Clone the repository locally and make an initial commit, e.g. by adding a readme file. +=== Step 1: Configuring the chart values -IMPORTANT: The following commands will fail in an empty Git repository, so make sure you have made at least one commit in this repository. - -Now use `git subtree` to get the required source files. The following commands may look a bit complicated, but in a nutshell, they are simply adding one folder (`deploy/`) from the `opendevstack/ods-pipeline` repository at the given revision (e.g. `master`) into your new local repository at the path `deploy`. The benefit of this approach is that it'll make updating ODS Pipeline simple. +Download the template and fill in the values according to the comments in that file. [source] ---- -pipelineGitRef=v0.13.2 # Pick the version you want to install - -git fetch --depth=1 https://github.com/opendevstack/ods-pipeline.git $pipelineGitRef:ods-pipeline-$pipelineGitRef && \ -git checkout ods-pipeline-$pipelineGitRef && \ -git subtree split --prefix=deploy -b subtree-split-branch-$pipelineGitRef && \ -git checkout - && \ -git subtree add --squash --prefix=deploy subtree-split-branch-$pipelineGitRef +curl -fsSL https://raw.githubusercontent.com/opendevstack/ods-pipeline/v0.14.0-preview.1/deploy/values.yaml.tmpl -o values.yaml ---- -Once this is done, change to the new folder `deploy` to configure the values of the Helm Chart. Run `cp values.yaml.tmpl values.yaml`, then edit `values.yaml` as described in the comments in that file. Commit and push before proceeding to the next step. +TIP: It is recommended to keep this file around after the installation so that it can be reused when updating ODS Pipeline to future versions. -=== Using Helm to install from the Git repository +=== Step 2: Running the install script -==== Option 1: With external API access - -If you have access to the OpenShift API from your local machine, you can simply login to the OpenShift cluster, then install ODS Pipeline by running: +Login to the OpenShift cluster in your terminal, then run: [source] ---- -./install.sh -n +curl -fsSL https://raw.githubusercontent.com/opendevstack/ods-pipeline/v0.14.0-preview.1/deploy/install.sh | bash -s -- -n= ---- -The script will interactively ask for credentials (such as Bitbucket access token) and will create corresponding K8s secrets. If you prefer to pass these secrets via flags, see `./install.sh --help` for all options. - -IMPORTANT: If tasks need to trust a private certificate, pass `--private-cert `. This will create a K8s secret containing the certificate from the specified host, which will then be mounted in pods during task runs. +The script will interactively ask for credentials (such as Bitbucket access token) and will create corresponding K8s secrets. If you prefer to pass these secrets via flags, use `--help` to see all options. -TIP: You may pass `--dry-run` to review what `install.sh` will do before actually running the script. +IMPORTANT: If tasks need to trust a private certificate, pass `--private-cert=`. This will create a K8s secret containing the certificate from the specified host, which will then be mounted in pods during task runs. -After you ran the install script, continue with the <> section. +TIP: If you want to review the changes first before applying them, supply `--dry-run`. -==== Option 2: Without external API access +TIP: If you do not have access to the OpenShift API from your local machine, you can use the https://docs.openshift.com/container-platform/latest/web_console/odc-about-web-terminal.html[OpenShift Web Terminal]. Open a web terminal in the target namespace and make sure the `values.yaml` file is present in the working directory there. Then run the installation script as described above. Note that you must either install the `helm-diff` plugin using `helm plugin install https://github.com/databus23/helm-diff --version "v3.3.2"` beforehand or supply `--no-diff` when running the install script. -If you do not have access to the OpenShift API from your local machine, you can use the https://docs.openshift.com/container-platform/latest/web_console/odc-about-web-terminal.html[OpenShift Web Terminal] to install ODS Pipeline. Open a web terminal in your `*-cd` namespace, then run: - -[source] ----- -curl -L https://raw.githubusercontent.com/opendevstack/ods-pipeline/master/scripts/web-terminal-install.sh | bash ----- - -This will install all prerequisites automatically. Then you can clone the repository and run `./install.sh -n ` in the terminal. - -`./install.sh` will interactively ask for credentials (such as Bitbucket access token) and will create corresponding K8s secrets. If you prefer to pass these secrets via flags, see `./install.sh --help` for all options. - -After you ran the install script, continue with the <> section. - -==== Finishing the installation +=== Step 3: Exposing a route to the pipeline manager Create an HTTPS route to expose the `ods-pipeline` service. You'll need the exposed URL (together with the webhook secret that is stored in the `ods-bitbucket-webhook` K8s secret) when you create webhooks in Bitbucket repositories later. +Done, now you are ready to link:add-to-repository.adoc[enable your repositories to use ODS pipeline]! + IMPORTANT: The `pipeline` serviceaccount needs `admin` permissions in the Kubernetes namespaces it deploys to (e.g. `foo-dev` and `foo-test`). You must create rolebindings for this manually. CAUTION: An important feature of ODS Pipeline is to retain pipeline run artifacts in Nexus and re-use them future pipeline runs (e.g. to promote built container images to another environment). For this purpose, you should create a few `raw` repositories in Nexus. These repositories should not allow re-deployment of artifacts. For example, you might want to have `ods-pipeline-dev`, `ods-pipeline-qa` and `ods-pipeline-prod` repositories, each with a different cleanup policy as fitting your needs. You can then use these repositories from your pipeline to store artifacts and enforce a progression of artifacts from DEV > QA > PROD. -Now you are ready to link:add-to-repository.adoc[enable your repositories to use ODS pipeline]! == Update Instructions -The update procedure consists of two steps: +The update procedure consists of two quick steps: -1. Updating the Git repository defining the configuration of the ODS Pipeline installation -2. Using Helm to install from the Git repository +1. Updating the chart values if required +2. Running the install script (which will deploy the Helm chart) -=== Updating the Git repository +=== Step 1: Updating the chart values -You may fetch updates (e.g. new versions) of `ods-pipeline` like this: +Ensure that the `values.yaml` file you used during installation is located in the working directory. Then check if any new values have been introduced in link:https://raw.githubusercontent.com/opendevstack/ods-pipeline/v0.14.0-preview.1/deploy/values.yaml.tmpl[`values.yaml.tmpl`] and update `values.yaml` accordingly. -[source] ----- -pipelineGitRef=v0.13.2 # Pick the version you want to update to - -git branch -D ods-pipeline-$pipelineGitRef subtree-split-branch-$pipelineGitRef || true && \ -git fetch --depth=1 https://github.com/opendevstack/ods-pipeline.git $pipelineGitRef:ods-pipeline-$pipelineGitRef && \ -git checkout ods-pipeline-$pipelineGitRef && \ -git subtree split --prefix=deploy -b subtree-split-branch-$pipelineGitRef && \ -git checkout - && \ -git subtree merge --prefix=deploy subtree-split-branch-$pipelineGitRef --squash ----- - -Afterwards, check if any new values have been introduced in `values.yaml.tmpl` and update `values.yaml` accordingly. Commit and push the result. +TIP: If you cannot find the `values.yaml` file from the installation, create it again using the values found in the current Helm installation. -=== Using Helm to install from the Git repository +=== Step 2: Running the install script -==== Option 1: With external API access - -If you have access to the OpenShift API from your local machine, you can simply login to the OpenShift cluster in your terminal, then update the ODS Pipeline installation by running: +Login to the OpenShift cluster in your terminal, then run: [source] ---- -./install.sh -n +curl -fsSL https://raw.githubusercontent.com/opendevstack/ods-pipeline/v0.14.0-preview.1/deploy/install.sh | bash -s -- -n= ---- -TIP: You may also use `--dry-run` to see the changes first. - -After you ran the script, continue with the <> section. - -TIP: By default, the credentials stored in the K8s secrets will not be updated. If you want to make a change, pass any new values as flags to `install.sh` (see `./install.sh --help` for all options) or update the secrets manually. - -==== Option 2: Without external API access - -If you do not have access to the OpenShift API from your local machine, you can use the https://docs.openshift.com/container-platform/latest/web_console/odc-about-web-terminal.html[OpenShift Web Terminal] to install ODS Pipeline. Open a web terminal in your `*-cd` namespace, then run: - -[source] ----- -curl -L https://raw.githubusercontent.com/opendevstack/ods-pipeline/master/scripts/web-terminal-install.sh | bash ----- - -This will install all prerequisites automatically and update your ODS Pipeline installation to the latest state of your Git repository. - -TIP: The credentials stored in the K8s secrets will not be updated. If you need to change those, update them manually. +TIP: If you want to review the changes first before applying them, supply `--dry-run`. -==== Finishing the update +TIP: By default, the credentials stored in the K8s secrets will not be updated. If you want to make a change, pass any new values as flags to the install script (supply `--help` to see all options) or update the secrets manually. -Once the resources in your namespace are updated, you likely have to update the `ods.yaml` files in your repository to point to the new tasks, e.g. changing `ods-build-go-v0-12-0` to `ods-build-go-v0-13-2`. +TIP: If you do not have access to the OpenShift API from your local machine, you can use the https://docs.openshift.com/container-platform/latest/web_console/odc-about-web-terminal.html[OpenShift Web Terminal]. Open a web terminal in the target namespace and make sure the `values.yaml` file is present in the working directory there. Then run the installation script as described above. Note that you must either install the `helm-diff` plugin using `helm plugin install https://github.com/databus23/helm-diff --version "v3.3.2"` beforehand or supply `--no-diff` when running the install script. diff --git a/scripts/install-inside-kind.sh b/scripts/install-inside-kind.sh index 308b46ea..1e206140 100755 --- a/scripts/install-inside-kind.sh +++ b/scripts/install-inside-kind.sh @@ -46,6 +46,7 @@ fi cd "${kind_deploy_path}" bash ./install.sh \ + --local-chart \ --bitbucket-auth "${bitbucket_auth}" \ --nexus-auth "${nexus_auth}" \ -f "${values_arg}" "$@" diff --git a/scripts/web-terminal-install.sh b/scripts/web-terminal-install.sh deleted file mode 100755 index dbd0bac5..00000000 --- a/scripts/web-terminal-install.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -set -eu - -HELM_PLUGIN_DIFF_VERSION=3.3.2 -REPOSITORY="" -NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) - -while [ "$#" -gt 0 ]; do - # shellcheck disable=SC2034 - case $1 in - - -v|--verbose) VERBOSE="true";; - - -r|--repository) REPOSITORY="$2"; shift;; - -r=*|--repository=*) REPOSITORY="${1#*=}";; - - *) echo "Unknown parameter passed: $1"; exit 1;; -esac; shift; done - -echo "Extending PATH to user-writable location ..." -mkdir -p bin -export PATH=/home/user/bin:$PATH - -echo "Installing Helm plugins ..." -NO_DIFF_FLAG="" -if [ "$(helm plugin list | grep ^diff)" != "" ]; then - echo "Plugin helm-diff is already installed." -else - if command -v tar; then - helm plugin install https://github.com/databus23/helm-diff --version "v${HELM_PLUGIN_DIFF_VERSION}" - else - NO_DIFF_FLAG="--no-diff" - fi -fi - -echo "Cloning Git repository ..." -if oc -n "${NAMESPACE}" get secrets/ods-bitbucket-auth &> /dev/null; then - repoBase=$(oc -n "${NAMESPACE}" get configmaps/ods-bitbucket -o jsonpath='{.data.repoBase}') - authToken=$(oc -n "${NAMESPACE}" get secrets/ods-bitbucket-auth -o jsonpath='{.data.password}' | base64 --decode) - if [ -z "${REPOSITORY}" ]; then - REPOSITORY="${repoBase}/${NAMESPACE%-cd}/${NAMESPACE}.git" - fi - repoName="${REPOSITORY##*/}" - rm -rf "${repoName%.git}" || true - git clone -c http.extraHeader="Authorization: Bearer ${authToken}" "${REPOSITORY}" -else - echo 'No secret ods-bitbucket-auth found.' - echo 'Most likely, there is no ODS Pipeline installation yet.' - echo 'Clone the Git repository and run install.sh manually.' - exit 1 -fi - -echo "Installing ..." -repoName="${REPOSITORY##*/}" -cd "${repoName%.git}/deploy" -./install.sh -n "${NAMESPACE}" ${NO_DIFF_FLAG}