Add Helm/Kubernetes integration tests (#120)

* Add helm/kubectl integration test

Signed-off-by: Jacob LeGrone <git@jacob.work>

* Use Chart type from integration tests

Signed-off-by: Jacob LeGrone <git@jacob.work>

* Update .circleci/config.yml

Co-Authored-By: jlegrone <jlegrone@users.noreply.github.com>
Signed-off-by: Jacob LeGrone <git@jacob.work>

* Remove extraneous test chart files

Signed-off-by: Jacob LeGrone <git@jacob.work>

* Fix shellcheck lint

Signed-off-by: Jacob LeGrone <git@jacob.work>

* Remove example kind config from e2e tests

Signed-off-by: Jacob LeGrone <git@jacob.work>

* Remove breaking semver test

This wasn't super clear and was no longer working properly
after the switch to git-worktree.

Signed-off-by: Jacob LeGrone <git@jacob.work>

* Bump kind version

Signed-off-by: Jacob LeGrone <git@jacob.work>

.circleci/config.yml

@@ -10,6 +10,30 @@ jobs:
           command: |
             shellcheck -x build.sh
             shellcheck -x tag.sh
+            shellcheck -x e2e-kind.sh
+  test:
+    machine:
+      enabled: true
+    working_directory: /home/circleci/.go_workspace/src/github.com/helm/chart-testing
+    steps:
+      - run:
+          name: Install tools
+          command: |
+            mkdir /home/circleci/.go_workspace/bin
+            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+            curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
+            chmod +x ./kubectl
+            sudo mv ./kubectl /usr/local/bin/kubectl
+            curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash
+            curl -sSLo kind "https://github.com/kubernetes-sigs/kind/releases/download/0.2.0/kind-linux-amd64"
+            chmod +x kind
+            sudo mv kind /usr/local/bin/kind
+      - checkout
+      - run:
+          name: Test
+          command: |
+            dep ensure -v
+            ./e2e-kind.sh
   build:
     docker:
       - image: golang:1.11.5-alpine3.9
@@ -50,6 +74,7 @@ workflows:
   untagged-build:
     jobs:
       - lint
+      - test
       - build
   tagged-build:
     jobs:

e2e-kind.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+readonly CLUSTER_NAME=chart-testing
+readonly K8S_VERSION=v1.13.2
+
+create_kind_cluster() {
+    kind create cluster --name "$CLUSTER_NAME" --image "kindest/node:$K8S_VERSION" --wait 10s
+    KUBECONFIG="$(kind get kubeconfig-path --name=$CLUSTER_NAME)"
+    export KUBECONFIG
+
+    kubectl cluster-info || kubectl cluster-info dump
+    echo
+
+    echo -n 'Waiting for cluster to be ready...'
+    until ! grep --quiet 'NotReady' <(kubectl get nodes --no-headers); do
+        printf '.'
+        sleep 1
+    done
+
+    echo '✔︎'
+    echo
+
+    kubectl get nodes
+    echo
+
+    echo 'Cluster ready!'
+    echo
+}
+
+install_tiller() {
+    echo 'Installing Tiller...'
+    kubectl --namespace kube-system --output yaml create serviceaccount tiller --dry-run | kubectl apply -f -
+    kubectl create --output yaml clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller --dry-run | kubectl apply -f -
+    helm init --service-account tiller --upgrade --wait
+    echo
+}
+
+install_local-path-provisioner() {
+    # kind doesn't support Dynamic PVC provisioning yet, this is one way to get it working
+    # https://github.com/rancher/local-path-provisioner
+
+    # Remove default storage class. It will be recreated by local-path-provisioner
+    kubectl delete storageclass standard
+
+    echo 'Installing local-path-provisioner...'
+    kubectl apply -f examples/kind/test/local-path-provisioner.yaml
+    echo
+}
+
+test_e2e() {
+    go test -cover -race -tags=integration ./...
+    echo
+}
+
+cleanup() {
+    kind delete cluster --name "$CLUSTER_NAME"
+    echo 'Done!'
+}
+
+main() {
+    trap cleanup EXIT
+
+    create_kind_cluster
+    install_local-path-provisioner
+    install_tiller
+    test_e2e
+}
+
+main

pkg/chart/chart_test.go

@@ -16,6 +16,8 @@ package chart
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
 	"strings"
 	"testing"
 
@@ -29,15 +31,21 @@ import (
 type fakeGit struct{}
 
 func (g fakeGit) FileExistsOnBranch(file string, remote string, branch string) bool {
-	return true
+	_, err := os.Open(computePreviousRevisionPath(file))
+	fmt.Println(err)
+	return err == nil
 }
 
 func (g fakeGit) Show(file string, remote string, branch string) (string, error) {
-	return "", nil
+	b, err := ioutil.ReadFile(computePreviousRevisionPath(file))
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
 }
 
 func (g fakeGit) MergeBase(commit1 string, commit2 string) (string, error) {
-	return "", nil
+	return "HEAD", nil
 }
 
 func (g fakeGit) ListChangedFilesInDirs(commit string, dirs ...string) ([]string, error) {
@@ -142,11 +150,18 @@ func TestComputeChangedChartDirectories(t *testing.T) {
 
 func TestReadAllChartDirectories(t *testing.T) {
 	actual, err := ct.ReadAllChartDirectories()
-	expected := []string{"test_charts/foo", "test_charts/bar", "test_chart_at_root"}
+	expected := []string{
+		"test_charts/foo",
+		"test_charts/bar",
+		"test_charts/must-pass-upgrade-install",
+		"test_charts/mutating-deployment-selector",
+		"test_charts/mutating-sfs-volumeclaim",
+		"test_chart_at_root",
+	}
 	for _, chart := range actual {
 		assert.Contains(t, expected, chart)
 	}
-	assert.Len(t, actual, 3)
+	assert.Len(t, actual, 6)
 	assert.Nil(t, err)
 }
 

pkg/chart/integration_test.go

@@ -0,0 +1,149 @@
+// Copyright The Helm Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build integration
+
+package chart
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/helm/chart-testing/pkg/config"
+	"github.com/helm/chart-testing/pkg/exec"
+	"github.com/helm/chart-testing/pkg/tool"
+	"github.com/helm/chart-testing/pkg/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func newTestingHelmIntegration(cfg config.Configuration) Testing {
+	fakeMockLinter := new(fakeLinter)
+	procExec := exec.NewProcessExecutor(true)
+	extraArgs := strings.Fields(cfg.HelmExtraArgs)
+	return Testing{
+		config:           cfg,
+		directoryLister:  util.DirectoryLister{},
+		git:              fakeGit{},
+		chartUtils:       util.ChartUtils{},
+		accountValidator: fakeAccountValidator{},
+		linter:           fakeMockLinter,
+		helm:             tool.NewHelm(procExec, extraArgs),
+		kubectl:          tool.NewKubectl(procExec),
+	}
+}
+
+func TestInstallChart(t *testing.T) {
+	type testCase struct {
+		name     string
+		cfg      config.Configuration
+		chartDir string
+		output   TestResult
+	}
+
+	cases := []testCase{
+		{
+			"install only in custom namespace",
+			config.Configuration{
+				Debug:        true,
+				Namespace:    "default",
+				ReleaseLabel: "app.kubernetes.io/instance",
+			},
+			"test_charts/must-pass-upgrade-install",
+			TestResult{mustNewChart("test_charts/must-pass-upgrade-install"), nil},
+		},
+		{
+			"install only in random namespace",
+			config.Configuration{
+				Debug: true,
+			},
+			"test_charts/must-pass-upgrade-install",
+			TestResult{mustNewChart("test_charts/must-pass-upgrade-install"), nil},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			ct := newTestingHelmIntegration(tc.cfg)
+			result := ct.InstallChart(mustNewChart(tc.chartDir))
+
+			if result.Error != tc.output.Error {
+				if result.Error != nil && tc.output.Error != nil {
+					assert.Equal(t, tc.output.Error.Error(), result.Error.Error())
+				} else {
+					assert.Equal(t, tc.output.Error, result.Error)
+				}
+			}
+		})
+	}
+}
+
+func TestUpgradeChart(t *testing.T) {
+	type testCase struct {
+		name string
+		old  string
+		new  string
+		err  error
+	}
+
+	cfg := config.Configuration{
+		Debug:   true,
+		Upgrade: true,
+	}
+	ct := newTestingHelmIntegration(cfg)
+	processError := fmt.Errorf("Error waiting for process: exit status 1")
+
+	cases := []testCase{
+		{
+			"upgrade nginx",
+			"test_charts/must-pass-upgrade-install",
+			"test_charts/must-pass-upgrade-install",
+			nil,
+		},
+		{
+			"change immutable deployment.spec.selector field",
+			"test_charts/mutating-deployment-selector",
+			"test_charts/mutating-deployment-selector",
+			processError,
+		},
+		{
+			"change immutable statefulset.spec.volumeClaimTemplates field",
+			"test_charts/mutating-sfs-volumeclaim",
+			"test_charts/mutating-sfs-volumeclaim",
+			processError,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := ct.doUpgrade(mustNewChart(tc.old), mustNewChart(tc.new), true)
+
+			if err != tc.err {
+				if err != nil && tc.err != nil {
+					assert.Equal(t, tc.err.Error(), err.Error())
+				} else {
+					assert.Equal(t, tc.err, err)
+				}
+			}
+		})
+	}
+}
+
+func mustNewChart(chartPath string) *Chart {
+	c, err := NewChart(chartPath)
+	if err != nil {
+		panic(err)
+	}
+	return c
+}

pkg/chart/test_charts/must-pass-upgrade-install/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: nginx
+version: 0.2.0

pkg/chart/test_charts/must-pass-upgrade-install/README.md

@@ -0,0 +1,3 @@
+Chart generated with `helm create nginx`, and is intended to pass upgrades and installs.
+
+A fake "previous revision" can be found at `./ct_prev_revision/must-pass-upgrade-install`.

pkg/chart/test_charts/must-pass-upgrade-install/ci/test-values.yaml

@@ -0,0 +1 @@
+replicaCount: 2

pkg/chart/test_charts/must-pass-upgrade-install/templates/_helpers.tpl

@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "nginx.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "nginx.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "nginx.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}