From b4f0afbe082678f388cc2dc3261b323bf43862a4 Mon Sep 17 00:00:00 2001 From: Nick Kubala Date: Thu, 14 May 2020 11:38:35 -0700 Subject: [PATCH] use kubectl's built-in kustomize when possible --- deploy/skaffold/Dockerfile | 2 +- deploy/skaffold/Dockerfile.deps | 10 +--- deploy/webhook/Dockerfile | 2 +- .../testdata/skaffold-in-cluster/Dockerfile | 2 +- pkg/skaffold/deploy/kubectl/cli.go | 5 ++ pkg/skaffold/deploy/kustomize.go | 39 ++++++++++--- pkg/skaffold/deploy/kustomize_test.go | 55 +++++++++++++------ 7 files changed, 78 insertions(+), 37 deletions(-) diff --git a/deploy/skaffold/Dockerfile b/deploy/skaffold/Dockerfile index dd0f271297e..8ea44ddefb1 100644 --- a/deploy/skaffold/Dockerfile +++ b/deploy/skaffold/Dockerfile @@ -13,7 +13,7 @@ # limitations under the License. # This base image has to be updated manually after running `make build_deps` -FROM gcr.io/k8s-skaffold/build_deps:d54e32f303a9bf5ce42978be0b04ae6cba235d37 as builder +FROM gcr.io/k8s-skaffold/build_deps:73e3714b2c30ecedb4df789ed285e9151e1ed832 as builder WORKDIR /skaffold COPY . . diff --git a/deploy/skaffold/Dockerfile.deps b/deploy/skaffold/Dockerfile.deps index f2233ffe799..0d574737012 100644 --- a/deploy/skaffold/Dockerfile.deps +++ b/deploy/skaffold/Dockerfile.deps @@ -14,7 +14,7 @@ # Download kubectl FROM alpine:3.10 as download-kubectl -ENV KUBECTL_VERSION v1.14.10 +ENV KUBECTL_VERSION v1.15.5 ENV KUBECTL_URL https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl RUN wget -O kubectl "${KUBECTL_URL}" RUN chmod +x kubectl @@ -26,13 +26,6 @@ ENV HELM_URL https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz RUN wget -O helm.tar.gz "${HELM_URL}" RUN tar -xvf helm.tar.gz --strip-components 1 -# Download kustomize -FROM alpine:3.10 as download-kustomize -ENV KUSTOMIZE_VERSION 3.5.4 -ENV KUSTOMIZE_URL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz -RUN wget -O kustomize.tar.gz "${KUSTOMIZE_URL}" -RUN tar -xvf kustomize.tar.gz - # Download kompose FROM alpine:3.10 as download-kompose ENV KOMPOSE_VERSION v1.21.0 @@ -78,7 +71,6 @@ RUN apt-get update && \ COPY --from=docker:18.09.6 /usr/local/bin/docker /usr/local/bin/ COPY --from=download-kubectl kubectl /usr/local/bin/ COPY --from=download-helm helm /usr/local/bin/ -COPY --from=download-kustomize kustomize /usr/local/bin/ COPY --from=download-kompose kompose /usr/local/bin/ COPY --from=download-container-structure-test container-structure-test /usr/local/bin/ COPY --from=download-bazel bazel /usr/local/bin/ diff --git a/deploy/webhook/Dockerfile b/deploy/webhook/Dockerfile index c97bf56d3f4..89b91883dc9 100644 --- a/deploy/webhook/Dockerfile +++ b/deploy/webhook/Dockerfile @@ -31,7 +31,7 @@ RUN wget -O- "${HUGO_URL}" | tar xz # Download kubectl FROM alpine:3.10 as download-kubectl -ENV KUBECTL_VERSION v1.12.0 +ENV KUBECTL_VERSION v1.15.5 ENV KUBECTL_URL https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl RUN wget -O kubectl "${KUBECTL_URL}" RUN chmod +x kubectl diff --git a/integration/testdata/skaffold-in-cluster/Dockerfile b/integration/testdata/skaffold-in-cluster/Dockerfile index 974ff0092fb..af0a229905a 100644 --- a/integration/testdata/skaffold-in-cluster/Dockerfile +++ b/integration/testdata/skaffold-in-cluster/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/gcp-runtimes/ubuntu_16_0_4 -ENV KUBECTL_VERSION v1.12.8 +ENV KUBECTL_VERSION v1.15.5 ENV KUBECTL_URL https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl RUN curl -O "${KUBECTL_URL}" RUN chmod +x kubectl diff --git a/pkg/skaffold/deploy/kubectl/cli.go b/pkg/skaffold/deploy/kubectl/cli.go index d13345acb77..0c4d4742f9a 100644 --- a/pkg/skaffold/deploy/kubectl/cli.go +++ b/pkg/skaffold/deploy/kubectl/cli.go @@ -73,6 +73,11 @@ func (c *CLI) Apply(ctx context.Context, out io.Writer, manifests ManifestList) return nil } +// Kustomize runs `kubectl kustomize` with the provided args +func (c *CLI) Kustomize(ctx context.Context, args []string) ([]byte, error) { + return c.RunOut(ctx, "kustomize", c.args(nil, args...)...) +} + // ReadManifests reads a list of manifests in yaml format. func (c *CLI) ReadManifests(ctx context.Context, manifests []string) (ManifestList, error) { var list []string diff --git a/pkg/skaffold/deploy/kustomize.go b/pkg/skaffold/deploy/kustomize.go index 05e628a0666..13ae673f372 100644 --- a/pkg/skaffold/deploy/kustomize.go +++ b/pkg/skaffold/deploy/kustomize.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/segmentio/textio" + "github.com/sirupsen/logrus" yaml "gopkg.in/yaml.v2" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build" @@ -88,18 +89,32 @@ type KustomizeDeployer struct { *latest.KustomizeDeploy kubectl deploy.CLI + useKubectl bool insecureRegistries map[string]bool BuildArgs []string } func NewKustomizeDeployer(runCtx *runcontext.RunContext) *KustomizeDeployer { + kubectl := deploy.CLI{ + CLI: kubectl.NewFromRunContext(runCtx), + Flags: runCtx.Cfg.Deploy.KustomizeDeploy.Flags, + ForceDeploy: runCtx.Opts.Force, + } + + // if user's kubectl version is >1.14, we can use the built-in kustomize command + var useKubectl bool + gt, err := kubectl.CompareVersionTo(context.Background(), 1, 14) + if err != nil { + logrus.Warnf("could not retrieve kubectl version: relying on standalone kustomize binary") + } + if gt == 1 { + useKubectl = true + } + return &KustomizeDeployer{ - KustomizeDeploy: runCtx.Cfg.Deploy.KustomizeDeploy, - kubectl: deploy.CLI{ - CLI: kubectl.NewFromRunContext(runCtx), - Flags: runCtx.Cfg.Deploy.KustomizeDeploy.Flags, - ForceDeploy: runCtx.Opts.Force, - }, + KustomizeDeploy: runCtx.Cfg.Deploy.KustomizeDeploy, + kubectl: kubectl, + useKubectl: useKubectl, insecureRegistries: runCtx.InsecureRegistries, BuildArgs: runCtx.Cfg.Deploy.KustomizeDeploy.BuildArgs, } @@ -339,8 +354,15 @@ func pathExistsLocally(filename string, workingDir string) (bool, os.FileMode) { func (k *KustomizeDeployer) readManifests(ctx context.Context) (deploy.ManifestList, error) { var manifests deploy.ManifestList for _, kustomizePath := range k.KustomizePaths { - cmd := exec.CommandContext(ctx, "kustomize", buildCommandArgs(k.BuildArgs, kustomizePath)...) - out, err := util.RunCmdOut(cmd) + var out []byte + var err error + if k.useKubectl { + out, err = k.kubectl.Kustomize(ctx, buildCommandArgs(k.BuildArgs, kustomizePath)) + } else { + cmd := exec.CommandContext(ctx, "kustomize", append([]string{"build"}, buildCommandArgs(k.BuildArgs, kustomizePath)...)...) + out, err = util.RunCmdOut(cmd) + } + if err != nil { return nil, fmt.Errorf("kustomize build: %w", err) } @@ -355,7 +377,6 @@ func (k *KustomizeDeployer) readManifests(ctx context.Context) (deploy.ManifestL func buildCommandArgs(buildArgs []string, kustomizePath string) []string { var args []string - args = append(args, "build") if len(buildArgs) > 0 { for _, v := range buildArgs { diff --git a/pkg/skaffold/deploy/kustomize_test.go b/pkg/skaffold/deploy/kustomize_test.go index 81e42703aa5..84d101258ba 100644 --- a/pkg/skaffold/deploy/kustomize_test.go +++ b/pkg/skaffold/deploy/kustomize_test.go @@ -87,6 +87,28 @@ func TestKustomizeDeploy(t *testing.T) { }, forceDeploy: true, }, + { + description: "built-in kubectl kustomize with newer version", + cfg: &latest.KustomizeDeploy{ + KustomizePaths: []string{"a", "b"}, + }, + commands: testutil. + CmdRunOut("kubectl version --client -ojson", kubectlVersion118). + AndRunOut("kubectl --context kubecontext --namespace testNamespace kustomize a", deploymentWebYAML). + AndRunOut("kubectl --context kubecontext --namespace testNamespace kustomize b", deploymentAppYAML). + AndRun("kubectl --context kubecontext --namespace testNamespace apply -f - --force --grace-period=0"), + builds: []build.Artifact{ + { + ImageName: "leeroy-web", + Tag: "leeroy-web:123", + }, + { + ImageName: "leeroy-app", + Tag: "leeroy-app:123", + }, + }, + forceDeploy: true, + }, } for _, test := range tests { testutil.Run(t, test.description, func(t *testutil.T) { @@ -131,7 +153,8 @@ func TestKustomizeCleanup(t *testing.T) { KustomizePaths: []string{tmpDir.Root()}, }, commands: testutil. - CmdRunOut("kustomize build "+tmpDir.Root(), deploymentWebYAML). + CmdRunOut("kubectl version --client -ojson", kubectlVersion112). + AndRunOut("kustomize build "+tmpDir.Root(), deploymentWebYAML). AndRun("kubectl --context kubecontext --namespace testNamespace delete --ignore-not-found=true -f -"), }, { @@ -140,7 +163,8 @@ func TestKustomizeCleanup(t *testing.T) { KustomizePaths: tmpDir.Paths("a", "b"), }, commands: testutil. - CmdRunOut("kustomize build "+tmpDir.Path("a"), deploymentWebYAML). + CmdRunOut("kubectl version --client -ojson", kubectlVersion112). + AndRunOut("kustomize build "+tmpDir.Path("a"), deploymentWebYAML). AndRunOut("kustomize build "+tmpDir.Path("b"), deploymentAppYAML). AndRun("kubectl --context kubecontext --namespace testNamespace delete --ignore-not-found=true -f -"), }, @@ -150,7 +174,8 @@ func TestKustomizeCleanup(t *testing.T) { KustomizePaths: []string{tmpDir.Root()}, }, commands: testutil. - CmdRunOut("kustomize build "+tmpDir.Root(), deploymentWebYAML). + CmdRunOut("kubectl version --client -ojson", kubectlVersion112). + AndRunOut("kustomize build "+tmpDir.Root(), deploymentWebYAML). AndRunErr("kubectl --context kubecontext --namespace testNamespace delete --ignore-not-found=true -f -", errors.New("BUG")), shouldErr: true, }, @@ -159,11 +184,9 @@ func TestKustomizeCleanup(t *testing.T) { cfg: &latest.KustomizeDeploy{ KustomizePaths: []string{tmpDir.Root()}, }, - commands: testutil.CmdRunOutErr( - "kustomize build "+tmpDir.Root(), - "", - errors.New("BUG"), - ), + commands: testutil. + CmdRunOut("kubectl version --client -ojson", kubectlVersion112). + AndRunOutErr("kustomize build "+tmpDir.Root(), "", errors.New("BUG")), shouldErr: true, }, } @@ -399,49 +422,49 @@ func TestKustomizeBuildCommandArgs(t *testing.T) { description: "no BuildArgs, empty KustomizePaths ", buildArgs: []string{}, kustomizePath: "", - expectedArgs: []string{"build"}, + expectedArgs: nil, }, { description: "One BuildArg, empty KustomizePaths", buildArgs: []string{"--foo"}, kustomizePath: "", - expectedArgs: []string{"build", "--foo"}, + expectedArgs: []string{"--foo"}, }, { description: "no BuildArgs, non-empty KustomizePaths", buildArgs: []string{}, kustomizePath: "foo", - expectedArgs: []string{"build", "foo"}, + expectedArgs: []string{"foo"}, }, { description: "One BuildArg, non-empty KustomizePaths", buildArgs: []string{"--foo"}, kustomizePath: "bar", - expectedArgs: []string{"build", "--foo", "bar"}, + expectedArgs: []string{"--foo", "bar"}, }, { description: "Multiple BuildArg, empty KustomizePaths", buildArgs: []string{"--foo", "--bar"}, kustomizePath: "", - expectedArgs: []string{"build", "--foo", "--bar"}, + expectedArgs: []string{"--foo", "--bar"}, }, { description: "Multiple BuildArg with spaces, empty KustomizePaths", buildArgs: []string{"--foo bar", "--baz"}, kustomizePath: "", - expectedArgs: []string{"build", "--foo", "bar", "--baz"}, + expectedArgs: []string{"--foo", "bar", "--baz"}, }, { description: "Multiple BuildArg with spaces, non-empty KustomizePaths", buildArgs: []string{"--foo bar", "--baz"}, kustomizePath: "barfoo", - expectedArgs: []string{"build", "--foo", "bar", "--baz", "barfoo"}, + expectedArgs: []string{"--foo", "bar", "--baz", "barfoo"}, }, { description: "Multiple BuildArg no spaces, non-empty KustomizePaths", buildArgs: []string{"--foo", "bar", "--baz"}, kustomizePath: "barfoo", - expectedArgs: []string{"build", "--foo", "bar", "--baz", "barfoo"}, + expectedArgs: []string{"--foo", "bar", "--baz", "barfoo"}, }, }