Skip to content

Commit

Permalink
use kubectl's built-in kustomize when possible
Browse files Browse the repository at this point in the history
  • Loading branch information
nkubala committed May 15, 2020
1 parent 650e618 commit b4f0afb
Show file tree
Hide file tree
Showing 7 changed files with 78 additions and 37 deletions.
2 changes: 1 addition & 1 deletion deploy/skaffold/Dockerfile
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.

# This base image has to be updated manually after running `make build_deps`
FROM gcr.io/k8s-skaffold/build_deps:d54e32f303a9bf5ce42978be0b04ae6cba235d37 as builder
FROM gcr.io/k8s-skaffold/build_deps:73e3714b2c30ecedb4df789ed285e9151e1ed832 as builder
WORKDIR /skaffold
COPY . .

Expand Down
10 changes: 1 addition & 9 deletions deploy/skaffold/Dockerfile.deps
Expand Up @@ -14,7 +14,7 @@

# Download kubectl
FROM alpine:3.10 as download-kubectl
ENV KUBECTL_VERSION v1.14.10
ENV KUBECTL_VERSION v1.15.5
ENV KUBECTL_URL https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl
RUN wget -O kubectl "${KUBECTL_URL}"
RUN chmod +x kubectl
Expand All @@ -26,13 +26,6 @@ ENV HELM_URL https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz
RUN wget -O helm.tar.gz "${HELM_URL}"
RUN tar -xvf helm.tar.gz --strip-components 1

# Download kustomize
FROM alpine:3.10 as download-kustomize
ENV KUSTOMIZE_VERSION 3.5.4
ENV KUSTOMIZE_URL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz
RUN wget -O kustomize.tar.gz "${KUSTOMIZE_URL}"
RUN tar -xvf kustomize.tar.gz

# Download kompose
FROM alpine:3.10 as download-kompose
ENV KOMPOSE_VERSION v1.21.0
Expand Down Expand Up @@ -78,7 +71,6 @@ RUN apt-get update && \
COPY --from=docker:18.09.6 /usr/local/bin/docker /usr/local/bin/
COPY --from=download-kubectl kubectl /usr/local/bin/
COPY --from=download-helm helm /usr/local/bin/
COPY --from=download-kustomize kustomize /usr/local/bin/
COPY --from=download-kompose kompose /usr/local/bin/
COPY --from=download-container-structure-test container-structure-test /usr/local/bin/
COPY --from=download-bazel bazel /usr/local/bin/
Expand Down
2 changes: 1 addition & 1 deletion deploy/webhook/Dockerfile
Expand Up @@ -31,7 +31,7 @@ RUN wget -O- "${HUGO_URL}" | tar xz

# Download kubectl
FROM alpine:3.10 as download-kubectl
ENV KUBECTL_VERSION v1.12.0
ENV KUBECTL_VERSION v1.15.5
ENV KUBECTL_URL https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl
RUN wget -O kubectl "${KUBECTL_URL}"
RUN chmod +x kubectl
Expand Down
2 changes: 1 addition & 1 deletion integration/testdata/skaffold-in-cluster/Dockerfile
@@ -1,6 +1,6 @@
FROM gcr.io/gcp-runtimes/ubuntu_16_0_4

ENV KUBECTL_VERSION v1.12.8
ENV KUBECTL_VERSION v1.15.5
ENV KUBECTL_URL https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl
RUN curl -O "${KUBECTL_URL}"
RUN chmod +x kubectl
Expand Down
5 changes: 5 additions & 0 deletions pkg/skaffold/deploy/kubectl/cli.go
Expand Up @@ -73,6 +73,11 @@ func (c *CLI) Apply(ctx context.Context, out io.Writer, manifests ManifestList)
return nil
}

// Kustomize runs `kubectl kustomize` with the provided args
func (c *CLI) Kustomize(ctx context.Context, args []string) ([]byte, error) {
return c.RunOut(ctx, "kustomize", c.args(nil, args...)...)
}

// ReadManifests reads a list of manifests in yaml format.
func (c *CLI) ReadManifests(ctx context.Context, manifests []string) (ManifestList, error) {
var list []string
Expand Down
39 changes: 30 additions & 9 deletions pkg/skaffold/deploy/kustomize.go
Expand Up @@ -27,6 +27,7 @@ import (
"strings"

"github.com/segmentio/textio"
"github.com/sirupsen/logrus"
yaml "gopkg.in/yaml.v2"

"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
Expand Down Expand Up @@ -88,18 +89,32 @@ type KustomizeDeployer struct {
*latest.KustomizeDeploy

kubectl deploy.CLI
useKubectl bool
insecureRegistries map[string]bool
BuildArgs []string
}

func NewKustomizeDeployer(runCtx *runcontext.RunContext) *KustomizeDeployer {
kubectl := deploy.CLI{
CLI: kubectl.NewFromRunContext(runCtx),
Flags: runCtx.Cfg.Deploy.KustomizeDeploy.Flags,
ForceDeploy: runCtx.Opts.Force,
}

// if user's kubectl version is >1.14, we can use the built-in kustomize command
var useKubectl bool
gt, err := kubectl.CompareVersionTo(context.Background(), 1, 14)
if err != nil {
logrus.Warnf("could not retrieve kubectl version: relying on standalone kustomize binary")
}
if gt == 1 {
useKubectl = true
}

return &KustomizeDeployer{
KustomizeDeploy: runCtx.Cfg.Deploy.KustomizeDeploy,
kubectl: deploy.CLI{
CLI: kubectl.NewFromRunContext(runCtx),
Flags: runCtx.Cfg.Deploy.KustomizeDeploy.Flags,
ForceDeploy: runCtx.Opts.Force,
},
KustomizeDeploy: runCtx.Cfg.Deploy.KustomizeDeploy,
kubectl: kubectl,
useKubectl: useKubectl,
insecureRegistries: runCtx.InsecureRegistries,
BuildArgs: runCtx.Cfg.Deploy.KustomizeDeploy.BuildArgs,
}
Expand Down Expand Up @@ -339,8 +354,15 @@ func pathExistsLocally(filename string, workingDir string) (bool, os.FileMode) {
func (k *KustomizeDeployer) readManifests(ctx context.Context) (deploy.ManifestList, error) {
var manifests deploy.ManifestList
for _, kustomizePath := range k.KustomizePaths {
cmd := exec.CommandContext(ctx, "kustomize", buildCommandArgs(k.BuildArgs, kustomizePath)...)
out, err := util.RunCmdOut(cmd)
var out []byte
var err error
if k.useKubectl {
out, err = k.kubectl.Kustomize(ctx, buildCommandArgs(k.BuildArgs, kustomizePath))
} else {
cmd := exec.CommandContext(ctx, "kustomize", append([]string{"build"}, buildCommandArgs(k.BuildArgs, kustomizePath)...)...)
out, err = util.RunCmdOut(cmd)
}

if err != nil {
return nil, fmt.Errorf("kustomize build: %w", err)
}
Expand All @@ -355,7 +377,6 @@ func (k *KustomizeDeployer) readManifests(ctx context.Context) (deploy.ManifestL

func buildCommandArgs(buildArgs []string, kustomizePath string) []string {
var args []string
args = append(args, "build")

if len(buildArgs) > 0 {
for _, v := range buildArgs {
Expand Down
55 changes: 39 additions & 16 deletions pkg/skaffold/deploy/kustomize_test.go
Expand Up @@ -87,6 +87,28 @@ func TestKustomizeDeploy(t *testing.T) {
},
forceDeploy: true,
},
{
description: "built-in kubectl kustomize with newer version",
cfg: &latest.KustomizeDeploy{
KustomizePaths: []string{"a", "b"},
},
commands: testutil.
CmdRunOut("kubectl version --client -ojson", kubectlVersion118).
AndRunOut("kubectl --context kubecontext --namespace testNamespace kustomize a", deploymentWebYAML).
AndRunOut("kubectl --context kubecontext --namespace testNamespace kustomize b", deploymentAppYAML).
AndRun("kubectl --context kubecontext --namespace testNamespace apply -f - --force --grace-period=0"),
builds: []build.Artifact{
{
ImageName: "leeroy-web",
Tag: "leeroy-web:123",
},
{
ImageName: "leeroy-app",
Tag: "leeroy-app:123",
},
},
forceDeploy: true,
},
}
for _, test := range tests {
testutil.Run(t, test.description, func(t *testutil.T) {
Expand Down Expand Up @@ -131,7 +153,8 @@ func TestKustomizeCleanup(t *testing.T) {
KustomizePaths: []string{tmpDir.Root()},
},
commands: testutil.
CmdRunOut("kustomize build "+tmpDir.Root(), deploymentWebYAML).
CmdRunOut("kubectl version --client -ojson", kubectlVersion112).
AndRunOut("kustomize build "+tmpDir.Root(), deploymentWebYAML).
AndRun("kubectl --context kubecontext --namespace testNamespace delete --ignore-not-found=true -f -"),
},
{
Expand All @@ -140,7 +163,8 @@ func TestKustomizeCleanup(t *testing.T) {
KustomizePaths: tmpDir.Paths("a", "b"),
},
commands: testutil.
CmdRunOut("kustomize build "+tmpDir.Path("a"), deploymentWebYAML).
CmdRunOut("kubectl version --client -ojson", kubectlVersion112).
AndRunOut("kustomize build "+tmpDir.Path("a"), deploymentWebYAML).
AndRunOut("kustomize build "+tmpDir.Path("b"), deploymentAppYAML).
AndRun("kubectl --context kubecontext --namespace testNamespace delete --ignore-not-found=true -f -"),
},
Expand All @@ -150,7 +174,8 @@ func TestKustomizeCleanup(t *testing.T) {
KustomizePaths: []string{tmpDir.Root()},
},
commands: testutil.
CmdRunOut("kustomize build "+tmpDir.Root(), deploymentWebYAML).
CmdRunOut("kubectl version --client -ojson", kubectlVersion112).
AndRunOut("kustomize build "+tmpDir.Root(), deploymentWebYAML).
AndRunErr("kubectl --context kubecontext --namespace testNamespace delete --ignore-not-found=true -f -", errors.New("BUG")),
shouldErr: true,
},
Expand All @@ -159,11 +184,9 @@ func TestKustomizeCleanup(t *testing.T) {
cfg: &latest.KustomizeDeploy{
KustomizePaths: []string{tmpDir.Root()},
},
commands: testutil.CmdRunOutErr(
"kustomize build "+tmpDir.Root(),
"",
errors.New("BUG"),
),
commands: testutil.
CmdRunOut("kubectl version --client -ojson", kubectlVersion112).
AndRunOutErr("kustomize build "+tmpDir.Root(), "", errors.New("BUG")),
shouldErr: true,
},
}
Expand Down Expand Up @@ -399,49 +422,49 @@ func TestKustomizeBuildCommandArgs(t *testing.T) {
description: "no BuildArgs, empty KustomizePaths ",
buildArgs: []string{},
kustomizePath: "",
expectedArgs: []string{"build"},
expectedArgs: nil,
},
{
description: "One BuildArg, empty KustomizePaths",
buildArgs: []string{"--foo"},
kustomizePath: "",
expectedArgs: []string{"build", "--foo"},
expectedArgs: []string{"--foo"},
},
{
description: "no BuildArgs, non-empty KustomizePaths",
buildArgs: []string{},
kustomizePath: "foo",
expectedArgs: []string{"build", "foo"},
expectedArgs: []string{"foo"},
},
{
description: "One BuildArg, non-empty KustomizePaths",
buildArgs: []string{"--foo"},
kustomizePath: "bar",
expectedArgs: []string{"build", "--foo", "bar"},
expectedArgs: []string{"--foo", "bar"},
},
{
description: "Multiple BuildArg, empty KustomizePaths",
buildArgs: []string{"--foo", "--bar"},
kustomizePath: "",
expectedArgs: []string{"build", "--foo", "--bar"},
expectedArgs: []string{"--foo", "--bar"},
},
{
description: "Multiple BuildArg with spaces, empty KustomizePaths",
buildArgs: []string{"--foo bar", "--baz"},
kustomizePath: "",
expectedArgs: []string{"build", "--foo", "bar", "--baz"},
expectedArgs: []string{"--foo", "bar", "--baz"},
},
{
description: "Multiple BuildArg with spaces, non-empty KustomizePaths",
buildArgs: []string{"--foo bar", "--baz"},
kustomizePath: "barfoo",
expectedArgs: []string{"build", "--foo", "bar", "--baz", "barfoo"},
expectedArgs: []string{"--foo", "bar", "--baz", "barfoo"},
},
{
description: "Multiple BuildArg no spaces, non-empty KustomizePaths",
buildArgs: []string{"--foo", "bar", "--baz"},
kustomizePath: "barfoo",
expectedArgs: []string{"build", "--foo", "bar", "--baz", "barfoo"},
expectedArgs: []string{"--foo", "bar", "--baz", "barfoo"},
},
}

Expand Down

0 comments on commit b4f0afb

Please sign in to comment.