From 9820c33fa4364a2b30a4c9551985ab5bec01f188 Mon Sep 17 00:00:00 2001 From: Andreas Sommer Date: Thu, 23 Jan 2020 15:37:12 +0100 Subject: [PATCH] Add option to render manifest from previous build result --- cmd/skaffold/app/cmd/deploy.go | 9 +- cmd/skaffold/app/cmd/flags.go | 8 + cmd/skaffold/app/cmd/render.go | 26 +- docs/content/en/docs/references/cli/_index.md | 9 +- integration/render_test.go | 335 +++++++++++++++++- integration/skaffold/helper.go | 5 + integration/testdata/render/build-output.json | 1 + pkg/skaffold/config/options.go | 42 ++- pkg/skaffold/deploy/deploy.go | 2 +- pkg/skaffold/deploy/deploy_mux.go | 4 +- pkg/skaffold/deploy/deploy_mux_test.go | 6 +- pkg/skaffold/deploy/helm.go | 24 +- pkg/skaffold/deploy/helm_test.go | 2 +- pkg/skaffold/deploy/kubectl.go | 42 ++- pkg/skaffold/deploy/kubectl_test.go | 8 +- pkg/skaffold/deploy/kustomize.go | 6 +- pkg/skaffold/deploy/kustomize_test.go | 5 +- pkg/skaffold/deploy/labels.go | 6 +- pkg/skaffold/runner/deploy.go | 2 +- pkg/skaffold/runner/render.go | 4 +- pkg/skaffold/runner/runner.go | 2 +- pkg/skaffold/runner/runner_test.go | 2 +- 22 files changed, 479 insertions(+), 71 deletions(-) create mode 100644 integration/testdata/render/build-output.json diff --git a/cmd/skaffold/app/cmd/deploy.go b/cmd/skaffold/app/cmd/deploy.go index 872579ec39a..5938a7d5423 100644 --- a/cmd/skaffold/app/cmd/deploy.go +++ b/cmd/skaffold/app/cmd/deploy.go @@ -32,8 +32,8 @@ import ( ) var ( - buildOutputFile flags.BuildOutputFileFlag - preBuiltImages flags.Images + deployFromBuildOutputFile flags.BuildOutputFileFlag + preBuiltImages flags.Images ) // NewCmdDeploy describes the CLI command to deploy artifacts. @@ -46,15 +46,14 @@ func NewCmdDeploy() *cobra.Command { WithCommonFlags(). WithFlags(func(f *pflag.FlagSet) { f.VarP(&preBuiltImages, "images", "i", "A list of pre-built images to deploy") - f.VarP(&buildOutputFile, "build-artifacts", "a", `Filepath containing build output. -E.g. build.out created by running skaffold build --quiet -o "{{json .}}" > build.out`) + f.VarP(&deployFromBuildOutputFile, "build-artifacts", "a", "File containing build result from a previous 'skaffold build --file-output'") }). NoArgs(doDeploy) } func doDeploy(ctx context.Context, out io.Writer) error { return withRunner(ctx, func(r runner.Runner, config *latest.SkaffoldConfig) error { - deployed, err := getArtifactsToDeploy(out, buildOutputFile.BuildArtifacts(), preBuiltImages.Artifacts(), config.Build.Artifacts) + deployed, err := getArtifactsToDeploy(out, deployFromBuildOutputFile.BuildArtifacts(), preBuiltImages.Artifacts(), config.Build.Artifacts) if err != nil { return err } diff --git a/cmd/skaffold/app/cmd/flags.go b/cmd/skaffold/app/cmd/flags.go index ce9f5570980..b5e1d379749 100644 --- a/cmd/skaffold/app/cmd/flags.go +++ b/cmd/skaffold/app/cmd/flags.go @@ -287,6 +287,14 @@ var FlagRegistry = []Flag{ FlagAddMethod: "BoolVar", DefinedOn: []string{"dev", "run", "debug", "deploy", "render", "build", "delete", "diagnose"}, }, + { + Name: "add-skaffold-labels", + Usage: "Add Skaffold-specific labels to rendered manifest. If false, custom labels are still applied. Helpful for GitOps model where Skaffold is not the deployer.", + Value: &opts.AddSkaffoldLabels, + DefValue: true, + FlagAddMethod: "BoolVar", + DefinedOn: []string{"render"}, + }, } func (fl *Flag) flag() *pflag.Flag { diff --git a/cmd/skaffold/app/cmd/render.go b/cmd/skaffold/app/cmd/render.go index aabab114090..c0d9d20af90 100644 --- a/cmd/skaffold/app/cmd/render.go +++ b/cmd/skaffold/app/cmd/render.go @@ -25,13 +25,17 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + "github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/flags" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" ) var ( - showBuild bool - renderOutputPath string + showBuild bool + renderOutputPath string + renderFromBuildOutputFile flags.BuildOutputFileFlag + offline bool ) // NewCmdRender describes the CLI command to build artifacts render Kubernetes manifests. @@ -42,6 +46,8 @@ func NewCmdRender() *cobra.Command { WithCommonFlags(). WithFlags(func(f *pflag.FlagSet) { f.BoolVar(&showBuild, "loud", false, "Show the build logs and output") + f.VarP(&renderFromBuildOutputFile, "build-artifacts", "a", "File containing build result from a previous 'skaffold build --file-output'") + f.BoolVar(&offline, "offline", false, `Do not connect to Kubernetes API server for manifest creation and validation. This is helpful when no Kubernetes cluster is available (e.g. GitOps model). No metadata.namespace attribute is injected in this case - the manifest content does not get changed.`) f.StringVar(&renderOutputPath, "output", "", "file to write rendered manifests to") f.StringVar(&opts.DigestSource, "digest-source", "local", "Set to 'local' to build images locally and use digests from built images; Set to 'remote' to resolve the digest of images by tag from the remote registry; Set to 'none' to use tags directly from the Kubernetes manifests") }). @@ -55,11 +61,19 @@ func doRender(ctx context.Context, out io.Writer) error { } return withRunner(ctx, func(r runner.Runner, config *latest.SkaffoldConfig) error { - bRes, err := r.BuildAndTest(ctx, buildOut, targetArtifacts(opts, config)) - if err != nil { - return fmt.Errorf("executing build: %w", err) + var bRes []build.Artifact + + if renderFromBuildOutputFile.String() != "" { + bRes = renderFromBuildOutputFile.BuildArtifacts() + } else { + var err error + bRes, err = r.BuildAndTest(ctx, buildOut, targetArtifacts(opts, config)) + if err != nil { + return fmt.Errorf("executing build: %w", err) + } } - if err := r.Render(ctx, out, bRes, renderOutputPath); err != nil { + + if err := r.Render(ctx, out, bRes, offline, renderOutputPath); err != nil { return fmt.Errorf("rendering manifests: %w", err) } return nil diff --git a/docs/content/en/docs/references/cli/_index.md b/docs/content/en/docs/references/cli/_index.md index 62064669be1..4d439ed101f 100644 --- a/docs/content/en/docs/references/cli/_index.md +++ b/docs/content/en/docs/references/cli/_index.md @@ -446,8 +446,7 @@ Examples: skaffold build -q | skaffold deploy --build-artifacts - Options: - -a, --build-artifacts=: Filepath containing build output. -E.g. build.out created by running skaffold build --quiet -o "{{json .}}" > build.out + -a, --build-artifacts=: File containing build result from a previous 'skaffold build --file-output' -c, --config='': File for global configurations (defaults to $HOME/.skaffold/config) -d, --default-repo='': Default repository value (overrides global config) --enable-rpc=false: Enable gRPC for exposing Skaffold events (true by default for `skaffold dev`) @@ -700,12 +699,15 @@ Examples: skaffold render --digest-source=remote Options: + --add-skaffold-labels=true: Add Skaffold-specific labels to rendered manifest. If false, custom labels are still applied. Helpful for GitOps model where Skaffold is not the deployer. + -a, --build-artifacts=: File containing build result from a previous 'skaffold build --file-output' -d, --default-repo='': Default repository value (overrides global config) --digest-source='local': Set to 'local' to build images locally and use digests from built images; Set to 'remote' to resolve the digest of images by tag from the remote registry; Set to 'none' to use tags directly from the Kubernetes manifests -f, --filename='skaffold.yaml': Path or URL to the Skaffold config file -l, --label=[]: Add custom labels to deployed objects. Set multiple times for multiple labels --loud=false: Show the build logs and output -n, --namespace='': Run deployments in the specified namespace + --offline=false: Do not connect to Kubernetes API server for manifest creation and validation. This is helpful when no Kubernetes cluster is available (e.g. GitOps model). No metadata.namespace attribute is injected in this case - the manifest content does not get changed. --output='': file to write rendered manifests to -p, --profile=[]: Activate profiles by name (prefixed with `-` to disable a profile) --profile-auto-activation=true: Set to false to disable profile auto activation @@ -719,12 +721,15 @@ Use "skaffold options" for a list of global command-line options (applies to all ``` Env vars: +* `SKAFFOLD_ADD_SKAFFOLD_LABELS` (same as `--add-skaffold-labels`) +* `SKAFFOLD_BUILD_ARTIFACTS` (same as `--build-artifacts`) * `SKAFFOLD_DEFAULT_REPO` (same as `--default-repo`) * `SKAFFOLD_DIGEST_SOURCE` (same as `--digest-source`) * `SKAFFOLD_FILENAME` (same as `--filename`) * `SKAFFOLD_LABEL` (same as `--label`) * `SKAFFOLD_LOUD` (same as `--loud`) * `SKAFFOLD_NAMESPACE` (same as `--namespace`) +* `SKAFFOLD_OFFLINE` (same as `--offline`) * `SKAFFOLD_OUTPUT` (same as `--output`) * `SKAFFOLD_PROFILE` (same as `--profile`) * `SKAFFOLD_PROFILE_AUTO_ACTIVATION` (same as `--profile-auto-activation`) diff --git a/integration/render_test.go b/integration/render_test.go index 0a22a383829..773081848af 100644 --- a/integration/render_test.go +++ b/integration/render_test.go @@ -19,9 +19,18 @@ package integration import ( "bytes" "context" + "io/ioutil" + "os" + "path" + "regexp" + "strconv" "testing" + yaml "gopkg.in/yaml.v2" + + "github.com/GoogleContainerTools/skaffold/integration/skaffold" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/config" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/runcontext" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" @@ -49,6 +58,8 @@ func TestKubectlRender(t *testing.T) { labels: []deploy.Labeller{}, input: `apiVersion: v1 kind: Pod +metadata: + name: my-pod-123 spec: containers: - image: gcr.io/k8s-skaffold/skaffold @@ -59,6 +70,7 @@ kind: Pod metadata: labels: skaffold.dev/deployer: kubectl + name: my-pod-123 namespace: default spec: containers: @@ -81,6 +93,8 @@ spec: labels: []deploy.Labeller{}, input: `apiVersion: v1 kind: Pod +metadata: + name: my-pod-123 spec: containers: - image: gcr.io/project/image1 @@ -93,6 +107,7 @@ kind: Pod metadata: labels: skaffold.dev/deployer: kubectl + name: my-pod-123 namespace: default spec: containers: @@ -116,6 +131,8 @@ spec: }, input: `apiVersion: v1 kind: Pod +metadata: + name: my-pod-123 spec: containers: - image: gcr.io/project/image1 @@ -123,6 +140,8 @@ spec: --- apiVersion: v1 kind: Pod +metadata: + name: my-pod-456 spec: containers: - image: gcr.io/project/image2 @@ -133,6 +152,7 @@ kind: Pod metadata: labels: skaffold.dev/deployer: kubectl + name: my-pod-123 namespace: default spec: containers: @@ -144,6 +164,7 @@ kind: Pod metadata: labels: skaffold.dev/deployer: kubectl + name: my-pod-456 namespace: default spec: containers: @@ -169,9 +190,12 @@ spec: }, }, }, + Opts: config.SkaffoldOptions{ + AddSkaffoldLabels: true, + }, }) var b bytes.Buffer - err := deployer.Render(context.Background(), &b, test.builds, test.labels, "") + err := deployer.Render(context.Background(), &b, test.builds, test.labels, false, "") t.CheckNoError(err) t.CheckDeepEqual(test.expectedOut, b.String()) @@ -359,10 +383,317 @@ spec: }, }) var b bytes.Buffer - err := deployer.Render(context.Background(), &b, test.builds, test.labels, "") + err := deployer.Render(context.Background(), &b, test.builds, test.labels, true, "") t.CheckNoError(err) t.CheckDeepEqual(test.expectedOut, b.String()) }) } } + +func TestRenderFromBuildOutput(t *testing.T) { + if testing.Short() || RunOnGCP() { + t.Skip("skipping kind integration test") + } + + tests := []struct { + description string + config string + buildOutputFilePath string + offline bool + addSkaffoldLabels bool + input map[string]string // file path => content + expectedOut string + }{ + { + description: "kubectl render from build output, online, no labels", + config: ` +apiVersion: skaffold/v2alpha1 +kind: Config + +# Irrelevant for rendering from previous build output +build: + artifacts: [] + +deploy: + kubectl: + manifests: + - deployment.yaml +`, + buildOutputFilePath: "testdata/render/build-output.json", + offline: false, + addSkaffoldLabels: false, + input: map[string]string{"deployment.yaml": ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-123 +spec: + containers: + - image: 12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a + name: a + - image: gcr.io/my/project-b + name: b +`}, + // `metadata.namespace` is injected by `kubectl create` in non-offline mode + expectedOut: `apiVersion: v1 +kind: Pod +metadata: + name: my-pod-123 + namespace: default +spec: + containers: + - image: 12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a:4da6a56988057d23f68a4e988f4905dd930ea438-dirty@sha256:d8a33c260c50385ea54077bc7032dba0a860dc8870464f6795fd0aa548d117bf + name: a + - image: gcr.io/my/project-b:764841f8bac17e625724adcbf0d28013f22d058f-dirty@sha256:79e160161fd8190acae2d04d8f296a27a562c8a59732c64ac71c99009a6e89bc + name: b +`, + }, + + { + description: "kubectl render from build output, offline, no labels", + config: ` +apiVersion: skaffold/v2alpha1 +kind: Config + +# Irrelevant for rendering from previous build output +build: + artifacts: [] + +deploy: + kubectl: + manifests: + - deployment.yaml +`, + buildOutputFilePath: "testdata/render/build-output.json", + offline: true, + addSkaffoldLabels: false, + input: map[string]string{"deployment.yaml": ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-123 +spec: + containers: + - image: 12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a + name: a + - image: gcr.io/my/project-b + name: b +`}, + // No `metadata.namespace` in offline mode + expectedOut: `apiVersion: v1 +kind: Pod +metadata: + name: my-pod-123 +spec: + containers: + - image: 12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a:4da6a56988057d23f68a4e988f4905dd930ea438-dirty@sha256:d8a33c260c50385ea54077bc7032dba0a860dc8870464f6795fd0aa548d117bf + name: a + - image: gcr.io/my/project-b:764841f8bac17e625724adcbf0d28013f22d058f-dirty@sha256:79e160161fd8190acae2d04d8f296a27a562c8a59732c64ac71c99009a6e89bc + name: b +`, + }, + + { + description: "kubectl render from build output, offline, with labels", + config: ` +apiVersion: skaffold/v2alpha1 +kind: Config + +# Irrelevant for rendering from previous build output +build: + artifacts: [] + +deploy: + kubectl: + manifests: + - deployment.yaml +`, + buildOutputFilePath: "testdata/render/build-output.json", + offline: true, + addSkaffoldLabels: true, + input: map[string]string{"deployment.yaml": ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-123 +spec: + containers: + - image: 12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a + name: a + - image: gcr.io/my/project-b + name: b +`}, + // No `metadata.namespace` in offline mode + expectedOut: `apiVersion: v1 +kind: Pod +metadata: + labels: + app.kubernetes.io/managed-by: SOMEDYNAMICVALUE + skaffold.dev/builder: local + skaffold.dev/cleanup: "true" + skaffold.dev/deployer: kubectl + skaffold.dev/docker-api-version: SOMEDYNAMICVALUE + skaffold.dev/run-id: SOMEDYNAMICVALUE + skaffold.dev/tag-policy: git-commit + name: my-pod-123 +spec: + containers: + - image: 12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a:4da6a56988057d23f68a4e988f4905dd930ea438-dirty@sha256:d8a33c260c50385ea54077bc7032dba0a860dc8870464f6795fd0aa548d117bf + name: a + - image: gcr.io/my/project-b:764841f8bac17e625724adcbf0d28013f22d058f-dirty@sha256:79e160161fd8190acae2d04d8f296a27a562c8a59732c64ac71c99009a6e89bc + name: b +`, + }, + + { + description: "kustomize render from build output, offline, no labels", + config: ` +apiVersion: skaffold/v2alpha1 +kind: Config + +# Irrelevant for rendering from previous build output +build: + artifacts: [] + +deploy: + kustomize: {} # defaults to current working directory +`, + buildOutputFilePath: "testdata/render/build-output.json", + offline: true, + addSkaffoldLabels: false, + input: map[string]string{"deployment.yaml": ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-123 +spec: + containers: + - image: 12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a + name: a + - image: gcr.io/my/project-b + name: b +`, + "kustomization.yaml": ` +commonLabels: + this-is-from: kustomization.yaml + +resources: + - deployment.yaml +`}, + // No `metadata.namespace` in offline mode + expectedOut: `apiVersion: v1 +kind: Pod +metadata: + labels: + this-is-from: kustomization.yaml + name: my-pod-123 +spec: + containers: + - image: 12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a:4da6a56988057d23f68a4e988f4905dd930ea438-dirty@sha256:d8a33c260c50385ea54077bc7032dba0a860dc8870464f6795fd0aa548d117bf + name: a + - image: gcr.io/my/project-b:764841f8bac17e625724adcbf0d28013f22d058f-dirty@sha256:79e160161fd8190acae2d04d8f296a27a562c8a59732c64ac71c99009a6e89bc + name: b +`, + }, + + { + description: "kustomize render from build output, offline, with labels", + config: ` +apiVersion: skaffold/v2alpha1 +kind: Config + +# Irrelevant for rendering from previous build output +build: + artifacts: [] + +deploy: + kustomize: {} # defaults to current working directory +`, + buildOutputFilePath: "testdata/render/build-output.json", + offline: true, + addSkaffoldLabels: true, + input: map[string]string{"deployment.yaml": ` +apiVersion: v1 +kind: Pod +metadata: + name: my-pod-123 +spec: + containers: + - image: 12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a + name: a + - image: gcr.io/my/project-b + name: b +`, + "kustomization.yaml": ` +commonLabels: + this-is-from: kustomization.yaml + +resources: + - deployment.yaml +`}, + // No `metadata.namespace` in offline mode + expectedOut: `apiVersion: v1 +kind: Pod +metadata: + labels: + app.kubernetes.io/managed-by: SOMEDYNAMICVALUE + skaffold.dev/builder: local + skaffold.dev/cleanup: "true" + skaffold.dev/deployer: kustomize + skaffold.dev/docker-api-version: SOMEDYNAMICVALUE + skaffold.dev/run-id: SOMEDYNAMICVALUE + skaffold.dev/tag-policy: git-commit + this-is-from: kustomization.yaml + name: my-pod-123 +spec: + containers: + - image: 12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a:4da6a56988057d23f68a4e988f4905dd930ea438-dirty@sha256:d8a33c260c50385ea54077bc7032dba0a860dc8870464f6795fd0aa548d117bf + name: a + - image: gcr.io/my/project-b:764841f8bac17e625724adcbf0d28013f22d058f-dirty@sha256:79e160161fd8190acae2d04d8f296a27a562c8a59732c64ac71c99009a6e89bc + name: b +`, + }, + } + + testDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + tmpDir := t.NewTempDir() + tmpDir.Write("skaffold.yaml", test.config) + + for filePath, content := range test.input { + tmpDir.Write(filePath, content) + } + + tmpDir.Chdir() + + args := []string{"--build-artifacts=" + path.Join(testDir, test.buildOutputFilePath), "--add-skaffold-labels=" + strconv.FormatBool(test.addSkaffoldLabels), "--output", "rendered.yaml"} + + if test.offline { + env := []string{"KUBECONFIG=not-supposed-to-be-used-in-offline-mode"} + args = append(args, "--offline") + skaffold.Render(args...).WithEnv(env).RunOrFail(t.T) + } else { + skaffold.Render(args...).RunOrFail(t.T) + } + + fileContent, err := ioutil.ReadFile("rendered.yaml") + t.RequireNoError(err) + + // Tests are written in a way that actual output is valid YAML + parsed := make(map[string]interface{}) + err = yaml.UnmarshalStrict(fileContent, parsed) + t.CheckNoError(err) + + fileContentReplaced := regexp.MustCompile("(?m)(app.kubernetes.io/managed-by|skaffold.dev/run-id|skaffold.dev/docker-api-version): .+$").ReplaceAll(fileContent, []byte("$1: SOMEDYNAMICVALUE")) + + t.RequireNoError(err) + t.CheckDeepEqual(test.expectedOut, string(fileContentReplaced)) + }) + } +} diff --git a/integration/skaffold/helper.go b/integration/skaffold/helper.go index 99f533f15e3..7d08977acfe 100644 --- a/integration/skaffold/helper.go +++ b/integration/skaffold/helper.go @@ -107,6 +107,11 @@ func Credits(args ...string) *RunBuilder { return &RunBuilder{command: "credits", args: args} } +// Render runs `skaffold render` with the given arguments. +func Render(args ...string) *RunBuilder { + return withDefaults("render", args) +} + func GeneratePipeline(args ...string) *RunBuilder { return withDefaults("generate-pipeline", args) } diff --git a/integration/testdata/render/build-output.json b/integration/testdata/render/build-output.json new file mode 100644 index 00000000000..52d099a3b58 --- /dev/null +++ b/integration/testdata/render/build-output.json @@ -0,0 +1 @@ +{"builds":[{"imageName":"12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a","tag":"12345.dkr.ecr.eu-central-1.amazonaws.com/my/project-a:4da6a56988057d23f68a4e988f4905dd930ea438-dirty@sha256:d8a33c260c50385ea54077bc7032dba0a860dc8870464f6795fd0aa548d117bf"},{"imageName":"gcr.io/my/project-b","tag":"gcr.io/my/project-b:764841f8bac17e625724adcbf0d28013f22d058f-dirty@sha256:79e160161fd8190acae2d04d8f296a27a562c8a59732c64ac71c99009a6e89bc"}]} \ No newline at end of file diff --git a/pkg/skaffold/config/options.go b/pkg/skaffold/config/options.go index 36f594578dd..d2dc6cc66a9 100644 --- a/pkg/skaffold/config/options.go +++ b/pkg/skaffold/config/options.go @@ -50,23 +50,31 @@ type SkaffoldOptions struct { RenderOnly bool ProfileAutoActivation bool DryRun bool - PortForward PortForwardOptions - CustomTag string - Namespace string - CacheFile string - Trigger string - KubeContext string - KubeConfig string - DigestSource string - WatchPollInterval int - DefaultRepo StringOrUndefined - CustomLabels []string - TargetImages []string - Profiles []string - InsecureRegistries []string - Command string - RPCPort int - RPCHTTPPort int + + // Add Skaffold-specific labels including runID, deployer labels, etc. + // `CustomLabels` are still applied if this is false. Must only be used in + // commands which don't deploy (e.g. `skaffold render`) since the runID + // label isn't available. + AddSkaffoldLabels bool + + PortForward PortForwardOptions + CustomTag string + Namespace string + CacheFile string + Trigger string + KubeContext string + KubeConfig string + DigestSource string + WatchPollInterval int + DefaultRepo StringOrUndefined + CustomLabels []string + TargetImages []string + Profiles []string + InsecureRegistries []string + Command string + RPCPort int + RPCHTTPPort int + // TODO(https://github.com/GoogleContainerTools/skaffold/issues/3668): // remove minikubeProfile from here and instead detect it by matching the // kubecontext API Server to minikube profiles diff --git a/pkg/skaffold/deploy/deploy.go b/pkg/skaffold/deploy/deploy.go index 43744323987..7f59b7c1333 100644 --- a/pkg/skaffold/deploy/deploy.go +++ b/pkg/skaffold/deploy/deploy.go @@ -41,7 +41,7 @@ type Deployer interface { // Render generates the Kubernetes manifests replacing the build results and // writes them to the given file path - Render(context.Context, io.Writer, []build.Artifact, []Labeller, string) error + Render(context.Context, io.Writer, []build.Artifact, []Labeller, bool, string) error } type Result struct { diff --git a/pkg/skaffold/deploy/deploy_mux.go b/pkg/skaffold/deploy/deploy_mux.go index 19814ae5daf..83a15414bf7 100644 --- a/pkg/skaffold/deploy/deploy_mux.go +++ b/pkg/skaffold/deploy/deploy_mux.go @@ -98,11 +98,11 @@ func (m DeployerMux) Cleanup(ctx context.Context, w io.Writer) error { return nil } -func (m DeployerMux) Render(ctx context.Context, w io.Writer, as []build.Artifact, ls []Labeller, filepath string) error { +func (m DeployerMux) Render(ctx context.Context, w io.Writer, as []build.Artifact, ls []Labeller, offline bool, filepath string) error { resources, buf := []string{}, &bytes.Buffer{} for _, deployer := range m { buf.Reset() - if err := deployer.Render(ctx, buf, as, ls, "" /* never write to files */); err != nil { + if err := deployer.Render(ctx, buf, as, ls, offline, "" /* never write to files */); err != nil { return err } resources = append(resources, buf.String()) diff --git a/pkg/skaffold/deploy/deploy_mux_test.go b/pkg/skaffold/deploy/deploy_mux_test.go index 9c525147354..d0297b0b5b6 100644 --- a/pkg/skaffold/deploy/deploy_mux_test.go +++ b/pkg/skaffold/deploy/deploy_mux_test.go @@ -86,7 +86,7 @@ func (m *MockDeployer) Deploy(context.Context, io.Writer, []build.Artifact, []La } } -func (m *MockDeployer) Render(_ context.Context, w io.Writer, _ []build.Artifact, _ []Labeller, _ string) error { +func (m *MockDeployer) Render(_ context.Context, w io.Writer, _ []build.Artifact, _ []Labeller, _ bool, _ string) error { w.Write([]byte(m.renderResult)) return m.renderErr } @@ -284,7 +284,7 @@ func TestDeployerMux_Render(t *testing.T) { }) buf := &bytes.Buffer{} - err := deployerMux.Render(context.Background(), buf, nil, nil, "") + err := deployerMux.Render(context.Background(), buf, nil, nil, true, "") testutil.CheckErrorAndDeepEqual(t, test.shouldErr, err, test.expectedRender, buf.String()) }) } @@ -300,7 +300,7 @@ func TestDeployerMux_Render(t *testing.T) { NewMockDeployer().WithRenderResult(test.render2).WithRenderErr(test.err2), }) - err := deployerMux.Render(context.Background(), nil, nil, nil, tmpDir.Path("render")) + err := deployerMux.Render(context.Background(), nil, nil, nil, true, tmpDir.Path("render")) testutil.CheckError(t, false, err) file, _ := os.Open(tmpDir.Path("render")) diff --git a/pkg/skaffold/deploy/helm.go b/pkg/skaffold/deploy/helm.go index 3fa5f2fd60b..57b362ee3e3 100644 --- a/pkg/skaffold/deploy/helm.go +++ b/pkg/skaffold/deploy/helm.go @@ -65,10 +65,11 @@ var ( type HelmDeployer struct { *latest.HelmDeploy - kubeContext string - kubeConfig string - namespace string - forceDeploy bool + kubeContext string + kubeConfig string + namespace string + forceDeploy bool + addSkaffoldLabels bool // packaging temporary directory, used for predictable test output pkgTmpDir string @@ -80,11 +81,12 @@ type HelmDeployer struct { // NewHelmDeployer returns a configured HelmDeployer func NewHelmDeployer(runCtx *runcontext.RunContext) *HelmDeployer { return &HelmDeployer{ - HelmDeploy: runCtx.Cfg.Deploy.HelmDeploy, - kubeContext: runCtx.KubeContext, - kubeConfig: runCtx.Opts.KubeConfig, - namespace: runCtx.Opts.Namespace, - forceDeploy: runCtx.Opts.Force, + HelmDeploy: runCtx.Cfg.Deploy.HelmDeploy, + kubeContext: runCtx.KubeContext, + kubeConfig: runCtx.Opts.KubeConfig, + namespace: runCtx.Opts.Namespace, + forceDeploy: runCtx.Opts.Force, + addSkaffoldLabels: runCtx.Opts.AddSkaffoldLabels, } } @@ -141,7 +143,7 @@ func (h *HelmDeployer) Deploy(ctx context.Context, out io.Writer, builds []build event.DeployComplete() - labels := merge(h, labellers...) + labels := merge(h.addSkaffoldLabels, h, labellers...) labelDeployResults(labels, dRes) // Collect namespaces in a string @@ -247,7 +249,7 @@ func (h *HelmDeployer) Cleanup(ctx context.Context, out io.Writer) error { } // Render generates the Kubernetes manifests and writes them out -func (h *HelmDeployer) Render(ctx context.Context, out io.Writer, builds []build.Artifact, labellers []Labeller, filepath string) error { +func (h *HelmDeployer) Render(ctx context.Context, out io.Writer, builds []build.Artifact, labellers []Labeller, offline bool, filepath string) error { hv, err := h.binVer(ctx) if err != nil { return fmt.Errorf(versionErrorString, err) diff --git a/pkg/skaffold/deploy/helm_test.go b/pkg/skaffold/deploy/helm_test.go index 5bbc41fc97d..ebf6c3ddefb 100644 --- a/pkg/skaffold/deploy/helm_test.go +++ b/pkg/skaffold/deploy/helm_test.go @@ -1123,7 +1123,7 @@ func TestHelmRender(t *testing.T) { deployer := NewHelmDeployer(test.runContext) t.Override(&util.DefaultExecCommand, test.commands) - err := deployer.Render(context.Background(), ioutil.Discard, test.builds, nil, file) + err := deployer.Render(context.Background(), ioutil.Discard, test.builds, nil, true, file) t.CheckError(test.shouldErr, err) if file != "" { diff --git a/pkg/skaffold/deploy/kubectl.go b/pkg/skaffold/deploy/kubectl.go index 52ee8cc4723..4ae51167ed7 100644 --- a/pkg/skaffold/deploy/kubectl.go +++ b/pkg/skaffold/deploy/kubectl.go @@ -19,8 +19,10 @@ package deploy import ( "bytes" "context" + "errors" "fmt" "io" + "io/ioutil" "strings" "github.com/segmentio/textio" @@ -48,6 +50,7 @@ type KubectlDeployer struct { defaultRepo *string kubectl deploy.CLI insecureRegistries map[string]bool + addSkaffoldLabels bool } // NewKubectlDeployer returns a new KubectlDeployer for a DeployConfig filled @@ -64,6 +67,7 @@ func NewKubectlDeployer(runCtx *runcontext.RunContext) *KubectlDeployer { ForceDeploy: runCtx.Opts.Force, }, insecureRegistries: runCtx.InsecureRegistries, + addSkaffoldLabels: runCtx.Opts.AddSkaffoldLabels, } } @@ -78,7 +82,7 @@ func (k *KubectlDeployer) Labels() map[string]string { func (k *KubectlDeployer) Deploy(ctx context.Context, out io.Writer, builds []build.Artifact, labellers []Labeller) *Result { event.DeployInProgress() - manifests, err := k.renderManifests(ctx, out, builds, labellers) + manifests, err := k.renderManifests(ctx, out, builds, labellers, false) if err != nil { event.DeployFailed(err) return NewDeployErrorResult(err) @@ -133,7 +137,7 @@ func (k *KubectlDeployer) manifestFiles(manifests []string) ([]string, error) { } // readManifests reads the manifests to deploy/delete. -func (k *KubectlDeployer) readManifests(ctx context.Context) (deploy.ManifestList, error) { +func (k *KubectlDeployer) readManifests(ctx context.Context, offline bool) (deploy.ManifestList, error) { // Get file manifests manifests, err := k.Dependencies() if err != nil { @@ -141,9 +145,11 @@ func (k *KubectlDeployer) readManifests(ctx context.Context) (deploy.ManifestLis } // Append URL manifests + hasURLManifest := false for _, manifest := range k.KubectlDeploy.Manifests { if util.IsURL(manifest) { manifests = append(manifests, manifest) + hasURLManifest = true } } @@ -151,7 +157,25 @@ func (k *KubectlDeployer) readManifests(ctx context.Context) (deploy.ManifestLis return deploy.ManifestList{}, nil } - return k.kubectl.ReadManifests(ctx, manifests) + if !offline { + return k.kubectl.ReadManifests(ctx, manifests) + } + + // In case no URLs are provided, we can stay offline - no need to run "kubectl create" which + // would try to connect to a cluster (https://github.com/kubernetes/kubernetes/issues/51475) + if hasURLManifest { + return nil, errors.New("cannot use offline mode if URL manifests are configured") + } + + var manifestList deploy.ManifestList + for _, manifestFilePath := range manifests { + manifestFileContent, err := ioutil.ReadFile(manifestFilePath) + if err != nil { + return nil, fmt.Errorf("reading manifest file %v: %w", manifestFilePath, err) + } + manifestList.Append(manifestFileContent) + } + return manifestList, nil } // readRemoteManifests will try to read manifests from the given kubernetes @@ -174,8 +198,8 @@ func (k *KubectlDeployer) readRemoteManifest(ctx context.Context, name string) ( return manifest.Bytes(), nil } -func (k *KubectlDeployer) Render(ctx context.Context, out io.Writer, builds []build.Artifact, labellers []Labeller, filepath string) error { - manifests, err := k.renderManifests(ctx, out, builds, labellers) +func (k *KubectlDeployer) Render(ctx context.Context, out io.Writer, builds []build.Artifact, labellers []Labeller, offline bool, filepath string) error { + manifests, err := k.renderManifests(ctx, out, builds, labellers, offline) if err != nil { return err } @@ -183,13 +207,13 @@ func (k *KubectlDeployer) Render(ctx context.Context, out io.Writer, builds []bu return outputRenderedManifests(manifests.String(), filepath, out) } -func (k *KubectlDeployer) renderManifests(ctx context.Context, out io.Writer, builds []build.Artifact, labellers []Labeller) (deploy.ManifestList, error) { +func (k *KubectlDeployer) renderManifests(ctx context.Context, out io.Writer, builds []build.Artifact, labellers []Labeller, offline bool) (deploy.ManifestList, error) { if err := k.kubectl.CheckVersion(ctx); err != nil { color.Default.Fprintln(out, "kubectl client version:", k.kubectl.Version(ctx)) color.Default.Fprintln(out, err) } - manifests, err := k.readManifests(ctx) + manifests, err := k.readManifests(ctx, offline) if err != nil { return nil, fmt.Errorf("reading manifests: %w", err) } @@ -239,7 +263,7 @@ func (k *KubectlDeployer) renderManifests(ctx context.Context, out io.Writer, bu } } - manifests, err = manifests.SetLabels(merge(k, labellers...)) + manifests, err = manifests.SetLabels(merge(k.addSkaffoldLabels, k, labellers...)) if err != nil { return nil, fmt.Errorf("setting labels in manifests: %w", err) } @@ -249,7 +273,7 @@ func (k *KubectlDeployer) renderManifests(ctx context.Context, out io.Writer, bu // Cleanup deletes what was deployed by calling Deploy. func (k *KubectlDeployer) Cleanup(ctx context.Context, out io.Writer) error { - manifests, err := k.readManifests(ctx) + manifests, err := k.readManifests(ctx, false) if err != nil { return fmt.Errorf("reading manifests: %w", err) } diff --git a/pkg/skaffold/deploy/kubectl_test.go b/pkg/skaffold/deploy/kubectl_test.go index ae1f25e2b5a..3e7ed5af31d 100644 --- a/pkg/skaffold/deploy/kubectl_test.go +++ b/pkg/skaffold/deploy/kubectl_test.go @@ -408,7 +408,8 @@ spec: }, KubeContext: testKubeContext, Opts: config.SkaffoldOptions{ - Namespace: testNamespace, + Namespace: testNamespace, + AddSkaffoldLabels: true, }, }) @@ -629,11 +630,14 @@ spec: }, }, }, + Opts: config.SkaffoldOptions{ + AddSkaffoldLabels: true, + }, KubeContext: testKubeContext, Opts: config.SkaffoldOptions{DefaultRepo: defaultRepo}, }) var b bytes.Buffer - err := deployer.Render(context.Background(), &b, test.builds, test.labels, "") + err := deployer.Render(context.Background(), &b, test.builds, test.labels, true, "") t.CheckNoError(err) t.CheckDeepEqual(test.expected, b.String()) }) diff --git a/pkg/skaffold/deploy/kustomize.go b/pkg/skaffold/deploy/kustomize.go index 464205ea54a..7b7ab4a62be 100644 --- a/pkg/skaffold/deploy/kustomize.go +++ b/pkg/skaffold/deploy/kustomize.go @@ -90,6 +90,7 @@ type KustomizeDeployer struct { kubectl deploy.CLI insecureRegistries map[string]bool BuildArgs []string + addSkaffoldLabels bool } func NewKustomizeDeployer(runCtx *runcontext.RunContext) *KustomizeDeployer { @@ -102,6 +103,7 @@ func NewKustomizeDeployer(runCtx *runcontext.RunContext) *KustomizeDeployer { }, insecureRegistries: runCtx.InsecureRegistries, BuildArgs: runCtx.Cfg.Deploy.KustomizeDeploy.BuildArgs, + addSkaffoldLabels: runCtx.Opts.AddSkaffoldLabels, } } @@ -169,7 +171,7 @@ func (k *KustomizeDeployer) renderManifests(ctx context.Context, out io.Writer, } } - manifests, err = manifests.SetLabels(merge(k, labellers...)) + manifests, err = manifests.SetLabels(merge(k.addSkaffoldLabels, k, labellers...)) if err != nil { return nil, fmt.Errorf("setting labels in manifests: %w", err) } @@ -204,7 +206,7 @@ func (k *KustomizeDeployer) Dependencies() ([]string, error) { return deps.toList(), nil } -func (k *KustomizeDeployer) Render(ctx context.Context, out io.Writer, builds []build.Artifact, labellers []Labeller, filepath string) error { +func (k *KustomizeDeployer) Render(ctx context.Context, out io.Writer, builds []build.Artifact, labellers []Labeller, offline bool, filepath string) error { manifests, err := k.renderManifests(ctx, out, builds, labellers) if err != nil { return err diff --git a/pkg/skaffold/deploy/kustomize_test.go b/pkg/skaffold/deploy/kustomize_test.go index 81e42703aa5..5bd0bccb97b 100644 --- a/pkg/skaffold/deploy/kustomize_test.go +++ b/pkg/skaffold/deploy/kustomize_test.go @@ -652,11 +652,12 @@ spec: }, KubeContext: testKubeContext, Opts: config.SkaffoldOptions{ - Namespace: testNamespace, + Namespace: testNamespace, + AddSkaffoldLabels: true, }, }) var b bytes.Buffer - err := k.Render(context.Background(), &b, test.builds, test.labels, "") + err := k.Render(context.Background(), &b, test.builds, test.labels, true, "") t.CheckError(test.shouldErr, err) t.CheckDeepEqual(test.expected, b.String()) }) diff --git a/pkg/skaffold/deploy/labels.go b/pkg/skaffold/deploy/labels.go index bf200f6afd3..d2cd3b9a945 100644 --- a/pkg/skaffold/deploy/labels.go +++ b/pkg/skaffold/deploy/labels.go @@ -48,7 +48,11 @@ type Labeller interface { } // merge merges the labels from multiple sources. -func merge(deployer Labeller, sources ...Labeller) map[string]string { +func merge(addSkaffoldLabels bool, deployer Labeller, sources ...Labeller) map[string]string { + if !addSkaffoldLabels { + return map[string]string{} + } + merged := deployer.Labels() for _, src := range sources { diff --git a/pkg/skaffold/runner/deploy.go b/pkg/skaffold/runner/deploy.go index 087c87d2bd7..20d8ec2d6b0 100644 --- a/pkg/skaffold/runner/deploy.go +++ b/pkg/skaffold/runner/deploy.go @@ -34,7 +34,7 @@ import ( func (r *SkaffoldRunner) Deploy(ctx context.Context, out io.Writer, artifacts []build.Artifact) error { if r.runCtx.Opts.RenderOnly { - return r.Render(ctx, out, artifacts, "") + return r.Render(ctx, out, artifacts, false, "") } color.Default.Fprintln(out, "Tags used in deployment:") diff --git a/pkg/skaffold/runner/render.go b/pkg/skaffold/runner/render.go index 5c6a5f70615..b58b4713c01 100644 --- a/pkg/skaffold/runner/render.go +++ b/pkg/skaffold/runner/render.go @@ -26,7 +26,7 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" ) -func (r *SkaffoldRunner) Render(ctx context.Context, out io.Writer, builds []build.Artifact, filepath string) error { +func (r *SkaffoldRunner) Render(ctx context.Context, out io.Writer, builds []build.Artifact, offline bool, filepath string) error { //Fetch the digest and append it to the tag with the format of "tag@digest" if r.runCtx.Opts.DigestSource == remoteDigestSource { for i, a := range builds { @@ -40,5 +40,5 @@ func (r *SkaffoldRunner) Render(ctx context.Context, out io.Writer, builds []bui if r.runCtx.Opts.DigestSource == noneDigestSource { color.Default.Fprintln(out, "--digest-source set to 'none', tags listed in Kubernetes manifests will be used for render") } - return r.deployer.Render(ctx, out, builds, r.labellers, filepath) + return r.deployer.Render(ctx, out, builds, r.labellers, offline, filepath) } diff --git a/pkg/skaffold/runner/runner.go b/pkg/skaffold/runner/runner.go index 9f75d49718e..e5c7fad47b8 100644 --- a/pkg/skaffold/runner/runner.go +++ b/pkg/skaffold/runner/runner.go @@ -46,7 +46,7 @@ type Runner interface { BuildAndTest(context.Context, io.Writer, []*latest.Artifact) ([]build.Artifact, error) DeployAndLog(context.Context, io.Writer, []build.Artifact) error GeneratePipeline(context.Context, io.Writer, *latest.SkaffoldConfig, []string, string) error - Render(context.Context, io.Writer, []build.Artifact, string) error + Render(context.Context, io.Writer, []build.Artifact, bool, string) error Cleanup(context.Context, io.Writer) error Prune(context.Context, io.Writer) error HasDeployed() bool diff --git a/pkg/skaffold/runner/runner_test.go b/pkg/skaffold/runner/runner_test.go index c0d19481b8c..a77ec54d3fd 100644 --- a/pkg/skaffold/runner/runner_test.go +++ b/pkg/skaffold/runner/runner_test.go @@ -163,7 +163,7 @@ func (t *TestBench) Deploy(_ context.Context, _ io.Writer, artifacts []build.Art return deploy.NewDeploySuccessResult(t.namespaces) } -func (t *TestBench) Render(context.Context, io.Writer, []build.Artifact, []deploy.Labeller, string) error { +func (t *TestBench) Render(context.Context, io.Writer, []build.Artifact, []deploy.Labeller, bool, string) error { return nil }