diff --git a/pkg/skaffold/initializer/analyze.go b/pkg/skaffold/initializer/analyze.go new file mode 100644 index 00000000000..520580d1c6c --- /dev/null +++ b/pkg/skaffold/initializer/analyze.go @@ -0,0 +1,170 @@ +/* +Copyright 2020 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package initializer + +import ( + "fmt" + "path/filepath" + "sort" + + "github.com/karrick/godirwalk" + "github.com/sirupsen/logrus" + + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/initializer/kubectl" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util" +) + +type analysis struct { + kubectlAnalyzer *kubectlAnalyzer + skaffoldAnalyzer *skaffoldConfigAnalyzer + builderAnalyzer *builderAnalyzer +} + +// analyzer is a generic Visitor that is called on every file in the directory +// It can manage state and react to walking events assuming a bread first search +type analyzer interface { + enterDir(dir string) + analyzeFile(file string) error + exitDir(dir string) +} + +type directoryAnalyzer struct { + currentDir string +} + +func (a *directoryAnalyzer) analyzeFile(filePath string) error { + return nil +} + +func (a *directoryAnalyzer) enterDir(dir string) { + a.currentDir = dir +} + +func (a *directoryAnalyzer) exitDir(dir string) { + //pass +} + +type kubectlAnalyzer struct { + directoryAnalyzer + kubernetesManifests []string +} + +func (a *kubectlAnalyzer) analyzeFile(filePath string) error { + if kubectl.IsKubernetesManifest(filePath) && !IsSkaffoldConfig(filePath) { + a.kubernetesManifests = append(a.kubernetesManifests, filePath) + } + return nil +} + +type skaffoldConfigAnalyzer struct { + directoryAnalyzer + force bool +} + +func (a *skaffoldConfigAnalyzer) analyzeFile(filePath string) error { + if !IsSkaffoldConfig(filePath) { + return nil + } + if !a.force { + return fmt.Errorf("pre-existing %s found (you may continue with --force)", filePath) + } + logrus.Debugf("%s is a valid skaffold configuration: continuing since --force=true", filePath) + return nil +} + +type builderAnalyzer struct { + directoryAnalyzer + enableJibInit bool + enableBuildpackInit bool + findBuilders bool + foundBuilders []InitBuilder + + parentDirToStopFindBuilders string +} + +func (a *builderAnalyzer) analyzeFile(filePath string) error { + if a.findBuilders && (a.parentDirToStopFindBuilders == "" || a.parentDirToStopFindBuilders == a.currentDir) { + builderConfigs, continueSearchingBuilders := detectBuilders(a.enableJibInit, a.enableBuildpackInit, filePath) + a.foundBuilders = append(a.foundBuilders, builderConfigs...) + if !continueSearchingBuilders { + a.parentDirToStopFindBuilders = a.currentDir + } + } + return nil +} + +func (a *builderAnalyzer) exitDir(dir string) { + if a.parentDirToStopFindBuilders == dir { + a.parentDirToStopFindBuilders = "" + } +} + +// analyze recursively walks a directory and returns the k8s configs and builder configs that it finds +func (a *analysis) analyze(dir string) error { + for _, analyzer := range a.analyzers() { + analyzer.enterDir(dir) + } + dirents, err := godirwalk.ReadDirents(dir, nil) + if err != nil { + return err + } + + var subdirectories []*godirwalk.Dirent + //this is for deterministic results - given the same directory structure + //init should have the same results + sort.Sort(dirents) + + // Traverse files + for _, file := range dirents { + if util.IsHiddenFile(file.Name()) || util.IsHiddenDir(file.Name()) { + continue + } + + // If we found a directory, keep track of it until we've gone through all the files first + if file.IsDir() { + subdirectories = append(subdirectories, file) + continue + } + + filePath := filepath.Join(dir, file.Name()) + for _, analyzer := range a.analyzers() { + if err := analyzer.analyzeFile(filePath); err != nil { + return err + } + } + } + + // Recurse into subdirectories + for _, subdir := range subdirectories { + if err = a.analyze(filepath.Join(dir, subdir.Name())); err != nil { + return err + } + } + + for _, analyzer := range a.analyzers() { + analyzer.exitDir(dir) + } + return nil +} + +func (a *analysis) analyzers() []analyzer { + return []analyzer{ + a.kubectlAnalyzer, + a.skaffoldAnalyzer, + a.builderAnalyzer, + } +} diff --git a/pkg/skaffold/initializer/builders.go b/pkg/skaffold/initializer/builders.go index 78e955473ba..e65bf2adf5b 100644 --- a/pkg/skaffold/initializer/builders.go +++ b/pkg/skaffold/initializer/builders.go @@ -25,7 +25,6 @@ import ( "strings" "github.com/sirupsen/logrus" - "gopkg.in/AlecAivazis/survey.v1" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/buildpacks" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/jib" @@ -222,19 +221,3 @@ func resolveBuilderImages(builderConfigs []InitBuilder, images []string, force b } return pairs, nil } - -func promptUserForBuildConfig(image string, choices []string) (string, error) { - var selectedBuildConfig string - options := append(choices, NoBuilder) - prompt := &survey.Select{ - Message: fmt.Sprintf("Choose the builder to build image %s", image), - Options: options, - PageSize: 15, - } - err := survey.AskOne(prompt, &selectedBuildConfig, nil) - if err != nil { - return "", err - } - - return selectedBuildConfig, nil -} diff --git a/pkg/skaffold/initializer/configs.go b/pkg/skaffold/initializer/config.go similarity index 92% rename from pkg/skaffold/initializer/configs.go rename to pkg/skaffold/initializer/config.go index be408079887..5dcdcadb79e 100644 --- a/pkg/skaffold/initializer/configs.go +++ b/pkg/skaffold/initializer/config.go @@ -23,13 +23,17 @@ import ( "strings" "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/warnings" ) -func generateSkaffoldConfig(k Initializer, buildConfigPairs []builderImagePair) ([]byte, error) { +var ( + // for testing + getWd = os.Getwd +) + +func generateSkaffoldConfig(k DeploymentInitializer, buildConfigPairs []builderImagePair) *latest.SkaffoldConfig { // if we're here, the user has no skaffold yaml so we need to generate one // if the user doesn't have any k8s yamls, generate one for each dockerfile logrus.Info("generating skaffold config") @@ -39,7 +43,7 @@ func generateSkaffoldConfig(k Initializer, buildConfigPairs []builderImagePair) warnings.Printf("Couldn't generate default config name: %s", err.Error()) } - return yaml.Marshal(&latest.SkaffoldConfig{ + return &latest.SkaffoldConfig{ APIVersion: latest.Version, Kind: "Config", Metadata: latest.Metadata{ @@ -51,11 +55,11 @@ func generateSkaffoldConfig(k Initializer, buildConfigPairs []builderImagePair) }, Deploy: k.GenerateDeployConfig(), }, - }) + } } func suggestConfigName() (string, error) { - cwd, err := os.Getwd() + cwd, err := getWd() if err != nil { return "", err } diff --git a/pkg/skaffold/initializer/config_test.go b/pkg/skaffold/initializer/config_test.go new file mode 100644 index 00000000000..a2736002fb2 --- /dev/null +++ b/pkg/skaffold/initializer/config_test.go @@ -0,0 +1,173 @@ +/* +Copyright 2020 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package initializer + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/buildpacks" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" + "github.com/GoogleContainerTools/skaffold/testutil" +) + +type stubDeploymentInitializer struct { + deployConfig latest.DeployConfig +} + +func (s stubDeploymentInitializer) GenerateDeployConfig() latest.DeployConfig { + return s.deployConfig +} + +func (s stubDeploymentInitializer) GetImages() []string { + panic("implement me") +} + +func TestGenerateSkaffoldConfig(t *testing.T) { + tests := []struct { + name string + expectedSkaffoldConfig *latest.SkaffoldConfig + deployConfig latest.DeployConfig + builderConfigPairs []builderImagePair + getWd func() (string, error) + }{ + { + name: "empty", + builderConfigPairs: []builderImagePair{}, + deployConfig: latest.DeployConfig{}, + getWd: func() (s string, err error) { + return filepath.Join("rootDir", "testConfig"), nil + }, + expectedSkaffoldConfig: &latest.SkaffoldConfig{ + APIVersion: latest.Version, + Kind: "Config", + Metadata: latest.Metadata{Name: "testconfig"}, + Pipeline: latest.Pipeline{ + Deploy: latest.DeployConfig{}, + }, + }, + }, + { + name: "root dir + builder image pairs", + builderConfigPairs: []builderImagePair{ + { + Builder: docker.ArtifactConfig{ + File: "testDir/Dockerfile", + }, + ImageName: "image1", + }, + }, + deployConfig: latest.DeployConfig{}, + getWd: func() (s string, err error) { + return string(filepath.Separator), nil + }, + expectedSkaffoldConfig: &latest.SkaffoldConfig{ + APIVersion: latest.Version, + Kind: "Config", + Metadata: latest.Metadata{}, + Pipeline: latest.Pipeline{ + Build: latest.BuildConfig{ + Artifacts: []*latest.Artifact{ + { + ImageName: "image1", + Workspace: "testDir", + }, + }, + }, + Deploy: latest.DeployConfig{}, + }, + }, + }, + { + name: "error working dir", + builderConfigPairs: []builderImagePair{}, + deployConfig: latest.DeployConfig{}, + getWd: func() (s string, err error) { + return "", errors.New("testError") + }, + expectedSkaffoldConfig: &latest.SkaffoldConfig{ + APIVersion: latest.Version, + Kind: "Config", + Metadata: latest.Metadata{}, + }, + }, + } + + for _, test := range tests { + testutil.Run(t, test.name, func(t *testutil.T) { + deploymentInitializer := stubDeploymentInitializer{ + test.deployConfig, + } + t.Override(&getWd, test.getWd) + config := generateSkaffoldConfig(deploymentInitializer, test.builderConfigPairs) + t.CheckDeepEqual(config, test.expectedSkaffoldConfig) + }) + } +} + +func TestArtifacts(t *testing.T) { + testutil.Run(t, "", func(t *testutil.T) { + artifacts := artifacts([]builderImagePair{ + { + ImageName: "image1", + Builder: docker.ArtifactConfig{ + File: "Dockerfile", + }, + }, + { + ImageName: "image2", + Builder: docker.ArtifactConfig{ + File: "front/Dockerfile2", + }, + }, + { + ImageName: "image3", + Builder: buildpacks.ArtifactConfig{ + File: "package.json", + }, + }, + }) + + expected := []*latest.Artifact{ + { + ImageName: "image1", + ArtifactType: latest.ArtifactType{}, + }, + { + ImageName: "image2", + Workspace: "front", + ArtifactType: latest.ArtifactType{ + DockerArtifact: &latest.DockerArtifact{ + DockerfilePath: "Dockerfile2", + }, + }, + }, + { + ImageName: "image3", + ArtifactType: latest.ArtifactType{ + BuildpackArtifact: &latest.BuildpackArtifact{ + Builder: "heroku/buildpacks", + }, + }, + }, + } + + t.CheckDeepEqual(expected, artifacts) + }) +} diff --git a/pkg/skaffold/initializer/init.go b/pkg/skaffold/initializer/init.go index d32758cf510..130759c2b5c 100644 --- a/pkg/skaffold/initializer/init.go +++ b/pkg/skaffold/initializer/init.go @@ -23,13 +23,10 @@ import ( "io" "io/ioutil" "os" - "path/filepath" - "sort" "strings" - "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" - "github.com/karrick/godirwalk" "github.com/pkg/errors" "github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/tips" @@ -37,22 +34,15 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/initializer/kubectl" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" - "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/warnings" ) -// For testing -var ( - promptUserForBuildConfigFunc = promptUserForBuildConfig -) - // NoBuilder allows users to specify they don't want to build // an image we parse out from a Kubernetes manifest const NoBuilder = "None (image not built from these sources)" -// Initializer is the Init API of skaffold and responsible for generating -// skaffold configuration file. -type Initializer interface { +// DeploymentInitializer detects a deployment type and is able to extract image names from it +type DeploymentInitializer interface { // GenerateDeployConfig generates Deploy Config for skaffold configuration. GenerateDeployConfig() latest.DeployConfig // GetImages fetches all the images defined in the manifest files. @@ -75,7 +65,7 @@ type InitBuilder interface { Path() string } -// Config defines the Initializer Config for Init API of skaffold. +// Config contains all the parameters for the initializer package type Config struct { ComposeFile string CliArtifacts []string @@ -104,17 +94,22 @@ func DoInit(ctx context.Context, out io.Writer, c Config) error { } a := &analysis{ - force: c.Force, - enableJibInit: c.EnableJibInit, - enableBuildpackInit: c.EnableBuildpackInit, - skipBuild: c.SkipBuild, + kubectlAnalyzer: &kubectlAnalyzer{}, + builderAnalyzer: &builderAnalyzer{ + findBuilders: !c.SkipBuild, + enableJibInit: c.EnableJibInit, + enableBuildpackInit: c.EnableBuildpackInit, + }, + skaffoldAnalyzer: &skaffoldConfigAnalyzer{ + force: c.Force, + }, } - if err := a.walk(rootDir); err != nil { + if err := a.analyze(rootDir); err != nil { return err } - k, err := kubectl.New(a.potentialConfigs) + k, err := kubectl.New(a.kubectlAnalyzer.kubernetesManifests) if err != nil { return err } @@ -137,7 +132,7 @@ func DoInit(ctx context.Context, out io.Writer, c Config) error { } // Determine which builders/images require prompting - pairs, unresolvedBuilderConfigs, unresolvedImages := autoSelectBuilders(a.foundBuilders, images) + pairs, unresolvedBuilderConfigs, unresolvedImages := autoSelectBuilders(a.builderAnalyzer.foundBuilders, images) if c.Analyze { // TODO: Remove backwards compatibility block @@ -150,7 +145,7 @@ func DoInit(ctx context.Context, out io.Writer, c Config) error { // conditionally generate build artifacts if !c.SkipBuild { - if len(a.foundBuilders) == 0 { + if len(a.builderAnalyzer.foundBuilders) == 0 { return errors.New("one or more valid builder configuration (Dockerfile or Jib configuration) must be present to build images with skaffold; please provide at least one build config and try again or run `skaffold init --skip-build`") } @@ -169,11 +164,10 @@ func DoInit(ctx context.Context, out io.Writer, c Config) error { } } - pipeline, err := generateSkaffoldConfig(k, pairs) + pipeline, err := yaml.Marshal(generateSkaffoldConfig(k, pairs)) if err != nil { return err } - if c.Opts.ConfigurationFile == "-" { out.Write(pipeline) return nil @@ -211,73 +205,3 @@ func DoInit(ctx context.Context, out io.Writer, c Config) error { return nil } - -type analysis struct { - force bool - enableJibInit bool - enableBuildpackInit bool - skipBuild bool - - potentialConfigs []string - foundBuilders []InitBuilder -} - -// walk recursively walks a directory and returns the k8s configs and builder configs that it finds -func (a *analysis) walk(dir string) error { - var searchConfigsAndBuilders func(path string, findBuilders bool) error - searchConfigsAndBuilders = func(path string, findBuilders bool) error { - dirents, err := godirwalk.ReadDirents(path, nil) - if err != nil { - return err - } - - var subdirectories []*godirwalk.Dirent - searchForBuildersInSubdirectories := findBuilders - sort.Sort(dirents) - - // Traverse files - for _, file := range dirents { - if util.IsHiddenFile(file.Name()) || util.IsHiddenDir(file.Name()) { - continue - } - - // If we found a directory, keep track of it until we've gone through all the files first - if file.IsDir() { - subdirectories = append(subdirectories, file) - continue - } - - // Check for skaffold.yaml/k8s manifest - filePath := filepath.Join(path, file.Name()) - isSkaffoldConfig := IsSkaffoldConfig(filePath) - isKubernetesManifest := false - if isSkaffoldConfig { - if !a.force { - return fmt.Errorf("pre-existing %s found (you may continue with --force)", filePath) - } - logrus.Debugf("%s is a valid skaffold configuration: continuing since --force=true", filePath) - } else if kubectl.IsKubernetesManifest(filePath) { - isKubernetesManifest = true - a.potentialConfigs = append(a.potentialConfigs, filePath) - } - - // Check for builder config - if !isSkaffoldConfig && !isKubernetesManifest && findBuilders { - builderConfigs, continueSearchingBuilders := detectBuilders(a.enableJibInit, a.enableBuildpackInit, filePath) - a.foundBuilders = append(a.foundBuilders, builderConfigs...) - searchForBuildersInSubdirectories = searchForBuildersInSubdirectories && continueSearchingBuilders - } - } - - // Recurse into subdirectories - for _, dir := range subdirectories { - if err = searchConfigsAndBuilders(filepath.Join(path, dir.Name()), searchForBuildersInSubdirectories); err != nil { - return err - } - } - - return nil - } - - return searchConfigsAndBuilders(dir, !a.skipBuild) -} diff --git a/pkg/skaffold/initializer/init_test.go b/pkg/skaffold/initializer/init_test.go index 0b0586c9155..c2bfd010f1f 100644 --- a/pkg/skaffold/initializer/init_test.go +++ b/pkg/skaffold/initializer/init_test.go @@ -27,7 +27,6 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/buildpacks" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/jib" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" - "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util" "github.com/GoogleContainerTools/skaffold/testutil" ) @@ -331,22 +330,28 @@ deploy: t.Override(&docker.Validate, fakeValidateDockerfile) t.Override(&jib.Validate, fakeValidateJibConfig) - a := analysis{ - force: test.force, - enableJibInit: test.enableJibInit, - enableBuildpackInit: test.enableBuildpackInit, + a := &analysis{ + kubectlAnalyzer: &kubectlAnalyzer{}, + builderAnalyzer: &builderAnalyzer{ + findBuilders: true, + enableJibInit: test.enableJibInit, + enableBuildpackInit: test.enableBuildpackInit, + }, + skaffoldAnalyzer: &skaffoldConfigAnalyzer{ + force: test.force, + }, } - err := a.walk(tmpDir.Root()) + err := a.analyze(tmpDir.Root()) t.CheckError(test.shouldErr, err) if test.shouldErr { return } - t.CheckDeepEqual(tmpDir.Paths(test.expectedConfigs...), a.potentialConfigs) - t.CheckDeepEqual(len(test.expectedPaths), len(a.foundBuilders)) - for i := range a.foundBuilders { - t.CheckDeepEqual(tmpDir.Path(test.expectedPaths[i]), a.foundBuilders[i].Path()) + t.CheckDeepEqual(tmpDir.Paths(test.expectedConfigs...), a.kubectlAnalyzer.kubernetesManifests) + t.CheckDeepEqual(len(test.expectedPaths), len(a.builderAnalyzer.foundBuilders)) + for i := range a.builderAnalyzer.foundBuilders { + t.CheckDeepEqual(tmpDir.Path(test.expectedPaths[i]), a.builderAnalyzer.foundBuilders[i].Path()) } }) } @@ -681,54 +686,3 @@ func TestRunKompose(t *testing.T) { }) } } - -func TestArtifacts(t *testing.T) { - testutil.Run(t, "", func(t *testutil.T) { - artifacts := artifacts([]builderImagePair{ - { - ImageName: "image1", - Builder: docker.ArtifactConfig{ - File: "Dockerfile", - }, - }, - { - ImageName: "image2", - Builder: docker.ArtifactConfig{ - File: "front/Dockerfile2", - }, - }, - { - ImageName: "image3", - Builder: buildpacks.ArtifactConfig{ - File: "package.json", - }, - }, - }) - - expected := []*latest.Artifact{ - { - ImageName: "image1", - ArtifactType: latest.ArtifactType{}, - }, - { - ImageName: "image2", - Workspace: "front", - ArtifactType: latest.ArtifactType{ - DockerArtifact: &latest.DockerArtifact{ - DockerfilePath: "Dockerfile2", - }, - }, - }, - { - ImageName: "image3", - ArtifactType: latest.ArtifactType{ - BuildpackArtifact: &latest.BuildpackArtifact{ - Builder: "heroku/buildpacks", - }, - }, - }, - } - - t.CheckDeepEqual(expected, artifacts) - }) -} diff --git a/pkg/skaffold/initializer/prompt.go b/pkg/skaffold/initializer/prompt.go new file mode 100644 index 00000000000..064df8c8866 --- /dev/null +++ b/pkg/skaffold/initializer/prompt.go @@ -0,0 +1,44 @@ +/* +Copyright 2020 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package initializer + +import ( + "fmt" + + "gopkg.in/AlecAivazis/survey.v1" +) + +// For testing +var ( + promptUserForBuildConfigFunc = promptUserForBuildConfig +) + +func promptUserForBuildConfig(image string, choices []string) (string, error) { + var selectedBuildConfig string + options := append(choices, NoBuilder) + prompt := &survey.Select{ + Message: fmt.Sprintf("Choose the builder to build image %s", image), + Options: options, + PageSize: 15, + } + err := survey.AskOne(prompt, &selectedBuildConfig, nil) + if err != nil { + return "", err + } + + return selectedBuildConfig, nil +}