diff --git a/README.rst b/README.rst index 3cd74cc79..020a21ec4 100644 --- a/README.rst +++ b/README.rst @@ -419,6 +419,12 @@ The following flags are accepted: | | | The ``repository_macro`` directive should be added to the WORKSPACE in order for future Gazelle calls to recognize the repos defined in the macro file. | +----------------------------------------------------------------------------------------------------------+----------------------------------------------+ +| :flag:`-prune true|false` | :value:`false` | ++----------------------------------------------------------------------------------------------------------+----------------------------------------------+ +| When true, Gazelle will remove `go_repository`_ rules that no longer have equivalent repos in the ``Gopkg.lock``/``go.mod`` file. | +| | +| This flag can only be used with ``-from_file``. | ++----------------------------------------------------------------------------------------------------------+----------------------------------------------+ | :flag:`-build_file_names file1,file2,...` | | +----------------------------------------------------------------------------------------------------------+----------------------------------------------+ | Sets the ``build_file_name`` attribute for the generated `go_repository`_ rule(s). | diff --git a/cmd/gazelle/integration_test.go b/cmd/gazelle/integration_test.go index 276397eec..e79e04fc7 100644 --- a/cmd/gazelle/integration_test.go +++ b/cmd/gazelle/integration_test.go @@ -1659,6 +1659,204 @@ def go_repositories(): }}) } +func TestPruneRepoRules(t *testing.T) { + files := []testtools.FileSpec{ + { + Path: "WORKSPACE", + Content: ` +http_archive( + name = "io_bazel_rules_go", + url = "https://github.com/bazelbuild/rules_go/releases/download/0.10.1/rules_go-0.10.1.tar.gz", + sha256 = "4b14d8dd31c6dbaf3ff871adcd03f28c3274e42abc855cb8fb4d01233c0154dc", +) + +http_archive( + name = "bazel_gazelle", + url = "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.10.0/bazel-gazelle-0.10.0.tar.gz", + sha256 = "6228d9618ab9536892aa69082c063207c91e777e51bd3c5544c9c060cafe1bd8", +) + +load("@io_bazel_rules_go//go:def.bzl", "go_register_toolchains", "go_rules_dependencies") + +go_rules_dependencies() + +go_register_toolchains() + +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") + +gazelle_dependencies() + +http_archive( + name = "com_github_go_yaml_yaml", + urls = ["https://example.com/yaml.tar.gz"], + sha256 = "1234", +) + +go_repository( + name = "pruneMe", + importpath = "pruneMe", +) + +# keep +go_repository( + name = "keepMe", + importpath = "keepMe", +) + +# gazelle:repository_macro repositories.bzl%go_repositories +# gazelle:repository_macro repositories.bzl%foo_repositories +`, + }, { + Path: "repositories.bzl", + Content: ` +load("@bazel_gazelle//:deps.bzl", "go_repository") + +def go_repositories(): + go_repository( + name = "org_golang_x_sys", + importpath = "golang.org/x/sys", + remote = "https://github.com/golang/sys", + ) + go_repository( + name = "foo", + importpath = "foo", + ) + +def foo_repositories(): + go_repository( + name = "org_golang_x_net", + importpath = "golang.org/x/net", + tag = "1.2", + ) + go_repository( + name = "bar", + importpath = "bar", + ) + + # keep + go_repository( + name = "stay", + importpath = "stay", + ) +`, + }, { + Path: "Gopkg.lock", + Content: `# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = ["context"] + revision = "66aacef3dd8a676686c7ae3716979581e8b03c47" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["unix"] + revision = "bb24a47a89eac6c1227fbcb2ae37a8b9ed323366" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "05c1cd69be2c917c0cc4b32942830c2acfa044d8200fdc94716aae48a8083702" + solver-name = "gps-cdcl" + solver-version = 1 +`, + }, + } + dir, cleanup := testtools.CreateFiles(t, files) + defer cleanup() + + args := []string{"update-repos", "-build_file_generation", "off", "-from_file", "Gopkg.lock", "-to_macro", "repositories.bzl%foo_repositories", "-prune"} + if err := runGazelle(dir, args); err != nil { + t.Fatal(err) + } + + testtools.CheckFiles(t, dir, []testtools.FileSpec{ + { + Path: "WORKSPACE", + Content: ` +http_archive( + name = "io_bazel_rules_go", + url = "https://github.com/bazelbuild/rules_go/releases/download/0.10.1/rules_go-0.10.1.tar.gz", + sha256 = "4b14d8dd31c6dbaf3ff871adcd03f28c3274e42abc855cb8fb4d01233c0154dc", +) + +http_archive( + name = "bazel_gazelle", + url = "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.10.0/bazel-gazelle-0.10.0.tar.gz", + sha256 = "6228d9618ab9536892aa69082c063207c91e777e51bd3c5544c9c060cafe1bd8", +) + +load("@io_bazel_rules_go//go:def.bzl", "go_register_toolchains", "go_rules_dependencies") + +go_rules_dependencies() + +go_register_toolchains() + +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") + +gazelle_dependencies() + +http_archive( + name = "com_github_go_yaml_yaml", + urls = ["https://example.com/yaml.tar.gz"], + sha256 = "1234", +) + +# keep +go_repository( + name = "keepMe", + importpath = "keepMe", +) + +# gazelle:repository_macro repositories.bzl%go_repositories +# gazelle:repository_macro repositories.bzl%foo_repositories +`, + }, { + Path: "repositories.bzl", + Content: ` +load("@bazel_gazelle//:deps.bzl", "go_repository") + +def go_repositories(): + go_repository( + name = "org_golang_x_sys", + build_file_generation = "off", + commit = "bb24a47a89eac6c1227fbcb2ae37a8b9ed323366", + importpath = "golang.org/x/sys", + ) + +def foo_repositories(): + go_repository( + name = "org_golang_x_net", + build_file_generation = "off", + commit = "66aacef3dd8a676686c7ae3716979581e8b03c47", + importpath = "golang.org/x/net", + ) + + # keep + go_repository( + name = "stay", + importpath = "stay", + ) + go_repository( + name = "com_github_pkg_errors", + build_file_generation = "off", + commit = "645ef00459ed84a119197bfb8d8205042c6df63d", + importpath = "github.com/pkg/errors", + ) +`, + }, + }) +} + func TestDeleteRulesInEmptyDir(t *testing.T) { files := []testtools.FileSpec{ {Path: "WORKSPACE"}, diff --git a/cmd/gazelle/update-repos.go b/cmd/gazelle/update-repos.go index fe5195be4..c020baf62 100644 --- a/cmd/gazelle/update-repos.go +++ b/cmd/gazelle/update-repos.go @@ -45,6 +45,7 @@ type updateReposConfig struct { buildTagsAttr string buildFileProtoModeAttr string buildExtraArgsAttr string + pruneRules bool } var validBuildExternalAttr = []string{"external", "vendored"} @@ -92,6 +93,7 @@ func (_ *updateReposConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *c fs.Var(&gzflag.AllowedStringFlag{Value: &uc.buildFileProtoModeAttr, Allowed: validBuildFileProtoModeAttr}, "build_file_proto_mode", "Sets the build_file_proto_mode attribute for the generated go_repository rule(s).") fs.StringVar(&uc.buildExtraArgsAttr, "build_extra_args", "", "Sets the build_extra_args attribute for the generated go_repository rule(s).") fs.Var(macroFlag{macroFileName: &uc.macroFileName, macroDefName: &uc.macroDefName}, "to_macro", "Tells Gazelle to write repository rules into a .bzl macro function rather than the WORKSPACE file. . The expected format is: macroFile%defName") + fs.BoolVar(&uc.pruneRules, "prune", false, "When enabled, Gazelle will remove rules that no longer have equivalent repos in the Gopkg.lock/go.mod file. Can only used with -from_file.") } func (_ *updateReposConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error { @@ -107,6 +109,9 @@ func (_ *updateReposConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) e if len(fs.Args()) == 0 { return fmt.Errorf("No repositories specified\nTry -help for more information.") } + if uc.pruneRules { + return fmt.Errorf("The -prune option can only be used with -from_file.") + } uc.fn = updateImportPaths uc.importPaths = fs.Args() } @@ -253,7 +258,7 @@ func updateImportPaths(c *updateReposConfig, workspace *rule.File, destFile *rul return nil, err } } - files := repo.MergeRules(genRules, reposByFile, destFile, kinds) + files := repo.MergeRules(genRules, reposByFile, destFile, kinds, false) return files, nil } @@ -271,7 +276,7 @@ func importFromLockFile(c *updateReposConfig, workspace *rule.File, destFile *ru for i := range genRules { applyBuildAttributes(c, genRules[i]) } - files := repo.MergeRules(genRules, reposByFile, destFile, kinds) + files := repo.MergeRules(genRules, reposByFile, destFile, kinds, c.pruneRules) return files, nil } diff --git a/language/go/kinds.go b/language/go/kinds.go index 8d5d849f6..fa8902372 100644 --- a/language/go/kinds.go +++ b/language/go/kinds.go @@ -83,8 +83,10 @@ var goKinds = map[string]rule.KindInfo{ ResolveAttrs: map[string]bool{"deps": true}, }, "go_repository": { - MatchAttrs: []string{"importpath"}, - NonEmptyAttrs: nil, // never empty + MatchAttrs: []string{"importpath"}, + NonEmptyAttrs: map[string]bool{ + "importpath": true, + }, MergeableAttrs: map[string]bool{ "commit": true, "importpath": true, diff --git a/repo/repo.go b/repo/repo.go index 27e409f45..8def07a87 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -114,16 +114,30 @@ func ImportRepoRules(filename string, repoCache *RemoteCache) ([]*rule.Rule, err // MergeRules merges a list of generated repo rules with the already defined repo rules, // and then updates each rule's underlying file. If the generated rule matches an existing // one, then it inherits the file where the existing rule was defined. If the rule is new then -// its file is set as the destFile parameter. A list of the updated files is returned. -func MergeRules(genRules []*rule.Rule, existingRules map[*rule.File][]string, destFile *rule.File, kinds map[string]rule.KindInfo) []*rule.File { +// its file is set as the destFile parameter. If pruneRules is set, then this function will prune +// any existing rules that no longer have an equivalent repo defined in the Gopkg.lock/go.mod file. +// A list of the updated files is returned. +func MergeRules(genRules []*rule.Rule, existingRules map[*rule.File][]string, destFile *rule.File, kinds map[string]rule.KindInfo, pruneRules bool) []*rule.File { sort.Stable(byRuleName(genRules)) + ruleMap := make(map[string]bool) + if pruneRules { + for _, r := range genRules { + ruleMap[r.Name()] = true + } + } + repoMap := make(map[string]*rule.File) + emptyRules := make([]*rule.Rule, 0) for file, repoNames := range existingRules { + // Avoid writing to the same file by matching destFile with its definition in existingRules if file.Path == destFile.Path && file.MacroName() != "" && file.MacroName() == destFile.MacroName() { file = destFile } for _, name := range repoNames { + if pruneRules && !ruleMap[name] { + emptyRules = append(emptyRules, rule.NewRule("go_repository", name)) + } repoMap[name] = file } } @@ -136,10 +150,17 @@ func MergeRules(genRules []*rule.Rule, existingRules map[*rule.File][]string, de } rulesByFile[dest] = append(rulesByFile[dest], rule) } + emptyRulesByFile := make(map[*rule.File][]*rule.Rule) + for _, rule := range emptyRules { + if file, ok := repoMap[rule.Name()]; ok { + emptyRulesByFile[file] = append(emptyRulesByFile[file], rule) + } + } updatedFiles := make(map[string]*rule.File) for f, rules := range rulesByFile { - merger.MergeFile(f, nil, rules, merger.PreResolve, kinds) + merger.MergeFile(f, emptyRulesByFile[f], rules, merger.PreResolve, kinds) + delete(emptyRulesByFile, f) f.Sync() if uf, ok := updatedFiles[f.Path]; ok { uf.SyncMacroFile(f) @@ -147,6 +168,17 @@ func MergeRules(genRules []*rule.Rule, existingRules map[*rule.File][]string, de updatedFiles[f.Path] = f } } + // Merge the remaining files that have empty rules, but no genRules + for f, rules := range emptyRulesByFile { + merger.MergeFile(f, rules, nil, merger.PreResolve, kinds) + f.Sync() + if uf, ok := updatedFiles[f.Path]; ok { + uf.SyncMacroFile(f) + } else { + updatedFiles[f.Path] = f + } + } + files := make([]*rule.File, 0, len(updatedFiles)) for _, f := range updatedFiles { files = append(files, f)