Skip to content

Commit

Permalink
Add support for pruning unneeded rules to update-repos (#514)
Browse files Browse the repository at this point in the history 
Adds the prune flag to update-repos. When enabled, Gazelle will remove go_repository rules that no longer have equivalent repos in the Gopkg.lock/go.mod file.

Updates #213
  • Loading branch information
@blico
blico authored and Jay Conrod committed May 9, 2019
1 parent 1d118af commit ddff739
Show file tree
Hide file tree
Showing 5 changed files with 250 additions and 7 deletions.
6 changes: 6 additions & 0 deletions README.rst
View file
Expand Up @@ -419,6 +419,12 @@ The following flags are accepted:
| |
| The ``repository_macro`` directive should be added to the WORKSPACE in order for future Gazelle calls to recognize the repos defined in the macro file. |
+----------------------------------------------------------------------------------------------------------+----------------------------------------------+
| :flag:`-prune true|false` | :value:`false` |
+----------------------------------------------------------------------------------------------------------+----------------------------------------------+
| When true, Gazelle will remove `go_repository`_ rules that no longer have equivalent repos in the ``Gopkg.lock``/``go.mod`` file. |
| |
| This flag can only be used with ``-from_file``. |
+----------------------------------------------------------------------------------------------------------+----------------------------------------------+
| :flag:`-build_file_names file1,file2,...` | |
+----------------------------------------------------------------------------------------------------------+----------------------------------------------+
| Sets the ``build_file_name`` attribute for the generated `go_repository`_ rule(s). |
Expand Down
198 changes: 198 additions & 0 deletions cmd/gazelle/integration_test.go
View file
Expand Up @@ -1659,6 +1659,204 @@ def go_repositories():
}})
}

func TestPruneRepoRules(t *testing.T) {
files := []testtools.FileSpec{
{
Path: "WORKSPACE",
Content: `
http_archive(
name = "io_bazel_rules_go",
url = "https://github.com/bazelbuild/rules_go/releases/download/0.10.1/rules_go-0.10.1.tar.gz",
sha256 = "4b14d8dd31c6dbaf3ff871adcd03f28c3274e42abc855cb8fb4d01233c0154dc",
)
http_archive(
name = "bazel_gazelle",
url = "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.10.0/bazel-gazelle-0.10.0.tar.gz",
sha256 = "6228d9618ab9536892aa69082c063207c91e777e51bd3c5544c9c060cafe1bd8",
)
load("@io_bazel_rules_go//go:def.bzl", "go_register_toolchains", "go_rules_dependencies")
go_rules_dependencies()
go_register_toolchains()
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
gazelle_dependencies()
http_archive(
name = "com_github_go_yaml_yaml",
urls = ["https://example.com/yaml.tar.gz"],
sha256 = "1234",
)
go_repository(
name = "pruneMe",
importpath = "pruneMe",
)
# keep
go_repository(
name = "keepMe",
importpath = "keepMe",
)
# gazelle:repository_macro repositories.bzl%go_repositories
# gazelle:repository_macro repositories.bzl%foo_repositories
`,
}, {
Path: "repositories.bzl",
Content: `
load("@bazel_gazelle//:deps.bzl", "go_repository")
def go_repositories():
go_repository(
name = "org_golang_x_sys",
importpath = "golang.org/x/sys",
remote = "https://github.com/golang/sys",
)
go_repository(
name = "foo",
importpath = "foo",
)
def foo_repositories():
go_repository(
name = "org_golang_x_net",
importpath = "golang.org/x/net",
tag = "1.2",
)
go_repository(
name = "bar",
importpath = "bar",
)
# keep
go_repository(
name = "stay",
importpath = "stay",
)
`,
}, {
Path: "Gopkg.lock",
Content: `# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
name = "github.com/pkg/errors"
packages = ["."]
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"
[[projects]]
branch = "master"
name = "golang.org/x/net"
packages = ["context"]
revision = "66aacef3dd8a676686c7ae3716979581e8b03c47"
[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = ["unix"]
revision = "bb24a47a89eac6c1227fbcb2ae37a8b9ed323366"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "05c1cd69be2c917c0cc4b32942830c2acfa044d8200fdc94716aae48a8083702"
solver-name = "gps-cdcl"
solver-version = 1
`,
},
}
dir, cleanup := testtools.CreateFiles(t, files)
defer cleanup()

args := []string{"update-repos", "-build_file_generation", "off", "-from_file", "Gopkg.lock", "-to_macro", "repositories.bzl%foo_repositories", "-prune"}
if err := runGazelle(dir, args); err != nil {
t.Fatal(err)
}

testtools.CheckFiles(t, dir, []testtools.FileSpec{
{
Path: "WORKSPACE",
Content: `
http_archive(
name = "io_bazel_rules_go",
url = "https://github.com/bazelbuild/rules_go/releases/download/0.10.1/rules_go-0.10.1.tar.gz",
sha256 = "4b14d8dd31c6dbaf3ff871adcd03f28c3274e42abc855cb8fb4d01233c0154dc",
)
http_archive(
name = "bazel_gazelle",
url = "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.10.0/bazel-gazelle-0.10.0.tar.gz",
sha256 = "6228d9618ab9536892aa69082c063207c91e777e51bd3c5544c9c060cafe1bd8",
)
load("@io_bazel_rules_go//go:def.bzl", "go_register_toolchains", "go_rules_dependencies")
go_rules_dependencies()
go_register_toolchains()
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
gazelle_dependencies()
http_archive(
name = "com_github_go_yaml_yaml",
urls = ["https://example.com/yaml.tar.gz"],
sha256 = "1234",
)
# keep
go_repository(
name = "keepMe",
importpath = "keepMe",
)
# gazelle:repository_macro repositories.bzl%go_repositories
# gazelle:repository_macro repositories.bzl%foo_repositories
`,
}, {
Path: "repositories.bzl",
Content: `
load("@bazel_gazelle//:deps.bzl", "go_repository")
def go_repositories():
go_repository(
name = "org_golang_x_sys",
build_file_generation = "off",
commit = "bb24a47a89eac6c1227fbcb2ae37a8b9ed323366",
importpath = "golang.org/x/sys",
)
def foo_repositories():
go_repository(
name = "org_golang_x_net",
build_file_generation = "off",
commit = "66aacef3dd8a676686c7ae3716979581e8b03c47",
importpath = "golang.org/x/net",
)
# keep
go_repository(
name = "stay",
importpath = "stay",
)
go_repository(
name = "com_github_pkg_errors",
build_file_generation = "off",
commit = "645ef00459ed84a119197bfb8d8205042c6df63d",
importpath = "github.com/pkg/errors",
)
`,
},
})
}

func TestDeleteRulesInEmptyDir(t *testing.T) {
files := []testtools.FileSpec{
{Path: "WORKSPACE"},
Expand Down
9 changes: 7 additions & 2 deletions cmd/gazelle/update-repos.go
View file
Expand Up @@ -45,6 +45,7 @@ type updateReposConfig struct {
buildTagsAttr string
buildFileProtoModeAttr string
buildExtraArgsAttr string
pruneRules bool
}

var validBuildExternalAttr = []string{"external", "vendored"}
Expand Down Expand Up @@ -92,6 +93,7 @@ func (_ *updateReposConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *c
fs.Var(&gzflag.AllowedStringFlag{Value: &uc.buildFileProtoModeAttr, Allowed: validBuildFileProtoModeAttr}, "build_file_proto_mode", "Sets the build_file_proto_mode attribute for the generated go_repository rule(s).")
fs.StringVar(&uc.buildExtraArgsAttr, "build_extra_args", "", "Sets the build_extra_args attribute for the generated go_repository rule(s).")
fs.Var(macroFlag{macroFileName: &uc.macroFileName, macroDefName: &uc.macroDefName}, "to_macro", "Tells Gazelle to write repository rules into a .bzl macro function rather than the WORKSPACE file. . The expected format is: macroFile%defName")
fs.BoolVar(&uc.pruneRules, "prune", false, "When enabled, Gazelle will remove rules that no longer have equivalent repos in the Gopkg.lock/go.mod file. Can only used with -from_file.")
}

func (_ *updateReposConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error {
Expand All @@ -107,6 +109,9 @@ func (_ *updateReposConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) e
if len(fs.Args()) == 0 {
return fmt.Errorf("No repositories specified\nTry -help for more information.")
}
if uc.pruneRules {
return fmt.Errorf("The -prune option can only be used with -from_file.")
}
uc.fn = updateImportPaths
uc.importPaths = fs.Args()
}
Expand Down Expand Up @@ -253,7 +258,7 @@ func updateImportPaths(c *updateReposConfig, workspace *rule.File, destFile *rul
return nil, err
}
}
files := repo.MergeRules(genRules, reposByFile, destFile, kinds)
files := repo.MergeRules(genRules, reposByFile, destFile, kinds, false)
return files, nil
}

Expand All @@ -271,7 +276,7 @@ func importFromLockFile(c *updateReposConfig, workspace *rule.File, destFile *ru
for i := range genRules {
applyBuildAttributes(c, genRules[i])
}
files := repo.MergeRules(genRules, reposByFile, destFile, kinds)
files := repo.MergeRules(genRules, reposByFile, destFile, kinds, c.pruneRules)
return files, nil
}

Expand Down
6 changes: 4 additions & 2 deletions language/go/kinds.go
View file
Expand Up @@ -83,8 +83,10 @@ var goKinds = map[string]rule.KindInfo{
ResolveAttrs: map[string]bool{"deps": true},
},
"go_repository": {
MatchAttrs: []string{"importpath"},
NonEmptyAttrs: nil, // never empty
MatchAttrs: []string{"importpath"},
NonEmptyAttrs: map[string]bool{
"importpath": true,
},
MergeableAttrs: map[string]bool{
"commit": true,
"importpath": true,
Expand Down
38 changes: 35 additions & 3 deletions repo/repo.go
View file
Expand Up @@ -114,16 +114,30 @@ func ImportRepoRules(filename string, repoCache *RemoteCache) ([]*rule.Rule, err
// MergeRules merges a list of generated repo rules with the already defined repo rules,
// and then updates each rule's underlying file. If the generated rule matches an existing
// one, then it inherits the file where the existing rule was defined. If the rule is new then
// its file is set as the destFile parameter. A list of the updated files is returned.
func MergeRules(genRules []*rule.Rule, existingRules map[*rule.File][]string, destFile *rule.File, kinds map[string]rule.KindInfo) []*rule.File {
// its file is set as the destFile parameter. If pruneRules is set, then this function will prune
// any existing rules that no longer have an equivalent repo defined in the Gopkg.lock/go.mod file.
// A list of the updated files is returned.
func MergeRules(genRules []*rule.Rule, existingRules map[*rule.File][]string, destFile *rule.File, kinds map[string]rule.KindInfo, pruneRules bool) []*rule.File {
sort.Stable(byRuleName(genRules))

ruleMap := make(map[string]bool)
if pruneRules {
for _, r := range genRules {
ruleMap[r.Name()] = true
}
}

repoMap := make(map[string]*rule.File)
emptyRules := make([]*rule.Rule, 0)
for file, repoNames := range existingRules {
// Avoid writing to the same file by matching destFile with its definition in existingRules
if file.Path == destFile.Path && file.MacroName() != "" && file.MacroName() == destFile.MacroName() {
file = destFile
}
for _, name := range repoNames {
if pruneRules && !ruleMap[name] {
emptyRules = append(emptyRules, rule.NewRule("go_repository", name))
}
repoMap[name] = file
}
}
Expand All @@ -136,17 +150,35 @@ func MergeRules(genRules []*rule.Rule, existingRules map[*rule.File][]string, de
}
rulesByFile[dest] = append(rulesByFile[dest], rule)
}
emptyRulesByFile := make(map[*rule.File][]*rule.Rule)
for _, rule := range emptyRules {
if file, ok := repoMap[rule.Name()]; ok {
emptyRulesByFile[file] = append(emptyRulesByFile[file], rule)
}
}

updatedFiles := make(map[string]*rule.File)
for f, rules := range rulesByFile {
merger.MergeFile(f, nil, rules, merger.PreResolve, kinds)
merger.MergeFile(f, emptyRulesByFile[f], rules, merger.PreResolve, kinds)
delete(emptyRulesByFile, f)
f.Sync()
if uf, ok := updatedFiles[f.Path]; ok {
uf.SyncMacroFile(f)
} else {
updatedFiles[f.Path] = f
}
}
// Merge the remaining files that have empty rules, but no genRules
for f, rules := range emptyRulesByFile {
merger.MergeFile(f, rules, nil, merger.PreResolve, kinds)
f.Sync()
if uf, ok := updatedFiles[f.Path]; ok {
uf.SyncMacroFile(f)
} else {
updatedFiles[f.Path] = f
}
}

files := make([]*rule.File, 0, len(updatedFiles))
for _, f := range updatedFiles {
files = append(files, f)
Expand Down

0 comments on commit ddff739

Please sign in to comment.