update-repos: add or update repository rules by import path (#107)

The update-repos command can now add or update go_repository rules by
import path. To use:

    gazelle update-repos example.com/repo1 example.com/repo2

This will automatically discover the remote, VCS, and latest commit
for each repository containing the named packages. If a repository
already exists in WORKSPACE, it will be updated in place.

Related #56

WORKSPACE

@@ -1,9 +1,9 @@
 workspace(name = "bazel_gazelle")
 
-git_repository(
+http_archive(
     name = "io_bazel_rules_go",
-    commit = "e7249a61c3a244513601d998a13df1fa835433eb", # master on 2018-01-04
-    remote = "https://github.com/bazelbuild/rules_go",
+    url = "https://github.com/bazelbuild/rules_go/releases/download/0.9.0/rules_go-0.9.0.tar.gz",
+    sha256 = "4d8d6244320dd751590f9100cf39fd7a4b75cd901e1f3ffdfd6f048328883695",
 )
 
 load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_toolchains")

cmd/gazelle/fix-update.go

@@ -152,7 +152,8 @@ func runFixUpdate(cmd command, args []string) error {
 	ruleIndex.Finish()
 
 	// Resolve dependencies.
-	resolver := resolve.NewResolver(uc.c, l, ruleIndex, uc.repos)
+	rc := repos.NewRemoteCache(uc.repos)
+	resolver := resolve.NewResolver(uc.c, l, ruleIndex, rc)
 	for i := range visits {
 		for j := range visits[i].rules {
 			visits[i].rules[j] = resolver.ResolveRule(visits[i].rules[j], visits[i].pkgRel)

cmd/gazelle/update-repos.go

@@ -22,16 +22,21 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sync"
 
 	"github.com/bazelbuild/bazel-gazelle/internal/merger"
 	"github.com/bazelbuild/bazel-gazelle/internal/repos"
 	"github.com/bazelbuild/bazel-gazelle/internal/wspace"
 	bf "github.com/bazelbuild/buildtools/build"
 )
 
+type updateReposFn func(c *updateReposConfiguration, oldFile *bf.File) (*bf.File, error)
+
 type updateReposConfiguration struct {
+	fn           updateReposFn
 	repoRoot     string
 	lockFilename string
+	importPaths  []string
 }
 
 func updateRepos(args []string) error {
@@ -50,12 +55,10 @@ func updateRepos(args []string) error {
 		return fmt.Errorf("error parsing %q: %v", workspacePath, err)
 	}
 
-	genRules, err := repos.ImportRepoRules(c.lockFilename)
+	mergedFile, err := c.fn(c, oldFile)
 	if err != nil {
 		return err
 	}
-
-	mergedFile, _ := merger.MergeFile(genRules, nil, oldFile, merger.RepoAttrs)
 	mergedFile = merger.FixLoads(mergedFile)
 	if err := ioutil.WriteFile(mergedFile.Path, bf.Format(mergedFile), 0666); err != nil {
 		return fmt.Errorf("error writing %q: %v", mergedFile.Path, err)
@@ -80,15 +83,8 @@ func newUpdateReposConfiguration(args []string) (*updateReposConfiguration, erro
 		// flag already prints the error; don't print it again.
 		return nil, errors.New("Try -help for more information")
 	}
-	if len(fs.Args()) != 0 {
-		return nil, fmt.Errorf("Got %d positional arguments; wanted 0.\nTry -help for more information.")
-	}
-
-	c.lockFilename = *fromFileFlag
-	if c.lockFilename == "" {
-		return nil, errors.New("-from_file not provided.\nTry -help for more information.")
-	}
 
+	// Handle general flags that apply to all subcommands.
 	c.repoRoot = *repoRootFlag
 	if c.repoRoot == "" {
 		if repoRoot, err := wspace.Find("."); err != nil {
@@ -98,18 +94,83 @@ func newUpdateReposConfiguration(args []string) (*updateReposConfiguration, erro
 		}
 	}
 
+	// Handle flags specific to each subcommand.
+	switch {
+	case *fromFileFlag != "":
+		if len(fs.Args()) != 0 {
+			return nil, fmt.Errorf("Got %d positional arguments with -from_file; wanted 0.\nTry -help for more information.", len(fs.Args()))
+		}
+		c.fn = importFromLockFile
+		c.lockFilename = *fromFileFlag
+
+	default:
+		if len(fs.Args()) == 0 {
+			return nil, fmt.Errorf("No repositories specified\nTry -help for more information.")
+		}
+		c.fn = updateImportPaths
+		c.importPaths = fs.Args()
+	}
+
 	return c, nil
 }
 
 func updateReposUsage(fs *flag.FlagSet) {
-	fmt.Fprintln(os.Stderr, `usage: gazelle update-repos -from_file file
+	fmt.Fprintln(os.Stderr, `usage:
 
-The update-repos command updates repository rules in the WORKSPACE file.
+# Add/update repositories by import path
+gazelle update-repos example.com/repo1 example.com/repo2
+
+# Import repositories from lock file
+gazelle update-repos -from_file=file
 
-Currently, update-repos can only import repository rules from the lock file
-of a vendoring tool, and only dep's Gopkg.lock is supported. More functionality
-is planned though, and you have to start somewhere.
+The update-repos command updates repository rules in the WORKSPACE file.
+update-repos can add or update repositories explicitly by import path.
+update-repos can also import repository rules from a vendoring tool's lock
+file (currently only deps' Gopkg.lock is supported).
 
 FLAGS:
 `)
 }
+
+func updateImportPaths(c *updateReposConfiguration, oldFile *bf.File) (*bf.File, error) {
+	rs := repos.ListRepositories(oldFile)
+	rc := repos.NewRemoteCache(rs)
+
+	genRules := make([]bf.Expr, len(c.importPaths))
+	errs := make([]error, len(c.importPaths))
+	var wg sync.WaitGroup
+	wg.Add(len(c.importPaths))
+	for i, imp := range c.importPaths {
+		go func(i int) {
+			defer wg.Done()
+			repo, err := repos.UpdateRepo(rc, imp)
+			if err != nil {
+				errs[i] = err
+				return
+			}
+			repo.Remote = "" // don't set these explicitly
+			repo.VCS = ""
+			rule := repos.GenerateRule(repo)
+			genRules[i] = rule
+		}(i)
+	}
+	wg.Wait()
+
+	for _, err := range errs {
+		if err != nil {
+			return nil, err
+		}
+	}
+	mergedFile, _ := merger.MergeFile(genRules, nil, oldFile, merger.RepoAttrs)
+	return mergedFile, nil
+}
+
+func importFromLockFile(c *updateReposConfiguration, oldFile *bf.File) (*bf.File, error) {
+	genRules, err := repos.ImportRepoRules(c.lockFilename)
+	if err != nil {
+		return nil, err
+	}
+
+	mergedFile, _ := merger.MergeFile(genRules, nil, oldFile, merger.RepoAttrs)
+	return mergedFile, nil
+}

internal/repos/BUILD.bazel

@@ -4,24 +4,31 @@ go_library(
     name = "go_default_library",
     srcs = [
         "dep.go",
+        "remote.go",
         "repo.go",
     ],
     importpath = "github.com/bazelbuild/bazel-gazelle/internal/repos",
     visibility = ["//visibility:public"],
     deps = [
         "//internal/label:go_default_library",
+        "//internal/pathtools:go_default_library",
         "//internal/rules:go_default_library",
         "//vendor/github.com/pelletier/go-toml:go_default_library",
         "@com_github_bazelbuild_buildtools//build:go_default_library",
+        "@org_golang_x_tools//go/vcs:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
     srcs = [
         "import_test.go",
+        "remote_test.go",
         "repo_test.go",
     ],
     embed = [":go_default_library"],
-    deps = ["@com_github_bazelbuild_buildtools//build:go_default_library"],
+    deps = [
+        "@com_github_bazelbuild_buildtools//build:go_default_library",
+        "@org_golang_x_tools//go/vcs:go_default_library",
+    ],
 )