diff --git a/cmd/zoekt-indexserver/config.go b/cmd/zoekt-indexserver/config.go index 07ee329c..72793b9e 100644 --- a/cmd/zoekt-indexserver/config.go +++ b/cmd/zoekt-indexserver/config.go @@ -40,6 +40,7 @@ type configEntry struct { ProjectType string Name string Exclude string + GitLabURL string } func randomize(entries []configEntry) []configEntry { @@ -199,6 +200,15 @@ func executeMirror(cfg []configEntry, repoDir string, pendingRepos chan<- string if c.Exclude != "" { cmd.Args = append(cmd.Args, "-exclude", c.Exclude) } + } else if c.GitLabURL != "" { + cmd = exec.Command("zoekt-mirror-gitlab", + "-dest", repoDir, "-url", c.GitLabURL) + if c.Name != "" { + cmd.Args = append(cmd.Args, "-name", c.Name) + } + if c.Exclude != "" { + cmd.Args = append(cmd.Args, "-exclude", c.Exclude) + } } stdout, _ := loggedRun(cmd) diff --git a/cmd/zoekt-mirror-gitlab/main.go b/cmd/zoekt-mirror-gitlab/main.go new file mode 100644 index 00000000..d602b69e --- /dev/null +++ b/cmd/zoekt-mirror-gitlab/main.go @@ -0,0 +1,210 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This binary fetches all repos for a user from gitlab. +// +// It is recommended to use a gitlab personal access token: +// https://docs.gitlab.com/ce/user/profile/personal_access_tokens.html. This +// token should be stored in a file and the --token option should be used. +// In addition, the token should be present in the ~/.netrc of the user running +// the mirror command. For example, the ~/.netrc may look like: +// +// machine gitlab.com +// login oauth +// password +// +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/google/zoekt/gitindex" + gitlab "github.com/xanzy/go-gitlab" +) + +func main() { + dest := flag.String("dest", "", "destination directory") + gitlabURL := flag.String("url", "https://gitlab.com/api/v4/", "Gitlab URL. If not set https://gitlab.com/api/v4/ will be used") + token := flag.String("token", + filepath.Join(os.Getenv("HOME"), ".gitlab-token"), + "file holding API token.") + isMember := flag.Bool("membership", false, "only mirror repos this user is a member of ") + deleteRepos := flag.Bool("delete", false, "delete missing repos") + namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.") + excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.") + flag.Parse() + + if *dest == "" { + log.Fatal("must set --dest") + } + + var host string + rootURL, err := url.Parse(*gitlabURL) + if err != nil { + log.Fatal(err) + } + host = rootURL.Host + + destDir := filepath.Join(*dest, host) + if err := os.MkdirAll(destDir, 0755); err != nil { + log.Fatal(err) + } + + content, err := ioutil.ReadFile(*token) + if err != nil { + log.Fatal(err) + } + apiToken := strings.TrimSpace(string(content)) + + client := gitlab.NewClient(nil, apiToken) + client.SetBaseURL(*gitlabURL) + + opt := &gitlab.ListProjectsOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: 10, + Page: 1, + }, + Membership: isMember, + } + + var gitlabProjects []*gitlab.Project + for { + projects, resp, err := client.Projects.ListProjects(opt) + + if err != nil { + log.Fatal(err) + } + + for _, project := range projects { + + // Skip projects without a default branch - these should be projects + // where the repository isn't enabled + if project.DefaultBranch == "" { + continue + } + + gitlabProjects = append(gitlabProjects, project) + } + + if resp.CurrentPage >= resp.TotalPages { + break + } + + opt.Page = resp.NextPage + } + + filter, err := gitindex.NewFilter(*namePattern, *excludePattern) + if err != nil { + log.Fatal(err) + } + + { + trimmed := gitlabProjects[:0] + for _, p := range gitlabProjects { + if filter.Include(p.NameWithNamespace) { + trimmed = append(trimmed, p) + } + } + gitlabProjects = trimmed + } + + fetchProjects(destDir, apiToken, gitlabProjects) + + if *deleteRepos { + if err := deleteStaleProjects(*dest, filter, gitlabProjects); err != nil { + log.Fatalf("deleteStaleProjects: %v", err) + } + } +} + +func deleteStaleProjects(destDir string, filter *gitindex.Filter, projects []*gitlab.Project) error { + + u, err := url.Parse(projects[0].HTTPURLToRepo) + u.Path = "" + if err != nil { + return err + } + + paths, err := gitindex.ListRepos(destDir, u) + if err != nil { + return err + } + + names := map[string]bool{} + for _, p := range projects { + u, err := url.Parse(p.HTTPURLToRepo) + if err != nil { + return err + } + + names[filepath.Join(u.Host, u.Path)] = true + } + + var toDelete []string + for _, p := range paths { + if filter.Include(p) && !names[p] { + toDelete = append(toDelete, p) + } + } + + if len(toDelete) > 0 { + log.Printf("deleting repos %v", toDelete) + } + + var errs []string + for _, d := range toDelete { + if err := os.RemoveAll(filepath.Join(destDir, d)); err != nil { + errs = append(errs, err.Error()) + } + } + if len(errs) > 0 { + return fmt.Errorf("errors: %v", errs) + } + return nil +} + +func fetchProjects(destDir, token string, projects []*gitlab.Project) { + + for _, p := range projects { + u, err := url.Parse(p.HTTPURLToRepo) + if err != nil { + log.Printf("Unable to parse project URL: %v", err) + continue + } + config := map[string]string{ + "zoekt.web-url-type": "gitlab", + "zoekt.web-url": p.WebURL, + "zoekt.name": filepath.Join(u.Hostname(), p.PathWithNamespace), + + "zoekt.gitlab-stars": strconv.Itoa(p.StarCount), + "zoekt.gitlab-forks": strconv.Itoa(p.ForksCount), + } + + cloneURL := p.HTTPURLToRepo + dest, err := gitindex.CloneRepo(destDir, p.PathWithNamespace, cloneURL, config) + if err != nil { + log.Printf("cloneRepos: %v", err) + continue + } + if dest != "" { + fmt.Println(dest) + } + } +} diff --git a/gitindex/index.go b/gitindex/index.go index e9af5550..a94d49be 100644 --- a/gitindex/index.go +++ b/gitindex/index.go @@ -143,6 +143,10 @@ func setTemplates(repo *zoekt.Repository, u *url.URL, typ string) error { repo.CommitURLTemplate = u.String() + "/commits/{{.Version}}" repo.FileURLTemplate = u.String() + "/{{.Path}}?at={{.Version}}" repo.LineFragmentTemplate = "#{{.LineNumber}}" + case "gitlab": + repo.CommitURLTemplate = u.String() + "/commit/{{.Version}}" + repo.FileURLTemplate = u.String() + "/blob/{{.Version}}/{{.Path}}" + repo.LineFragmentTemplate = "#L{{.LineNumber}}" default: return fmt.Errorf("URL scheme type %q unknown", typ) } diff --git a/go.mod b/go.mod index 03559c93..e7fab0d9 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,12 @@ require ( github.com/fsnotify/fsnotify v1.4.7 github.com/gfleury/go-bitbucket-v1 v0.0.0-20181102191809-4910839b609e github.com/google/go-github v17.0.0+incompatible - github.com/google/go-querystring v1.0.0 // indirect github.com/google/slothfs v0.0.0-20170112234537-ecdd255f653d github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/xanzy/go-gitlab v0.13.0 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba golang.org/x/sync v0.0.0-20181108010431-42b317875d0f - google.golang.org/appengine v1.3.0 // indirect gopkg.in/src-d/go-git.v4 v4.8.0 ) diff --git a/go.sum b/go.sum index 6f0470cb..18fcccdc 100644 --- a/go.sum +++ b/go.sum @@ -54,14 +54,18 @@ github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/xanzy/go-gitlab v0.13.0 h1:vBxlISwRackWHqZb4IaMDycTrlfJ0918ZlpZjL20Zyk= +github.com/xanzy/go-gitlab v0.13.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro= github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba h1:YDkOrzGLLYybtuP6ZgebnO4OWYEYVMFSniazXsxrFN8= golang.org/x/oauth2 v0.0.0-20181120190819-8f65e3013eba/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=