From 491396f41034a05a22cb7ff44a3ef84de7f5256b Mon Sep 17 00:00:00 2001 From: Guido Josquin Date: Sun, 7 Jul 2019 22:21:13 +0200 Subject: [PATCH] #2 Attempt to use go-scm to add support for other repos --- README.md | 20 ++++-- cmd/drone-tree-config/main.go | 6 +- go.mod | 3 +- go.sum | 6 ++ plugin/plugin.go | 128 ++++++++++++++++++---------------- plugin/plugin_test.go | 5 ++ 6 files changed, 99 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index 853d2a4..d6afa23 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,13 @@ +# Disclaimer + +This is a fork of drone-tree-config attempting to add other SCM systems to the plugin. +It is not yet finished so it will not work if you try to use it. + +Issues I ran into during implementation: + +0. The go-scm library cannot list directory contents, only files. +0. The go-scm library cannot call the compare API yet. + # Drone Tree Config This is a Drone extension to support mono repositories with multiple `.drone.yml`. @@ -8,7 +18,7 @@ There is an official Docker image: https://hub.docker.com/r/bitsbeats/drone-tree ## Limitations -Currently supports only Github. +Currently supports only repositories supported by [go-scm](https://github.com/drone/go-scm). ## Usage @@ -20,8 +30,8 @@ Environment variables: - `PLUGIN_DEBUG`: Set this to `true` to enable debug messages. - `PLUGIN_ADDRESS`: Listen address for the plugins webserver. Defaults to `:3000`. - `PLUGIN_SECRET`: Shared secret with drone. You can generate the token using `openssl rand -hex 16`. -- `GITHUB_TOKEN`: Github personal access token. Only needs repo rights. See [here][1]. -- `GITHUB_SERVER`: Custom Github server for Github Enterprise +- `SCM_TOKEN`: SCM personal access token. Only needs repo rights. See [here][1]. +- `SCM_SERVER`: Custom SCM server for Github Enterprise If `PLUGIN_CONCAT` is not set, the first `.drone.yml` will be used. @@ -62,9 +72,9 @@ services: - PLUGIN_CONCAT=true - PLUGIN_FALLBACK=true - PLUGIN_SECRET= - - GITHUB_TOKEN= + - SCM_TOKEN= ``` -Edit the Secrets (`***`), `` and `` to your needs. `` is used between Drone and drone-tree-config. +Edit the Secrets (`***`), `` and `` to your needs. `` is used between Drone and drone-tree-config. `` is an access token for e.g. GitHub or BitBucket. [1]: https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line diff --git a/cmd/drone-tree-config/main.go b/cmd/drone-tree-config/main.go index 9611d0f..0cb0976 100644 --- a/cmd/drone-tree-config/main.go +++ b/cmd/drone-tree-config/main.go @@ -18,8 +18,8 @@ type ( Debug bool `envconfig:"PLUGIN_DEBUG"` Address string `envconfig:"PLUGIN_ADDRESS" default:":3000"` Secret string `envconfig:"PLUGIN_SECRET"` - Token string `envconfig:"GITHUB_TOKEN"` - Server string `envconfig:"GITHUB_SERVER"` + Token string `envconfig:"SCM_TOKEN"` + Server string `envconfig:"SCM_SERVER"` } ) @@ -36,7 +36,7 @@ func main() { logrus.Fatalln("missing secret key") } if spec.Token == "" { - logrus.Warnln("missing github token") + logrus.Warnln("missing scm token") } if spec.Address == "" { spec.Address = ":3000" diff --git a/go.mod b/go.mod index f835d0e..7298c2d 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,11 @@ go 1.12 require ( github.com/drone/drone-go v1.0.4 - github.com/google/go-github v17.0.0+incompatible + github.com/drone/go-scm v1.5.0 github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.1.1 github.com/kelseyhightower/envconfig v1.3.0 + github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 // indirect github.com/sirupsen/logrus v1.4.1 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 // indirect golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a diff --git a/go.sum b/go.sum index 779bf02..2566abb 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/drone/drone-go v1.0.4 h1:Yom1lix1Lmk3KmKIsBSQJF1bw0YR2lDGaFQrXxqHMko= github.com/drone/drone-go v1.0.4/go.mod h1:GxyeGClYohaKNYJv/ZpsmVHtMJ7WhoT+uDaJNcDIrk4= +github.com/drone/go-scm v1.5.0 h1:Hn3bFYsUgOEUCx2wt2II9CxkTfev2h+tPheuYHp7ehg= +github.com/drone/go-scm v1.5.0/go.mod h1:YT4FxQ3U/ltdCrBJR9B0tRpJ1bYA/PM3NyaLE/rYIvw= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= @@ -15,10 +17,14 @@ github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASu github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/h2non/gock v1.0.9 h1:17gCehSo8ZOgEsFKpQgqHiR7VLyjxdAG3lkhVvO9QZU= +github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM= github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= diff --git a/plugin/plugin.go b/plugin/plugin.go index d116411..16f1dcd 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "path" "regexp" "strconv" @@ -11,11 +12,11 @@ import ( "github.com/drone/drone-go/drone" "github.com/drone/drone-go/plugin/config" - - "github.com/google/go-github/github" + "github.com/drone/go-scm/scm" + "github.com/drone/go-scm/scm/driver/github" + "github.com/drone/go-scm/scm/transport" "github.com/google/uuid" "github.com/sirupsen/logrus" - "golang.org/x/oauth2" "gopkg.in/yaml.v2" ) @@ -47,38 +48,41 @@ type ( request struct { *config.Request UUID uuid.UUID - Client *github.Client + Client *scm.Client } ) var dedupRegex = regexp.MustCompile(`(?ms)(---[\s]*){2,}`) -// Find is called by dorne +// Find is called by drone func (p *plugin) Find(ctx context.Context, droneRequest *config.Request) (*drone.Config, error) { - uuid := uuid.New() - logrus.Infof("%s %s/%s started", uuid, droneRequest.Repo.Namespace, droneRequest.Repo.Name) - defer logrus.Infof("%s finished", uuid) - - // connect to github - trans := oauth2.NewClient(ctx, oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: p.token}, - )) - var client *github.Client + requestUuid := uuid.New() + logrus.Infof("%s %s/%s started", requestUuid, droneRequest.Repo.Namespace, droneRequest.Repo.Name) + defer logrus.Infof("%s finished", requestUuid) + + // connect to SCM + var client *scm.Client if p.server == "" { - client = github.NewClient(trans) + client = github.NewDefault() } else { var err error - client, err = github.NewEnterpriseClient(p.server, p.server, trans) + client, err = github.New(p.server) if err != nil { - logrus.Errorf("%s Unable to connect to Github: '%v'", uuid, err) + logrus.Errorf("%s Unable to connect to SCM: '%v'", requestUuid, err) return nil, err } } - req := request{droneRequest, uuid, client} + client.Client = &http.Client{ + Transport: &transport.BearerToken{ + Token: p.token, + }, + } + + req := request{droneRequest, requestUuid, client} // get changed files - changedFiles, err := p.getGithubChanges(ctx, &req) + changedFiles, err := p.getScmChanges(ctx, &req) if err != nil { return nil, err } @@ -86,7 +90,7 @@ func (p *plugin) Find(ctx context.Context, droneRequest *config.Request) (*drone // get drone.yml for changed files or all of them if no changes/cron configData := "" if changedFiles != nil { - configData, err = p.getGithubConfigData(ctx, &req, changedFiles) + configData, err = p.getScmConfigData(ctx, &req, changedFiles) } else if req.Build.Trigger == "@cron" { logrus.Warnf("%s @cron, rebuilding all", req.UUID) configData, err = p.getAllConfigData(ctx, &req, "/", 0) @@ -110,8 +114,8 @@ func (p *plugin) Find(ctx context.Context, droneRequest *config.Request) (*drone return &drone.Config{Data: configData}, nil } -// getGithubChanges tries to get a list of changed files from github -func (p *plugin) getGithubChanges(ctx context.Context, req *request) ([]string, error) { +// getScmChanges tries to get a list of changed files from scm +func (p *plugin) getScmChanges(ctx context.Context, req *request) ([]string, error) { var changedFiles []string if req.Build.Trigger == "@cron" { @@ -124,14 +128,14 @@ func (p *plugin) getGithubChanges(ctx context.Context, req *request) ([]string, logrus.Errorf("%s unable to get pull request id %v", req.UUID, err) return nil, err } - opts := github.ListOptions{} - files, _, err := req.Client.PullRequests.ListFiles(ctx, req.Repo.Namespace, req.Repo.Name, pullRequestID, &opts) + opts := scm.ListOptions{} + files, _, err := req.Client.PullRequests.ListChanges(ctx, req.Repo.Slug, pullRequestID, opts) if err != nil { logrus.Errorf("%s unable to fetch diff for Pull request %v", req.UUID, err) return nil, err } for _, file := range files { - changedFiles = append(changedFiles, *file.Filename) + changedFiles = append(changedFiles, file.Path) } } else { // use diff to get changed files @@ -139,13 +143,15 @@ func (p *plugin) getGithubChanges(ctx context.Context, req *request) ([]string, if before == "0000000000000000000000000000000000000000" || before == "" { before = fmt.Sprintf("%s~1", req.Build.After) } - changes, _, err := req.Client.Repositories.CompareCommits(ctx, req.Repo.Namespace, req.Repo.Name, before, req.Build.After) + opts := scm.ListOptions{} + // TODO verify that ListChanges is functionally equivalent to the /compare API + changes, _, err := req.Client.Git.ListChanges(ctx, req.Repo.Slug, req.Build.After, opts) if err != nil { logrus.Errorf("%s unable to fetch diff: '%v'", req.UUID, err) return nil, err } - for _, file := range changes.Files { - changedFiles = append(changedFiles, *file.Filename) + for _, file := range changes { + changedFiles = append(changedFiles, file.Path) } } @@ -158,23 +164,23 @@ func (p *plugin) getGithubChanges(ctx context.Context, req *request) ([]string, return changedFiles, nil } -// getGithubFile downloads a file from github -func (p *plugin) getGithubFile(ctx context.Context, req *request, file string) (content string, err error) { +// getScmFile downloads a file from scm +func (p *plugin) getScmFile(ctx context.Context, req *request, file string) (content string, err error) { logrus.Debugf("%s checking %s/%s %s", req.UUID, req.Repo.Namespace, req.Repo.Name, file) - ref := github.RepositoryContentGetOptions{Ref: req.Build.After} - data, _, _, err := req.Client.Repositories.GetContents(ctx, req.Repo.Namespace, req.Repo.Name, file, &ref) + + data, _, err := req.Client.Contents.Find(ctx, req.Repo.Slug, file, req.Build.After) if data == nil { err = fmt.Errorf("failed to get %s: is not a file", file) } if err != nil { return "", err } - return data.GetContent() + return string(data.Data), nil } -// getGithubDroneConfig downloads a drone config and validates it -func (p *plugin) getGithubDroneConfig(ctx context.Context, req *request, file string) (configData string, critical bool, err error) { - fileContent, err := p.getGithubFile(ctx, req, file) +// getScmDroneConfig downloads a drone config and validates it +func (p *plugin) getScmDroneConfig(ctx context.Context, req *request, file string) (configData string, critical bool, err error) { + fileContent, err := p.getScmFile(ctx, req, file) if err != nil { logrus.Debugf("%s skipping: unable to load file: %s %v", req.UUID, file, err) return "", false, err @@ -196,8 +202,8 @@ func (p *plugin) getGithubDroneConfig(ctx context.Context, req *request, file st return fileContent, false, nil } -// getGithubConfigData scans a repository based on the changed files -func (p *plugin) getGithubConfigData(ctx context.Context, req *request, changedFiles []string) (configData string, err error) { +// getScmConfigData scans a repository based on the changed files +func (p *plugin) getScmConfigData(ctx context.Context, req *request, changedFiles []string) (configData string, err error) { // collect drone.yml files configData = "" cache := map[string]bool{} @@ -222,7 +228,7 @@ func (p *plugin) getGithubConfigData(ctx context.Context, req *request, changedF } // download file from git - fileContent, critical, err := p.getGithubDroneConfig(ctx, req, file) + fileContent, critical, err := p.getScmDroneConfig(ctx, req, file) if err != nil { if critical { return "", err @@ -243,8 +249,7 @@ func (p *plugin) getGithubConfigData(ctx context.Context, req *request, changedF // getAllConfigData searches for all or fist 'drone.yml' in the repo func (p *plugin) getAllConfigData(ctx context.Context, req *request, dir string, depth int) (configData string, err error) { - ref := github.RepositoryContentGetOptions{Ref: req.Build.After} - _, ls, _, err := req.Client.Repositories.GetContents(ctx, req.Repo.Namespace, req.Repo.Name, dir, &ref) + ls, _, err := req.Client.Contents.Find(ctx, req.Repo.Slug, dir, req.Build.After) if err != nil { return "", err } @@ -257,26 +262,29 @@ func (p *plugin) getAllConfigData(ctx context.Context, req *request, dir string, // check recursivly for drone.yml configData = "" - for _, f := range ls { - var fileContent string - if *f.Type == "dir" { - fileContent, _ = p.getAllConfigData(ctx, req, *f.Path, depth) - } else if *f.Type == "file" && *f.Name == req.Repo.Config { - var critical bool - fileContent, critical, err = p.getGithubDroneConfig(ctx, req, *f.Path) - if critical { - return "", err - } - } - // append - configData = p.droneConfigAppend(configData, fileContent) - if !p.concat { - logrus.Infof("%s concat is disabled. Using just first .drone.yml.", req.UUID) - break - } - } - return configData, nil + // TODO this will always crash because go-scm cannot handle a /contents request on a directory + err2 := errors.New(string(ls.Data)) + //for _, f := range ls.Data { + // var fileContent string + // if f. == "dir" { + // fileContent, _ = p.getAllConfigData(ctx, req, *f.Path, depth) + // } else if *f.Type == "file" && *f.Name == req.Repo.Config { + // var critical bool + // fileContent, critical, err = p.getScmDroneConfig(ctx, req, *f.Path) + // if critical { + // return "", err + // } + // } + // // append + // configData = p.droneConfigAppend(configData, fileContent) + // if !p.concat { + // logrus.Infof("%s concat is disabled. Using just first .drone.yml.", req.UUID) + // break + // } + //} + + return configData, err2 } diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go index fd87a45..728ef44 100644 --- a/plugin/plugin_test.go +++ b/plugin/plugin_test.go @@ -171,6 +171,11 @@ func testMux() *http.ServeMux { f, _ := os.Open("testdata/compare.json") _, _ = io.Copy(w, f) }) + mux.HandleFunc("/repos/foosinn/dronetest/commits/8ecad91991d5da985a2a8dd97cc19029dc1c2899", + func(w http.ResponseWriter, r *http.Request) { + f, _ := os.Open("testdata/compare.json") + _, _ = io.Copy(w, f) + }) mux.HandleFunc("/repos/foosinn/dronetest/contents/a/b/.drone.yml", func(w http.ResponseWriter, r *http.Request) { f, _ := os.Open("testdata/a_b_.drone.yml.json")