bitsbeats#2 Attempt to use go-scm to add support for other repos

README.md

@@ -1,3 +1,13 @@
+# Disclaimer
+
+This is a fork of drone-tree-config attempting to add other SCM systems to the plugin.
+It is not yet finished so it will not work if you try to use it.
+
+Issues I ran into during implementation:
+
+0. The go-scm library cannot list directory contents, only files.
+0. The go-scm library cannot call the compare API yet. 
+
 # Drone Tree Config
 
 This is a Drone extension to support mono repositories with multiple `.drone.yml`.
@@ -8,7 +18,7 @@ There is an official Docker image: https://hub.docker.com/r/bitsbeats/drone-tree
 
 ## Limitations
 
-Currently supports only Github.
+Currently supports only repositories supported by [go-scm](https://github.com/drone/go-scm).
 
 ## Usage
 
@@ -20,8 +30,8 @@ Environment variables:
 - `PLUGIN_DEBUG`: Set this to `true` to enable debug messages.
 - `PLUGIN_ADDRESS`: Listen address for the plugins webserver. Defaults to `:3000`.
 - `PLUGIN_SECRET`: Shared secret with drone. You can generate the token using `openssl rand -hex 16`.
-- `GITHUB_TOKEN`: Github personal access token. Only needs repo rights. See [here][1].
-- `GITHUB_SERVER`: Custom Github server for Github Enterprise
+- `SCM_TOKEN`: SCM personal access token. Only needs repo rights. See [here][1].
+- `SCM_SERVER`: Custom SCM server for Github Enterprise
 
 If `PLUGIN_CONCAT` is not set, the first `.drone.yml` will be used.
 
@@ -62,9 +72,9 @@ services:
       - PLUGIN_CONCAT=true
       - PLUGIN_FALLBACK=true
       - PLUGIN_SECRET=<SECRET>
-      - GITHUB_TOKEN=<GITHUB_TOKEN>
+      - SCM_TOKEN=<SCM_TOKEN>
 ```
 
-Edit the Secrets (`***`), `<SECRET>` and `<GITHUB_TOKEN>` to your needs. `<SECRET>` is used between Drone and drone-tree-config.
+Edit the Secrets (`***`), `<SECRET>` and `<SCM_TOKEN>` to your needs. `<SECRET>` is used between Drone and drone-tree-config. `<SCM_TOKEN>` is an access token for e.g. GitHub or BitBucket.
 
 [1]: https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line

cmd/drone-tree-config/main.go

@@ -18,8 +18,8 @@ type (
 		Debug    bool   `envconfig:"PLUGIN_DEBUG"`
 		Address  string `envconfig:"PLUGIN_ADDRESS" default:":3000"`
 		Secret   string `envconfig:"PLUGIN_SECRET"`
-		Token    string `envconfig:"GITHUB_TOKEN"`
-		Server   string `envconfig:"GITHUB_SERVER"`
+		Token    string `envconfig:"SCM_TOKEN"`
+		Server   string `envconfig:"SCM_SERVER"`
 	}
 )
 
@@ -36,7 +36,7 @@ func main() {
 		logrus.Fatalln("missing secret key")
 	}
 	if spec.Token == "" {
-		logrus.Warnln("missing github token")
+		logrus.Warnln("missing scm token")
 	}
 	if spec.Address == "" {
 		spec.Address = ":3000"

go.mod

@@ -4,10 +4,11 @@ go 1.12
 
 require (
 	github.com/drone/drone-go v1.0.4
-	github.com/google/go-github v17.0.0+incompatible
+	github.com/drone/go-scm v1.5.0
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/uuid v1.1.1
 	github.com/kelseyhightower/envconfig v1.3.0
+	github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 // indirect
 	github.com/sirupsen/logrus v1.4.1
 	golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 // indirect
 	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a

go.sum

@@ -5,6 +5,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/drone/drone-go v1.0.4 h1:Yom1lix1Lmk3KmKIsBSQJF1bw0YR2lDGaFQrXxqHMko=
 github.com/drone/drone-go v1.0.4/go.mod h1:GxyeGClYohaKNYJv/ZpsmVHtMJ7WhoT+uDaJNcDIrk4=
+github.com/drone/go-scm v1.5.0 h1:Hn3bFYsUgOEUCx2wt2II9CxkTfev2h+tPheuYHp7ehg=
+github.com/drone/go-scm v1.5.0/go.mod h1:YT4FxQ3U/ltdCrBJR9B0tRpJ1bYA/PM3NyaLE/rYIvw=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
@@ -15,10 +17,14 @@ github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASu
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/h2non/gock v1.0.9 h1:17gCehSo8ZOgEsFKpQgqHiR7VLyjxdAG3lkhVvO9QZU=
+github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE=
 github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM=
 github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=

plugin/plugin.go

@@ -4,18 +4,19 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"path"
 	"regexp"
 	"strconv"
 	"strings"
 
 	"github.com/drone/drone-go/drone"
 	"github.com/drone/drone-go/plugin/config"
-
-	"github.com/google/go-github/github"
+	"github.com/drone/go-scm/scm"
+	"github.com/drone/go-scm/scm/driver/github"
+	"github.com/drone/go-scm/scm/transport"
 	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
-	"golang.org/x/oauth2"
 	"gopkg.in/yaml.v2"
 )
 
@@ -47,46 +48,49 @@ type (
 	request struct {
 		*config.Request
 		UUID   uuid.UUID
-		Client *github.Client
+		Client *scm.Client
 	}
 )
 
 var dedupRegex = regexp.MustCompile(`(?ms)(---[\s]*){2,}`)
 
-// Find is called by dorne
+// Find is called by drone
 func (p *plugin) Find(ctx context.Context, droneRequest *config.Request) (*drone.Config, error) {
-	uuid := uuid.New()
-	logrus.Infof("%s %s/%s started", uuid, droneRequest.Repo.Namespace, droneRequest.Repo.Name)
-	defer logrus.Infof("%s finished", uuid)
-
-	// connect to github
-	trans := oauth2.NewClient(ctx, oauth2.StaticTokenSource(
-		&oauth2.Token{AccessToken: p.token},
-	))
-	var client *github.Client
+	requestUuid := uuid.New()
+	logrus.Infof("%s %s/%s started", requestUuid, droneRequest.Repo.Namespace, droneRequest.Repo.Name)
+	defer logrus.Infof("%s finished", requestUuid)
+
+	// connect to SCM
+	var client *scm.Client
 	if p.server == "" {
-		client = github.NewClient(trans)
+		client = github.NewDefault()
 	} else {
 		var err error
-		client, err = github.NewEnterpriseClient(p.server, p.server, trans)
+		client, err = github.New(p.server)
 		if err != nil {
-			logrus.Errorf("%s Unable to connect to Github: '%v'", uuid, err)
+			logrus.Errorf("%s Unable to connect to SCM: '%v'", requestUuid, err)
 			return nil, err
 		}
 	}
 
-	req := request{droneRequest, uuid, client}
+	client.Client = &http.Client{
+		Transport: &transport.BearerToken{
+			Token: p.token,
+		},
+	}
+
+	req := request{droneRequest, requestUuid, client}
 
 	// get changed files
-	changedFiles, err := p.getGithubChanges(ctx, &req)
+	changedFiles, err := p.getScmChanges(ctx, &req)
 	if err != nil {
 		return nil, err
 	}
 
 	// get drone.yml for changed files or all of them if no changes/cron
 	configData := ""
 	if changedFiles != nil {
-		configData, err = p.getGithubConfigData(ctx, &req, changedFiles)
+		configData, err = p.getScmConfigData(ctx, &req, changedFiles)
 	} else if req.Build.Trigger == "@cron" {
 		logrus.Warnf("%s @cron, rebuilding all", req.UUID)
 		configData, err = p.getAllConfigData(ctx, &req, "/", 0)
@@ -110,8 +114,8 @@ func (p *plugin) Find(ctx context.Context, droneRequest *config.Request) (*drone
 	return &drone.Config{Data: configData}, nil
 }
 
-// getGithubChanges tries to get a list of changed files from github
-func (p *plugin) getGithubChanges(ctx context.Context, req *request) ([]string, error) {
+// getScmChanges tries to get a list of changed files from scm
+func (p *plugin) getScmChanges(ctx context.Context, req *request) ([]string, error) {
 	var changedFiles []string
 
 	if req.Build.Trigger == "@cron" {
@@ -124,28 +128,30 @@ func (p *plugin) getGithubChanges(ctx context.Context, req *request) ([]string,
 			logrus.Errorf("%s unable to get pull request id %v", req.UUID, err)
 			return nil, err
 		}
-		opts := github.ListOptions{}
-		files, _, err := req.Client.PullRequests.ListFiles(ctx, req.Repo.Namespace, req.Repo.Name, pullRequestID, &opts)
+		opts := scm.ListOptions{}
+		files, _, err := req.Client.PullRequests.ListChanges(ctx, req.Repo.Slug, pullRequestID, opts)
 		if err != nil {
 			logrus.Errorf("%s unable to fetch diff for Pull request %v", req.UUID, err)
 			return nil, err
 		}
 		for _, file := range files {
-			changedFiles = append(changedFiles, *file.Filename)
+			changedFiles = append(changedFiles, file.Path)
 		}
 	} else {
 		// use diff to get changed files
 		before := req.Build.Before
 		if before == "0000000000000000000000000000000000000000" || before == "" {
 			before = fmt.Sprintf("%s~1", req.Build.After)
 		}
-		changes, _, err := req.Client.Repositories.CompareCommits(ctx, req.Repo.Namespace, req.Repo.Name, before, req.Build.After)
+		opts := scm.ListOptions{}
+		// TODO verify that ListChanges is functionally equivalent to the /compare API
+		changes, _, err := req.Client.Git.ListChanges(ctx, req.Repo.Slug, req.Build.After, opts)
 		if err != nil {
 			logrus.Errorf("%s unable to fetch diff: '%v'", req.UUID, err)
 			return nil, err
 		}
-		for _, file := range changes.Files {
-			changedFiles = append(changedFiles, *file.Filename)
+		for _, file := range changes {
+			changedFiles = append(changedFiles, file.Path)
 		}
 	}
 
@@ -158,23 +164,23 @@ func (p *plugin) getGithubChanges(ctx context.Context, req *request) ([]string,
 	return changedFiles, nil
 }
 
-// getGithubFile downloads a file from github
-func (p *plugin) getGithubFile(ctx context.Context, req *request, file string) (content string, err error) {
+// getScmFile downloads a file from scm
+func (p *plugin) getScmFile(ctx context.Context, req *request, file string) (content string, err error) {
 	logrus.Debugf("%s checking %s/%s %s", req.UUID, req.Repo.Namespace, req.Repo.Name, file)
-	ref := github.RepositoryContentGetOptions{Ref: req.Build.After}
-	data, _, _, err := req.Client.Repositories.GetContents(ctx, req.Repo.Namespace, req.Repo.Name, file, &ref)
+
+	data, _, err := req.Client.Contents.Find(ctx, req.Repo.Slug, file, req.Build.After)
 	if data == nil {
 		err = fmt.Errorf("failed to get %s: is not a file", file)
 	}
 	if err != nil {
 		return "", err
 	}
-	return data.GetContent()
+	return string(data.Data), nil
 }
 
-// getGithubDroneConfig downloads a drone config and validates it
-func (p *plugin) getGithubDroneConfig(ctx context.Context, req *request, file string) (configData string, critical bool, err error) {
-	fileContent, err := p.getGithubFile(ctx, req, file)
+// getScmDroneConfig downloads a drone config and validates it
+func (p *plugin) getScmDroneConfig(ctx context.Context, req *request, file string) (configData string, critical bool, err error) {
+	fileContent, err := p.getScmFile(ctx, req, file)
 	if err != nil {
 		logrus.Debugf("%s skipping: unable to load file: %s %v", req.UUID, file, err)
 		return "", false, err
@@ -196,8 +202,8 @@ func (p *plugin) getGithubDroneConfig(ctx context.Context, req *request, file st
 	return fileContent, false, nil
 }
 
-// getGithubConfigData scans a repository based on the changed files
-func (p *plugin) getGithubConfigData(ctx context.Context, req *request, changedFiles []string) (configData string, err error) {
+// getScmConfigData scans a repository based on the changed files
+func (p *plugin) getScmConfigData(ctx context.Context, req *request, changedFiles []string) (configData string, err error) {
 	// collect drone.yml files
 	configData = ""
 	cache := map[string]bool{}
@@ -222,7 +228,7 @@ func (p *plugin) getGithubConfigData(ctx context.Context, req *request, changedF
 			}
 
 			// download file from git
-			fileContent, critical, err := p.getGithubDroneConfig(ctx, req, file)
+			fileContent, critical, err := p.getScmDroneConfig(ctx, req, file)
 			if err != nil {
 				if critical {
 					return "", err
@@ -243,8 +249,7 @@ func (p *plugin) getGithubConfigData(ctx context.Context, req *request, changedF
 
 // getAllConfigData searches for all or fist 'drone.yml' in the repo
 func (p *plugin) getAllConfigData(ctx context.Context, req *request, dir string, depth int) (configData string, err error) {
-	ref := github.RepositoryContentGetOptions{Ref: req.Build.After}
-	_, ls, _, err := req.Client.Repositories.GetContents(ctx, req.Repo.Namespace, req.Repo.Name, dir, &ref)
+	ls, _, err := req.Client.Contents.Find(ctx, req.Repo.Slug, dir, req.Build.After)
 	if err != nil {
 		return "", err
 	}
@@ -257,26 +262,29 @@ func (p *plugin) getAllConfigData(ctx context.Context, req *request, dir string,
 
 	// check recursivly for drone.yml
 	configData = ""
-	for _, f := range ls {
-		var fileContent string
-		if *f.Type == "dir" {
-			fileContent, _ = p.getAllConfigData(ctx, req, *f.Path, depth)
-		} else if *f.Type == "file" && *f.Name == req.Repo.Config {
-			var critical bool
-			fileContent, critical, err = p.getGithubDroneConfig(ctx, req, *f.Path)
-			if critical {
-				return "", err
-			}
-		}
-		// append
-		configData = p.droneConfigAppend(configData, fileContent)
-		if !p.concat {
-			logrus.Infof("%s concat is disabled. Using just first .drone.yml.", req.UUID)
-			break
-		}
-	}
 
-	return configData, nil
+	// TODO this will always crash because go-scm cannot handle a /contents request on a directory
+	err2 := errors.New(string(ls.Data))
+	//for _, f := range ls.Data {
+	//	var fileContent string
+	//	if f. == "dir" {
+	//		fileContent, _ = p.getAllConfigData(ctx, req, *f.Path, depth)
+	//	} else if *f.Type == "file" && *f.Name == req.Repo.Config {
+	//		var critical bool
+	//		fileContent, critical, err = p.getScmDroneConfig(ctx, req, *f.Path)
+	//		if critical {
+	//			return "", err
+	//		}
+	//	}
+	//	// append
+	//	configData = p.droneConfigAppend(configData, fileContent)
+	//	if !p.concat {
+	//		logrus.Infof("%s concat is disabled. Using just first .drone.yml.", req.UUID)
+	//		break
+	//	}
+	//}
+
+	return configData, err2
 
 }
 

plugin/plugin_test.go

@@ -171,6 +171,11 @@ func testMux() *http.ServeMux {
 			f, _ := os.Open("testdata/compare.json")
 			_, _ = io.Copy(w, f)
 		})
+	mux.HandleFunc("/repos/foosinn/dronetest/commits/8ecad91991d5da985a2a8dd97cc19029dc1c2899",
+		func(w http.ResponseWriter, r *http.Request) {
+			f, _ := os.Open("testdata/compare.json")
+			_, _ = io.Copy(w, f)
+		})
 	mux.HandleFunc("/repos/foosinn/dronetest/contents/a/b/.drone.yml",
 		func(w http.ResponseWriter, r *http.Request) {
 			f, _ := os.Open("testdata/a_b_.drone.yml.json")