plugin: use Cloudsmith to download plugins (#1174)

- update default configuration pointing to Cloudsmith
- change path to download plugin
- allow custom path templates for plugin dist and sig files

Co-authored-by: Ricardo Silva <1945557+ricardolyn@users.noreply.github.com>

plugin/central.go

@@ -10,7 +10,10 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"runtime"
 	"strings"
+	"text/template"
+	"time"
 
 	"github.com/ethereum/go-ethereum/log"
 )
@@ -54,54 +57,50 @@ func (cc *CentralClient) getNewSecureDialer() Dialer {
 	}
 }
 
-// Get the public key from central
+// Get the public key from central. PublicKeyURI can be relative to the base URL
+// so we need to parse and make sure finally URL is resolved.
 func (cc *CentralClient) PublicKey() ([]byte, error) {
-	target := fmt.Sprintf("%s/%s", cc.config.BaseURL, cc.config.PublicKeyURI)
-	log.Debug("downloading public key", "url", target)
-	readCloser, err := cc.get(target)
+	target, err := cc.toURL(cc.config.PublicKeyURI)
 	if err != nil {
 		return nil, err
 	}
-	defer func() {
-		_ = readCloser.Close()
-	}()
-	return ioutil.ReadAll(readCloser)
+	log.Debug("downloading public key", "url", target)
+	buf := new(bytes.Buffer)
+	if err := cc.download(target, buf); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
 }
 
 // retrieve plugin signature
 func (cc *CentralClient) PluginSignature(definition *PluginDefinition) ([]byte, error) {
-	target := fmt.Sprintf("%s/%s/%s", cc.config.BaseURL, definition.RemotePath(), definition.SignatureFileName())
-	log.Debug("downloading plugin signature file", "url", target)
-	readCloser, err := cc.get(target)
+	target, err := cc.toURLFromTemplate(cc.config.PluginSigPathTemplate, definition)
 	if err != nil {
 		return nil, err
 	}
-	defer func() {
-		_ = readCloser.Close()
-	}()
-	return ioutil.ReadAll(readCloser)
+	log.Debug("downloading plugin signature file", "url", target)
+	buf := new(bytes.Buffer)
+	if err := cc.download(target, buf); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
 }
 
 // retrieve plugin distribution file
 func (cc *CentralClient) PluginDistribution(definition *PluginDefinition, outFilePath string) error {
-	target := fmt.Sprintf("%s/%s/%s", cc.config.BaseURL, definition.RemotePath(), definition.DistFileName())
-	log.Debug("downloading plugin zip file", "url", target)
-	outFile, err := os.Create(outFilePath)
+	target, err := cc.toURLFromTemplate(cc.config.PluginDistPathTemplate, definition)
 	if err != nil {
 		return err
 	}
-	defer func() {
-		_ = outFile.Close()
-	}()
-	readCloser, err := cc.get(target)
+	log.Debug("downloading plugin zip file", "url", target)
+	outFile, err := os.Create(outFilePath)
 	if err != nil {
 		return err
 	}
 	defer func() {
-		_ = readCloser.Close()
+		_ = outFile.Close()
 	}()
-	_, err = io.Copy(outFile, readCloser)
-	return err
+	return cc.download(target, outFile)
 }
 
 // perform HTTP GET
@@ -125,6 +124,59 @@ func (cc *CentralClient) get(target string) (io.ReadCloser, error) {
 	return res.Body, nil
 }
 
+// return full URL using config.BaseURL
+func (cc *CentralClient) toURL(relativePath string) (string, error) {
+	base, err := url.Parse(cc.config.BaseURL)
+	if err != nil {
+		return "", err
+	}
+	u, err := base.Parse(relativePath)
+	if err != nil {
+		return "", err
+	}
+	return u.String(), nil
+}
+
+// return full URL using config.BaseURL from given template
+func (cc *CentralClient) toURLFromTemplate(pathTemplate string, definition *PluginDefinition) (string, error) {
+	t, err := template.New("").Parse(pathTemplate)
+	if err != nil {
+		return "", err
+	}
+	path := new(bytes.Buffer)
+	if err := t.Execute(path, struct {
+		Name    string
+		Version string
+		OS      string
+		Arch    string
+	}{
+		Name:    definition.Name,
+		Version: string(definition.Version),
+		OS:      runtime.GOOS,
+		Arch:    runtime.GOARCH,
+	}); err != nil {
+		return "", err
+	}
+	return cc.toURL(path.String())
+}
+
+// peform http GET to the target URL and write output to out
+func (cc *CentralClient) download(target string, out io.Writer) (err error) {
+	defer func(start time.Time) {
+		log.Debug("download completed", "url", target, "err", err, "took", time.Since(start))
+	}(time.Now())
+	var readCloser io.ReadCloser
+	readCloser, err = cc.get(target)
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = readCloser.Close()
+	}()
+	_, err = io.Copy(out, readCloser)
+	return err
+}
+
 // An adapter function for tls.Dial with CA verification & SSL Pinning support.
 type Dialer func(network, addr string) (net.Conn, error)
 

plugin/central_test.go

@@ -2,12 +2,14 @@ package plugin
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"path"
+	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -31,6 +33,24 @@ func TestCentralClient_PublicKey(t *testing.T) {
 	assert.Equal(t, arbitraryPubKey, actualValue)
 }
 
+func TestCentralClient_PublicKey_RelativePath(t *testing.T) {
+	arbitraryServer := newTestServer("/aa/"+DefaultPublicKeyFile, arbitraryPubKey)
+	defer arbitraryServer.Close()
+	arbitraryConfig := &PluginCentralConfiguration{
+		BaseURL:      arbitraryServer.URL + "/aa/bb/cc/", // without postfix /, the relative path would be diff
+		PublicKeyURI: "../../" + DefaultPublicKeyFile,
+	}
+
+	testObject := NewPluginCentralClient(arbitraryConfig)
+
+	actualValue, err := testObject.PublicKey()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, arbitraryPubKey, actualValue)
+}
+
 func TestCentralClient_PublicKey_withSSL(t *testing.T) {
 	arbitraryServer := httptest.NewTLSServer(newMux("/"+DefaultPublicKeyFile, arbitraryPubKey))
 	defer arbitraryServer.Close()
@@ -56,11 +76,13 @@ func TestCentralClient_PluginSignature(t *testing.T) {
 		Name:    "arbitrary-plugin",
 		Version: "1.0.0",
 	}
-	arbitraryServer := newTestServer("/"+arbitraryDef.RemotePath()+"/"+arbitraryDef.SignatureFileName(), validSignature)
+	expectedPath := fmt.Sprintf("/maven/bin/%s/%s/%s-%s-%s-%s-sha256.checksum.asc", arbitraryDef.Name, arbitraryDef.Version, arbitraryDef.Name, arbitraryDef.Version, runtime.GOOS, runtime.GOARCH)
+	arbitraryServer := newTestServer(expectedPath, validSignature)
 	defer arbitraryServer.Close()
 	arbitraryConfig := &PluginCentralConfiguration{
 		BaseURL: arbitraryServer.URL,
 	}
+	arbitraryConfig.SetDefaults()
 
 	testObject := NewPluginCentralClient(arbitraryConfig)
 
@@ -85,11 +107,13 @@ func TestCentralClient_PluginDistribution(t *testing.T) {
 		Version: "1.0.0",
 	}
 	arbitraryData := []byte("arbitrary data")
-	arbitraryServer := newTestServer("/"+arbitraryDef.RemotePath()+"/"+arbitraryDef.DistFileName(), arbitraryData)
+	expectedPath := fmt.Sprintf("/maven/bin/%s/%s/%s-%s-%s-%s.zip", arbitraryDef.Name, arbitraryDef.Version, arbitraryDef.Name, arbitraryDef.Version, runtime.GOOS, runtime.GOARCH)
+	arbitraryServer := newTestServer(expectedPath, arbitraryData)
 	defer arbitraryServer.Close()
 	arbitraryConfig := &PluginCentralConfiguration{
 		BaseURL: arbitraryServer.URL,
 	}
+	arbitraryConfig.SetDefaults()
 
 	testObject := NewPluginCentralClient(arbitraryConfig)
 

plugin/downloader_test.go

@@ -1,9 +1,11 @@
 package plugin
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
+	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -17,7 +19,7 @@ func TestDownloader_Download_whenPluginIsAvailableLocally(t *testing.T) {
 	defer func() {
 		_ = os.RemoveAll(tmpDir)
 	}()
-	arbitraryPluginDistPath := path.Join(tmpDir, "arbitrary-plugin-1.0.0.zip")
+	arbitraryPluginDistPath := path.Join(tmpDir, fmt.Sprintf("arbitrary-plugin-1.0.0-%s-%s.zip", runtime.GOOS, runtime.GOARCH))
 	if err := ioutil.WriteFile(arbitraryPluginDistPath, []byte{}, 0644); err != nil {
 		t.Fatal(err)
 	}

plugin/local_verifier.go

@@ -7,7 +7,9 @@ import (
 	"path"
 )
 
-const DefaultPublicKeyFile = "Central.pgp.pk"
+// For Cloudsmith, this references to the latest GPG key
+// being setup in the repo
+const DefaultPublicKeyFile = "gpg.key"
 
 // Local Implementation of plugin.Verifier
 type LocalVerifier struct {

plugin/service_test.go

@@ -1,6 +1,8 @@
 package plugin
 
 import (
+	"fmt"
+	"runtime"
 	"testing"
 
 	"github.com/hashicorp/go-plugin"
@@ -47,8 +49,8 @@ func TestPluginManager_ProvidersPopulation(t *testing.T) {
 	}, false, false, "")
 
 	testifyassert.NoError(t, err)
-	testifyassert.Equal(t, "arbitrary-helloWorld-1.0.0", testObject.initializedPlugins[HelloWorldPluginInterfaceName].(*basePlugin).pluginDefinition.FullName())
-	testifyassert.Equal(t, "foo-bar-2.0.0", testObject.initializedPlugins[arbitraryPluginInterfaceName].(*basePlugin).pluginDefinition.FullName())
+	testifyassert.Equal(t, fmt.Sprintf("arbitrary-helloWorld-1.0.0-%s-%s", runtime.GOOS, runtime.GOARCH), testObject.initializedPlugins[HelloWorldPluginInterfaceName].(*basePlugin).pluginDefinition.FullName())
+	testifyassert.Equal(t, fmt.Sprintf("foo-bar-2.0.0-%s-%s", runtime.GOOS, runtime.GOARCH), testObject.initializedPlugins[arbitraryPluginInterfaceName].(*basePlugin).pluginDefinition.FullName())
 }
 
 func TestPluginManager_GetPluginTemplate_whenTypical(t *testing.T) {

plugin/settings.go

@@ -80,10 +80,12 @@ var (
 
 	// this is the place holder for future solution of the plugin central
 	quorumPluginCentralConfiguration = &PluginCentralConfiguration{
-		CertFingerprint:       "",
-		BaseURL:               "https://dl.bintray.com/quorumengineering/quorum-plugins",
-		PublicKeyURI:          "/.pgp/" + DefaultPublicKeyFile,
-		InsecureSkipTLSVerify: false,
+		CertFingerprint:        "",
+		BaseURL:                "https://artifacts.consensys.net/public/quorum-go-plugins/",
+		PublicKeyURI:           DefaultPublicKeyFile,
+		InsecureSkipTLSVerify:  false,
+		PluginDistPathTemplate: "maven/bin/{{.Name}}/{{.Version}}/{{.Name}}-{{.Version}}-{{.OS}}-{{.Arch}}.zip",
+		PluginSigPathTemplate:  "maven/bin/{{.Name}}/{{.Version}}/{{.Name}}-{{.Version}}-{{.OS}}-{{.Arch}}-sha256.checksum.asc",
 	}
 )
 
@@ -146,24 +148,17 @@ func ReadMultiFormatConfig(config interface{}) ([]byte, error) {
 	}
 }
 
-// return remote folder storing the plugin distribution file and signature file
-//
-// e.g.: my-plugin/v1.0.0/darwin-amd64
-func (m *PluginDefinition) RemotePath() string {
-	return fmt.Sprintf("%s/v%s/%s-%s", m.Name, m.Version, runtime.GOOS, runtime.GOARCH)
-}
-
-// return plugin name and version
+// return plugin distribution name. i.e.: <Name>-<Version>-<OS>-<Arch>
 func (m *PluginDefinition) FullName() string {
-	return fmt.Sprintf("%s-%s", m.Name, m.Version)
+	return fmt.Sprintf("%s-%s-%s-%s", m.Name, m.Version, runtime.GOOS, runtime.GOARCH)
 }
 
-// return plugin distribution file name
+// return plugin distribution file name stored locally
 func (m *PluginDefinition) DistFileName() string {
 	return fmt.Sprintf("%s.zip", m.FullName())
 }
 
-// return plugin distribution signature file name
+// return plugin distribution signature file name stored locally
 func (m *PluginDefinition) SignatureFileName() string {
 	return fmt.Sprintf("%s.sha256sum.asc", m.DistFileName())
 }
@@ -215,6 +210,8 @@ func (s *Settings) GetPluginDefinition(name PluginInterfaceName) (*PluginDefinit
 func (s *Settings) SetDefaults() {
 	if s.CentralConfig == nil {
 		s.CentralConfig = quorumPluginCentralConfiguration
+	} else {
+		s.CentralConfig.SetDefaults()
 	}
 }
 
@@ -248,6 +245,29 @@ type PluginCentralConfiguration struct {
 	BaseURL               string `json:"baseURL" toml:""`
 	PublicKeyURI          string `json:"publicKeyURI" toml:""`
 	InsecureSkipTLSVerify bool   `json:"insecureSkipTLSVerify" toml:""`
+
+	// URL path template to the plugin distribution file.
+	// It uses Golang text template.
+	PluginDistPathTemplate string `json:"pluginDistPathTemplate" toml:""`
+	// URL path template to the plugin sha256 checksum signature file.
+	// It uses Golang text template.
+	PluginSigPathTemplate string `json:"pluginSigPathTemplate" toml:""`
+}
+
+// populate default values from quorumPluginCentralConfiguration
+func (c *PluginCentralConfiguration) SetDefaults() {
+	if len(c.BaseURL) == 0 {
+		c.BaseURL = quorumPluginCentralConfiguration.BaseURL
+	}
+	if len(c.PublicKeyURI) == 0 {
+		c.PublicKeyURI = quorumPluginCentralConfiguration.PublicKeyURI
+	}
+	if len(c.PluginDistPathTemplate) == 0 {
+		c.PluginDistPathTemplate = quorumPluginCentralConfiguration.PluginDistPathTemplate
+	}
+	if len(c.PluginSigPathTemplate) == 0 {
+		c.PluginSigPathTemplate = quorumPluginCentralConfiguration.PluginSigPathTemplate
+	}
 }
 
 // support URI format with 'env' scheme during JSON/TOML/TEXT unmarshalling