feat(cmd/gossamer): implement --telemetry-url parameter (ChainSafe#1890)

* Implement --telemetry-url Parameter

Fixes ChainSafe#1502

* Added tests for --telemetry-url parameter

* Fixed tests failing because of nil telemetry endpoints

* Addressed some small review comments

* Fixing tests by using []TelemetryEndpoint(nil)

as TelemetryURLs

* tests for telemetry-url with some failure cases

* replace telemetry endpoints instead of appending them

* trying to increase coverage

* Revert "trying to increase coverage"

This reverts commit ee3b096.

chain/dev/defaults.go

@@ -17,6 +17,7 @@
 package dev
 
 import (
+	"github.com/ChainSafe/gossamer/lib/genesis"
 	"github.com/ChainSafe/gossamer/lib/runtime/wasmer"
 	log "github.com/ChainSafe/log15"
 )
@@ -44,6 +45,9 @@ var (
 	// DefaultRetainBlocks is the default retained blocks
 	DefaultRetainBlocks = int64(512)
 
+	// DefaultTelemetryURLs is the default URL of the telemetry server to connect to.
+	DefaultTelemetryURLs []genesis.TelemetryEndpoint
+
 	// InitConfig
 
 	// DefaultGenesis is the default genesis configuration path

chain/gssmr/defaults.go

@@ -19,6 +19,7 @@ package gssmr
 import (
 	"time"
 
+	"github.com/ChainSafe/gossamer/lib/genesis"
 	"github.com/ChainSafe/gossamer/lib/runtime/wasmer"
 	log "github.com/ChainSafe/log15"
 )
@@ -46,6 +47,9 @@ var (
 	// DefaultRetainBlocks is the default retained blocks
 	DefaultRetainBlocks = int64(512)
 
+	// DefaultTelemetryURLs is the default URL of the telemetry server to connect to.
+	DefaultTelemetryURLs []genesis.TelemetryEndpoint
+
 	// InitConfig
 
 	// DefaultGenesis is the default genesis configuration path

chain/kusama/defaults.go

@@ -17,6 +17,7 @@
 package kusama
 
 import (
+	"github.com/ChainSafe/gossamer/lib/genesis"
 	"github.com/ChainSafe/gossamer/lib/runtime/wasmer"
 	log "github.com/ChainSafe/log15"
 )
@@ -44,6 +45,9 @@ var (
 	// DefaultRetainBlocks is the default retained blocks
 	DefaultRetainBlocks = int64(512)
 
+	// DefaultTelemetryURLs is the default URL of the telemetry server to connect to.
+	DefaultTelemetryURLs []genesis.TelemetryEndpoint
+
 	// InitConfig
 
 	// DefaultGenesis is the default genesis configuration path

chain/polkadot/defaults.go

@@ -17,6 +17,7 @@
 package polkadot
 
 import (
+	"github.com/ChainSafe/gossamer/lib/genesis"
 	"github.com/ChainSafe/gossamer/lib/runtime/wasmer"
 	log "github.com/ChainSafe/log15"
 )
@@ -41,6 +42,9 @@ var (
 	// DefaultRetainBlocks is the default pruning mode
 	DefaultRetainBlocks = int64(512)
 
+	// DefaultTelemetryURLs is the default URL of the telemetry server to connect to.
+	DefaultTelemetryURLs []genesis.TelemetryEndpoint
+
 	// InitConfig
 
 	// DefaultGenesis is the default genesis configuration path

cmd/gossamer/config.go

@@ -421,7 +421,9 @@ func setDotInitConfig(ctx *cli.Context, tomlCfg ctoml.InitConfig, cfg *dot.InitC
 
 func setDotGlobalConfig(ctx *cli.Context, tomlConfig *ctoml.Config, cfg *dot.GlobalConfig) error {
 	setDotGlobalConfigFromToml(tomlConfig, cfg)
-	setDotGlobalConfigFromFlags(ctx, cfg)
+	if err := setDotGlobalConfigFromFlags(ctx, cfg); err != nil {
+		return fmt.Errorf("could not set global config from flags: %w", err)
+	}
 
 	if err := setDotGlobalConfigName(ctx, tomlConfig, cfg); err != nil {
 		return fmt.Errorf("could not set global node name: %w", err)
@@ -460,7 +462,7 @@ func setDotGlobalConfigFromToml(tomlCfg *ctoml.Config, cfg *dot.GlobalConfig) {
 }
 
 // setDotGlobalConfigFromFlags sets dot.GlobalConfig using flag values from the cli context
-func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) {
+func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) error {
 	// check --basepath flag and update node configuration
 	if basepath := ctx.GlobalString(BasePathFlag.Name); basepath != "" {
 		cfg.BasePath = basepath
@@ -488,6 +490,28 @@ func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) {
 	cfg.RetainBlocks = ctx.Int64(RetainBlockNumberFlag.Name)
 	cfg.Pruning = pruner.Mode(ctx.String(PruningFlag.Name))
 	cfg.NoTelemetry = ctx.Bool("no-telemetry")
+
+	var telemetryEndpoints []genesis.TelemetryEndpoint
+	for _, telemetryURL := range ctx.GlobalStringSlice(TelemetryURLFlag.Name) {
+		splits := strings.Split(telemetryURL, " ")
+		if len(splits) != 2 {
+			return fmt.Errorf("%s must be in the format 'URL VERBOSITY'", TelemetryURLFlag.Name)
+		}
+
+		verbosity, err := strconv.Atoi(splits[1])
+		if err != nil {
+			return fmt.Errorf("could not parse verbosity from %s: %w", TelemetryURLFlag.Name, err)
+		}
+
+		telemetryEndpoints = append(telemetryEndpoints, genesis.TelemetryEndpoint{
+			Endpoint:  splits[0],
+			Verbosity: verbosity,
+		})
+	}
+
+	cfg.TelemetryURLs = telemetryEndpoints
+
+	return nil
 }
 
 func setDotGlobalConfigName(ctx *cli.Context, tomlCfg *ctoml.Config, cfg *dot.GlobalConfig) error {

cmd/gossamer/config_test.go

@@ -245,6 +245,28 @@ func TestGlobalConfigFromFlags(t *testing.T) {
 				NoTelemetry:    true,
 			},
 		},
+		{
+			"Test gossamer --telemetry-url",
+			[]string{"config", "telemetry-url", "name"},
+			[]interface{}{
+				testCfgFile.Name(),
+				[]string{"ws://localhost:8001/submit 0", "ws://foo/bar 0"},
+				testCfg.Global.Name,
+			},
+			dot.GlobalConfig{
+				Name:           testCfg.Global.Name,
+				ID:             testCfg.Global.ID,
+				BasePath:       testCfg.Global.BasePath,
+				LogLvl:         log.LvlInfo,
+				PublishMetrics: testCfg.Global.PublishMetrics,
+				MetricsPort:    testCfg.Global.MetricsPort,
+				NoTelemetry:    false,
+				TelemetryURLs: []genesis.TelemetryEndpoint{
+					{Endpoint: "ws://localhost:8001/submit", Verbosity: 0},
+					{Endpoint: "ws://foo/bar", Verbosity: 0},
+				},
+			},
+		},
 	}
 
 	for _, c := range testcases {
@@ -260,6 +282,58 @@ func TestGlobalConfigFromFlags(t *testing.T) {
 	}
 }
 
+func TestGlobalConfigFromFlagsFails(t *testing.T) {
+	testCfg, testCfgFile := newTestConfigWithFile(t)
+	require.NotNil(t, testCfg)
+	require.NotNil(t, testCfgFile)
+
+	defer utils.RemoveTestDir(t)
+
+	testApp := cli.NewApp()
+	testApp.Writer = ioutil.Discard
+
+	testcases := []struct {
+		description string
+		flags       []string
+		values      []interface{}
+		err         string
+	}{
+		{
+			"Test gossamer --telemetry-url invalid format",
+			[]string{"config", "telemetry-url", "name"},
+			[]interface{}{
+				testCfgFile.Name(),
+				[]string{"ws://localhost:8001/submit"},
+				testCfg.Global.Name,
+			},
+			"could not set global config from flags: telemetry-url must be in the format 'URL VERBOSITY'",
+		},
+		{
+			"Test gossamer invalid --telemetry-url invalid verbosity",
+			[]string{"config", "telemetry-url", "name"},
+			[]interface{}{
+				testCfgFile.Name(),
+				[]string{"ws://foo/bar k"},
+				testCfg.Global.Name,
+			},
+			"could not set global config from flags: could not parse verbosity from telemetry-url: strconv.Atoi: parsing \"k\": invalid syntax",
+		},
+	}
+
+	for _, c := range testcases {
+		c := c // bypass scopelint false positive
+		t.Run(c.description, func(t *testing.T) {
+			ctx, err := newTestContext(c.description, c.flags, c.values)
+			require.Nil(t, err)
+
+			cfg, err := createDotConfig(ctx)
+			require.NotNil(t, err)
+			require.Nil(t, cfg)
+			require.Equal(t, c.err, err.Error())
+		})
+	}
+}
+
 // TestAccountConfigFromFlags tests createDotAccountConfig using relevant account flags
 func TestAccountConfigFromFlags(t *testing.T) {
 	testCfg, testCfgFile := newTestConfigWithFile(t)
@@ -696,6 +770,7 @@ func TestUpdateConfigFromGenesisJSON(t *testing.T) {
 			LogLvl:         testCfg.Global.LogLvl,
 			PublishMetrics: testCfg.Global.PublishMetrics,
 			MetricsPort:    testCfg.Global.MetricsPort,
+			TelemetryURLs:  testCfg.Global.TelemetryURLs,
 		},
 		Log: dot.LogConfig{
 			CoreLvl:           log.LvlInfo,
@@ -749,6 +824,7 @@ func TestUpdateConfigFromGenesisJSON_Default(t *testing.T) {
 			LogLvl:         testCfg.Global.LogLvl,
 			PublishMetrics: testCfg.Global.PublishMetrics,
 			MetricsPort:    testCfg.Global.MetricsPort,
+			TelemetryURLs:  testCfg.Global.TelemetryURLs,
 		},
 		Log: dot.LogConfig{
 			CoreLvl:           log.LvlInfo,
@@ -798,6 +874,7 @@ func TestUpdateConfigFromGenesisData(t *testing.T) {
 			LogLvl:         testCfg.Global.LogLvl,
 			PublishMetrics: testCfg.Global.PublishMetrics,
 			MetricsPort:    testCfg.Global.MetricsPort,
+			TelemetryURLs:  testCfg.Global.TelemetryURLs,
 		},
 		Log: dot.LogConfig{
 			CoreLvl:           log.LvlInfo,

cmd/gossamer/flags.go

@@ -105,6 +105,19 @@ var (
 		Name:  "no-telemetry",
 		Usage: "Disable connecting to the Substrate telemetry server",
 	}
+
+	// TelemetryURLFlag is URL of the telemetry server to connect to.
+	// This flag can be passed multiple times as a means to specify multiple
+	// telemetry endpoints. Verbosity levels range from 0-9, with 0 denoting the
+	// least verbosity.
+	// Expected format is 'URL VERBOSITY', e.g. `--telemetry-url 'wss://foo/bar 0'`.
+	TelemetryURLFlag = cli.StringSliceFlag{
+		Name: "telemetry-url",
+		Usage: `The URL of the telemetry server to connect to, this flag can be
+		passed multiple times, the verbosity levels range from 0-9, with 0 denoting
+		least verbosity.
+		Expected format --telemetry-url 'wss://foo/bar 0'`,
+	}
 )
 
 // Initialization-only flags
@@ -374,6 +387,7 @@ var (
 
 		// telemetry flags
 		NoTelemetryFlag,
+		TelemetryURLFlag,
 
 		// BABE flags
 		BABELeadFlag,

cmd/gossamer/utils.go

@@ -95,6 +95,7 @@ func newTestConfig(t *testing.T) *dot.Config {
 			MetricsPort:    dot.GssmrConfig().Global.MetricsPort,
 			RetainBlocks:   dot.GssmrConfig().Global.RetainBlocks,
 			Pruning:        dot.GssmrConfig().Global.Pruning,
+			TelemetryURLs:  dot.GssmrConfig().Global.TelemetryURLs,
 		},
 		Log: dot.LogConfig{
 			CoreLvl:           log.LvlInfo,

cmd/gossamer/utils_test.go

@@ -29,6 +29,11 @@ import (
 
 // newTestContext creates a cli context for a test given a set of flags and values
 func newTestContext(description string, flags []string, values []interface{}) (*cli.Context, error) {
+	if len(flags) != len(values) {
+		return nil, fmt.Errorf("number of flags and values are not same, number of flags: %d, number of values: %d", len(flags), len(values))
+	}
+
+	// Define flags with its name and default value
 	set := flag.NewFlagSet(description, 0)
 	for i := range values {
 		switch v := values[i].(type) {
@@ -40,6 +45,8 @@ func newTestContext(description string, flags []string, values []interface{}) (*
 			set.Uint(flags[i], v, "")
 		case int64:
 			set.Int64(flags[i], v, "")
+		case []string:
+			set.Var(&cli.StringSlice{}, flags[i], "")
 		default:
 			return nil, fmt.Errorf("unexpected cli value type: %T", values[i])
 		}
@@ -50,31 +57,31 @@ func newTestContext(description string, flags []string, values []interface{}) (*
 	for i := range values {
 		switch v := values[i].(type) {
 		case bool:
-			if v {
-				err := ctx.Set(flags[i], "true")
-				if err != nil {
-					return nil, fmt.Errorf("failed to set cli flag: %T", flags[i])
-				}
-			} else {
-				err := ctx.Set(flags[i], "false")
-				if err != nil {
-					return nil, fmt.Errorf("failed to set cli flag: %T", flags[i])
-				}
+			err := ctx.Set(flags[i], strconv.FormatBool(v))
+			if err != nil {
+				return nil, fmt.Errorf("failed to set cli flag: %T, err: %w", flags[i], err)
 			}
 		case string:
 			err := ctx.Set(flags[i], values[i].(string))
 			if err != nil {
-				return nil, fmt.Errorf("failed to set cli flag: %T", flags[i])
+				return nil, fmt.Errorf("failed to set cli flag: %T, err: %w", flags[i], err)
 			}
 		case uint:
 			err := ctx.Set(flags[i], strconv.Itoa(int(values[i].(uint))))
 			if err != nil {
-				return nil, fmt.Errorf("failed to set cli flag: %T", flags[i])
+				return nil, fmt.Errorf("failed to set cli flag: %T, err: %w", flags[i], err)
 			}
 		case int64:
 			err := ctx.Set(flags[i], strconv.Itoa(int(values[i].(int64))))
 			if err != nil {
-				return nil, fmt.Errorf("failed to set cli flag: %T", flags[i])
+				return nil, fmt.Errorf("failed to set cli flag: %T, err: %w", flags[i], err)
+			}
+		case []string:
+			for _, str := range values[i].([]string) {
+				err := ctx.Set(flags[i], str)
+				if err != nil {
+					return nil, fmt.Errorf("failed to set cli flag: %T, err: %w", flags[i], err)
+				}
 			}
 		default:
 			return nil, fmt.Errorf("unexpected cli value type: %T", values[i])