Skip to content

Commit dace0f9

Browse files
authored
cli to push metrics to aggregator service (#8835)
* new prometheus metrics for client-stats metrics * adds client-stats types beacause they are used by some of the prometheus collection code. * new prometheus collector for db disk size * new prometheus collector for web3 client connection status * adds client-stats api push cli in cmd/client-stats * adds api metadata to client-stats collector posts * appease deepsource * mop up copypasta * use prysm assert package for testing
1 parent a6e1ff0 commit dace0f9

File tree

16 files changed

+869
-4
lines changed

16 files changed

+869
-4
lines changed

beacon-chain/powchain/service.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@ func (s *Service) Status() error {
312312
return nil
313313
}
314314

315-
func (s *Service) updateBeaconnodeStats() {
315+
func (s *Service) updateBeaconNodeStats() {
316316
bs := clientstats.BeaconNodeStats{}
317317
if len(s.httpEndpoints) > 1 {
318318
bs.SyncEth1FallbackConfigured = true
@@ -329,12 +329,12 @@ func (s *Service) updateBeaconnodeStats() {
329329

330330
func (s *Service) updateCurrHttpEndpoint(endpoint httputils.Endpoint) {
331331
s.currHttpEndpoint = endpoint
332-
s.updateBeaconnodeStats()
332+
s.updateBeaconNodeStats()
333333
}
334334

335335
func (s *Service) updateConnectedETH1(state bool) {
336336
s.connectedETH1 = state
337-
s.updateBeaconnodeStats()
337+
s.updateBeaconNodeStats()
338338
}
339339

340340
// IsConnectedToETH1 checks if the beacon node is connected to a ETH1 Node.

cmd/client-stats/BUILD.bazel

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_binary")
2+
load("@prysm//tools/go:def.bzl", "go_library")
3+
4+
go_library(
5+
name = "go_default_library",
6+
srcs = [
7+
"log.go",
8+
"main.go",
9+
"usage.go",
10+
],
11+
importpath = "github.com/prysmaticlabs/prysm/cmd/client-stats",
12+
visibility = ["//visibility:private"],
13+
deps = [
14+
"//cmd/client-stats/flags:go_default_library",
15+
"//shared/clientstats:go_default_library",
16+
"//shared/cmd:go_default_library",
17+
"//shared/journald:go_default_library",
18+
"//shared/logutil:go_default_library",
19+
"//shared/version:go_default_library",
20+
"@com_github_joonix_log//:go_default_library",
21+
"@com_github_sirupsen_logrus//:go_default_library",
22+
"@com_github_urfave_cli_v2//:go_default_library",
23+
"@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
24+
],
25+
)
26+
27+
go_binary(
28+
name = "client-stats",
29+
embed = [":go_default_library"],
30+
visibility = ["//visibility:public"],
31+
)

cmd/client-stats/flags/BUILD.bazel

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
load("@prysm//tools/go:def.bzl", "go_library")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = ["flags.go"],
6+
importpath = "github.com/prysmaticlabs/prysm/cmd/client-stats/flags",
7+
visibility = ["//visibility:public"],
8+
deps = ["@com_github_urfave_cli_v2//:go_default_library"],
9+
)

cmd/client-stats/flags/flags.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Package flags contains all configuration runtime flags for
2+
// the client-stats daemon.
3+
package flags
4+
5+
import (
6+
"github.com/urfave/cli/v2"
7+
)
8+
9+
var (
10+
// BeaconCertFlag defines a flag for the beacon api certificate.
11+
ValidatorMetricsURLFlag = &cli.StringFlag{
12+
Name: "validator-metrics-url",
13+
Usage: "Full URL to the validator /metrics prometheus endpoint to scrape. eg http://localhost:8081/metrics",
14+
}
15+
// BeaconRPCProviderFlag defines a flag for the beacon host ip or address.
16+
BeaconnodeMetricsURLFlag = &cli.StringFlag{
17+
Name: "beacon-node-metrics-url",
18+
Usage: "Full URL to the beacon-node /metrics prometheus endpoint to scrape. eg http://localhost:8080/metrics",
19+
}
20+
// CertFlag defines a flag for the node's TLS certificate.
21+
ClientStatsAPIURLFlag = &cli.StringFlag{
22+
Name: "clientstats-api-url",
23+
Usage: "Full URL to the client stats endpoint where collected metrics should be sent.",
24+
}
25+
// CertFlag defines a flag for the node's TLS certificate.
26+
ScrapeIntervalFlag = &cli.DurationFlag{
27+
Name: "scrape-interval",
28+
Usage: "Frequency of scraping expressed as a duration, eg 2m or 1m5s. Default is 60s.",
29+
}
30+
)

cmd/client-stats/log.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package main
2+
3+
import "github.com/sirupsen/logrus"
4+
5+
var log = logrus.WithField("prefix", "main")

cmd/client-stats/main.go

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
runtimeDebug "runtime/debug"
7+
"time"
8+
9+
joonix "github.com/joonix/log"
10+
"github.com/prysmaticlabs/prysm/cmd/client-stats/flags"
11+
"github.com/prysmaticlabs/prysm/shared/clientstats"
12+
"github.com/prysmaticlabs/prysm/shared/cmd"
13+
"github.com/prysmaticlabs/prysm/shared/journald"
14+
"github.com/prysmaticlabs/prysm/shared/logutil"
15+
"github.com/prysmaticlabs/prysm/shared/version"
16+
"github.com/sirupsen/logrus"
17+
"github.com/urfave/cli/v2"
18+
prefixed "github.com/x-cray/logrus-prefixed-formatter"
19+
)
20+
21+
var appFlags = []cli.Flag{
22+
cmd.VerbosityFlag,
23+
cmd.LogFormat,
24+
cmd.LogFileName,
25+
cmd.ConfigFileFlag,
26+
flags.BeaconnodeMetricsURLFlag,
27+
flags.ValidatorMetricsURLFlag,
28+
flags.ClientStatsAPIURLFlag,
29+
flags.ScrapeIntervalFlag,
30+
}
31+
var scrapeInterval = 60 * time.Second
32+
33+
func main() {
34+
app := cli.App{}
35+
app.Name = "client-stats"
36+
app.Usage = "daemon to scrape client-stats from prometheus and ship to a remote endpoint"
37+
app.Action = run
38+
app.Version = version.Version()
39+
40+
app.Flags = appFlags
41+
42+
// logging/config setup cargo-culted from beaconchain
43+
app.Before = func(ctx *cli.Context) error {
44+
// Load flags from config file, if specified.
45+
if err := cmd.LoadFlagsFromConfig(ctx, app.Flags); err != nil {
46+
return err
47+
}
48+
49+
verbosity := ctx.String(cmd.VerbosityFlag.Name)
50+
level, err := logrus.ParseLevel(verbosity)
51+
if err != nil {
52+
return err
53+
}
54+
logrus.SetLevel(level)
55+
56+
format := ctx.String(cmd.LogFormat.Name)
57+
switch format {
58+
case "text":
59+
formatter := new(prefixed.TextFormatter)
60+
formatter.TimestampFormat = "2006-01-02 15:04:05"
61+
formatter.FullTimestamp = true
62+
// If persistent log files are written - we disable the log messages coloring because
63+
// the colors are ANSI codes and seen as gibberish in the log files.
64+
formatter.DisableColors = ctx.String(cmd.LogFileName.Name) != ""
65+
logrus.SetFormatter(formatter)
66+
case "fluentd":
67+
f := joonix.NewFormatter()
68+
if err := joonix.DisableTimestampFormat(f); err != nil {
69+
panic(err)
70+
}
71+
logrus.SetFormatter(f)
72+
case "json":
73+
logrus.SetFormatter(&logrus.JSONFormatter{})
74+
case "journald":
75+
if err := journald.Enable(); err != nil {
76+
return err
77+
}
78+
default:
79+
return fmt.Errorf("unknown log format %s", format)
80+
}
81+
82+
logFileName := ctx.String(cmd.LogFileName.Name)
83+
if logFileName != "" {
84+
if err := logutil.ConfigurePersistentLogging(logFileName); err != nil {
85+
log.WithError(err).Error("Failed to configuring logging to disk.")
86+
}
87+
}
88+
return nil
89+
}
90+
91+
defer func() {
92+
if x := recover(); x != nil {
93+
log.Errorf("Runtime panic: %v\n%v", x, string(runtimeDebug.Stack()))
94+
panic(x)
95+
}
96+
}()
97+
98+
if err := app.Run(os.Args); err != nil {
99+
log.Error(err.Error())
100+
}
101+
}
102+
103+
func run(ctx *cli.Context) error {
104+
if ctx.IsSet(flags.ScrapeIntervalFlag.Name) {
105+
scrapeInterval = ctx.Duration(flags.ScrapeIntervalFlag.Name)
106+
}
107+
108+
var upd clientstats.Updater
109+
if ctx.IsSet(flags.ClientStatsAPIURLFlag.Name) {
110+
u := ctx.String(flags.ClientStatsAPIURLFlag.Name)
111+
upd = clientstats.NewClientStatsHTTPPostUpdater(u)
112+
} else {
113+
log.Warn("No --clientstats-api-url flag set, writing to stdout as default metrics sink.")
114+
upd = clientstats.NewGenericClientStatsUpdater(os.Stdout)
115+
}
116+
117+
scrapers := make([]clientstats.Scraper, 0)
118+
if ctx.IsSet(flags.BeaconnodeMetricsURLFlag.Name) {
119+
u := ctx.String(flags.BeaconnodeMetricsURLFlag.Name)
120+
scrapers = append(scrapers, clientstats.NewBeaconNodeScraper(u))
121+
}
122+
if ctx.IsSet(flags.ValidatorMetricsURLFlag.Name) {
123+
u := ctx.String(flags.ValidatorMetricsURLFlag.Name)
124+
scrapers = append(scrapers, clientstats.NewValidatorScraper(u))
125+
}
126+
127+
ticker := time.NewTicker(scrapeInterval)
128+
for {
129+
select {
130+
case <-ticker.C:
131+
for _, s := range scrapers {
132+
r, err := s.Scrape()
133+
if err != nil {
134+
log.Errorf("Scraper error: %s", err)
135+
continue
136+
}
137+
err = upd.Update(r)
138+
if err != nil {
139+
log.Errorf("client-stats collector error: %s", err)
140+
continue
141+
}
142+
}
143+
case <-ctx.Done():
144+
ticker.Stop()
145+
return nil
146+
}
147+
}
148+
}

cmd/client-stats/usage.go

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// This code was adapted from https://github.com/ethereum/go-ethereum/blob/master/cmd/geth/usage.go
2+
package main
3+
4+
import (
5+
"io"
6+
"sort"
7+
8+
"github.com/prysmaticlabs/prysm/cmd/client-stats/flags"
9+
"github.com/prysmaticlabs/prysm/shared/cmd"
10+
"github.com/urfave/cli/v2"
11+
)
12+
13+
var appHelpTemplate = `NAME:
14+
{{.App.Name}} - {{.App.Usage}}
15+
USAGE:
16+
{{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}}
17+
{{if .App.Version}}
18+
AUTHOR:
19+
{{range .App.Authors}}{{ . }}{{end}}
20+
{{end}}{{if .App.Commands}}
21+
GLOBAL OPTIONS:
22+
{{range .App.Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
23+
{{end}}{{end}}{{if .FlagGroups}}
24+
{{range .FlagGroups}}{{.Name}} OPTIONS:
25+
{{range .Flags}}{{.}}
26+
{{end}}
27+
{{end}}{{end}}{{if .App.Copyright }}
28+
COPYRIGHT:
29+
{{.App.Copyright}}
30+
VERSION:
31+
{{.App.Version}}
32+
{{end}}{{if len .App.Authors}}
33+
{{end}}
34+
`
35+
36+
type flagGroup struct {
37+
Name string
38+
Flags []cli.Flag
39+
}
40+
41+
var appHelpFlagGroups = []flagGroup{
42+
{
43+
Name: "cmd",
44+
Flags: []cli.Flag{
45+
cmd.VerbosityFlag,
46+
cmd.LogFormat,
47+
cmd.LogFileName,
48+
cmd.ConfigFileFlag,
49+
},
50+
},
51+
{
52+
Name: "client-stats",
53+
Flags: []cli.Flag{
54+
flags.BeaconnodeMetricsURLFlag,
55+
flags.ValidatorMetricsURLFlag,
56+
flags.ClientStatsAPIURLFlag,
57+
flags.ScrapeIntervalFlag,
58+
},
59+
},
60+
}
61+
62+
func init() {
63+
cli.AppHelpTemplate = appHelpTemplate
64+
65+
type helpData struct {
66+
App interface{}
67+
FlagGroups []flagGroup
68+
}
69+
70+
originalHelpPrinter := cli.HelpPrinter
71+
cli.HelpPrinter = func(w io.Writer, tmpl string, data interface{}) {
72+
if tmpl == appHelpTemplate {
73+
for _, group := range appHelpFlagGroups {
74+
sort.Sort(cli.FlagsByName(group.Flags))
75+
}
76+
originalHelpPrinter(w, tmpl, helpData{data, appHelpFlagGroups})
77+
} else {
78+
originalHelpPrinter(w, tmpl, data)
79+
}
80+
}
81+
}

deps.bzl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2669,6 +2669,13 @@ def prysm_deps():
26692669
sum = "h1:Uehi/mxLK0eiUc0H0++5tpMGTexB8wZ598MIgU8VpDM=",
26702670
version = "v0.3.0",
26712671
)
2672+
go_repository(
2673+
name = "com_github_prometheus_prom2json",
2674+
importpath = "github.com/prometheus/prom2json",
2675+
sum = "h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y=",
2676+
version = "v1.3.0",
2677+
)
2678+
26722679
go_repository(
26732680
name = "com_github_prometheus_tsdb",
26742681
importpath = "github.com/prometheus/tsdb",

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,9 @@ require (
8282
github.com/pkg/errors v0.9.1
8383
github.com/prestonvanloon/go-recaptcha v0.0.0-20190217191114-0834cef6e8bd
8484
github.com/prometheus/client_golang v1.9.0
85+
github.com/prometheus/client_model v0.2.0
8586
github.com/prometheus/procfs v0.3.0 // indirect
87+
github.com/prometheus/prom2json v1.3.0
8688
github.com/prometheus/tsdb v0.10.0 // indirect
8789
github.com/prysmaticlabs/eth2-types v0.0.0-20210219172114-1da477c09a06
8890
github.com/prysmaticlabs/ethereumapis v0.0.0-20210311175904-cf9f64632dd4

0 commit comments

Comments
 (0)