diff --git a/cmd/sentry/main.go b/cmd/sentry/main.go index d73089de799..c77b0e8e71b 100644 --- a/cmd/sentry/main.go +++ b/cmd/sentry/main.go @@ -6,13 +6,14 @@ import ( "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/datadir" + "github.com/spf13/cobra" + "github.com/ledgerwatch/erigon/cmd/sentry/sentry" "github.com/ledgerwatch/erigon/cmd/utils" "github.com/ledgerwatch/erigon/common/paths" "github.com/ledgerwatch/erigon/turbo/debug" logging2 "github.com/ledgerwatch/erigon/turbo/logging" node2 "github.com/ledgerwatch/erigon/turbo/node" - "github.com/spf13/cobra" ) // generate the messages @@ -33,6 +34,7 @@ var ( maxPeers int maxPendPeers int healthCheck bool + metrics bool ) func init() { @@ -52,10 +54,15 @@ func init() { rootCmd.Flags().IntVar(&maxPeers, utils.MaxPeersFlag.Name, utils.MaxPeersFlag.Value, utils.MaxPeersFlag.Usage) rootCmd.Flags().IntVar(&maxPendPeers, utils.MaxPendingPeersFlag.Name, utils.MaxPendingPeersFlag.Value, utils.MaxPendingPeersFlag.Usage) rootCmd.Flags().BoolVar(&healthCheck, utils.HealthCheckFlag.Name, false, utils.HealthCheckFlag.Usage) + rootCmd.Flags().BoolVar(&metrics, utils.MetricsEnabledFlag.Name, false, utils.MetricsEnabledFlag.Usage) if err := rootCmd.MarkFlagDirname(utils.DataDirFlag.Name); err != nil { panic(err) } + + if err := debug.SetCobraFlagsFromConfigFile(rootCmd); err != nil { + panic(err) + } } var rootCmd = &cobra.Command{ @@ -85,6 +92,7 @@ var rootCmd = &cobra.Command{ uint(port), protocol, allowedPorts, + metrics, ) if err != nil { return err diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 57eab187215..0355ff9b39c 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -905,6 +905,7 @@ func NewP2PConfig( port uint, protocol uint, allowedPorts []uint, + metricsEnabled bool, ) (*p2p.Config, error) { var enodeDBPath string switch protocol { @@ -935,6 +936,7 @@ func NewP2PConfig( NodeDatabase: enodeDBPath, AllowedPorts: allowedPorts, TmpDir: dirs.Tmp, + MetricsEnabled: metricsEnabled, } if netRestrict != "" { cfg.NetRestrict = new(netutil.Netlist) @@ -1098,6 +1100,10 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config, nodeName, datadir string) { cfg.DiscoveryV5 = ctx.Bool(DiscoveryV5Flag.Name) } + if ctx.IsSet(MetricsEnabledFlag.Name) { + cfg.MetricsEnabled = ctx.Bool(MetricsEnabledFlag.Name) + } + ethPeers := cfg.MaxPeers cfg.Name = nodeName log.Info("Maximum peer count", "ETH", ethPeers, "total", cfg.MaxPeers) diff --git a/go.mod b/go.mod index d8cbf5632d5..414c13b3b70 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230313075942-ae07c2744483 + github.com/ledgerwatch/erigon-lib v0.0.0-20230313125449-2155965bd62d github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 4c1f3d288a8..d2c6cb86784 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230313075942-ae07c2744483 h1:59UeMgiynV+JflMPfQCH+I4qxVIer0wSgtEJvX+sUV8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230313075942-ae07c2744483/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= +github.com/ledgerwatch/erigon-lib v0.0.0-20230313125449-2155965bd62d h1:TEbIpIPOXtLvBd9WLCGFRHmSGAF+GJ0y1TX3g5L78jo= +github.com/ledgerwatch/erigon-lib v0.0.0-20230313125449-2155965bd62d/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= diff --git a/metrics/init_test.go b/metrics/init_test.go deleted file mode 100644 index 6191feb1880..00000000000 --- a/metrics/init_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package metrics - -import ( - "github.com/ledgerwatch/erigon-lib/common/metrics" -) - -func init() { - metrics.Enabled = true -} diff --git a/p2p/peer.go b/p2p/peer.go index 89cd68ec6e1..05930e032e5 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -26,14 +26,14 @@ import ( "time" metrics2 "github.com/VictoriaMetrics/metrics" - "github.com/ledgerwatch/erigon-lib/common/metrics" + "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/common/debug" "github.com/ledgerwatch/erigon/common/mclock" "github.com/ledgerwatch/erigon/event" "github.com/ledgerwatch/erigon/p2p/enode" "github.com/ledgerwatch/erigon/p2p/enr" "github.com/ledgerwatch/erigon/rlp" - "github.com/ledgerwatch/log/v3" ) var ( @@ -117,16 +117,17 @@ type Peer struct { disc chan DiscReason // events receives message send / receive events if set - events *event.Feed - pubkey [64]byte + events *event.Feed + pubkey [64]byte + metricsEnabled bool } // NewPeer returns a peer for testing purposes. -func NewPeer(id enode.ID, pubkey [64]byte, name string, caps []Cap) *Peer { +func NewPeer(id enode.ID, pubkey [64]byte, name string, caps []Cap, metricsEnabled bool) *Peer { pipe, _ := net.Pipe() node := enode.SignNull(new(enr.Record), id) conn := &conn{fd: pipe, transport: nil, node: node, caps: caps, name: name} - peer := newPeer(log.Root(), conn, nil, pubkey) + peer := newPeer(log.Root(), conn, nil, pubkey, metricsEnabled) close(peer.closed) // ensures Disconnect doesn't block return peer } @@ -209,17 +210,18 @@ func (p *Peer) Inbound() bool { return p.rw.is(inboundConn) } -func newPeer(logger log.Logger, conn *conn, protocols []Protocol, pubkey [64]byte) *Peer { +func newPeer(logger log.Logger, conn *conn, protocols []Protocol, pubkey [64]byte, metricsEnabled bool) *Peer { protomap := matchProtocols(protocols, conn.caps, conn) p := &Peer{ - rw: conn, - running: protomap, - created: mclock.Now(), - disc: make(chan DiscReason), - protoErr: make(chan error, len(protomap)+1), // protocols + pingLoop - closed: make(chan struct{}), - log: logger.New("id", conn.node.ID(), "conn", conn.flags), - pubkey: pubkey, + rw: conn, + running: protomap, + created: mclock.Now(), + disc: make(chan DiscReason), + protoErr: make(chan error, len(protomap)+1), // protocols + pingLoop + closed: make(chan struct{}), + log: logger.New("id", conn.node.ID(), "conn", conn.flags), + pubkey: pubkey, + metricsEnabled: metricsEnabled, } return p } @@ -335,7 +337,7 @@ func (p *Peer) handle(msg Msg) error { if err != nil { return fmt.Errorf("msg code out of range: %v", msg.Code) } - if metrics.Enabled { + if p.metricsEnabled { m := fmt.Sprintf("%s_%s_%d_%#02x", ingressMeterName, proto.Name, proto.Version, msg.Code-proto.offset) metrics2.GetOrCreateCounter(m).Set(uint64(msg.meterSize)) metrics2.GetOrCreateCounter(m + "_packets").Set(1) diff --git a/p2p/peer_test.go b/p2p/peer_test.go index 637a3f19ca3..10131a3b9b7 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -28,9 +28,10 @@ import ( "testing" "time" + "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/p2p/enode" "github.com/ledgerwatch/erigon/p2p/enr" - "github.com/ledgerwatch/log/v3" ) var discard = Protocol{ @@ -98,7 +99,7 @@ func testPeer(protos []Protocol) (func(), *conn, *Peer, <-chan error) { c2.caps = append(c2.caps, p.cap()) } - peer := newPeer(log.Root(), c1, protos, [64]byte{1}) + peer := newPeer(log.Root(), c1, protos, [64]byte{1}, true) errc := make(chan error, 1) go func() { _, err := peer.run() @@ -250,7 +251,7 @@ func TestNewPeer(t *testing.T) { name := "nodename" caps := []Cap{{"foo", 2}, {"bar", 3}} id := randomID() - p := NewPeer(id, [64]byte{1}, name, caps) + p := NewPeer(id, [64]byte{1}, name, caps, true) if p.ID() != id { t.Errorf("ID mismatch: got %v, expected %v", p.ID(), id) } diff --git a/p2p/server.go b/p2p/server.go index 362995b2237..f9243e506b6 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -32,6 +32,8 @@ import ( "golang.org/x/sync/semaphore" + "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/common/debug" "github.com/ledgerwatch/erigon/common/mclock" @@ -42,7 +44,6 @@ import ( "github.com/ledgerwatch/erigon/p2p/enr" "github.com/ledgerwatch/erigon/p2p/nat" "github.com/ledgerwatch/erigon/p2p/netutil" - "github.com/ledgerwatch/log/v3" ) const ( @@ -171,6 +172,8 @@ type Config struct { clock mclock.Clock //nolint:structcheck TmpDir string + + MetricsEnabled bool } // Server manages all peer connections. @@ -1057,7 +1060,7 @@ func (srv *Server) checkpoint(c *conn, stage chan<- *conn) error { } func (srv *Server) launchPeer(c *conn, pubkey [64]byte) *Peer { - p := newPeer(srv.log, c, srv.Protocols, pubkey) + p := newPeer(srv.log, c, srv.Protocols, pubkey, srv.MetricsEnabled) if srv.EnableMsgEvents { // If message events are enabled, pass the peerFeed // to the peer. diff --git a/turbo/debug/flags.go b/turbo/debug/flags.go index 19bbdbe4dd4..7dd6a48fda0 100644 --- a/turbo/debug/flags.go +++ b/turbo/debug/flags.go @@ -17,19 +17,24 @@ package debug import ( + "errors" "fmt" "net/http" _ "net/http/pprof" //nolint:gosec + "os" + "path/filepath" metrics2 "github.com/VictoriaMetrics/metrics" - "github.com/ledgerwatch/erigon-lib/common/metrics" + "github.com/ledgerwatch/log/v3" + "github.com/pelletier/go-toml" + "github.com/spf13/cobra" + "github.com/urfave/cli/v2" + "gopkg.in/yaml.v2" + "github.com/ledgerwatch/erigon/common/fdlimit" "github.com/ledgerwatch/erigon/diagnostics" "github.com/ledgerwatch/erigon/metrics/exp" "github.com/ledgerwatch/erigon/turbo/logging" - "github.com/ledgerwatch/log/v3" - "github.com/spf13/cobra" - "github.com/urfave/cli/v2" ) var ( @@ -39,6 +44,12 @@ var ( Usage: "Per-module verbosity: comma-separated list of = (e.g. eth/*=5,p2p=4)", Value: "", } + configFlag = cli.StringFlag{ + Name: "config", + } + metricsEnabledFlag = cli.BoolFlag{ + Name: "metrics", + } metricsAddrFlag = cli.StringFlag{ Name: "metrics.addr", } @@ -77,6 +88,10 @@ var Flags = []cli.Flag{ } func SetupCobra(cmd *cobra.Command) error { + // ensure we've read in config file details before setting up metrics etc. + if err := SetCobraFlagsFromConfigFile(cmd); err != nil { + log.Warn("failed setting config flags from yaml/toml file", "err", err) + } RaiseFdLimit() flags := cmd.Flags() @@ -117,6 +132,10 @@ func SetupCobra(cmd *cobra.Command) error { return err } + metricsEnabled, err := flags.GetBool(metricsEnabledFlag.Name) + if err != nil { + return err + } metricsAddr, err := flags.GetString(metricsAddrFlag.Name) if err != nil { return err @@ -126,12 +145,12 @@ func SetupCobra(cmd *cobra.Command) error { return err } - if metrics.Enabled && metricsAddr != "" { + if metricsEnabled && metricsAddr != "" { address := fmt.Sprintf("%s:%d", metricsAddr, metricsPort) exp.Setup(address) } - withMetrics := metrics.Enabled && metricsAddr == "" + withMetrics := metricsEnabled && metricsAddr == "" if pprof { // metrics and pprof server StartPProf(fmt.Sprintf("%s:%d", pprofAddr, pprofPort), withMetrics) @@ -142,6 +161,11 @@ func SetupCobra(cmd *cobra.Command) error { // Setup initializes profiling and logging based on the CLI flags. // It should be called as early as possible in the program. func Setup(ctx *cli.Context) error { + // ensure we've read in config file details before setting up metrics etc. + if err := SetFlagsFromConfigFile(ctx); err != nil { + log.Warn("failed setting config flags from yaml/toml file", "err", err) + } + RaiseFdLimit() _ = logging.GetLoggerCtx("debug", ctx) @@ -158,9 +182,10 @@ func Setup(ctx *cli.Context) error { } } pprofEnabled := ctx.Bool(pprofFlag.Name) + metricsEnabled := ctx.Bool(metricsEnabledFlag.Name) metricsAddr := ctx.String(metricsAddrFlag.Name) - if metrics.Enabled && (!pprofEnabled || metricsAddr != "") { + if metricsEnabled && (!pprofEnabled || metricsAddr != "") { metricsPort := ctx.Int(metricsPortFlag.Name) address := fmt.Sprintf("%s:%d", metricsAddr, metricsPort) exp.Setup(address) @@ -175,7 +200,7 @@ func Setup(ctx *cli.Context) error { address := fmt.Sprintf("%s:%d", pprofHost, pprofPort) // This context value ("metrics.addr") represents the utils.MetricsHTTPFlag.Name. // It cannot be imported because it will cause a cyclical dependency. - withMetrics := metrics.Enabled && metricsAddr == "" + withMetrics := metricsEnabled && metricsAddr == "" StartPProf(address, withMetrics) } return nil @@ -218,3 +243,87 @@ func RaiseFdLimit() { log.Error("Failed to raise file descriptor allowance", "err", err) } } + +var ( + metricsConfigs = []string{metricsEnabledFlag.Name, metricsAddrFlag.Name, metricsPortFlag.Name} +) + +func SetFlagsFromConfigFile(ctx *cli.Context) error { + filePath := ctx.String(configFlag.Name) + if filePath == "" { + return nil + } + + fileConfig, err := readConfigAsMap(filePath) + if err != nil { + return err + } + + for _, flag := range metricsConfigs { + if v, ok := fileConfig[flag]; ok { + err = ctx.Set(flag, fmt.Sprintf("%v", v)) + if err != nil { + return err + } + } + } + + return nil +} + +func SetCobraFlagsFromConfigFile(cmd *cobra.Command) error { + flags := cmd.Flags() + filePath, err := flags.GetString(configFlag.Name) + if err != nil { + return err + } + if filePath == "" { + return nil + } + + fileConfig, err := readConfigAsMap(filePath) + if err != nil { + return err + } + + for _, flag := range metricsConfigs { + if v, ok := fileConfig[flag]; ok { + err = flags.Set(flag, fmt.Sprintf("%v", v)) + if err != nil { + return err + } + } + } + + return nil +} + +func readConfigAsMap(filePath string) (map[string]interface{}, error) { + fileExtension := filepath.Ext(filePath) + + fileConfig := make(map[string]interface{}) + + if fileExtension == ".yaml" { + yamlFile, err := os.ReadFile(filePath) + if err != nil { + return fileConfig, err + } + err = yaml.Unmarshal(yamlFile, fileConfig) + if err != nil { + return fileConfig, err + } + } else if fileExtension == ".toml" { + tomlFile, err := os.ReadFile(filePath) + if err != nil { + return fileConfig, err + } + err = toml.Unmarshal(tomlFile, &fileConfig) + if err != nil { + return fileConfig, err + } + } else { + return fileConfig, errors.New("config files only accepted are .yaml and .toml") + } + + return fileConfig, nil +}