[Ingest Manager] Log level reloadable from fleet (#22690) (#22729)

[Ingest Manager] Log level reloadable from fleet (#22690)

libbeat/logp/core.go

@@ -79,7 +79,7 @@ func ConfigureWithOutputs(cfg Config, outputs ...zapcore.Core) error {
 
 	// Build a single output (stderr has priority if more than one are enabled).
 	if cfg.toObserver {
-		sink, observedLogs = observer.New(cfg.Level.zapLevel())
+		sink, observedLogs = observer.New(cfg.Level.ZapLevel())
 	} else {
 		sink, err = createLogOutput(cfg)
 	}
@@ -201,24 +201,24 @@ func makeOptions(cfg Config) []zap.Option {
 
 func makeStderrOutput(cfg Config) (zapcore.Core, error) {
 	stderr := zapcore.Lock(os.Stderr)
-	return newCore(cfg, buildEncoder(cfg), stderr, cfg.Level.zapLevel()), nil
+	return newCore(cfg, buildEncoder(cfg), stderr, cfg.Level.ZapLevel()), nil
 }
 
 func makeDiscardOutput(cfg Config) (zapcore.Core, error) {
 	discard := zapcore.AddSync(ioutil.Discard)
-	return newCore(cfg, buildEncoder(cfg), discard, cfg.Level.zapLevel()), nil
+	return newCore(cfg, buildEncoder(cfg), discard, cfg.Level.ZapLevel()), nil
 }
 
 func makeSyslogOutput(cfg Config) (zapcore.Core, error) {
-	core, err := newSyslog(buildEncoder(cfg), cfg.Level.zapLevel())
+	core, err := newSyslog(buildEncoder(cfg), cfg.Level.ZapLevel())
 	if err != nil {
 		return nil, err
 	}
 	return wrappedCore(cfg, core), nil
 }
 
 func makeEventLogOutput(cfg Config) (zapcore.Core, error) {
-	core, err := newEventLog(cfg.Beat, buildEncoder(cfg), cfg.Level.zapLevel())
+	core, err := newEventLog(cfg.Beat, buildEncoder(cfg), cfg.Level.ZapLevel())
 	if err != nil {
 		return nil, err
 	}
@@ -244,7 +244,7 @@ func makeFileOutput(cfg Config) (zapcore.Core, error) {
 		return nil, errors.Wrap(err, "failed to create file rotator")
 	}
 
-	return newCore(cfg, buildEncoder(cfg), rotator, cfg.Level.zapLevel()), nil
+	return newCore(cfg, buildEncoder(cfg), rotator, cfg.Level.ZapLevel()), nil
 }
 
 func newCore(cfg Config, enc zapcore.Encoder, ws zapcore.WriteSyncer, enab zapcore.LevelEnabler) zapcore.Core {

libbeat/logp/level.go

@@ -101,7 +101,8 @@ func (l Level) MarshalJSON() ([]byte, error) {
 	return nil, errors.Errorf("invalid level '%d'", l)
 }
 
-func (l Level) zapLevel() zapcore.Level {
+// ZapLevel returns zap alternative to logp.Level.
+func (l Level) ZapLevel() zapcore.Level {
 	z, found := zapLevels[l]
 	if found {
 		return z

x-pack/elastic-agent/CHANGELOG.next.asciidoc

@@ -51,3 +51,4 @@
 - Removed `install-service.ps1` and `uninstall-service.ps1` from Windows .zip packaging {pull}21694[21694]
 - Add `priority` to `AddOrUpdate` on dynamic composable input providers communication channel {pull}22352[22352]
 - Ship `endpoint-security` logs to elasticsearch {pull}22526[22526]
+- Log level reloadable from fleet {pull}22690[22690]

x-pack/elastic-agent/pkg/agent/application/application.go

@@ -31,7 +31,7 @@ type upgraderControl interface {
 }
 
 // New creates a new Agent and bootstrap the required subsystem.
-func New(log *logger.Logger, pathConfigFile string, reexec reexecManager, uc upgraderControl) (Application, error) {
+func New(log *logger.Logger, pathConfigFile string, reexec reexecManager, uc upgraderControl, agentInfo *info.AgentInfo) (Application, error) {
 	// Load configuration from disk to understand in which mode of operation
 	// we must start the elastic-agent, the mode of operation cannot be changed without restarting the
 	// elastic-agent.
@@ -44,7 +44,7 @@ func New(log *logger.Logger, pathConfigFile string, reexec reexecManager, uc upg
 		return nil, err
 	}
 
-	return createApplication(log, pathConfigFile, rawConfig, reexec, uc)
+	return createApplication(log, pathConfigFile, rawConfig, reexec, uc, agentInfo)
 }
 
 func createApplication(
@@ -53,6 +53,7 @@ func createApplication(
 	rawConfig *config.Config,
 	reexec reexecManager,
 	uc upgraderControl,
+	agentInfo *info.AgentInfo,
 ) (Application, error) {
 	warn.LogNotGA(log)
 	log.Info("Detecting execution mode")
@@ -63,16 +64,16 @@ func createApplication(
 		return nil, err
 	}
 
-	if isStandalone(cfg.Fleet) {
+	if IsStandalone(cfg.Fleet) {
 		log.Info("Agent is managed locally")
-		return newLocal(ctx, log, pathConfigFile, rawConfig, reexec, uc)
+		return newLocal(ctx, log, pathConfigFile, rawConfig, reexec, uc, agentInfo)
 	}
 
 	log.Info("Agent is managed by Fleet")
-	return newManaged(ctx, log, rawConfig, reexec)
+	return newManaged(ctx, log, rawConfig, reexec, agentInfo)
 }
 
-// missing of fleet.enabled: true or fleet.{access_token,kibana} will place Elastic Agent into standalone mode.
-func isStandalone(cfg *configuration.FleetAgentConfig) bool {
+// IsStandalone decides based on missing of fleet.enabled: true or fleet.{access_token,kibana} will place Elastic Agent into standalone mode.
+func IsStandalone(cfg *configuration.FleetAgentConfig) bool {
 	return cfg == nil || !cfg.Enabled
 }

x-pack/elastic-agent/pkg/agent/application/config_test.go

@@ -70,7 +70,7 @@ func testMgmtMode(t *testing.T) {
 		err := c.Unpack(&m)
 		require.NoError(t, err)
 		assert.Equal(t, false, m.Fleet.Enabled)
-		assert.Equal(t, true, isStandalone(m.Fleet))
+		assert.Equal(t, true, IsStandalone(m.Fleet))
 
 	})
 
@@ -80,7 +80,7 @@ func testMgmtMode(t *testing.T) {
 		err := c.Unpack(&m)
 		require.NoError(t, err)
 		assert.Equal(t, true, m.Fleet.Enabled)
-		assert.Equal(t, false, isStandalone(m.Fleet))
+		assert.Equal(t, false, IsStandalone(m.Fleet))
 	})
 }
 

x-pack/elastic-agent/pkg/agent/application/handler_action_settings.go

@@ -0,0 +1,52 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package application
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi"
+)
+
+// handlerSettings handles settings change coming from fleet and updates log level.
+type handlerSettings struct {
+	log       *logger.Logger
+	reexec    reexecManager
+	agentInfo *info.AgentInfo
+}
+
+// Handle handles SETTINGS action.
+func (h *handlerSettings) Handle(ctx context.Context, a action, acker fleetAcker) error {
+	h.log.Debugf("handlerUpgrade: action '%+v' received", a)
+	action, ok := a.(*fleetapi.ActionSettings)
+	if !ok {
+		return fmt.Errorf("invalid type, expected ActionSettings and received %T", a)
+	}
+
+	if !isSupportedLogLevel(action.LogLevel) {
+		return fmt.Errorf("invalid log level, expected debug|info|warning|error and received '%s'", action.LogLevel)
+	}
+
+	if err := h.agentInfo.LogLevel(action.LogLevel); err != nil {
+		return errors.New("failed to update log level", err)
+	}
+
+	if err := acker.Ack(ctx, a); err != nil {
+		h.log.Errorf("failed to acknowledge SETTINGS action with id '%s'", action.ActionID)
+	} else if err := acker.Commit(ctx); err != nil {
+		h.log.Errorf("failed to commit acker after acknowledging action with id '%s'", action.ActionID)
+	}
+
+	h.reexec.ReExec()
+	return nil
+}
+
+func isSupportedLogLevel(level string) bool {
+	return level == "error" || level == "debug" || level == "info" || level == "warning"
+}

x-pack/elastic-agent/pkg/agent/application/info/agent_id.go

@@ -26,8 +26,11 @@ const agentInfoKey = "agent"
 // defaultAgentActionStoreFile is the file that will contains the action that can be replayed after restart.
 const defaultAgentActionStoreFile = "action_store.yml"
 
+const defaultLogLevel = "info"
+
 type persistentAgentInfo struct {
-	ID string `json:"id" yaml:"id" config:"id"`
+	ID       string `json:"id" yaml:"id" config:"id"`
+	LogLevel string `json:"logging.level,omitempty" yaml:"logging.level,omitempty" config:"logging.level,omitempty"`
 }
 
 type ioStore interface {
@@ -45,6 +48,25 @@ func AgentActionStoreFile() string {
 	return filepath.Join(paths.Home(), defaultAgentActionStoreFile)
 }
 
+// updateLogLevel updates log level and persists it to disk.
+func updateLogLevel(level string) error {
+	ai, err := loadAgentInfo(false, defaultLogLevel)
+	if err != nil {
+		return err
+	}
+
+	if ai.LogLevel == level {
+		// no action needed
+		return nil
+	}
+
+	agentConfigFile := AgentConfigFile()
+	s := storage.NewDiskStore(agentConfigFile)
+
+	ai.LogLevel = level
+	return updateAgentInfo(s, ai)
+}
+
 func generateAgentID() (string, error) {
 	uid, err := uuid.NewV4()
 	if err != nil {
@@ -54,11 +76,11 @@ func generateAgentID() (string, error) {
 	return uid.String(), nil
 }
 
-func loadAgentInfo(forceUpdate bool) (*persistentAgentInfo, error) {
+func loadAgentInfo(forceUpdate bool, logLevel string) (*persistentAgentInfo, error) {
 	agentConfigFile := AgentConfigFile()
 	s := storage.NewDiskStore(agentConfigFile)
 
-	agentinfo, err := getInfoFromStore(s)
+	agentinfo, err := getInfoFromStore(s, logLevel)
 	if err != nil {
 		return nil, err
 	}
@@ -79,7 +101,7 @@ func loadAgentInfo(forceUpdate bool) (*persistentAgentInfo, error) {
 	return agentinfo, nil
 }
 
-func getInfoFromStore(s ioStore) (*persistentAgentInfo, error) {
+func getInfoFromStore(s ioStore, logLevel string) (*persistentAgentInfo, error) {
 	agentConfigFile := AgentConfigFile()
 	reader, err := s.Load()
 	if err != nil {
@@ -104,15 +126,19 @@ func getInfoFromStore(s ioStore) (*persistentAgentInfo, error) {
 
 	agentInfoSubMap, found := configMap[agentInfoKey]
 	if !found {
-		return &persistentAgentInfo{}, nil
+		return &persistentAgentInfo{
+			LogLevel: logLevel,
+		}, nil
 	}
 
 	cc, err := config.NewConfigFrom(agentInfoSubMap)
 	if err != nil {
 		return nil, errors.New(err, "failed to create config from agent info submap")
 	}
 
-	pid := &persistentAgentInfo{}
+	pid := &persistentAgentInfo{
+		LogLevel: logLevel,
+	}
 	if err := cc.Unpack(&pid); err != nil {
 		return nil, errors.New(err, "failed to unpack stored config to map")
 	}

x-pack/elastic-agent/pkg/agent/application/info/agent_info.go

@@ -4,42 +4,51 @@
 
 package info
 
-import "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
+import (
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
+)
 
 // AgentInfo is a collection of information about agent.
 type AgentInfo struct {
-	agentID string
+	agentID  string
+	logLevel string
 }
 
-// NewAgentInfo creates a new agent information.
+// NewAgentInfoWithLog creates a new agent information.
 // In case when agent ID was already created it returns,
 // this created ID otherwise it generates
 // new unique identifier for agent.
 // If agent config file does not exist it gets created.
-func NewAgentInfo() (*AgentInfo, error) {
-	agentInfo, err := loadAgentInfo(false)
+// Initiates log level to predefined value.
+func NewAgentInfoWithLog(level string) (*AgentInfo, error) {
+	agentInfo, err := loadAgentInfo(false, level)
 	if err != nil {
 		return nil, err
 	}
 
 	return &AgentInfo{
-		agentID: agentInfo.ID,
+		agentID:  agentInfo.ID,
+		logLevel: agentInfo.LogLevel,
 	}, nil
 }
 
-// ForceNewAgentInfo creates a new agent information.
-// Generates new unique identifier for agent regardless
-// of any existing ID.
+// NewAgentInfo creates a new agent information.
+// In case when agent ID was already created it returns,
+// this created ID otherwise it generates
+// new unique identifier for agent.
 // If agent config file does not exist it gets created.
-func ForceNewAgentInfo() (*AgentInfo, error) {
-	agentInfo, err := loadAgentInfo(true)
-	if err != nil {
-		return nil, err
+func NewAgentInfo() (*AgentInfo, error) {
+	return NewAgentInfoWithLog(defaultLogLevel)
+}
+
+// LogLevel updates log level of agent.
+func (i *AgentInfo) LogLevel(level string) error {
+	if err := updateLogLevel(level); err != nil {
+		return err
 	}
 
-	return &AgentInfo{
-		agentID: agentInfo.ID,
-	}, nil
+	i.logLevel = level
+	return nil
 }
 
 // AgentID returns an agent identifier.

x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go

@@ -40,6 +40,9 @@ type AgentECSMeta struct {
 	BuildOriginal string `json:"build.original"`
 	// Upgradeable is a flag specifying if it is possible for agent to be upgraded.
 	Upgradeable bool `json:"upgradeable"`
+	// LogLevel describes currently set log level.
+	// Possible values: "debug"|"info"|"warning"|"error"
+	LogLevel string `json:"log_level"`
 }
 
 // SystemECSMeta is a collection of operating system metadata in ECS compliant object form.
@@ -140,6 +143,7 @@ func (i *AgentInfo) ECSMetadata() (*ECSMeta, error) {
 				// only upgradeable if running from Agent installer and running under the
 				// control of the system supervisor (or built specifically with upgrading enabled)
 				Upgradeable: release.Upgradeable() || (install.RunningInstalled() && install.RunningUnderSupervisor()),
+				LogLevel:    i.logLevel,
 			},
 		},
 		Host: &HostECSMeta{

x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go

@@ -46,7 +46,7 @@ func (c *InspectConfigCmd) inspectConfig() error {
 		return err
 	}
 
-	if isStandalone(cfg.Fleet) {
+	if IsStandalone(cfg.Fleet) {
 		return printConfig(rawConfig)
 	}
 

x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go

@@ -66,7 +66,7 @@ func (c *InspectOutputCmd) inspectOutputs(agentInfo *info.AgentInfo) error {
 		return err
 	}
 
-	if isStandalone(cfg.Fleet) {
+	if IsStandalone(cfg.Fleet) {
 		return listOutputsFromConfig(l, agentInfo, rawConfig)
 	}
 
@@ -119,7 +119,7 @@ func (c *InspectOutputCmd) inspectOutput(agentInfo *info.AgentInfo) error {
 		return err
 	}
 
-	if isStandalone(cfg.Fleet) {
+	if IsStandalone(cfg.Fleet) {
 		return printOutputFromConfig(l, agentInfo, c.output, c.program, rawConfig)
 	}
 

x-pack/elastic-agent/pkg/agent/application/local_mode.go

@@ -63,6 +63,7 @@ func newLocal(
 	rawConfig *config.Config,
 	reexec reexecManager,
 	uc upgraderControl,
+	agentInfo *info.AgentInfo,
 ) (*Local, error) {
 	cfg, err := configuration.NewFromConfig(rawConfig)
 	if err != nil {
@@ -75,10 +76,6 @@ func newLocal(
 			return nil, err
 		}
 	}
-	agentInfo, err := info.NewAgentInfo()
-	if err != nil {
-		return nil, err
-	}
 
 	logR := logreporter.NewReporter(log)