Disable elastic agent self monitoring by default

Makefile

@@ -71,6 +71,9 @@ test-stack-command-8x:
 test-stack-command-with-apm-server:
 	APM_SERVER_ENABLED=true ./scripts/test-stack-command.sh
 
+test-stack-command-with-self-monitor:
+	SELF_MONITOR_ENABLED=true ./scripts/test-stack-command.sh
+
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x test-stack-command-with-apm-server
 
 test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-logstash

README.md

@@ -620,6 +620,8 @@ The following settings are available per profile:
 * `stack.logstash_enabled` can be set to true to start Logstash and configure it as the
   default output for tests using elastic-package. Supported only by the compose provider.
   Defaults to false.
+* `stack.self_monitor_enabled` enables monitoring and the system package for the default
+  policy assigned to the managed Elastic Agent. Defaults to false.
 * `stack.serverless.type` selects the type of serverless project to start when using
   the serverless stack provider.
 * `stack.serverless.region` can be used to select the region to use when starting

internal/serverless/project.go

@@ -232,36 +232,49 @@ func (p *Project) getFleetHealth(ctx context.Context) error {
 	return nil
 }
 
-func (p *Project) CreateAgentPolicy(stackVersion string, kibanaClient *kibana.Client, outputId string) error {
-	systemPackages, err := registry.Production.Revisions("system", registry.SearchOptions{
-		KibanaVersion: strings.TrimSuffix(stackVersion, kibana.SNAPSHOT_SUFFIX),
-	})
-	if err != nil {
-		return fmt.Errorf("could not get the system package version for Kibana %v: %w", stackVersion, err)
-	}
-	if len(systemPackages) != 1 {
-		return fmt.Errorf("unexpected number of system package versions for Kibana %s - found %d expected 1", stackVersion, len(systemPackages))
-	}
-	logger.Debugf("Found %s package - version %s", systemPackages[0].Name, systemPackages[0].Version)
-
+func (p *Project) CreateAgentPolicy(kibanaClient *kibana.Client, stackVersion string, outputId string, selfMonitor bool) error {
 	policy := kibana.Policy{
 		ID:                "elastic-agent-managed-ep",
 		Name:              "Elastic-Agent (elastic-package)",
 		Description:       "Policy created by elastic-package",
 		Namespace:         "default",
-		MonitoringEnabled: []string{"logs", "metrics"},
+		MonitoringEnabled: []string{},
 		DataOutputID:      outputId,
 	}
+	if selfMonitor {
+		policy.MonitoringEnabled = []string{"logs", "metrics"}
+	}
 
 	newPolicy, err := kibanaClient.CreatePolicy(policy)
 	if err != nil {
 		return fmt.Errorf("error while creating agent policy: %w", err)
 	}
 
+	if selfMonitor {
+		err := p.createSystemPackagePolicy(kibanaClient, stackVersion, newPolicy.ID, newPolicy.Namespace)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (p *Project) createSystemPackagePolicy(kibanaClient *kibana.Client, stackVersion, agentPolicyID, namespace string) error {
+	systemPackages, err := registry.Production.Revisions("system", registry.SearchOptions{
+		KibanaVersion: strings.TrimSuffix(stackVersion, kibana.SNAPSHOT_SUFFIX),
+	})
+	if err != nil {
+		return fmt.Errorf("could not get the system package version for Kibana %v: %w", stackVersion, err)
+	}
+	if len(systemPackages) != 1 {
+		return fmt.Errorf("unexpected number of system package versions for Kibana %s - found %d expected 1", stackVersion, len(systemPackages))
+	}
+	logger.Debugf("Found %s package - version %s", systemPackages[0].Name, systemPackages[0].Version)
 	packagePolicy := kibana.PackagePolicy{
 		Name:      "system-1",
-		PolicyID:  newPolicy.ID,
-		Namespace: newPolicy.Namespace,
+		PolicyID:  agentPolicyID,
+		Namespace: namespace,
 	}
 	packagePolicy.Package.Name = "system"
 	packagePolicy.Package.Version = systemPackages[0].Version

internal/stack/_static/kibana.yml.tmpl

@@ -47,10 +47,13 @@ xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012"
 xpack.cloudSecurityPosture.enabled: true
 {{ end }}
 
+{{ $self_monitor_enabled := fact "self_monitor_enabled" }}
 {{ if not (semverLessThan $version "8.0.0") }}
 xpack.fleet.packages:
+  {{ if eq $self_monitor_enabled "true" }}
   - name: system
     version: latest
+  {{ end }}
   - name: elastic_agent
     version: latest
   - name: fleet_server
@@ -65,6 +68,7 @@ xpack.fleet.agentPolicies:
     is_default: true
     is_managed: false
     namespace: default
+  {{ if eq $self_monitor_enabled "true" }}
     monitoring_enabled:
       - logs
       - metrics
@@ -73,6 +77,10 @@ xpack.fleet.agentPolicies:
         id: default-system
         package:
           name: system
+  {{ else }}
+    monitoring_enabled: []
+    package_policies: []
+  {{ end }}
   - name: Fleet Server (elastic-package)
     id: fleet-server-policy
     is_default_fleet_server: true

internal/stack/resources.go

@@ -51,6 +51,11 @@ const (
 
 	elasticsearchUsername = "elastic"
 	elasticsearchPassword = "changeme"
+
+	configAPMEnabled         = "stack.apm_enabled"
+	configGeoIPDir           = "stack.geoip_dir"
+	configLogstashEnabled    = "stack.logstash_enabled"
+	configSelfMonitorEnabled = "stack.self_monitor_enabled"
 )
 
 var (
@@ -130,9 +135,10 @@ func applyResources(profile *profile.Profile, stackVersion string) error {
 		"username": elasticsearchUsername,
 		"password": elasticsearchPassword,
 
-		"geoip_dir":        profile.Config("stack.geoip_dir", "./ingest-geoip"),
-		"apm_enabled":      profile.Config("stack.apm_enabled", "false"),
-		"logstash_enabled": profile.Config("stack.logstash_enabled", "false"),
+		"apm_enabled":          profile.Config(configAPMEnabled, "false"),
+		"geoip_dir":            profile.Config(configGeoIPDir, "./ingest-geoip"),
+		"logstash_enabled":     profile.Config(configLogstashEnabled, "false"),
+		"self_monitor_enabled": profile.Config(configSelfMonitorEnabled, "false"),
 	})
 
 	os.MkdirAll(stackDir, 0755)

internal/stack/serverless.go

@@ -29,7 +29,6 @@ const (
 	configRegion          = "stack.serverless.region"
 	configProjectType     = "stack.serverless.type"
 	configElasticCloudURL = "stack.elastic_cloud.host"
-	configLogstashEnabled = "stack.logstash_enabled"
 
 	defaultRegion      = "aws-us-east-1"
 	defaultProjectType = "observability"
@@ -57,6 +56,7 @@ type projectSettings struct {
 
 	StackVersion    string
 	LogstashEnabled bool
+	SelfMonitor     bool
 }
 
 func (sp *serverlessProvider) createProject(settings projectSettings, options Options, conf Config) (Config, error) {
@@ -211,6 +211,7 @@ func getProjectSettings(options Options) (projectSettings, error) {
 		Region:          options.Profile.Config(configRegion, defaultRegion),
 		StackVersion:    options.StackVersion,
 		LogstashEnabled: options.Profile.Config(configLogstashEnabled, "false") == "true",
+		SelfMonitor:     options.Profile.Config(configSelfMonitorEnabled, "false") == "true",
 	}
 
 	return s, nil
@@ -276,7 +277,7 @@ func (sp *serverlessProvider) BootUp(options Options) error {
 		}
 
 		logger.Infof("Creating agent policy")
-		err = project.CreateAgentPolicy(options.StackVersion, sp.kibanaClient, outputID)
+		err = project.CreateAgentPolicy(sp.kibanaClient, options.StackVersion, outputID, settings.SelfMonitor)
 
 		if err != nil {
 			return fmt.Errorf("failed to create agent policy: %w", err)

scripts/test-stack-command.sh

@@ -4,6 +4,7 @@ set -euxo pipefail
 
 VERSION=${1:-default}
 APM_SERVER_ENABLED=${APM_SERVER_ENABLED:-false}
+SELF_MONITOR_ENABLED=${SELF_MONITOR_ENABLED:-false}
 
 cleanup() {
   r=$?
@@ -15,10 +16,13 @@ cleanup() {
   elastic-package stack down -v
 
   if [ "${APM_SERVER_ENABLED}" = true ]; then
-    # Create an apm-server profile and use it
     elastic-package profiles delete with-apm-server
   fi
 
+  if [ "${SELF_MONITOR_ENABLED}" = true ]; then
+    elastic-package profiles delete with-self-monitor
+  fi
+
   exit $r
 }
 
@@ -40,7 +44,11 @@ if [ "${VERSION}" != "default" ]; then
   EXPECTED_VERSION=${VERSION}
 fi
 
+OUTPUT_PATH_STATUS="build/elastic-stack-status/${VERSION}"
+
 if [ "${APM_SERVER_ENABLED}" = true ]; then
+  OUTPUT_PATH_STATUS="build/elastic-stack-status/${VERSION}_with_apm_server"
+
   # Create an apm-server profile and use it
   profile=with-apm-server
   elastic-package profiles create -v ${profile}
@@ -52,10 +60,17 @@ stack.apm_enabled: true
 EOF
 fi
 
-OUTPUT_PATH_STATUS="build/elastic-stack-status/${VERSION}"
-if [ "${APM_SERVER_ENABLED}" = true ]; then
-  OUTPUT_PATH_STATUS="build/elastic-stack-status/${VERSION}_with_apm_server"
+if [ "${SELF_MONITOR_ENABLED}" = true ]; then
+  # Create a self-monitor profile and use it
+  profile=with-self-monitor
+  elastic-package profiles create -v ${profile}
+  elastic-package profiles use ${profile}
+
+  cat ~/.elastic-package/profiles/${profile}/config.yml.example - <<EOF > ~/.elastic-package/profiles/${profile}/config.yml
+stack.self_monitor_enabled: true
+EOF
 fi
+
 mkdir -p "${OUTPUT_PATH_STATUS}"
 
 # Initial status empty
@@ -96,4 +111,12 @@ if [ "${APM_SERVER_ENABLED}" = true ]; then
   curl http://localhost:8200/
 fi
 
+if [ "${SELF_MONITOR_ENABLED}" = true ]; then
+  # Check that there is some data in the system indexes.
+  curl -s -S --retry 5 --retry-all-errors --fail \
+    -u "${ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME}:${ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD}" \
+    --cacert "${ELASTIC_PACKAGE_CA_CERT}" \
+    -f "${ELASTIC_PACKAGE_ELASTICSEARCH_HOST}/metrics-system.*/_search?allow_no_indices=false&size=0"
+fi
+
 diff -q "${OUTPUT_PATH_STATUS}/running_no_spaces.txt" "${OUTPUT_PATH_STATUS}/expected_no_spaces.txt"

tools/readme/readme.md.tmpl

@@ -173,6 +173,8 @@ The following settings are available per profile:
 * `stack.logstash_enabled` can be set to true to start Logstash and configure it as the
   default output for tests using elastic-package. Supported only by the compose provider.
   Defaults to false.
+* `stack.self_monitor_enabled` enables monitoring and the system package for the default
+  policy assigned to the managed Elastic Agent. Defaults to false.
 * `stack.serverless.type` selects the type of serverless project to start when using
   the serverless stack provider.
 * `stack.serverless.region` can be used to select the region to use when starting