Add ignore_service_error flag to selectively ignore test service errored containers

docs/howto/system_testing.md

@@ -409,6 +409,7 @@ for system tests.
 | Option | Type | Required | Description |
 |---|---|---|---|
 | data_stream.vars | dictionary |  | Data stream level variables to set (i.e. declared in `package_root/data_stream/$data_stream/manifest.yml`). If not specified the defaults from the manifest are used. |
+| ignore_service_error | boolean | no | If `true`, it will ignore any failures in the deployed test services. Defaults to `false`. |
 | input | string | yes | Input type to test (e.g. logfile, httpjson, etc). Defaults to the input used by the first stream in the data stream manifest. |
 | numeric_keyword_fields | []string |  | List of fields to ignore during validation that are mapped as `keyword` in Elasticsearch, but their JSON data type is a number. |
 | policy_template | string |  | Name of policy template associated with the data stream and input. Required when multiple policy templates include the input being tested. |

internal/compose/compose.go

@@ -318,8 +318,7 @@ func (p *Project) Logs(opts CommandOptions) ([]byte, error) {
 func (p *Project) WaitForHealthy(opts CommandOptions) error {
 	// Read container IDs
 	args := p.baseArgs()
-	args = append(args, "ps")
-	args = append(args, "-q")
+	args = append(args, "ps", "-a", "-q")
 
 	var b bytes.Buffer
 	if err := p.runDockerComposeCmd(dockerComposeOptions{args: args, env: opts.Env, stdout: &b}); err != nil {
@@ -387,6 +386,40 @@ func (p *Project) WaitForHealthy(opts CommandOptions) error {
 	return nil
 }
 
+// ServiceExitCode returns true if the specified service is exited with an error.
+func (p *Project) ServiceExitCode(service string, opts CommandOptions) (bool, int, error) {
+	// Read container IDs
+	args := p.baseArgs()
+	args = append(args, "ps", "-a", "-q", service)
+
+	var b bytes.Buffer
+	if err := p.runDockerComposeCmd(dockerComposeOptions{args: args, env: opts.Env, stdout: &b}); err != nil {
+		return false, -1, err
+	}
+
+	containerIDs := strings.Fields(b.String())
+	if len(containerIDs) != 1 {
+		return false, -1, fmt.Errorf("expected to find one service container named: %s, found: %d", service, len(containerIDs))
+	}
+	containerID := containerIDs[0]
+
+	containerDescriptions, err := docker.InspectContainers(containerID)
+	if err != nil {
+		return false, -1, err
+	}
+	if len(containerDescriptions) != 1 {
+		return false, -1, fmt.Errorf("expected to get one service status, found: %d", len(containerIDs))
+	}
+	containerDescription := containerDescriptions[0]
+
+	// Container exited with code > 0
+	if containerDescription.State.Status == "exited" {
+		return true, containerDescription.State.ExitCode, nil
+	}
+
+	return false, -1, nil
+}
+
 func (p *Project) baseArgs() []string {
 	var args []string
 	for _, path := range p.composeFilePaths {

internal/servicedeployer/compose.go

@@ -155,6 +155,22 @@ func (s *dockerComposeDeployedService) Signal(signal string) error {
 	return nil
 }
 
+// ExitCode returns true if the service is exited and its exit code.
+func (s *dockerComposeDeployedService) ExitCode(service string) (bool, int, error) {
+	p, err := compose.NewProject(s.project, s.ymlPaths...)
+	if err != nil {
+		return false, -1, fmt.Errorf("could not create Docker Compose project for service: %w", err)
+	}
+
+	opts := compose.CommandOptions{
+		Env: append(
+			s.env,
+			s.variant.Env...),
+	}
+
+	return p.ServiceExitCode(service, opts)
+}
+
 // TearDown tears down the service.
 func (s *dockerComposeDeployedService) TearDown() error {
 	logger.Debugf("tearing down service using Docker Compose runner")

internal/servicedeployer/deployed_service.go

@@ -4,6 +4,10 @@
 
 package servicedeployer
 
+import "errors"
+
+var ErrNotSupported error = errors.New("not supported")
+
 // DeployedService defines the interface for interacting with a service that has been deployed.
 type DeployedService interface {
 	// TearDown implements the logic for tearing down a service.
@@ -17,4 +21,7 @@ type DeployedService interface {
 
 	// SetContext sets the current context for the service.
 	SetContext(str ServiceContext) error
+
+	// ExitCode returns true if the service is exited and its exit code.
+	ExitCode(service string) (bool, int, error)
 }

internal/servicedeployer/kubernetes.go

@@ -8,7 +8,6 @@ import (
 	"bytes"
 	_ "embed"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -57,7 +56,11 @@ func (s kubernetesDeployedService) TearDown() error {
 }
 
 func (s kubernetesDeployedService) Signal(_ string) error {
-	return errors.New("signal is not supported")
+	return ErrNotSupported
+}
+
+func (s kubernetesDeployedService) ExitCode(_ string) (bool, int, error) {
+	return false, -1, ErrNotSupported
 }
 
 func (s kubernetesDeployedService) Context() ServiceContext {

internal/testrunner/runners/system/runner.go

@@ -753,6 +753,16 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 		return result.WithError(err)
 	}
 
+	if config.Service != "" && !config.IgnoreServiceError {
+		exited, code, err := service.ExitCode(config.Service)
+		if err != nil && !errors.Is(err, servicedeployer.ErrNotSupported) {
+			return result.WithError(err)
+		}
+		if exited && code > 0 {
+			result.FailureMsg = fmt.Sprintf("the test service %s unexpectedly exited with code %d", config.Service, code)
+		}
+	}
+
 	return result.WithSuccess()
 }
 

internal/testrunner/runners/system/test_config.go

@@ -32,6 +32,7 @@ type testConfig struct {
 	PolicyTemplate      string        `config:"policy_template"` // Policy template associated with input. Required when multiple policy templates include the input being tested.
 	Service             string        `config:"service"`
 	ServiceNotifySignal string        `config:"service_notify_signal"` // Signal to send when the agent policy is applied.
+	IgnoreServiceError  bool          `config:"ignore_service_error"`
 	WaitForDataTimeout  time.Duration `config:"wait_for_data_timeout"`
 
 	Vars       common.MapStr `config:"vars"`

test/packages/false_positives/docker_failing_test_service.expected_errors

@@ -0,0 +1 @@
+<testcase name=\"system test: fail\" classname=\"docker_failing_test_service.log\" time=\".*\"> * <failure>the test service failing unexpectedly exited with code 1</failure>

test/packages/false_positives/docker_failing_test_service/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@v8.9.0

test/packages/false_positives/docker_failing_test_service/_dev/build/docs/README.md

@@ -0,0 +1 @@
+# Test integration

test/packages/false_positives/docker_failing_test_service/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "999.999.999"
+  changes:
+    - description: Test
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/99999

test/packages/false_positives/docker_failing_test_service/data_stream/log/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '2.3'
+services:
+  failing:
+    image: alpine
+    volumes:
+      - ./logs:/logs:ro
+      - ${SERVICE_LOGS_DIR}:/var/log
+    command: /bin/sh -c "echo \"Sleep 10s...\"; sleep 10; echo \"Copying files...\"; cp /logs/* /var/log/; echo \"Done.\"; sleep 5; exit 1"

test/packages/false_positives/docker_failing_test_service/data_stream/log/_dev/deploy/docker/logs/logs.log

@@ -0,0 +1,2 @@
+foo
+bar

test/packages/false_positives/docker_failing_test_service/data_stream/log/_dev/test/system/test-fail-config.yml

@@ -0,0 +1,9 @@
+input: logfile
+service: failing
+ignore_service_error: false
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/*.log"
+assert:
+  hit_count: 2

test/packages/false_positives/docker_failing_test_service/data_stream/log/agent/stream/stream.yml.hbs

@@ -0,0 +1,5 @@
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+exclude_files: [".gz$"]

test/packages/false_positives/docker_failing_test_service/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,6 @@
+---
+description: "Pipeline for Cisco ASA logs"
+processors:
+  - set:
+      field: ecs.version
+      value: '8.9.0'