Add logstash to stack as an optional service

.buildkite/pipeline.trigger.integration.tests.sh

@@ -34,6 +34,7 @@ CHECK_PACKAGES_TESTS=(
     test-check-packages-with-kind
     test-check-packages-with-custom-agent
     test-check-packages-benchmarks
+    test-check-packages-with-logstash
 )
 for test in ${CHECK_PACKAGES_TESTS[@]}; do
     echo "      - label: \":go: Running integration test: ${test}\""

Makefile

@@ -70,7 +70,7 @@ test-stack-command-8x:
 
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x
 
-test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives
+test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-logstash
 
 test-check-packages-with-kind:
 	PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh
@@ -81,6 +81,9 @@ test-check-packages-other:
 test-check-packages-false-positives:
 	PACKAGE_TEST_TYPE=false_positives ./scripts/test-check-false-positives.sh
 
+test-check-packages-with-logstash:
+	PACKAGE_TEST_TYPE=with-logstash ./scripts/test-check-packages.sh
+
 test-check-packages-benchmarks:
 	PACKAGE_TEST_TYPE=benchmarks ./scripts/test-check-packages.sh
 

cmd/stack.go

@@ -24,6 +24,7 @@ var availableServices = map[string]struct{}{
 	"fleet-server":     {},
 	"kibana":           {},
 	"package-registry": {},
+	"logstash":         {},
 }
 
 const stackLongDescription = `Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version <version>.

docs/howto/custom_images.md

@@ -20,6 +20,7 @@ The current images that could be overwritten are:
 | Elasticsearch | ELASTICSEARCH_IMAGE_REF_OVERRIDE | elasticsearch |
 | Kibana | KIBANA_IMAGE_REF_OVERRIDE | kibana |
 | Elastic Agent | ELASTIC_AGENT_IMAGE_REF_OVERRIDE | elastic-agent |
+| Logstash | LOGSTASH_IMAGE_REF_OVERRIDE | logstash |
 
 
 For the following two examples, it will be used as example overwriting elastic-agent image.

docs/howto/system_testing.md

@@ -579,6 +579,25 @@ Example `expected_errors` file content:
 <testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>
 ```
 
+### System testing with logstash
+
+It is possible to test packages that output to Logstash which in turn publishes events to Elasticsearch.
+A profile config option `stack.logstash_enabled` has been added to profile configuration.
+
+When this profile config is enabled
+- Logstash output is added in Fleet with id `fleet-logstash-output`
+- Logstash service is created in the stack which reads from `elastic-agent` input and outputs to `elasticsearch`.
+- Logstash is also configured with `elastic-integration` plugin. Once configured to point to an Elasticsearch cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, auto-detecting the event’s data-stream and its default pipeline.
+
+A sample workflow would look like:
+
+- You can [create](https://github.com/elastic/elastic-package#elastic-package-profiles-create) a new profile / [use existing profile](https://github.com/elastic/elastic-package#elastic-package-profiles-use) to test this.
+- Navigate to `~/.elastic-package/profiles/<profilename>/`.
+- Rename `config.yml.example` to `config.yml` [ If config is not used before ]
+- Add the following line (or uncomment if present) `stack.logstash_enabled: true`
+- Run `elastic-package stack up -d -v`
+- Navigate to the package folder in integrations and run `elastic-package test system -v`
+
 ## Continuous Integration
 
 `elastic-package` runs a set of system tests on some [dummy packages](https://github.com/elastic/elastic-package/tree/main/test/packages) to ensure it's functionalities work as expected. This allows to test changes affecting package testing within `elastic-package` before merging and releasing the changes.

internal/install/application_configuration.go

@@ -29,6 +29,7 @@ const (
 	elasticAgentCompleteImageName       = "docker.elastic.co/elastic-agent/elastic-agent-complete"
 	elasticsearchImageName              = "docker.elastic.co/elasticsearch/elasticsearch"
 	kibanaImageName                     = "docker.elastic.co/kibana/kibana"
+	logstashImageName                   = "docker.elastic.co/logstash/logstash"
 
 	applicationConfigurationYmlFile = "config.yml"
 )
@@ -87,6 +88,7 @@ func (s stack) ImageRefOverridesForVersion(version string) ImageRefs {
 		ElasticAgent:  checkImageRefOverride("ELASTIC_AGENT_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.ElasticAgent, "")),
 		Elasticsearch: checkImageRefOverride("ELASTICSEARCH_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.Elasticsearch, "")),
 		Kibana:        checkImageRefOverride("KIBANA_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.Kibana, "")),
+		Logstash:      checkImageRefOverride("LOGSTASH_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.Logstash, "")),
 	}
 }
 
@@ -95,6 +97,7 @@ type ImageRefs struct {
 	ElasticAgent  string `yaml:"elastic-agent"`
 	Elasticsearch string `yaml:"elasticsearch"`
 	Kibana        string `yaml:"kibana"`
+	Logstash      string `yaml:"logstash"`
 }
 
 // AsEnv method returns key=value representation of image refs.
@@ -103,6 +106,7 @@ func (ir ImageRefs) AsEnv() []string {
 	vars = append(vars, "ELASTIC_AGENT_IMAGE_REF="+ir.ElasticAgent)
 	vars = append(vars, "ELASTICSEARCH_IMAGE_REF="+ir.Elasticsearch)
 	vars = append(vars, "KIBANA_IMAGE_REF="+ir.Kibana)
+	vars = append(vars, "LOGSTASH_IMAGE_REF="+ir.Logstash)
 	return vars
 }
 
@@ -112,6 +116,7 @@ func (ac *ApplicationConfiguration) StackImageRefs(version string) ImageRefs {
 	refs.ElasticAgent = stringOrDefault(refs.ElasticAgent, fmt.Sprintf("%s:%s", selectElasticAgentImageName(version), version))
 	refs.Elasticsearch = stringOrDefault(refs.Elasticsearch, fmt.Sprintf("%s:%s", elasticsearchImageName, version))
 	refs.Kibana = stringOrDefault(refs.Kibana, fmt.Sprintf("%s:%s", kibanaImageName, version))
+	refs.Logstash = stringOrDefault(refs.Logstash, fmt.Sprintf("%s:%s", logstashImageName, version))
 	return refs
 }
 

internal/kibana/policies.go

@@ -21,6 +21,7 @@ type Policy struct {
 	Revision           int      `json:"revision,omitempty"`
 	MonitoringEnabled  []string `json:"monitoring_enabled,omitempty"`
 	MonitoringOutputID string   `json:"monitoring_output_id,omitempty"`
+	DataOutputID       string   `json:"data_output_id,omitempty"`
 }
 
 // CreatePolicy persists the given Policy in Fleet.

internal/profile/_static/config.yml.example

@@ -10,3 +10,7 @@
 # Region where the Serverless project is going to be created
 # stack.serverless.region: aws-us-east-1
 
+## Enable logstash for testing
+# Flag to enable logstash in elastic-package stack profile config
+# stack.logstash_enabled: true
+

internal/profile/_testdata/config.yml

@@ -1,5 +1,6 @@
 # An expected setting.
 stack.geoip_dir: "/home/foo/Documents/ingest-geoip"
+stack.logstash_enabled: true
 
 # An empty string, should exist, but return empty.
 other.empty: ""

internal/profile/config_test.go

@@ -47,6 +47,11 @@ func TestLoadProfileConfig(t *testing.T) {
 			expected: "false",
 			found:    true,
 		},
+		{
+			name:     "stack.logstash_enabled",
+			expected: "true",
+			found:    true,
+		},
 		{
 			name:  "not.present",
 			found: false,

internal/stack/_static/docker-compose-stack.yml.tmpl

@@ -147,3 +147,37 @@ services:
     depends_on:
       elastic-agent:
         condition: service_healthy
+
+{{ $logstash_enabled := fact "logstash_enabled" }}
+{{ if eq $logstash_enabled "true" }}
+  logstash:
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    image: ${LOGSTASH_IMAGE_REF}
+    healthcheck:
+      test: bin/logstash -t
+      interval: 60s
+      timeout: 50s
+      retries: 5
+    command: bash -c "bin/logstash-plugin install logstash-filter-elastic_integration && logstash -f /usr/share/logstash/pipeline/logstash.conf"
+    volumes:
+      - "../certs/logstash:/usr/share/logstash/config/certs"
+      - "../certs/elasticsearch/cert.pem:/usr/share/logstash/config/certs/elasticsearch.pem"
+      - "./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro"
+    ports:
+       - "127.0.0.1:5044:5044"
+    environment:
+      - xpack.monitoring.enabled=false
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=changeme
+      - ELASTIC_HOSTS=https://127.0.0.1:9200
+
+  logstash_is_ready:
+    image: tianon/true
+    depends_on:
+      logstash:
+        condition: service_healthy
+{{ end }}

internal/stack/_static/kibana.yml.tmpl

@@ -80,4 +80,12 @@ xpack.fleet.outputs:
     ca_trusted_fingerprint: "${ELASTIC_PACKAGE_CA_TRUSTED_FINGERPRINT}"
     is_default: true
     is_default_monitoring: true
+
+  {{ $logstash_enabled := fact "logstash_enabled" }}
+  {{ if eq $logstash_enabled "true" }}
+  - id: fleet-logstash-output
+    name: logstash-output
+    type: logstash
+    hosts: [ logstash:5044 ]
+  {{ end }}
 {{ end }}

internal/stack/_static/logstash.conf.tmpl

@@ -0,0 +1,33 @@
+input {
+  elastic_agent {
+    port => 5044
+    ssl_enabled => false
+    ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca-cert.pem"]
+    ssl_certificate => "/usr/share/logstash/config/certs/cert.pem"
+    ssl_key => "/usr/share/logstash/config/certs/key.pem"
+  }
+}
+
+
+filter {
+  elastic_integration {
+    remove_field => ['@version']
+    hosts => ["https://elasticsearch:9200"]
+    username => {{ fact "username" }}
+    password => {{ fact "password" }}
+    ssl_enabled => true
+    ssl_verification_mode => "none"
+  }
+}
+
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    user => {{ fact "username" }}
+    password => {{ fact "password" }}
+    ssl_enabled => true
+    ssl_certificate_authorities => "/usr/share/logstash/config/certs/elasticsearch.pem"
+    data_stream => "true"
+  }
+}

internal/stack/certs.go

@@ -23,6 +23,7 @@ var tlsServices = []string{
 	"kibana",
 	"package-registry",
 	"fleet-server",
+	"logstash",
 }
 
 var (

internal/stack/resources.go

@@ -32,6 +32,9 @@ const (
 	// KibanaConfigFile is the kibana config file.
 	KibanaConfigFile = "kibana.yml"
 
+	// LogstashConfigFile is the logstash config file.
+	LogstashConfigFile = "logstash.conf"
+
 	// KibanaHealthcheckFile is the kibana healthcheck.
 	KibanaHealthcheckFile = "kibana_healthcheck.sh"
 
@@ -91,6 +94,10 @@ var (
 			Path:    KibanaConfigFile,
 			Content: staticSource.Template("_static/kibana.yml.tmpl"),
 		},
+		&resource.File{
+			Path:    LogstashConfigFile,
+			Content: staticSource.Template("_static/logstash.conf.tmpl"),
+		},
 		&resource.File{
 			Path:    KibanaHealthcheckFile,
 			Content: staticSource.Template("_static/kibana_healthcheck.sh.tmpl"),
@@ -122,7 +129,8 @@ func applyResources(profile *profile.Profile, stackVersion string) error {
 		"username": elasticsearchUsername,
 		"password": elasticsearchPassword,
 
-		"geoip_dir": profile.Config("stack.geoip_dir", "./ingest-geoip"),
+		"geoip_dir":        profile.Config("stack.geoip_dir", "./ingest-geoip"),
+		"logstash_enabled": profile.Config("stack.logstash_enabled", "false"),
 	})
 
 	os.MkdirAll(stackDir, 0755)

internal/testrunner/runners/system/runner.go

@@ -516,11 +516,17 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 	// Configure package (single data stream) via Ingest Manager APIs.
 	logger.Debug("creating test policy...")
 	testTime := time.Now().Format("20060102T15:04:05Z")
+
 	p := kibana.Policy{
 		Name:        fmt.Sprintf("ep-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime),
 		Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream),
 		Namespace:   "ep",
 	}
+	// Assign the data_output_id to the agent policy to configure the output to logstash. The value is inferred from stack/_static/kibana.yml.tmpl
+	if r.options.Profile.Config("stack.logstash_enabled", "false") == "true" {
+		p.DataOutputID = "fleet-logstash-output"
+	}
+
 	policy, err := r.options.KibanaClient.CreatePolicy(p)
 	if err != nil {
 		return result.WithError(fmt.Errorf("could not create test policy: %w", err))

scripts/test-check-packages.sh

@@ -22,6 +22,11 @@ cleanup() {
   # Take down the stack
   elastic-package stack down -v
 
+  if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then
+    # Delete the logstash profile
+    elastic-package profiles delete logstash -v
+  fi
+
   # Clean used resources
   for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
     (
@@ -47,6 +52,18 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
 done
 cd -
 
+if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then
+  # Create a logstash profile and use it
+  elastic-package profiles create logstash -v
+  elastic-package profiles use logstash
+
+  # Rename the config.yml.example to config.yml
+  mv ~/.elastic-package/profiles/logstash/config.yml.example ~/.elastic-package/profiles/logstash/config.yml
+
+  # Append config to enable logstash
+  echo "stack.logstash_enabled: true" >> ~/.elastic-package/profiles/logstash/config.yml
+fi
+
 # Update the stack
 elastic-package stack update -v
 

test/packages/with-logstash/ti_misp/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: "git@v8.10.0"

test/packages/with-logstash/ti_misp/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "2.3"
+services:
+  misp:
+    image: docker.elastic.co/observability/stream:v0.6.1
+    ports:
+      - 8080
+    volumes:
+      - ./files:/files:ro
+    environment:
+      PORT: 8080
+    command:
+      - http-server
+      - --addr=:8080
+      - --config=/files/config.yml