Add Terraform Outputs to ServiceContext to use them in agent policy test config

.buildkite/hooks/pre-command

@@ -46,7 +46,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =
     export GCP_PROJECT_ID=${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-aws" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" == "integration-parallel-aws" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs") ]]; then
     export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 

docs/howto/system_testing.md

@@ -262,6 +262,65 @@ data "aws_ami" "latest-amzn" {
 
 Notice the use of the `TEST_RUN_ID` variable. It contains a unique ID, which can help differentiate resources created in potential concurrent test runs.
 
+#### Terraform Outputs
+
+The outputs generated by the terraform service deployer can be accessed in the system test config using handlebars template.
+For example, if a `SQS queue` is configured in terraform and if the `queue_url` is configured as output , it can be used in the test config as a handlebars template `{{TF_OUTPUT_queue_url}}`
+
+Sample Terraform definition
+
+```
+resource "aws_sqs_queue" "test" {
+
+}
+
+output "queue_url"{
+  value = aws_sqs_queue.test.url
+}
+```
+
+Sample system test config
+
+``` yaml
+data_stream:
+  vars:
+    period: 5m
+    latency: 10m
+    queue_url: '{{TF_OUTPUT_queue_url}}'
+    tags_filter: |-
+      - key: Name
+        value: "elastic-package-test-{{TEST_RUN_ID}}"
+```
+
+For complex outputs from terraform you can use `{{TF_OUTPUT_root_key.nested_key}}`
+
+```
+output "root_key"{
+  value = someoutput.nested_key_value
+}
+```
+``` json
+{
+  "root_key": {
+    "sensitive": false,
+    "type": [
+      "object",
+      {
+        "nested_key": "string"
+      }
+    ],
+    "value": {
+      "nested_key": "this is a nested key"
+    }
+  }
+}
+```
+``` yaml
+data_stream:
+  vars:
+    queue_url: '{{TF_OUTPUT_root_key.nested_key}}'
+```
+
 #### Environment variables
 
 To use environment variables within the Terraform service deployer a `env.yml` file is required.

internal/configuration/locations/locations.go

@@ -32,6 +32,7 @@ var (
 
 	serviceLogsDir        = filepath.Join(temporaryDir, "service_logs")
 	kubernetesDeployerDir = filepath.Join(deployerDir, "kubernetes")
+	serviceOutputDir      = filepath.Join(temporaryDir, "output")
 )
 
 // LocationManager maintains an instance of a config path location
@@ -90,6 +91,11 @@ func (loc LocationManager) ServiceLogDir() string {
 	return filepath.Join(loc.stackPath, serviceLogsDir)
 }
 
+// ServiceOutputDir returns the output directory
+func (loc LocationManager) ServiceOutputDir() string {
+	return filepath.Join(loc.stackPath, serviceOutputDir)
+}
+
 // FieldsCacheDir returns the directory with cached fields
 func (loc LocationManager) FieldsCacheDir() string {
 	return filepath.Join(loc.stackPath, fieldsCachedDir)

internal/testrunner/runners/system/runner.go

@@ -197,6 +197,13 @@ func (r *runner) runTestPerVariant(result *testrunner.ResultComposer, locationMa
 	ctxt.Logs.Folder.Local = locationManager.ServiceLogDir()
 	ctxt.Logs.Folder.Agent = ServiceLogsAgentDir
 	ctxt.Test.RunID = createTestRunID()
+
+	outputDir, err := createOutputDir(locationManager, ctxt.Test.RunID)
+	if err != nil {
+		return nil, fmt.Errorf("could not create output dir for terraform deployer %w", err)
+	}
+	ctxt.OutputDir = outputDir
+
 	testConfig, err := newConfig(filepath.Join(r.options.TestFolder.Path, cfgFile), ctxt, variantName)
 	if err != nil {
 		return result.WithError(errors.Wrapf(err, "unable to load system test case file '%s'", cfgFile))
@@ -224,6 +231,14 @@ func (r *runner) runTestPerVariant(result *testrunner.ResultComposer, locationMa
 	return partial, nil
 }
 
+func createOutputDir(locationManager *locations.LocationManager, runId string) (string, error) {
+	outputDir := filepath.Join(locationManager.ServiceOutputDir(), runId)
+	if err := os.MkdirAll(outputDir, 0755); err != nil {
+		return "", fmt.Errorf("failed to create output directory: %w", err)
+	}
+	return outputDir, nil
+}
+
 func createTestRunID() string {
 	return fmt.Sprintf("%d", rand.Intn(testRunMaxID-testRunMinID)+testRunMinID)
 }

internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml

@@ -13,3 +13,4 @@ services:
       - TF_VAR_REPO=${REPO:-unknown}
     volumes:
       - ${TF_DIR}:/stage
+      - ${TF_OUTPUT_DIR}:/output

internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh

@@ -22,6 +22,8 @@ terraform init
 terraform plan
 terraform apply -auto-approve && touch /tmp/tf-applied
 
+terraform output -json > /output/tfOutputValues.json
+
 echo "Terraform definitions applied."
 
 set +x

internal/testrunner/runners/system/servicedeployer/compose.go

@@ -158,6 +158,10 @@ func (s *dockerComposeDeployedService) TearDown() error {
 		if err != nil {
 			logger.Errorf("could not remove the service logs (path: %s)", s.ctxt.Logs.Folder.Local)
 		}
+		// Remove the outputs generated by the service container
+		if err = os.RemoveAll(s.ctxt.OutputDir); err != nil {
+			logger.Errorf("could not remove the temporary output files %w", err)
+		}
 	}()
 
 	p, err := compose.NewProject(s.project, s.ymlPaths...)

internal/testrunner/runners/system/servicedeployer/context.go

@@ -60,6 +60,9 @@ type ServiceContext struct {
 
 	// CustomProperties store additional data used to boot up the service, e.g. AWS credentials.
 	CustomProperties map[string]interface{}
+
+	// Directory to store any outputs generated
+	OutputDir string
 }
 
 // Aliases method returned aliases to properties of the service context.

internal/testrunner/runners/system/servicedeployer/terraform.go

@@ -6,6 +6,7 @@ package servicedeployer
 
 import (
 	_ "embed"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -26,6 +27,8 @@ const (
 	terraformDeployerYml        = "terraform-deployer.yml"
 	terraformDeployerDockerfile = "Dockerfile"
 	terraformDeployerRun        = "run.sh"
+	terraformOutputPrefix       = "TF_OUTPUT_"
+	terraformOutputJsonFile     = "tfOutputValues.json"
 )
 
 //go:embed _static/terraform_deployer.yml
@@ -42,6 +45,43 @@ type TerraformServiceDeployer struct {
 	definitionsDir string
 }
 
+// addTerraformOutputs method reads the terraform outputs generated in the json format and
+// adds them to the custom properties of ServiceContext and can be used in the handlebars template
+// like `{{TF_OUTPUT_queue_url}}` where `queue_url` is the output configured
+func addTerraformOutputs(outCtxt ServiceContext) error {
+	// Read the `output.json` file where terraform outputs are generated
+	outputFile := filepath.Join(outCtxt.OutputDir, terraformOutputJsonFile)
+	content, err := os.ReadFile(outputFile)
+	if err != nil {
+		return fmt.Errorf("failed to read terraform output file: %w", err)
+	}
+
+	// https://github.com/hashicorp/terraform/blob/v1.4.6/internal/command/views/output.go#L217-L222
+	type OutputMeta struct {
+		Value interface{} `json:"value"`
+	}
+
+	// Unmarshall the data into `terraformOutputs`
+	logger.Debug("Unmarshalling terraform output json")
+	var terraformOutputs map[string]OutputMeta
+	if err = json.Unmarshal(content, &terraformOutputs); err != nil {
+		return fmt.Errorf("error during json Unmarshal %w", err)
+	}
+
+	if len(terraformOutputs) == 0 {
+		return nil
+	}
+
+	if outCtxt.CustomProperties == nil {
+		outCtxt.CustomProperties = make(map[string]any, len(terraformOutputs))
+	}
+	// Prefix variables names with TF_OUTPUT_
+	for k, outputs := range terraformOutputs {
+		outCtxt.CustomProperties[terraformOutputPrefix+k] = outputs.Value
+	}
+	return nil
+}
+
 // NewTerraformServiceDeployer creates an instance of TerraformServiceDeployer.
 func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeployer, error) {
 	return &TerraformServiceDeployer{
@@ -117,6 +157,11 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic
 	}
 
 	outCtxt.Agent.Host.NamePrefix = "docker-fleet-agent"
+
+	err = addTerraformOutputs(outCtxt)
+	if err != nil {
+		return nil, fmt.Errorf("could not handle terraform output %w", err)
+	}
 	service.ctxt = outCtxt
 	return &service, nil
 }

internal/testrunner/runners/system/servicedeployer/terraform_env.go

@@ -14,6 +14,7 @@ import (
 
 const (
 	tfDir       = "TF_DIR"
+	tfOutputDir = "TF_OUTPUT_DIR"
 	tfTestRunID = "TF_VAR_TEST_RUN_ID"
 
 	envYmlFile = "env.yml"
@@ -24,6 +25,7 @@ func (tsd TerraformServiceDeployer) buildTerraformExecutorEnvironment(ctxt Servi
 	vars[serviceLogsDirEnv] = ctxt.Logs.Folder.Local
 	vars[tfTestRunID] = ctxt.Test.RunID
 	vars[tfDir] = tsd.definitionsDir
+	vars[tfOutputDir] = ctxt.OutputDir
 
 	var pairs []string
 	for k, v := range vars {

internal/testrunner/runners/system/servicedeployer/terraform_test.go

@@ -0,0 +1,128 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package servicedeployer
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAddTerraformOutputs(t *testing.T) {
+	var testCases = []struct {
+		testName      string
+		err           string
+		ctxt          ServiceContext
+		runId         string
+		content       []byte
+		expectedProps map[string]interface{}
+	}{
+		{
+			testName: "single_value_output",
+			runId:    "99999",
+			ctxt: ServiceContext{
+				Test: struct{ RunID string }{"99999"},
+			},
+			content: []byte(
+				`{
+				"queue_url": {
+				  "sensitive": false,
+				  "type": "string",
+				  "value": "https://sqs.us-east-1.amazonaws.com/1234654/elastic-package-aws-logs-queue-someId"
+				}
+			   }`,
+			),
+			expectedProps: map[string]interface{}{
+				"TF_OUTPUT_queue_url": "https://sqs.us-east-1.amazonaws.com/1234654/elastic-package-aws-logs-queue-someId",
+			},
+		},
+		{
+			testName: "multiple_value_output",
+			runId:    "23465",
+			ctxt: ServiceContext{
+				Test: struct{ RunID string }{"23465"},
+			},
+			content: []byte(
+				`{
+				"queue_url": {
+				  "sensitive": false,
+				  "type": "string",
+				  "value": "https://sqs.us-east-1.amazonaws.com/1234654/elastic-package-aws-logs-queue-someId"
+				},
+				"instance_id": {
+					"sensitive": false,
+					"type": "string",
+					"value": "some-random-id"
+				  }
+			   }`,
+			),
+			expectedProps: map[string]interface{}{
+				"TF_OUTPUT_queue_url":   "https://sqs.us-east-1.amazonaws.com/1234654/elastic-package-aws-logs-queue-someId",
+				"TF_OUTPUT_instance_id": "some-random-id",
+			},
+		},
+		{
+			testName: "complex_value_output",
+			runId:    "078907890",
+			ctxt: ServiceContext{
+				Test: struct{ RunID string }{"078907890"},
+			},
+			content: []byte(
+				`{
+					"queue_url": {
+					  "sensitive": false,
+					  "type": "string",
+					  "value": "https://sqs.us-east-1.amazonaws.com/1234654/elastic-package-aws-logs-queue-someId"
+					},
+					"triangle_output": {
+					  "sensitive": false,
+					  "type": [
+						"object",
+						{
+						  "description": "string",
+						  "s_one": "number",
+						  "s_three": "number",
+						  "s_two": "number"
+						}
+					  ],
+					  "value": {
+						"value": "this is a triangle",
+						"s_one": 1,
+						"s_three": 2.5,
+						"s_two": 2.5
+					  }
+					}
+				}`,
+			),
+			expectedProps: map[string]interface{}{
+				"TF_OUTPUT_queue_url": "https://sqs.us-east-1.amazonaws.com/1234654/elastic-package-aws-logs-queue-someId",
+				"TF_OUTPUT_triangle_output": map[string]any{
+					"s_one":   1.0,
+					"s_three": 2.5,
+					"s_two":   2.5,
+					"value":   "this is a triangle",
+				},
+			},
+		},
+	}
+
+	t.Parallel()
+	for _, tc := range testCases {
+
+		t.Run(tc.testName, func(t *testing.T) {
+			tc.ctxt.CustomProperties = make(map[string]interface{})
+			tc.ctxt.OutputDir = t.TempDir()
+
+			if err := os.WriteFile(tc.ctxt.OutputDir+"/tfOutputValues.json", tc.content, 0777); err != nil {
+				t.Fatal(err)
+			}
+
+			// Test that the terraform output values are generated correctly
+			addTerraformOutputs(tc.ctxt)
+			assert.Equal(t, tc.expectedProps, tc.ctxt.CustomProperties)
+		})
+	}
+}

test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf

@@ -31,3 +31,7 @@ data "aws_ami" "latest-amzn" {
     values = ["amzn2-ami-minimal-hvm-*-ebs"]
   }
 }
+
+output "instance_id" {
+  value = aws_instance.i.id
+}