Record with go-vcr instead of doing our custom recording

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
 	golang.org/x/tools v0.15.0
+	gopkg.in/dnaeon/go-vcr.v3 v3.1.2
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/gotestsum v1.11.0
 	helm.sh/helm/v3 v3.13.2

go.sum

@@ -906,6 +906,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/dnaeon/go-vcr.v3 v3.1.2 h1:F1smfXBqQqwpVifDfUBQG6zzaGjzT+EnVZakrOdr5wA=
+gopkg.in/dnaeon/go-vcr.v3 v3.1.2/go.mod h1:2IMOnnlx9I6u9x+YBsM3tAMx6AlOxnJ0pWxQAzZ79Ag=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/hjson/hjson-go.v3 v3.0.1/go.mod h1:X6zrTSVeImfwfZLfgQdInl9mWjqPqgH90jom9nym/lw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=

internal/dump/agentpolicies_test.go

@@ -24,19 +24,30 @@ func TestDumpAgentPolicies(t *testing.T) {
 	// - Configure environment variables for this stack (eval "$(elastic-package stack shellinit)").
 	// - Run tests.
 	// - Check that recorded files make sense and commit them.
+	// To update the suite:
+	// - Reproduce the scenario as described in the comments.
+	// - Remove the files that you want to update.
+	// - Follow the same steps to create a new suite.
+	// - Check if the changes are the expected ones and commit them.
 	suites := []*agentPoliciesDumpSuite{
 		&agentPoliciesDumpSuite{
+			// To reproduce this scenario:
+			// - Start stack with version 7.17.0.
+			// - Install nginx package.
 			AgentPolicy:        "499b5aa7-d214-5b5d-838b-3cd76469844e",
 			PackageName:        "nginx",
-			RecordDir:          "./testdata/fleet-7-mock-dump-all",
+			Record:             "./testdata/fleet-7-mock-dump-all",
 			DumpDirAll:         "./testdata/fleet-7-dump/all",
 			DumpDirPackage:     "./testdata/fleet-7-dump/package",
 			DumpDirAgentPolicy: "./testdata/fleet-7-dump/agentpolicy",
 		},
 		&agentPoliciesDumpSuite{
+			// To reproduce this scenario:
+			// - Start stack with version 8.0.0.
+			// - Install nginx package.
 			AgentPolicy:        "fleet-server-policy",
 			PackageName:        "nginx",
-			RecordDir:          "./testdata/fleet-8-mock-dump-all",
+			Record:             "./testdata/fleet-8-mock-dump-all",
 			DumpDirAll:         "./testdata/fleet-8-dump/all",
 			DumpDirPackage:     "./testdata/fleet-8-dump/package",
 			DumpDirAgentPolicy: "./testdata/fleet-8-dump/agentpolicy",
@@ -57,8 +68,8 @@ type agentPoliciesDumpSuite struct {
 	// AgentPolicy is the name of the agent policy to look for.
 	PackageName string
 
-	// RecordDir is where responses from Kibana are recorded.
-	RecordDir string
+	// Record is where responses from Kibana are recorded.
+	Record string
 
 	// DumpDirAll is where the expected dumped files are stored when looking for all agent policies.
 	DumpDirAll string
@@ -111,7 +122,7 @@ func (s *agentPoliciesDumpSuite) SetupTest() {
 }
 
 func (s *agentPoliciesDumpSuite) TestDumpAll() {
-	client := kibanatest.NewClient(s.T(), s.RecordDir)
+	client := kibanatest.NewClient(s.T(), s.Record)
 
 	outputDir := s.T().TempDir()
 	dumper := NewAgentPoliciesDumper(client)
@@ -128,7 +139,7 @@ func (s *agentPoliciesDumpSuite) TestDumpAll() {
 }
 
 func (s *agentPoliciesDumpSuite) TestDumpByPackage() {
-	client := kibanatest.NewClient(s.T(), s.RecordDir)
+	client := kibanatest.NewClient(s.T(), s.Record)
 
 	outputDir := s.T().TempDir()
 	dumper := NewAgentPoliciesDumper(client)
@@ -145,7 +156,7 @@ func (s *agentPoliciesDumpSuite) TestDumpByPackage() {
 }
 
 func (s *agentPoliciesDumpSuite) TestDumpByName() {
-	client := kibanatest.NewClient(s.T(), s.RecordDir)
+	client := kibanatest.NewClient(s.T(), s.Record)
 
 	outputDir := s.T().TempDir()
 	dumper := NewAgentPoliciesDumper(client)

internal/dump/indextemplates.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/http"
 	"slices"
 
 	"github.com/elastic/elastic-package/internal/elasticsearch"
@@ -77,6 +78,10 @@ func getIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, pa
 	}
 	defer resp.Body.Close()
 
+	if resp.StatusCode == http.StatusNotFound {
+		// Some packages don't have index templates.
+		return nil, nil
+	}
 	if resp.IsError() {
 		return nil, fmt.Errorf("failed to get index templates: %s", resp.String())
 	}

internal/dump/ingestpipelines.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/http"
 	"slices"
 
 	"github.com/elastic/elastic-package/internal/elasticsearch"
@@ -71,6 +72,10 @@ func getIngestPipelineByID(ctx context.Context, api *elasticsearch.API, id strin
 	}
 	defer resp.Body.Close()
 
+	// Ingest templates referenced by other templates may not exist.
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, nil
+	}
 	if resp.IsError() {
 		return nil, fmt.Errorf("failed to get ingest pipeline %s: %s", id, resp.String())
 	}

internal/dump/installedobjects.go

@@ -223,7 +223,7 @@ func (e *InstalledObjectsDumper) getIngestPipelines(ctx context.Context) ([]Inge
 		names := getIngestPipelinesFromTemplates(templates)
 		ingestPipelines, err := getIngestPipelines(ctx, e.client, names...)
 		if err != nil {
-			return nil, fmt.Errorf("failed to get ingest pipelines: %w", err)
+			return nil, fmt.Errorf("failed to get ingest pipelines from templates: %w", err)
 		}
 		e.ingestPipelines = ingestPipelines
 	}

internal/dump/installedobjects_test.go

@@ -30,20 +30,35 @@ func TestDumpInstalledObjects(t *testing.T) {
 	// - Configure environment variables for this stack (eval "$(elastic-package stack shellinit)").
 	// - Run tests.
 	// - Check that recorded files make sense and commit them.
+	// To update a suite:
+	// - Reproduce the scenario as described in the comments.
+	// - Remove the files that you want to update.
+	// - Follow the same steps to create a new suite.
+	// - Check if the changes are the expected ones and commit them.
 	suites := []*installedObjectsDumpSuite{
 		&installedObjectsDumpSuite{
+			// To reproduce the scenario:
+			// - Start the stack with version 7.16.2.
+			// - Install apache package (1.3.4).
 			PackageName: "apache",
-			RecordDir:   "./testdata/elasticsearch-7-mock-dump-apache",
+			Record:      "./testdata/elasticsearch-7-mock-dump-apache",
 			DumpDir:     "./testdata/elasticsearch-7-apache-dump-all",
 		},
 		&installedObjectsDumpSuite{
+			// To reproduce the scenario:
+			// - Start the stack with version 8.1.0.
+			// - Install apache package (1.3.6).
 			PackageName: "apache",
-			RecordDir:   "./testdata/elasticsearch-8-mock-dump-apache",
+			Record:      "./testdata/elasticsearch-8-mock-dump-apache",
 			DumpDir:     "./testdata/elasticsearch-8-apache-dump-all",
 		},
 		&installedObjectsDumpSuite{
+			// To reproduce the scenario:
+			// - Start the stack with version 8.9.0.
+			// - Install dga package (2.1.0).
+			// - Manually replace the `compressed_definition` fields with "//REDACTED//".
 			PackageName: "dga",
-			RecordDir:   "./testdata/elasticsearch-8-mock-dump-dga",
+			Record:      "./testdata/elasticsearch-8-mock-dump-dga",
 			DumpDir:     "./testdata/elasticsearch-8-dga-dump-all",
 		},
 	}
@@ -59,8 +74,8 @@ type installedObjectsDumpSuite struct {
 	// PackageName is the name of the package.
 	PackageName string
 
-	// RecordDir is where responses from Elasticsearch are recorded.
-	RecordDir string
+	// Record is where responses from Elasticsearch are recorded.
+	Record string
 
 	// DumpDir is where the expected dumped files are stored.
 	DumpDir string
@@ -82,7 +97,7 @@ func (s *installedObjectsDumpSuite) SetupTest() {
 }
 
 func (s *installedObjectsDumpSuite) TestDumpAll() {
-	client := estest.NewClient(s.T(), s.RecordDir)
+	client := estest.NewClient(s.T(), s.Record)
 
 	outputDir := s.T().TempDir()
 	dumper := NewInstalledObjectsDumper(client.API, s.PackageName)
@@ -99,7 +114,7 @@ func (s *installedObjectsDumpSuite) TestDumpAll() {
 }
 
 func (s *installedObjectsDumpSuite) TestDumpSome() {
-	client := estest.NewClient(s.T(), s.RecordDir)
+	client := estest.NewClient(s.T(), s.Record)
 	dumper := NewInstalledObjectsDumper(client.API, s.PackageName)
 
 	// In a map so order of execution is randomized.

internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/logs.json

@@ -1,6 +1,6 @@
 {
   "version": 1,
-  "modified_date": "2022-01-25T18:01:46.058Z",
+  "modified_date": "2023-11-27T13:55:33.162Z",
   "policy": {
     "phases": {
       "hot": {
@@ -20,10 +20,10 @@
   },
   "in_use_by": {
     "indices": [
-      ".ds-logs-elastic_agent-default-2022.01.25-000001",
-      ".ds-logs-elastic_agent.metricbeat-default-2022.01.25-000001",
-      ".ds-logs-elastic_agent.filebeat-default-2022.01.25-000001",
-      ".ds-logs-elastic_agent.fleet_server-default-2022.01.25-000001"
+      ".ds-logs-elastic_agent.metricbeat-default-2023.11.27-000001",
+      ".ds-logs-elastic_agent.fleet_server-default-2023.11.27-000001",
+      ".ds-logs-elastic_agent.filebeat-default-2023.11.27-000001",
+      ".ds-logs-elastic_agent-default-2023.11.27-000001"
     ],
     "data_streams": [
       "logs-elastic_agent-default",
@@ -33,12 +33,15 @@
     ],
     "composable_templates": [
       "logs-apache.access",
+      "logs-elastic_agent.cloudbeat",
       "logs-elastic_agent.apm_server",
+      "logs-elastic_agent.cloud_defend",
       "logs-system.security",
       "logs-system.auth",
       "logs-elastic_agent.metricbeat",
       "logs-elastic_agent.filebeat",
       "logs-elastic_agent.packetbeat",
+      "logs-elastic_agent.filebeat_input",
       "logs-elastic_agent.endpoint_security",
       "logs-elastic_agent.fleet_server",
       "logs-apache.error",

internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/metrics.json

@@ -1,6 +1,6 @@
 {
   "version": 1,
-  "modified_date": "2022-01-25T18:01:48.410Z",
+  "modified_date": "2023-11-27T13:55:33.210Z",
   "policy": {
     "phases": {
       "hot": {
@@ -20,21 +20,21 @@
   },
   "in_use_by": {
     "indices": [
-      ".ds-metrics-system.socket_summary-default-2022.01.25-000001",
-      ".ds-metrics-system.cpu-default-2022.01.25-000001",
-      ".ds-metrics-elastic_agent.metricbeat-default-2022.01.25-000001",
-      ".ds-metrics-system.uptime-default-2022.01.25-000001",
-      ".ds-metrics-system.process-default-2022.01.25-000001",
-      ".ds-metrics-system.memory-default-2022.01.25-000001",
-      ".ds-metrics-system.diskio-default-2022.01.25-000001",
-      ".ds-metrics-elastic_agent.fleet_server-default-2022.01.25-000001",
-      ".ds-metrics-elastic_agent.filebeat-default-2022.01.25-000001",
-      ".ds-metrics-system.load-default-2022.01.25-000001",
-      ".ds-metrics-system.process.summary-default-2022.01.25-000001",
-      ".ds-metrics-elastic_agent.elastic_agent-default-2022.01.25-000001",
-      ".ds-metrics-system.filesystem-default-2022.01.25-000001",
-      ".ds-metrics-system.network-default-2022.01.25-000001",
-      ".ds-metrics-system.fsstat-default-2022.01.25-000001"
+      ".ds-metrics-system.process.summary-default-2023.11.27-000001",
+      ".ds-metrics-system.fsstat-default-2023.11.27-000001",
+      ".ds-metrics-system.uptime-default-2023.11.27-000001",
+      ".ds-metrics-system.network-default-2023.11.27-000001",
+      ".ds-metrics-system.filesystem-default-2023.11.27-000001",
+      ".ds-metrics-elastic_agent.elastic_agent-default-2023.11.27-000001",
+      ".ds-metrics-system.socket_summary-default-2023.11.27-000001",
+      ".ds-metrics-system.diskio-default-2023.11.27-000001",
+      ".ds-metrics-elastic_agent.filebeat-default-2023.11.27-000001",
+      ".ds-metrics-system.process-default-2023.11.27-000001",
+      ".ds-metrics-system.cpu-default-2023.11.27-000001",
+      ".ds-metrics-elastic_agent.fleet_server-default-2023.11.27-000001",
+      ".ds-metrics-elastic_agent.metricbeat-default-2023.11.27-000001",
+      ".ds-metrics-system.memory-default-2023.11.27-000001",
+      ".ds-metrics-system.load-default-2023.11.27-000001"
     ],
     "data_streams": [
       "metrics-system.filesystem-default",
@@ -68,11 +68,13 @@
       "metrics-system.load",
       "metrics-system.core",
       "metrics-elastic_agent.filebeat",
+      "metrics-elastic_agent.filebeat_input",
       "metrics-system.uptime",
       "metrics-system.process.summary",
       "metrics-system.cpu",
       "metrics-elastic_agent.heartbeat",
       "metrics-system.diskio",
+      "metrics-elastic_agent.cloudbeat",
       "metrics-elastic_agent.metricbeat",
       "metrics-elastic_agent.auditbeat",
       "metrics-system.network",

internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.access.json

@@ -137,7 +137,8 @@
                     "properties": {
                       "name": {
                         "ignore_above": 1024,
-                        "type": "keyword"
+                        "type": "keyword",
+                        "fields": {}
                       }
                     }
                   }
@@ -267,7 +268,8 @@
             "properties": {
               "path": {
                 "ignore_above": 1024,
-                "type": "keyword"
+                "type": "keyword",
+                "fields": {}
               }
             }
           },
@@ -468,29 +470,33 @@
             "properties": {
               "name": {
                 "ignore_above": 1024,
-                "type": "keyword"
+                "type": "keyword",
+                "fields": {}
               }
             }
           },
           "user_agent": {
             "properties": {
               "original": {
                 "ignore_above": 1024,
-                "type": "keyword"
+                "type": "keyword",
+                "fields": {}
               },
               "os": {
                 "properties": {
                   "name": {
                     "ignore_above": 1024,
-                    "type": "keyword"
+                    "type": "keyword",
+                    "fields": {}
                   },
                   "version": {
                     "ignore_above": 1024,
                     "type": "keyword"
                   },
                   "full": {
                     "ignore_above": 1024,
-                    "type": "keyword"
+                    "type": "keyword",
+                    "fields": {}
                   }
                 }
               },