System test runner MVP (#64)

* Unexporting const

* Fleshing out system test runner a bit

* Adding godoc and TODOs

* Exporting Ctx

* Try to boot up test cluster

* Using logger

* WIP

* WIP: rename service to servicedeployer

* WIP: start talking to Kibana APIs

* Fixing compile errors

* Delete policy when test ends

* WIP: Adding idea TODOs

* More WIP TODO

* Fixing syntax error

* WIP: Fleshing out Kibana API call for adding package to policy

* Fixing compile errors

* WIP: call Kibana API to add datastream to policy

* Allow for scalar or list var types

* Add elastic-agent serice to snapshot docker-compose file

* Adding healthcheck for elastic-agent service

* Adding methods to assign policy to agent

* Reassign agent to original policy so test policy can be cleaned up

* Overlay test config vars

* WIP: mounting hackery in progress

* Fixing rebase errors

* Fleshing out docker-compose service deployer

* Adding bind mount for temp dir in agent container

* Adding TODO + debug log for ctxt

* Using consts for log file names

* Creating function stub and moving TODO

* Apply context to config

* Updating fleet ES URL in Kibana config

* Add Service.Hostname context var

* Wait until true

* Update root service in dependency tree

* Renaming per feedback

* Move STDOUT/STDERR cleanup to defers

* Consistent naming

* Adding godoc explaining context

* Breaking up ServiceRunner into ServiceDeployer and DeployedService interfaces

* Strictly type context

* Sharing ES client across test runs

* Renaming files to be consistent across test runners

* Adding some godoc + unexporting unnecessary exports

* Adding godoc

* Adding license headers

* Changing local dir to /tmp/service_logs

* Moving service context to servicedeployer package

* Read ES and Kibana hostnames from Elastic Stack Docker Compose configuration file

* Adding test timestamp to policy name

* Bumping up default version of Elastic Stack to latest released version

* Renaming method per language idiom

* Fixing comment that I missed earlier

* Prefixing shellinit hostnames with http://

* Refactoring: introducing dockerComposeOptions struct

* Bumping up default stack version to 7.10.0-SNAPSHOT

* Updating health check command for Elastic Agent

cmd/testrunner.go

@@ -12,6 +12,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/elasticsearch"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/testrunner"
 	_ "github.com/elastic/elastic-package/internal/testrunner/runners" // register all test runners
@@ -82,22 +83,29 @@ func testTypeCommandActionFactory(testType testrunner.TestType) cobraext.Command
 			return errors.Wrap(err, "locating package root failed")
 		}
 
-		testFolderPaths, err := testrunner.FindTestFolders(packageRootPath, testType, datasets)
+		testFolders, err := testrunner.FindTestFolders(packageRootPath, testType, datasets)
 		if err != nil {
 			return errors.Wrap(err, "unable to determine test folder paths")
 		}
 
-		if failOnMissing && len(testFolderPaths) == 0 {
+		if failOnMissing && len(testFolders) == 0 {
 			if len(datasets) > 0 {
 				return fmt.Errorf("no %s tests found for %s dataset(s)", testType, strings.Join(datasets, ","))
 			}
 			return fmt.Errorf("no %s tests found", testType)
 		}
 
-		for _, path := range testFolderPaths {
+		esClient, err := elasticsearch.Client()
+		if err != nil {
+			return errors.Wrap(err, "fetching Elasticsearch client instance failed")
+		}
+
+		for _, folder := range testFolders {
 			if err := testrunner.Run(testType, testrunner.TestOptions{
-				TestFolderPath:     path,
+				TestFolder:         folder,
+				PackageRootPath:    packageRootPath,
 				GenerateTestResult: generateTestResult,
+				ESClient:           esClient,
 			}); err != nil {
 				return errors.Wrapf(err, "error running package %s tests", testType)
 			}

go.mod

@@ -5,6 +5,7 @@ go 1.14
 require (
 	github.com/AlecAivazis/survey/v2 v2.1.0
 	github.com/Masterminds/semver v1.5.0
+	github.com/aymerick/raymond v2.0.2+incompatible
 	github.com/elastic/go-elasticsearch/v7 v7.9.0
 	github.com/elastic/package-spec/code/go v0.0.0-20200826192647-f45b05c887b5
 	github.com/go-git/go-billy/v5 v5.0.0
@@ -14,7 +15,10 @@ require (
 	github.com/magefile/mage v1.10.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.0.0
+	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de // indirect
 	golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	golang.org/x/sys v0.0.0-20200817155316-9781c653f443 // indirect
+	golang.org/x/text v0.3.3 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 )

go.sum

@@ -19,6 +19,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/aymerick/raymond v2.0.2+incompatible h1:VEp3GpgdAnv9B2GFyTvqgcKvY+mfKMjPOA3SbKLtnU0=
+github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -189,6 +191,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig=
+golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -206,7 +210,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -230,9 +233,13 @@ golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200817155316-9781c653f443 h1:X18bCaipMcoJGm27Nv7zr4XYPKGUy92GtqboKC2Hxaw=
+golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

internal/builder/dashboards.go

@@ -10,6 +10,8 @@ import (
 	"path/filepath"
 
 	"github.com/pkg/errors"
+
+	"github.com/elastic/elastic-package/internal/common"
 )
 
 var fieldsToEncode = []string{
@@ -53,15 +55,15 @@ func encodeDashboards(destinationDir string) error {
 // The reason is that for versioning it is much nicer to have the full
 // json so only on packaging this is changed.
 func encodedSavedObject(data []byte) ([]byte, bool, error) {
-	savedObject := mapStr{}
+	savedObject := common.MapStr{}
 	err := json.Unmarshal(data, &savedObject)
 	if err != nil {
 		return nil, false, errors.Wrapf(err, "unmarshalling saved object failed")
 	}
 
 	var changed bool
 	for _, v := range fieldsToEncode {
-		out, err := savedObject.getValue(v)
+		out, err := savedObject.GetValue(v)
 		// This means the key did not exists, no conversion needed.
 		if err != nil {
 			continue
@@ -79,11 +81,11 @@ func encodedSavedObject(data []byte) ([]byte, bool, error) {
 		if err != nil {
 			return nil, false, err
 		}
-		_, err = savedObject.put(v, string(r))
+		_, err = savedObject.Put(v, string(r))
 		if err != nil {
 			return nil, false, errors.Wrapf(err, "can't put value to the saved object")
 		}
 		changed = true
 	}
-	return []byte(savedObject.stringToPrint()), changed, nil
+	return []byte(savedObject.StringToPrint()), changed, nil
 }

internal/builder/mapstr.go → internal/common/mapstr.go

@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package builder
+package common
 
 // WARNING: This code is copied from https://github.com/elastic/beats/blob/master/libbeat/common/mapstr.go
 // This was done to not have to import the full common package and all its dependencies
@@ -21,13 +21,13 @@ var (
 	errKeyNotFound = errors.New("key not found")
 )
 
-// mapStr is a map[string]interface{} wrapper with utility methods for common
+// MapStr is a map[string]interface{} wrapper with utility methods for common
 // map operations like converting to JSON.
-type mapStr map[string]interface{}
+type MapStr map[string]interface{}
 
 // GetValue gets a value from the map. If the key does not exist then an error
 // is returned.
-func (m mapStr) getValue(key string) (interface{}, error) {
+func (m MapStr) GetValue(key string) (interface{}, error) {
 	_, _, v, found, err := mapFind(key, m, false)
 	if err != nil {
 		return nil, err
@@ -45,7 +45,7 @@ func (m mapStr) getValue(key string) (interface{}, error) {
 //
 // If you need insert keys containing dots then you must use bracket notation
 // to insert values (e.g. m[key] = value).
-func (m mapStr) put(key string, value interface{}) (interface{}, error) {
+func (m MapStr) Put(key string, value interface{}) (interface{}, error) {
 	// XXX `safemapstr.Put` mimics this implementation, both should be updated to have similar behavior
 	k, d, old, _, err := mapFind(key, m, true)
 	if err != nil {
@@ -56,38 +56,38 @@ func (m mapStr) put(key string, value interface{}) (interface{}, error) {
 	return old, nil
 }
 
-// StringToPrint returns the mapStr as pretty JSON.
-func (m mapStr) stringToPrint() string {
-	json, err := json.MarshalIndent(m, "", "  ")
+// StringToPrint returns the MapStr as pretty JSON.
+func (m MapStr) StringToPrint() string {
+	j, err := json.MarshalIndent(m, "", "  ")
 	if err != nil {
 		return fmt.Sprintf("Not valid json: %v", err)
 	}
-	return string(json)
+	return string(j)
 }
 
-// tomapStr performs a type assertion on v and returns a mapStr. v can be either
-// a mapStr or a map[string]interface{}. If it's any other type or nil then
+// tomapStr performs a type assertion on v and returns a MapStr. v can be either
+// a MapStr or a map[string]interface{}. If it's any other type or nil then
 // an error is returned.
-func toMapStr(v interface{}) (mapStr, error) {
+func toMapStr(v interface{}) (MapStr, error) {
 	m, ok := tryToMapStr(v)
 	if !ok {
 		return nil, errors.Errorf("expected map but type is %T", v)
 	}
 	return m, nil
 }
 
-func tryToMapStr(v interface{}) (mapStr, bool) {
+func tryToMapStr(v interface{}) (MapStr, bool) {
 	switch m := v.(type) {
-	case mapStr:
+	case MapStr:
 		return m, true
 	case map[string]interface{}:
-		return mapStr(m), true
+		return MapStr(m), true
 	default:
 		return nil, false
 	}
 }
 
-// mapFind iterates a mapStr based on a the given dotted key, finding the final
+// mapFind iterates a MapStr based on a the given dotted key, finding the final
 // subMap and subKey to operate on.
 // An error is returned if some intermediate is no map or the key doesn't exist.
 // If createMissing is set to true, intermediate maps are created.
@@ -97,9 +97,9 @@ func tryToMapStr(v interface{}) (mapStr, bool) {
 // the original value.
 func mapFind(
 	key string,
-	data mapStr,
+	data MapStr,
 	createMissing bool,
-) (subKey string, subMap mapStr, oldValue interface{}, present bool, err error) {
+) (subKey string, subMap MapStr, oldValue interface{}, present bool, err error) {
 	// XXX `safemapstr.mapFind` mimics this implementation, both should be updated to have similar behavior
 
 	for {
@@ -117,7 +117,7 @@ func mapFind(
 		d, exists := data[k]
 		if !exists {
 			if createMissing {
-				d = mapStr{}
+				d = MapStr{}
 				data[k] = d
 			} else {
 				return "", nil, nil, false, errKeyNotFound