Separate management and functions in Functionbeat

.gitignore

@@ -10,6 +10,7 @@
 *beat/build
 *beat/logs
 *beat/data
+x-pack/functionbeat/pkg
 
 # Files
 .DS_Store
@@ -21,6 +22,8 @@ coverage.out
 beat.db
 *.keystore
 mage_output_file.go
+x-pack/functionbeat/*/fields.yml
+x-pack/functionbeat/provider/*/functionbeat-*
 
 # Editor swap files
 *.swp

dev-tools/mage/build.go

@@ -33,6 +33,7 @@ import (
 // "go build" is invoked.
 type BuildArgs struct {
 	Name       string // Name of binary. (On Windows '.exe' is appended.)
+	InputFiles []string
 	OutputDir  string
 	CGO        bool
 	Static     bool
@@ -143,6 +144,10 @@ func Build(params BuildArgs) error {
 		args = append(args, MustExpand(strings.Join(ldflags, " ")))
 	}
 
+	if len(params.InputFiles) > 0 {
+		args = append(args, params.InputFiles...)
+	}
+
 	log.Println("Adding build environment vars:", env)
 	return sh.RunWith(env, "go", args...)
 }

dev-tools/mage/gotest.go

@@ -53,6 +53,12 @@ type GoTestArgs struct {
 	CoverageProfileFile string            // Test coverage profile file (enables -cover).
 }
 
+// TestBinaryArgs are the arguments used when building binary for testing.
+type TestBinaryArgs struct {
+	Name       string // Name of the binary to build
+	InputFiles []string
+}
+
 func makeGoTestArgs(name string) GoTestArgs {
 	fileName := fmt.Sprintf("build/TEST-go-%s", strings.Replace(strings.ToLower(name), " ", "_", -1))
 	params := GoTestArgs{
@@ -80,6 +86,14 @@ func DefaultGoTestIntegrationArgs() GoTestArgs {
 	return args
 }
 
+// DefaultTestBinaryArgs returns the default arguments for building
+// a binary for testing.
+func DefaultTestBinaryArgs() TestBinaryArgs {
+	return TestBinaryArgs{
+		Name: BeatName,
+	}
+}
+
 // GoTest invokes "go test" and reports the results to stdout. It returns an
 // error if there was any failure executing the tests or if there were any
 // test failures.
@@ -329,15 +343,24 @@ func (s *GoTestSummary) String() string {
 	return strings.TrimRight(b.String(), "\n")
 }
 
-// BuildSystemTestBinary build a binary for testing that is instrumented for
+// BuildSystemTestBinary runs BuildSystemTestGoBinary with default values.
+func BuildSystemTestBinary() error {
+	return BuildSystemTestGoBinary(DefaultTestBinaryArgs())
+}
+
+// BuildSystemTestGoBinary build a binary for testing that is instrumented for
 // testing and measuring code coverage. The binary is only instrumented for
 // coverage when TEST_COVERAGE=true (default is false).
-func BuildSystemTestBinary() error {
+func BuildSystemTestGoBinary(binArgs TestBinaryArgs) error {
 	args := []string{
 		"test", "-c",
+		"-o", binArgs.Name + ".test",
 	}
 	if TestCoverage {
 		args = append(args, "-coverpkg", "./...")
 	}
+	if len(binArgs.InputFiles) > 0 {
+		args = append(args, binArgs.InputFiles...)
+	}
 	return sh.RunV("go", args...)
 }

dev-tools/mage/pkgtypes.go

@@ -280,6 +280,10 @@ func (s PackageSpec) Clone() PackageSpec {
 	for k, v := range s.Files {
 		clone.Files[k] = v
 	}
+	clone.ExtraVars = make(map[string]string, len(s.ExtraVars))
+	for k, v := range s.ExtraVars {
+		clone.ExtraVars[k] = v
+	}
 	return clone
 }
 
@@ -343,6 +347,10 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec {
 		return MustExpand(in, args...)
 	}
 
+	if s.evalContext == nil {
+		s.evalContext = map[string]interface{}{}
+	}
+
 	for k, v := range s.ExtraVars {
 		s.evalContext[k] = mustExpand(v)
 	}
@@ -375,9 +383,6 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec {
 	} else {
 		s.packageDir = filepath.Clean(mustExpand(s.packageDir))
 	}
-	if s.evalContext == nil {
-		s.evalContext = map[string]interface{}{}
-	}
 	s.evalContext["PackageDir"] = s.packageDir
 
 	evaluatedFiles := make(map[string]PackageFile, len(s.Files))

dev-tools/make/xpack.mk

@@ -24,6 +24,8 @@ check: mage
 clean: mage
 	mage clean
 
+fix-permissions:
+
 .PHONY: fmt
 fmt: mage
 	mage fmt
@@ -38,6 +40,8 @@ help:
 release: mage
 	mage package
 
+stop-environment:
+
 .PHONY: testsuite
 testsuite: mage
 	-rm build/TEST-go-integration.out

libbeat/cmd/root.go

@@ -76,7 +76,7 @@ func GenRootCmdWithSettings(beatCreator beat.Creator, settings instance.Settings
 	rootCmd.TestCmd = genTestCmd(settings, beatCreator)
 	rootCmd.SetupCmd = genSetupCmd(settings, beatCreator)
 	rootCmd.KeystoreCmd = genKeystoreCmd(settings)
-	rootCmd.VersionCmd = genVersionCmd(settings)
+	rootCmd.VersionCmd = GenVersionCmd(settings)
 	rootCmd.CompletionCmd = genCompletionCmd(settings, rootCmd)
 
 	// Root command is an alias for run

libbeat/cmd/version.go

@@ -28,7 +28,8 @@ import (
 	"github.com/elastic/beats/libbeat/version"
 )
 
-func genVersionCmd(settings instance.Settings) *cobra.Command {
+// GenVersionCmd generates the command version for a Beat.
+func GenVersionCmd(settings instance.Settings) *cobra.Command {
 	return &cobra.Command{
 		Use:   "version",
 		Short: "Show current version info",

x-pack/functionbeat/Makefile

@@ -1,22 +1,11 @@
-BEAT_NAME?=functionbeat
-LICENSE=Elastic
-BEAT_TITLE?=Functionbeat
-SYSTEM_TESTS?=true
-BEAT_PATH?=github.com/elastic/beats/x-pack/${BEAT_NAME}
-TEST_ENVIRONMENT?=true
+#
+# Variables
+#
 GOX_FLAGS=-arch="amd64 386 arm ppc64 ppc64le"
 ES_BEATS?=../../
-FIELDS_FILE_PATH=module
-XPACK_ONLY?=true
 
-# Path to the libbeat Makefile
-include $(ES_BEATS)/libbeat/scripts/Makefile
+#
+# Includes
+#
+include $(ES_BEATS)/dev-tools/make/xpack.mk
 
-# Runs all collection steps and updates afterwards
-.PHONY: collect
-collect:
-
-# Generate an artifact to be push on serverless provider.
-.PHONY: linux
-linux:
-	GOOS=linux go build -o pkg/functionbeat

x-pack/functionbeat/config/config.go

@@ -14,29 +14,28 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 )
 
+// ConfigOverrides overrides the defaults provided by libbeat.
 var (
 	functionPattern = "^[A-Za-z][A-Za-z0-9\\-]{0,139}$"
 	functionRE      = regexp.MustCompile(functionPattern)
+	ConfigOverrides = common.MustNewConfigFrom(map[string]interface{}{
+		"path.data":              "/tmp",
+		"path.logs":              "/tmp/logs",
+		"logging.to_stderr":      true,
+		"logging.to_files":       false,
+		"setup.template.enabled": true,
+		"queue.mem": map[string]interface{}{
+			"events":           "${output.elasticsearch.bulk_max_size}",
+			"flush.min_events": 10,
+			"flush.timeout":    "0.01s",
+		},
+		"output.elasticsearch.bulk_max_size": 50,
+	})
 )
 
-// ConfigOverrides overrides the defaults provided by libbeat.
-var ConfigOverrides = common.MustNewConfigFrom(map[string]interface{}{
-	"path.data":              "/tmp",
-	"path.logs":              "/tmp/logs",
-	"logging.to_stderr":      true,
-	"logging.to_files":       false,
-	"setup.template.enabled": true,
-	"queue.mem": map[string]interface{}{
-		"events":           "${output.elasticsearch.bulk_max_size}",
-		"flush.min_events": 10,
-		"flush.timeout":    "0.01s",
-	},
-	"output.elasticsearch.bulk_max_size": 50,
-})
-
 // Config default configuration for Functionbeat.
 type Config struct {
-	Provider *common.ConfigNamespace `config:"provider" validate:"required"`
+	Provider *common.Config `config:"provider" validate:"required"`
 }
 
 // ProviderConfig is a generic configured used by providers.

x-pack/functionbeat/config/config_test.go

@@ -2,6 +2,9 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
+// Config is put into a different package to prevent cyclic imports in case
+// it is needed in several locations
+
 package config
 
 import (

x-pack/functionbeat/dev-tools/packaging/packages.yml

@@ -41,9 +41,6 @@ shared:
       source: '{{.BeatName}}.yml'
       mode: 0600
       config: true
-    kibana:
-      source: _meta/kibana.generated
-      mode: 0644
 
   # Binary package spec (tar.gz for linux/darwin)
   - &binary_spec
@@ -61,12 +58,12 @@ shared:
         source: '{{ repo.RootDir }}/licenses/ELASTIC-LICENSE.txt'
         mode: 0644
   #
-  # Binaries used to run the function.
+  # Binaries used to run functions.
   #
   - &functionbeat_binaries
     files:
-      pkg/functionbeat:
-        source: 'build/golang-crossbuild/{{.BeatName}}-linux-amd64'
+      pkg/functionbeat-aws:
+        source: 'provider/aws/build/golang-crossbuild/aws-linux-amd64'
         mode: 0755
 # specs is a list of named packaging "flavors".
 specs:

x-pack/functionbeat/docker-compose.yml

@@ -4,8 +4,6 @@ services:
     build: ${PWD}/.
     depends_on:
       - proxy_dep
-    env_file:
-      - ${PWD}/build/test.env
     working_dir: /go/src/github.com/elastic/beats/x-pack/functionbeat
     volumes:
       - ${PWD}/../..:/go/src/github.com/elastic/beats/

x-pack/functionbeat/beater/functionbeat.go → x-pack/functionbeat/function/beater/functionbeat.go

@@ -18,9 +18,8 @@ import (
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/processors"
 	"github.com/elastic/beats/x-pack/functionbeat/config"
-	"github.com/elastic/beats/x-pack/functionbeat/core"
-	_ "github.com/elastic/beats/x-pack/functionbeat/include" // imports features
-	"github.com/elastic/beats/x-pack/functionbeat/provider"
+	"github.com/elastic/beats/x-pack/functionbeat/function/core"
+	"github.com/elastic/beats/x-pack/functionbeat/function/provider"
 	"github.com/elastic/beats/x-pack/libbeat/licenser"
 )
 
@@ -52,7 +51,7 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
 		return nil, fmt.Errorf("error reading config file: %+v", err)
 	}
 
-	provider, err := provider.NewProvider(c)
+	provider, err := getProvider(c.Provider)
 	if err != nil {
 		return nil, err
 	}
@@ -68,6 +67,23 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
 	return bt, nil
 }
 
+func getProvider(cfg *common.Config) (provider.Provider, error) {
+	providers, err := provider.List()
+	if err != nil {
+		return nil, err
+	}
+	if len(providers) != 1 {
+		return nil, fmt.Errorf("too many providers are available, expected one, got: %s", providers)
+	}
+
+	providerCfg, err := cfg.Child(providers[0], -1)
+	if err != nil {
+		return nil, err
+	}
+
+	return provider.NewProvider(providers[0], providerCfg)
+}
+
 // Run starts functionbeat.
 func (bt *Functionbeat) Run(b *beat.Beat) error {
 	defer bt.cancel()
@@ -108,6 +124,7 @@ func (bt *Functionbeat) Run(b *beat.Beat) error {
 	return nil
 }
 
+// enabledFunctions returns the enabled function types
 func (bt *Functionbeat) enabledFunctions() (values []string) {
 	raw, found := os.LookupEnv("ENABLED_FUNCTIONS")
 	if !found {

x-pack/functionbeat/function/cmd/root.go

@@ -0,0 +1,54 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/libbeat/cfgfile"
+	"github.com/elastic/beats/libbeat/cmd"
+	"github.com/elastic/beats/libbeat/cmd/instance"
+	"github.com/elastic/beats/x-pack/functionbeat/config"
+)
+
+// FunctionCmd is the command of the function.
+type FunctionCmd struct {
+	*cobra.Command
+	VersionCmd *cobra.Command
+}
+
+// NewFunctionCmd return a new initialized function command.
+func NewFunctionCmd(name string, beatCreator beat.Creator) *FunctionCmd {
+	settings := instance.Settings{
+		Name:            name,
+		IndexPrefix:     name,
+		ConfigOverrides: config.ConfigOverrides,
+	}
+
+	err := cfgfile.ChangeDefaultCfgfileFlag(settings.Name)
+	if err != nil {
+		panic(fmt.Errorf("failed to set default config file path: %v", err))
+	}
+
+	rootCmd := &FunctionCmd{
+		&cobra.Command{
+			Run: func(cmd *cobra.Command, args []string) {
+				err := instance.Run(settings, beatCreator)
+				if err != nil {
+					os.Exit(1)
+				}
+			},
+		},
+		cmd.GenVersionCmd(settings),
+	}
+
+	rootCmd.AddCommand(rootCmd.VersionCmd)
+
+	return rootCmd
+}