From b581b17ed536a9dd24cc8505b094aa217918f7f0 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 25 Mar 2020 04:15:17 -0700 Subject: [PATCH] [7.x] Separate ES client code from ES output code (#16150) (#17222) * Separate ES client code from ES output code (#16150) * Basic extraction of ES client code from ES output code * Move test * Removing duplicate function in monitoring reporter * Break import cycle * Guard onConnect callback execution * Replace use of field with getter * Moving common bulk API response processing into esclientleg * Moving API integration tests * Fixing references in tests * Adding developer CHANGELOG entry * Move LoadJSON method * Move callbacks to own file * Move client-related constructors into client.go file * Reducing global logging usage * Use new constructor in test * Passing logger in test * Use logger in test * Use struct fieldnames when initializing * Use constructor in test * Fixing typos * Replace esclient.ParseProxyURL with generic function in common * Imports formatting * Moving more fields from ES output client to esclientleg.Connection * Moving more fields * Update test code * Use new TLS package * Extracting common test code into eslegtest package * Replace uses of elasticsearch output client with esclientleg.NewConnection * Replacing uses of ES output client struct with esclientleg.Connection * Handle callbacks * Fixing formatting * Fixing import cycle * Fixing import and package name * Fixing imports * More fixes * Breaking import cycle * Removing unused function * Adding back missing statement * Fixing param * Fixing package name * Include ES output plugin so it's registered * Proxy handling * Let Connection handle ProxyDisable setting * Only parse proxy field from config if set * Cast timeout ints * Parse proxy URL * Fixing proxy integration test * Fixing ILM test * Updating expected request count in test * Fixing package names * Lots more refactoring!!! * Move timeout field * More fixes * Adding missing files * No need to pass HTTP any more * Simplifying Bulk API usage * Removing unused code * Remove bulk state from Connection * Removing empty file * Moving Bulk API response streaming parsing code back into ES output package * Don't make monitoring bulk parsing code use streaming parser * Replacing old HTTP struct passing * Removing HTTP use * Adding build tag * Fixing up tests * Allow default scheme to be configurable * Adding versions to import paths * Remove redundant check * Undoing unnecessary heartbeat import change * Forgot to resolve conflicts * Fixing imports * Running go mod tidy * Revert "Remove redundant check" This reverts commit c5fde6ff3be765a89c0bc20f9cae8f697d08d47e. * Fixing args order * Removing extraneous parameter * Removing wrong errors package import * Fixing order of arguments * Fixing package name * Instantiating logger for tests * Making streaming JSON parser private to ES output package * Detect and try to fix scheme before parsing URL * Making Connection private to ES output Client * Update test * Replace client.Ping() calls with client.Connect() calls in test code * Updating tests * Removing usage of ES output from monitoring code! * Using strings.Index instead of strings.SplitN * Return default config via function call * Removing "escape hatch" method to expose underlying connection from ES output client * Using client connection in tests * Re-implement Test() method for ES output client * Adding back missing import / sorting imports * Removing unused import * Fixing up developer CHANGELOG * Clean up rebase * Rebase cleanup * Making 7.x specific adaptations (ML setup in Filebeat) * Updating go.mod and go.sum files * Running go mod tidy --- CHANGELOG-developer.next.asciidoc | 1 + filebeat/beater/filebeat.go | 30 +- filebeat/fileset/factory.go | 3 +- filebeat/fileset/modules_integration_test.go | 41 +- filebeat/fileset/pipelines_test.go | 14 +- libbeat/cmd/instance/beat.go | 17 +- libbeat/common/transport/wrap.go | 4 +- libbeat/common/url.go | 30 + libbeat/common/url_test.go | 53 ++ .../eslegclient}/api.go | 2 +- .../eslegclient}/api_integration_test.go | 28 +- .../eslegclient}/api_mock_test.go | 8 +- .../eslegclient}/api_test.go | 24 +- .../eslegclient}/bulkapi.go | 94 ++- .../eslegclient}/bulkapi_integration_test.go | 8 +- .../eslegclient}/bulkapi_mock_test.go | 14 +- libbeat/esleg/eslegclient/config.go | 78 +++ libbeat/esleg/eslegclient/connection.go | 435 ++++++++++++ .../connection_integration_test.go | 177 +++++ .../eslegclient}/enc.go | 12 +- .../eslegclient}/enc_test.go | 6 +- .../estest.go => esleg/eslegclient/errors.go} | 31 +- .../eslegclient}/url.go | 18 +- .../eslegclient}/url_test.go | 54 +- .../testing.go => esleg/eslegtest/util.go} | 32 +- .../ilm/client_handler_integration_test.go | 10 +- .../ml-importer/importer_integration_test.go | 26 +- .../monitoring/report/elasticsearch/client.go | 60 +- .../report/elasticsearch/elasticsearch.go | 32 +- libbeat/outputs/elasticsearch/bulk.go | 155 +++++ libbeat/outputs/elasticsearch/bulk_test.go | 131 ++++ libbeat/outputs/elasticsearch/callbacks.go | 111 +++ libbeat/outputs/elasticsearch/client.go | 639 +++--------------- .../elasticsearch/client_integration_test.go | 90 +-- .../elasticsearch/client_proxy_test.go | 13 +- libbeat/outputs/elasticsearch/client_test.go | 254 ++----- libbeat/outputs/elasticsearch/config.go | 3 +- .../outputs/elasticsearch/elasticsearch.go | 228 +------ .../elasticsearch/elasticsearch_test.go | 14 +- libbeat/outputs/elasticsearch/json_read.go | 73 +- .../logstash/logstash_integration_test.go | 12 +- libbeat/template/load_integration_test.go | 29 +- x-pack/libbeat/licenser/elastic_fetcher.go | 8 +- .../elastic_fetcher_integration_test.go | 12 +- .../libbeat/licenser/elastic_fetcher_test.go | 11 +- x-pack/libbeat/licenser/es_callback.go | 3 +- 46 files changed, 1759 insertions(+), 1369 deletions(-) rename libbeat/{outputs/elasticsearch => esleg/eslegclient}/api.go (99%) rename libbeat/{outputs/elasticsearch => esleg/eslegclient}/api_integration_test.go (81%) rename libbeat/{outputs/elasticsearch => esleg/eslegclient}/api_mock_test.go (96%) rename libbeat/{outputs/elasticsearch => esleg/eslegclient}/api_test.go (90%) rename libbeat/{outputs/elasticsearch => esleg/eslegclient}/bulkapi.go (71%) rename libbeat/{outputs/elasticsearch => esleg/eslegclient}/bulkapi_integration_test.go (95%) rename libbeat/{outputs/elasticsearch => esleg/eslegclient}/bulkapi_mock_test.go (91%) create mode 100644 libbeat/esleg/eslegclient/config.go create mode 100644 libbeat/esleg/eslegclient/connection.go create mode 100644 libbeat/esleg/eslegclient/connection_integration_test.go rename libbeat/{outputs/elasticsearch => esleg/eslegclient}/enc.go (95%) rename libbeat/{outputs/elasticsearch => esleg/eslegclient}/enc_test.go (95%) rename libbeat/{outputs/elasticsearch/estest/estest.go => esleg/eslegclient/errors.go} (53%) rename libbeat/{outputs/elasticsearch => esleg/eslegclient}/url.go (82%) rename libbeat/{outputs/elasticsearch => esleg/eslegclient}/url_test.go (64%) rename libbeat/{outputs/elasticsearch/internal/testing.go => esleg/eslegtest/util.go} (80%) create mode 100644 libbeat/outputs/elasticsearch/bulk.go create mode 100644 libbeat/outputs/elasticsearch/bulk_test.go create mode 100644 libbeat/outputs/elasticsearch/callbacks.go diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index c2a189b8d08..d02b3caa7e7 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -33,6 +33,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - The disk spool types `spool.Spool` and `spool.Settings` have been renamed to the internal types `spool.diskSpool` and `spool.settings`. {pull}16693[16693] - `queue.Eventer` has been renamed to `queue.ACKListener` {pull}16691[16691] - Require logger as first parameter for `outputs.elasticsearch.client#BulkReadItemStatus`. {pull}16761[16761] +- Extract Elasticsearch client logic from `outputs/elasticsearch` package into new `esclientleg` package. {pull}16150[16150] ==== Bugfixes diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index a01a5cde554..5bae97cdd04 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -22,31 +22,29 @@ import ( "fmt" "strings" - "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/joeshaw/multierror" "github.com/pkg/errors" + fbautodiscover "github.com/elastic/beats/v7/filebeat/autodiscover" + "github.com/elastic/beats/v7/filebeat/channel" + cfg "github.com/elastic/beats/v7/filebeat/config" + "github.com/elastic/beats/v7/filebeat/fileset" + _ "github.com/elastic/beats/v7/filebeat/include" + "github.com/elastic/beats/v7/filebeat/input" + "github.com/elastic/beats/v7/filebeat/registrar" "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/common/reload" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/kibana" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - fbautodiscover "github.com/elastic/beats/v7/filebeat/autodiscover" - "github.com/elastic/beats/v7/filebeat/channel" - cfg "github.com/elastic/beats/v7/filebeat/config" - "github.com/elastic/beats/v7/filebeat/fileset" - "github.com/elastic/beats/v7/filebeat/input" - "github.com/elastic/beats/v7/filebeat/registrar" - - _ "github.com/elastic/beats/v7/filebeat/include" - // Add filebeat level processors _ "github.com/elastic/beats/v7/filebeat/processor/add_kubernetes_metadata" _ "github.com/elastic/beats/v7/libbeat/processors/decode_csv_fields" @@ -157,7 +155,7 @@ func (fb *Filebeat) setupPipelineLoaderCallback(b *beat.Beat) error { overwritePipelines := true b.OverwritePipelinesCallback = func(esConfig *common.Config) error { - esClient, err := elasticsearch.NewConnectedClient(esConfig) + esClient, err := eslegclient.NewConnectedClient(esConfig) if err != nil { return err } @@ -191,7 +189,7 @@ func (fb *Filebeat) loadModulesPipelines(b *beat.Beat) error { // register pipeline loading to happen every time a new ES connection is // established - callback := func(esClient *elasticsearch.Client) error { + callback := func(esClient *eslegclient.Connection) error { return fb.moduleRegistry.LoadPipelines(esClient, overwritePipelines) } _, err := elasticsearch.RegisterConnectCallback(callback) @@ -211,7 +209,7 @@ func (fb *Filebeat) loadModulesML(b *beat.Beat, kibanaConfig *common.Config) err } esConfig := b.Config.Output.Config() - esClient, err := elasticsearch.NewConnectedClient(esConfig) + esClient, err := eslegclient.NewConnectedClient(esConfig) if err != nil { return errors.Errorf("Error creating Elasticsearch client: %v", err) } @@ -273,7 +271,7 @@ func (fb *Filebeat) loadModulesML(b *beat.Beat, kibanaConfig *common.Config) err return errs.Err() } -func setupMLBasedOnVersion(reg *fileset.ModuleRegistry, esClient *elasticsearch.Client, kibanaClient *kibana.Client) error { +func setupMLBasedOnVersion(reg *fileset.ModuleRegistry, esClient *eslegclient.Connection, kibanaClient *kibana.Client) error { if isElasticsearchLoads(kibanaClient.GetVersion()) { return reg.LoadML(esClient) } @@ -457,7 +455,7 @@ func (fb *Filebeat) Stop() { // Create a new pipeline loader (es client) factory func newPipelineLoaderFactory(esConfig *common.Config) fileset.PipelineLoaderFactory { pipelineLoaderFactory := func() (fileset.PipelineLoader, error) { - esClient, err := elasticsearch.NewConnectedClient(esConfig) + esClient, err := eslegclient.NewConnectedClient(esConfig) if err != nil { return nil, errors.Wrap(err, "Error creating Elasticsearch client") } diff --git a/filebeat/fileset/factory.go b/filebeat/fileset/factory.go index e7e6d30515d..58b79eeb2b3 100644 --- a/filebeat/fileset/factory.go +++ b/filebeat/fileset/factory.go @@ -26,6 +26,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" @@ -138,7 +139,7 @@ func (p *inputsRunner) Start() { } // Register callback to try to load pipelines when connecting to ES. - callback := func(esClient *elasticsearch.Client) error { + callback := func(esClient *eslegclient.Connection) error { return p.moduleRegistry.LoadPipelines(esClient, p.overwritePipelines) } p.pipelineCallbackID, err = elasticsearch.RegisterConnectCallback(callback) diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index ea6d646ede2..0f620110296 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -26,12 +26,12 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch/estest" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" + "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" ) func TestLoadPipeline(t *testing.T) { - client := estest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -69,7 +69,7 @@ func TestLoadPipeline(t *testing.T) { checkUploadedPipeline(t, client, "describe pipeline 2") } -func checkUploadedPipeline(t *testing.T, client *elasticsearch.Client, expectedDescription string) { +func checkUploadedPipeline(t *testing.T, client *eslegclient.Connection, expectedDescription string) { status, response, err := client.Request("GET", "/_ingest/pipeline/my-pipeline-id", "", nil, nil) assert.NoError(t, err) assert.Equal(t, 200, status) @@ -82,7 +82,7 @@ func checkUploadedPipeline(t *testing.T, client *elasticsearch.Client, expectedD } func TestSetupNginx(t *testing.T) { - client := estest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -114,7 +114,7 @@ func TestSetupNginx(t *testing.T) { } func TestAvailableProcessors(t *testing.T) { - client := estest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -139,18 +139,18 @@ func TestAvailableProcessors(t *testing.T) { assert.Contains(t, err.Error(), "ingest-hello") } -func hasIngest(client *elasticsearch.Client) bool { +func hasIngest(client *eslegclient.Connection) bool { v := client.GetVersion() return v.Major >= 5 } -func hasIngestPipelineProcessor(client *elasticsearch.Client) bool { +func hasIngestPipelineProcessor(client *eslegclient.Connection) bool { v := client.GetVersion() return v.Major > 6 || (v.Major == 6 && v.Minor >= 5) } func TestLoadMultiplePipelines(t *testing.T) { - client := estest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -195,7 +195,7 @@ func TestLoadMultiplePipelines(t *testing.T) { } func TestLoadMultiplePipelinesWithRollback(t *testing.T) { - client := estest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -237,3 +237,24 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { status, _, _ = client.Request("GET", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-plain_logs_bad", "", nil, nil) assert.Equal(t, 404, status) } + +func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { + conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ + URL: eslegtest.GetURL(), + Timeout: 0, + }) + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + conn.Encoder = eslegclient.NewJSONEncoder(nil, false) + + err = conn.Connect() + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + return conn +} diff --git a/filebeat/fileset/pipelines_test.go b/filebeat/fileset/pipelines_test.go index d808639cda8..648e82a1c2e 100644 --- a/filebeat/fileset/pipelines_test.go +++ b/filebeat/fileset/pipelines_test.go @@ -23,11 +23,12 @@ import ( "net/http" "net/http/httptest" "testing" - - "github.com/stretchr/testify/assert" + "time" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" + + "github.com/stretchr/testify/assert" ) func TestLoadPipelinesWithMultiPipelineFileset(t *testing.T) { @@ -87,9 +88,10 @@ func TestLoadPipelinesWithMultiPipelineFileset(t *testing.T) { })) defer testESServer.Close() - testESClient, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ - URL: testESServer.URL, - }, nil) + testESClient, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ + URL: testESServer.URL, + Timeout: 90 * time.Second, + }) assert.NoError(t, err) err = testESClient.Connect() diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 9ab8af14a13..d8a713b8be5 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -33,16 +33,10 @@ import ( "strings" "time" - "github.com/elastic/beats/v7/libbeat/kibana" - "github.com/gofrs/uuid" errw "github.com/pkg/errors" "go.uber.org/zap" - sysinfo "github.com/elastic/go-sysinfo" - "github.com/elastic/go-sysinfo/types" - ucfg "github.com/elastic/go-ucfg" - "github.com/elastic/beats/v7/libbeat/api" "github.com/elastic/beats/v7/libbeat/asset" "github.com/elastic/beats/v7/libbeat/beat" @@ -54,8 +48,10 @@ import ( "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/common/seccomp" "github.com/elastic/beats/v7/libbeat/dashboards" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/beats/v7/libbeat/kibana" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/logp/configure" "github.com/elastic/beats/v7/libbeat/management" @@ -71,6 +67,9 @@ import ( "github.com/elastic/beats/v7/libbeat/publisher/processing" svc "github.com/elastic/beats/v7/libbeat/service" "github.com/elastic/beats/v7/libbeat/version" + sysinfo "github.com/elastic/go-sysinfo" + "github.com/elastic/go-sysinfo/types" + ucfg "github.com/elastic/go-ucfg" ) // Beat provides the runnable and configurable instance of a beat. @@ -498,7 +497,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er if outCfg.Name() != "elasticsearch" { return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled") } - esClient, err := elasticsearch.NewConnectedClient(outCfg.Config()) + esClient, err := eslegclient.NewConnectedClient(outCfg.Config()) if err != nil { return err } @@ -811,7 +810,7 @@ func (b *Beat) registerESIndexManagement() error { } func (b *Beat) indexSetupCallback() elasticsearch.ConnectCallback { - return func(esClient *elasticsearch.Client) error { + return func(esClient *eslegclient.Connection) error { m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields)) return m.Setup(idxmgmt.LoadModeEnabled, idxmgmt.LoadModeEnabled) } @@ -857,7 +856,7 @@ func (b *Beat) clusterUUIDFetchingCallback() (elasticsearch.ConnectCallback, err elasticsearchRegistry := stateRegistry.NewRegistry("outputs.elasticsearch") clusterUUIDRegVar := monitoring.NewString(elasticsearchRegistry, "cluster_uuid") - callback := func(esClient *elasticsearch.Client) error { + callback := func(esClient *eslegclient.Connection) error { var response struct { ClusterUUID string `json:"cluster_uuid"` } diff --git a/libbeat/common/transport/wrap.go b/libbeat/common/transport/wrap.go index 7bde4735354..7192899652b 100644 --- a/libbeat/common/transport/wrap.go +++ b/libbeat/common/transport/wrap.go @@ -17,7 +17,9 @@ package transport -import "net" +import ( + "net" +) func ConnWrapper(d Dialer, w func(net.Conn) net.Conn) Dialer { return DialerFunc(func(network, addr string) (net.Conn, error) { diff --git a/libbeat/common/url.go b/libbeat/common/url.go index 327edeef996..949c4631edf 100644 --- a/libbeat/common/url.go +++ b/libbeat/common/url.go @@ -83,3 +83,33 @@ func EncodeURLParams(url string, params url.Values) string { return strings.Join([]string{url, "?", params.Encode()}, "") } + +type ParseHint func(raw string) string + +// ParseURL tries to parse a URL and return the parsed result. +func ParseURL(raw string, hints ...ParseHint) (*url.URL, error) { + if raw == "" { + return nil, nil + } + + if len(hints) == 0 { + hints = append(hints, WithDefaultScheme("http")) + } + + if strings.Index(raw, "://") == -1 { + for _, hint := range hints { + raw = hint(raw) + } + } + + return url.Parse(raw) +} + +func WithDefaultScheme(scheme string) ParseHint { + return func(raw string) string { + if !strings.Contains(raw, "://") { + return scheme + "://" + raw + } + return raw + } +} diff --git a/libbeat/common/url_test.go b/libbeat/common/url_test.go index e0ea76c9348..aaded710f83 100644 --- a/libbeat/common/url_test.go +++ b/libbeat/common/url_test.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGetUrl(t *testing.T) { @@ -114,3 +115,55 @@ func TestURLParamsEncode(t *testing.T) { assert.Equal(t, output, urlWithParams) } } + +func TestParseURL(t *testing.T) { + tests := map[string]struct { + input string + hints []ParseHint + expected string + errorAssertFunc require.ErrorAssertionFunc + }{ + "http": { + "http://host:1234/path", + nil, + "http://host:1234/path", + require.NoError, + }, + "https": { + "https://host:1234/path", + nil, + "https://host:1234/path", + require.NoError, + }, + "no_scheme": { + "host:1234/path", + nil, + "http://host:1234/path", + require.NoError, + }, + "default_scheme_https": { + "host:1234/path", + []ParseHint{WithDefaultScheme("https")}, + "https://host:1234/path", + require.NoError, + }, + "invalid": { + "foobar:port", + nil, + "", + require.Error, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + u, err := ParseURL(test.input, test.hints...) + test.errorAssertFunc(t, err) + if test.expected != "" { + require.Equal(t, test.expected, u.String()) + } else { + require.Nil(t, u) + } + }) + } +} diff --git a/libbeat/outputs/elasticsearch/api.go b/libbeat/esleg/eslegclient/api.go similarity index 99% rename from libbeat/outputs/elasticsearch/api.go rename to libbeat/esleg/eslegclient/api.go index d267fb6a98a..ba15d0ad26a 100644 --- a/libbeat/outputs/elasticsearch/api.go +++ b/libbeat/esleg/eslegclient/api.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package elasticsearch +package eslegclient import ( "encoding/json" diff --git a/libbeat/outputs/elasticsearch/api_integration_test.go b/libbeat/esleg/eslegclient/api_integration_test.go similarity index 81% rename from libbeat/outputs/elasticsearch/api_integration_test.go rename to libbeat/esleg/eslegclient/api_integration_test.go index 787cb7a6a42..4fdd5e2f659 100644 --- a/libbeat/outputs/elasticsearch/api_integration_test.go +++ b/libbeat/esleg/eslegclient/api_integration_test.go @@ -17,7 +17,7 @@ // +build integration -package elasticsearch +package eslegclient import ( "encoding/json" @@ -36,7 +36,7 @@ func TestIndex(t *testing.T) { index := fmt.Sprintf("beats-test-index-%d", os.Getpid()) - client := getTestingElasticsearch(t) + conn := getTestingElasticsearch(t) body := map[string]interface{}{ "user": "test", @@ -46,7 +46,7 @@ func TestIndex(t *testing.T) { params := map[string]string{ "refresh": "true", } - _, resp, err := client.Index(index, "test", "1", params, body) + _, resp, err := conn.Index(index, "test", "1", params, body) if err != nil { t.Fatalf("Index() returns error: %s", err) } @@ -59,7 +59,7 @@ func TestIndex(t *testing.T) { "match_all": map[string]interface{}{}, }, } - _, result, err := client.SearchURIWithBody(index, "", nil, map[string]interface{}{}) + _, result, err := conn.SearchURIWithBody(index, "", nil, map[string]interface{}{}) if err != nil { t.Fatalf("SearchUriWithBody() returns an error: %s", err) } @@ -70,7 +70,7 @@ func TestIndex(t *testing.T) { params = map[string]string{ "q": "user:test", } - _, result, err = client.SearchURI(index, "test", params) + _, result, err = conn.SearchURI(index, "test", params) if err != nil { t.Fatalf("SearchUri() returns an error: %s", err) } @@ -78,7 +78,7 @@ func TestIndex(t *testing.T) { t.Errorf("Wrong number of search results: %d", result.Hits.Total.Value) } - _, resp, err = client.Delete(index, "test", "1", nil) + _, resp, err = conn.Delete(index, "test", "1", nil) if err != nil { t.Errorf("Delete() returns error: %s", err) } @@ -103,17 +103,17 @@ func TestIngest(t *testing.T) { }, } - client := getTestingElasticsearch(t) - if client.Connection.version.Major < 5 { + conn := getTestingElasticsearch(t) + if conn.GetVersion().Major < 5 { t.Skip("Skipping tests as pipeline not available in <5.x releases") } - status, _, err := client.DeletePipeline(pipeline, nil) + status, _, err := conn.DeletePipeline(pipeline, nil) if err != nil && status != http.StatusNotFound { t.Fatal(err) } - exists, err := client.PipelineExists(pipeline) + exists, err := conn.PipelineExists(pipeline) if err != nil { t.Fatal(err) } @@ -121,7 +121,7 @@ func TestIngest(t *testing.T) { t.Fatalf("Test expected PipelineExists to return false for %v", pipeline) } - _, resp, err := client.CreatePipeline(pipeline, nil, pipelineBody) + _, resp, err := conn.CreatePipeline(pipeline, nil, pipelineBody) if err != nil { t.Fatal(err) } @@ -129,7 +129,7 @@ func TestIngest(t *testing.T) { t.Fatalf("Test pipeline %v not created", pipeline) } - exists, err = client.PipelineExists(pipeline) + exists, err = conn.PipelineExists(pipeline) if err != nil { t.Fatal(err) } @@ -138,7 +138,7 @@ func TestIngest(t *testing.T) { } params := map[string]string{"refresh": "true"} - _, resp, err = client.Ingest(index, "test", pipeline, "1", params, obj{ + _, resp, err = conn.Ingest(index, "test", pipeline, "1", params, obj{ "testfield": "TEST", }) if err != nil { @@ -149,7 +149,7 @@ func TestIngest(t *testing.T) { } // get _source field from indexed document - _, docBody, err := client.apiCall("GET", index, "", "_source/1", "", nil, nil) + _, docBody, err := conn.apiCall("GET", index, "", "_source/1", "", nil, nil) if err != nil { t.Fatal(err) } diff --git a/libbeat/outputs/elasticsearch/api_mock_test.go b/libbeat/esleg/eslegclient/api_mock_test.go similarity index 96% rename from libbeat/outputs/elasticsearch/api_mock_test.go rename to libbeat/esleg/eslegclient/api_mock_test.go index 65e68754833..cfb9b9b722b 100644 --- a/libbeat/outputs/elasticsearch/api_mock_test.go +++ b/libbeat/esleg/eslegclient/api_mock_test.go @@ -17,7 +17,7 @@ // +build !integration -package elasticsearch +package eslegclient import ( "encoding/json" @@ -63,7 +63,7 @@ func TestOneHostSuccessResp(t *testing.T) { server := ElasticsearchMock(200, expectedResp) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) params := map[string]string{ "refresh": "true", @@ -89,7 +89,7 @@ func TestOneHost500Resp(t *testing.T) { server := ElasticsearchMock(http.StatusInternalServerError, []byte("Something wrong happened")) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) err := client.Connect() if err != nil { t.Fatalf("Failed to connect: %v", err) @@ -121,7 +121,7 @@ func TestOneHost503Resp(t *testing.T) { server := ElasticsearchMock(503, []byte("Something wrong happened")) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) params := map[string]string{ "refresh": "true", diff --git a/libbeat/outputs/elasticsearch/api_test.go b/libbeat/esleg/eslegclient/api_test.go similarity index 90% rename from libbeat/outputs/elasticsearch/api_test.go rename to libbeat/esleg/eslegclient/api_test.go index 3706dd64511..9055eb1f942 100644 --- a/libbeat/outputs/elasticsearch/api_test.go +++ b/libbeat/esleg/eslegclient/api_test.go @@ -16,17 +16,15 @@ // under the License. // Need for unit and integration tests -package elasticsearch +package eslegclient import ( "encoding/json" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/libbeat/outputs/outil" ) func GetValidQueryResult() QueryResult { @@ -172,23 +170,19 @@ func TestReadSearchResult_invalid(t *testing.T) { assert.Error(t, err) } -func newTestClient(url string) *Client { - client, err := NewClient(ClientSettings{ - URL: url, - Index: outil.MakeSelector(), - Timeout: 60 * time.Second, - CompressionLevel: 3, - }, nil) - if err != nil { - panic(err) - } - return client +func newTestConnection(url string) *Connection { + conn, _ := NewConnection(ConnectionSettings{ + URL: url, + Timeout: 0, + }) + conn.Encoder = NewJSONEncoder(nil, false) + return conn } func (r QueryResult) String() string { out, err := json.Marshal(r) if err != nil { - logp.NewLogger(logSelector).Warnf("failed to marshal QueryResult (%+v): %#v", err, r) + logp.L().Warnf("failed to marshal QueryResult (%+v): %#v", err, r) return "ERROR" } return string(out) diff --git a/libbeat/outputs/elasticsearch/bulkapi.go b/libbeat/esleg/eslegclient/bulkapi.go similarity index 71% rename from libbeat/outputs/elasticsearch/bulkapi.go rename to libbeat/esleg/eslegclient/bulkapi.go index 325b54e9bef..86b518eeea1 100644 --- a/libbeat/outputs/elasticsearch/bulkapi.go +++ b/libbeat/esleg/eslegclient/bulkapi.go @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. -package elasticsearch +package eslegclient import ( "bytes" "encoding/json" + "errors" "io" "io/ioutil" "net/http" @@ -29,8 +30,24 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) -// MetaBuilder creates meta data for bulk requests -type MetaBuilder func(interface{}) interface{} +var ( + ErrTempBulkFailure = errors.New("temporary bulk send failure") +) + +type BulkIndexAction struct { + Index BulkMeta `json:"index" struct:"index"` +} + +type BulkCreateAction struct { + Create BulkMeta `json:"create" struct:"create"` +} + +type BulkMeta struct { + Index string `json:"_index" struct:"_index"` + DocType string `json:"_type,omitempty" struct:"_type,omitempty"` + Pipeline string `json:"pipeline,omitempty" struct:"pipeline,omitempty"` + ID string `json:"_id,omitempty" struct:"_id,omitempty"` +} type bulkRequest struct { requ *http.Request @@ -44,40 +61,23 @@ type BulkResult json.RawMessage func (conn *Connection) Bulk( index, docType string, params map[string]string, body []interface{}, -) (BulkResult, error) { - return conn.BulkWith(index, docType, params, nil, body) -} - -// BulkWith creates a HTTP request containing a bunch of operations and send -// them to Elasticsearch. The request is retransmitted up to max_retries before -// returning an error. -func (conn *Connection) BulkWith( - index string, - docType string, - params map[string]string, - metaBuilder MetaBuilder, - body []interface{}, -) (BulkResult, error) { +) (int, BulkResult, error) { if len(body) == 0 { - return nil, nil + return 0, nil, nil } - enc := conn.encoder + enc := conn.Encoder enc.Reset() - if err := bulkEncode(conn.log, enc, metaBuilder, body); err != nil { - return nil, err + if err := bulkEncode(conn.log, enc, body); err != nil { + return 0, nil, err } requ, err := newBulkRequest(conn.URL, index, docType, params, enc) if err != nil { - return nil, err + return 0, nil, err } - _, result, err := conn.sendBulkRequest(requ) - if err != nil { - return nil, err - } - return result, nil + return conn.sendBulkRequest(requ) } // SendMonitoringBulk creates a HTTP request to the X-Pack Monitoring API containing a bunch of @@ -91,9 +91,9 @@ func (conn *Connection) SendMonitoringBulk( return nil, nil } - enc := conn.encoder + enc := conn.Encoder enc.Reset() - if err := bulkEncode(conn.log, enc, nil, body); err != nil { + if err := bulkEncode(conn.log, enc, body); err != nil { return nil, err } @@ -103,7 +103,7 @@ func (conn *Connection) SendMonitoringBulk( } } - requ, err := newMonitoringBulkRequest(conn.version, conn.URL, params, enc) + requ, err := newMonitoringBulkRequest(conn.GetVersion(), conn.URL, params, enc) if err != nil { return nil, err } @@ -119,7 +119,7 @@ func newBulkRequest( urlStr string, index, docType string, params map[string]string, - body bodyEncoder, + body BodyEncoder, ) (*bulkRequest, error) { path, err := makePath(index, docType, "_bulk") if err != nil { @@ -133,7 +133,7 @@ func newMonitoringBulkRequest( esVersion common.Version, urlStr string, params map[string]string, - body bodyEncoder, + body BodyEncoder, ) (*bulkRequest, error) { var path string var err error @@ -154,7 +154,7 @@ func newBulkRequestWithPath( urlStr string, path string, params map[string]string, - body bodyEncoder, + body BodyEncoder, ) (*bulkRequest, error) { url := addToURL(urlStr, path, "", params) @@ -172,12 +172,15 @@ func newBulkRequestWithPath( body.AddHeader(&requ.Header) } - return &bulkRequest{ + r := bulkRequest{ requ: requ, - }, nil + } + r.reset(body) + + return &r, nil } -func (r *bulkRequest) Reset(body bodyEncoder) { +func (r *bulkRequest) reset(body BodyEncoder) { bdy := body.Reader() rc, ok := bdy.(io.ReadCloser) @@ -205,20 +208,11 @@ func (conn *Connection) sendBulkRequest(requ *bulkRequest) (int, BulkResult, err return status, BulkResult(resp), err } -func bulkEncode(log *logp.Logger, out bulkWriter, metaBuilder MetaBuilder, body []interface{}) error { - if metaBuilder == nil { - for _, obj := range body { - if err := out.AddRaw(obj); err != nil { - log.Debugf("Failed to encode message: %+v", err) - return err - } - } - } else { - for _, obj := range body { - meta := metaBuilder(obj) - if err := out.Add(meta, obj); err != nil { - log.Debugf("Failed to encode event (dropping event): %+v", err) - } +func bulkEncode(log *logp.Logger, out BulkWriter, body []interface{}) error { + for _, obj := range body { + if err := out.AddRaw(obj); err != nil { + log.Debugf("Failed to encode message: %s", err) + return err } } return nil diff --git a/libbeat/outputs/elasticsearch/bulkapi_integration_test.go b/libbeat/esleg/eslegclient/bulkapi_integration_test.go similarity index 95% rename from libbeat/outputs/elasticsearch/bulkapi_integration_test.go rename to libbeat/esleg/eslegclient/bulkapi_integration_test.go index 7599a407f49..ec7ba4a1c4d 100644 --- a/libbeat/outputs/elasticsearch/bulkapi_integration_test.go +++ b/libbeat/esleg/eslegclient/bulkapi_integration_test.go @@ -17,7 +17,7 @@ // +build integration -package elasticsearch +package eslegclient import ( "fmt" @@ -54,7 +54,7 @@ func TestBulk(t *testing.T) { params := map[string]string{ "refresh": "true", } - _, err := client.Bulk(index, "type1", params, body) + _, _, err := client.Bulk(index, "type1", params, body) if err != nil { t.Fatalf("Bulk() returned error: %s", err) } @@ -87,7 +87,7 @@ func TestEmptyBulk(t *testing.T) { params := map[string]string{ "refresh": "true", } - resp, err := client.Bulk(index, "type1", params, body) + _, resp, err := client.Bulk(index, "type1", params, body) if err != nil { t.Fatalf("Bulk() returned error: %s", err) } @@ -155,7 +155,7 @@ func TestBulkMoreOperations(t *testing.T) { params := map[string]string{ "refresh": "true", } - resp, err := client.Bulk(index, "type1", params, body) + _, resp, err := client.Bulk(index, "type1", params, body) if err != nil { t.Fatalf("Bulk() returned error: %s [%s]", err, resp) } diff --git a/libbeat/outputs/elasticsearch/bulkapi_mock_test.go b/libbeat/esleg/eslegclient/bulkapi_mock_test.go similarity index 91% rename from libbeat/outputs/elasticsearch/bulkapi_mock_test.go rename to libbeat/esleg/eslegclient/bulkapi_mock_test.go index a87d0d1046c..1fbd53d9425 100644 --- a/libbeat/outputs/elasticsearch/bulkapi_mock_test.go +++ b/libbeat/esleg/eslegclient/bulkapi_mock_test.go @@ -17,7 +17,7 @@ // +build !integration -package elasticsearch +package eslegclient import ( "fmt" @@ -55,12 +55,12 @@ func TestOneHostSuccessResp_Bulk(t *testing.T) { server := ElasticsearchMock(200, expectedResp) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) params := map[string]string{ "refresh": "true", } - _, err := client.Bulk(index, "type1", params, body) + _, _, err := client.Bulk(index, "type1", params, body) if err != nil { t.Errorf("Bulk() returns error: %s", err) } @@ -91,12 +91,12 @@ func TestOneHost500Resp_Bulk(t *testing.T) { server := ElasticsearchMock(http.StatusInternalServerError, []byte("Something wrong happened")) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) params := map[string]string{ "refresh": "true", } - _, err := client.Bulk(index, "type1", params, body) + _, _, err := client.Bulk(index, "type1", params, body) if err == nil { t.Errorf("Bulk() should return error.") } @@ -131,12 +131,12 @@ func TestOneHost503Resp_Bulk(t *testing.T) { server := ElasticsearchMock(503, []byte("Something wrong happened")) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) params := map[string]string{ "refresh": "true", } - _, err := client.Bulk(index, "type1", params, body) + _, _, err := client.Bulk(index, "type1", params, body) if err == nil { t.Errorf("Bulk() should return error.") } diff --git a/libbeat/esleg/eslegclient/config.go b/libbeat/esleg/eslegclient/config.go new file mode 100644 index 00000000000..5c171a4eb2b --- /dev/null +++ b/libbeat/esleg/eslegclient/config.go @@ -0,0 +1,78 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package eslegclient + +import ( + "fmt" + "time" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" +) + +type config struct { + Hosts []string `config:"hosts" validate:"required"` + Protocol string `config:"protocol"` + Path string `config:"path"` + Params map[string]string `config:"parameters"` + Headers map[string]string `config:"headers"` + + TLS *tlscommon.Config `config:"ssl"` + + ProxyURL string `config:"proxy_url"` + ProxyDisable bool `config:"proxy_disable"` + + Username string `config:"username"` + Password string `config:"password"` + APIKey string `config:"api_key"` + + CompressionLevel int `config:"compression_level" validate:"min=0, max=9"` + EscapeHTML bool `config:"escape_html"` + Timeout time.Duration `config:"timeout"` +} + +func defaultConfig() config { + return config{ + Protocol: "", + Path: "", + ProxyURL: "", + ProxyDisable: false, + Params: nil, + Username: "", + Password: "", + APIKey: "", + Timeout: 90 * time.Second, + CompressionLevel: 0, + EscapeHTML: false, + TLS: nil, + } +} + +func (c *config) Validate() error { + if c.ProxyURL != "" && !c.ProxyDisable { + if _, err := common.ParseURL(c.ProxyURL); err != nil { + return err + } + } + + if c.APIKey != "" && (c.Username != "" || c.Password != "") { + return fmt.Errorf("cannot set both api_key and username/password") + } + + return nil +} diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go new file mode 100644 index 00000000000..b591307c444 --- /dev/null +++ b/libbeat/esleg/eslegclient/connection.go @@ -0,0 +1,435 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package eslegclient + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "time" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/testing" +) + +// Connection manages the connection for a given client. +type Connection struct { + ConnectionSettings + + Encoder BodyEncoder + HTTP *http.Client + + version common.Version + log *logp.Logger +} + +// ConnectionSettings are the settings needed for a Connection +type ConnectionSettings struct { + URL string + Proxy *url.URL + ProxyDisable bool + + Username string + Password string + APIKey string + Headers map[string]string + + TLS *tlscommon.TLSConfig + + OnConnectCallback func() error + Observer transport.IOStatser + + Parameters map[string]string + CompressionLevel int + EscapeHTML bool + Timeout time.Duration +} + +// NewConnection returns a new Elasticsearch client +func NewConnection(s ConnectionSettings) (*Connection, error) { + u, err := url.Parse(s.URL) + if err != nil { + return nil, fmt.Errorf("failed to parse elasticsearch URL: %v", err) + } + + if u.User != nil { + s.Username = u.User.Username() + s.Password, _ = u.User.Password() + u.User = nil + + // Re-write URL without credentials. + s.URL = u.String() + } + logp.Info("elasticsearch url: %s", s.URL) + + // TODO: add socks5 proxy support + var dialer, tlsDialer transport.Dialer + + dialer = transport.NetDialer(s.Timeout) + tlsDialer, err = transport.TLSDialer(dialer, s.TLS, s.Timeout) + if err != nil { + return nil, err + } + + if st := s.Observer; st != nil { + dialer = transport.StatsDialer(dialer, st) + tlsDialer = transport.StatsDialer(tlsDialer, st) + } + + var encoder BodyEncoder + compression := s.CompressionLevel + if compression == 0 { + encoder = NewJSONEncoder(nil, s.EscapeHTML) + } else { + encoder, err = NewGzipEncoder(compression, nil, s.EscapeHTML) + if err != nil { + return nil, err + } + } + + var proxy func(*http.Request) (*url.URL, error) + if !s.ProxyDisable { + proxy = http.ProxyFromEnvironment + if s.Proxy != nil { + proxy = http.ProxyURL(s.Proxy) + } + } + + return &Connection{ + ConnectionSettings: s, + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: dialer.Dial, + DialTLS: tlsDialer.Dial, + TLSClientConfig: s.TLS.ToConfig(), + Proxy: proxy, + }, + Timeout: s.Timeout, + }, + Encoder: encoder, + log: logp.NewLogger("esclientleg"), + }, nil +} + +// NewClients returns a list of Elasticsearch clients based on the given +// configuration. It accepts the same configuration parameters as the Elasticsearch +// output, except for the output specific configuration options. If multiple hosts +// are defined in the configuration, a client is returned for each of them. +func NewClients(cfg *common.Config) ([]Connection, error) { + config := defaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, err + } + + tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS) + if err != nil { + return nil, err + } + + var proxyURL *url.URL + if !config.ProxyDisable { + proxyURL, err = common.ParseURL(config.ProxyURL) + if err != nil { + return nil, err + } + if proxyURL != nil { + logp.Info("using proxy URL: %s", proxyURL) + } + } + + params := config.Params + if len(params) == 0 { + params = nil + } + + clients := []Connection{} + for _, host := range config.Hosts { + esURL, err := common.MakeURL(config.Protocol, config.Path, host, 9200) + if err != nil { + logp.Err("invalid host param set: %s, Error: %v", host, err) + return nil, err + } + + client, err := NewConnection(ConnectionSettings{ + URL: esURL, + Proxy: proxyURL, + ProxyDisable: config.ProxyDisable, + TLS: tlsConfig, + Username: config.Username, + Password: config.Password, + APIKey: config.APIKey, + Parameters: params, + Headers: config.Headers, + Timeout: config.Timeout, + CompressionLevel: config.CompressionLevel, + }) + if err != nil { + return clients, err + } + clients = append(clients, *client) + } + if len(clients) == 0 { + return clients, fmt.Errorf("no hosts defined in the config") + } + return clients, nil +} + +func NewConnectedClient(cfg *common.Config) (*Connection, error) { + clients, err := NewClients(cfg) + if err != nil { + return nil, err + } + + errors := []string{} + + for _, client := range clients { + err = client.Connect() + if err != nil { + const errMsg = "error connecting to Elasticsearch at %v: %v" + client.log.Errorf(errMsg, client.URL, err) + err = fmt.Errorf(errMsg, client.URL, err) + errors = append(errors, err.Error()) + continue + } + return &client, nil + } + return nil, fmt.Errorf("couldn't connect to any of the configured Elasticsearch hosts. Errors: %v", errors) +} + +// Connect connects the client. It runs a GET request against the root URL of +// the configured host, updates the known Elasticsearch version and calls +// globally configured handlers. +func (conn *Connection) Connect() error { + if err := conn.getVersion(); err != nil { + return err + } + + if conn.OnConnectCallback != nil { + if err := conn.OnConnectCallback(); err != nil { + return fmt.Errorf("Connection marked as failed because the onConnect callback failed: %v", err) + } + } + + return nil +} + +// Ping sends a GET request to the Elasticsearch. +func (conn *Connection) Ping() (string, error) { + conn.log.Debugf("ES Ping(url=%v)", conn.URL) + + status, body, err := conn.execRequest("GET", conn.URL, nil) + if err != nil { + conn.log.Debugf("Ping request failed with: %v", err) + return "", err + } + + if status >= 300 { + return "", fmt.Errorf("Non 2xx response code: %d", status) + } + + var response struct { + Version struct { + Number string + } + } + + err = json.Unmarshal(body, &response) + if err != nil { + return "", fmt.Errorf("Failed to parse JSON response: %v", err) + } + + conn.log.Debugf("Ping status code: %v", status) + conn.log.Infof("Attempting to connect to Elasticsearch version %s", response.Version.Number) + return response.Version.Number, nil +} + +// Close closes a connection. +func (conn *Connection) Close() error { + return nil +} + +func (conn *Connection) Test(d testing.Driver) { + d.Run("elasticsearch: "+conn.URL, func(d testing.Driver) { + u, err := url.Parse(conn.URL) + d.Fatal("parse url", err) + + address := u.Host + + d.Run("connection", func(d testing.Driver) { + netDialer := transport.TestNetDialer(d, conn.Timeout) + _, err = netDialer.Dial("tcp", address) + d.Fatal("dial up", err) + }) + + if u.Scheme != "https" { + d.Warn("TLS", "secure connection disabled") + } else { + d.Run("TLS", func(d testing.Driver) { + netDialer := transport.NetDialer(conn.Timeout) + tlsDialer, err := transport.TestTLSDialer(d, netDialer, conn.TLS, conn.Timeout) + _, err = tlsDialer.Dial("tcp", address) + d.Fatal("dial up", err) + }) + } + + err = conn.Connect() + d.Fatal("talk to server", err) + version := conn.GetVersion() + d.Info("version", version.String()) + }) +} + +// Request sends a request via the connection. +func (conn *Connection) Request( + method, path string, + pipeline string, + params map[string]string, + body interface{}, +) (int, []byte, error) { + + url := addToURL(conn.URL, path, pipeline, params) + conn.log.Debugf("%s %s %s %v", method, url, pipeline, body) + + return conn.RequestURL(method, url, body) +} + +// RequestURL sends a request with the connection object to an alternative url +func (conn *Connection) RequestURL( + method, url string, + body interface{}, +) (int, []byte, error) { + + if body == nil { + return conn.execRequest(method, url, nil) + } + + if err := conn.Encoder.Marshal(body); err != nil { + conn.log.Warnf("Failed to json encode body (%v): %#v", err, body) + return 0, nil, ErrJSONEncodeFailed + } + return conn.execRequest(method, url, conn.Encoder.Reader()) +} + +func (conn *Connection) execRequest( + method, url string, + body io.Reader, +) (int, []byte, error) { + req, err := http.NewRequest(method, url, body) + if err != nil { + conn.log.Warnf("Failed to create request %+v", err) + return 0, nil, err + } + if body != nil { + conn.Encoder.AddHeader(&req.Header) + } + return conn.execHTTPRequest(req) +} + +// GetVersion returns the elasticsearch version the client is connected to. +func (conn *Connection) GetVersion() common.Version { + if !conn.version.IsValid() { + conn.getVersion() + } + + return conn.version +} + +func (conn *Connection) getVersion() error { + versionString, err := conn.Ping() + if err != nil { + return err + } + + if version, err := common.NewVersion(versionString); err != nil { + conn.log.Errorf("Invalid version from Elasticsearch: %v", versionString) + conn.version = common.Version{} + } else { + conn.version = *version + } + + return nil +} + +// LoadJSON creates a PUT request based on a JSON document. +func (conn *Connection) LoadJSON(path string, json map[string]interface{}) ([]byte, error) { + status, body, err := conn.Request("PUT", path, "", nil, json) + if err != nil { + return body, fmt.Errorf("couldn't load json. Error: %s", err) + } + if status > 300 { + return body, fmt.Errorf("couldn't load json. Status: %v", status) + } + + return body, nil +} + +func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) { + req.Header.Add("Accept", "application/json") + + if conn.Username != "" || conn.Password != "" { + req.SetBasicAuth(conn.Username, conn.Password) + } + + if conn.APIKey != "" { + req.Header.Add("Authorization", "ApiKey "+conn.APIKey) + } + + for name, value := range conn.Headers { + req.Header.Add(name, value) + } + + // The stlib will override the value in the header based on the configured `Host` + // on the request which default to the current machine. + // + // We use the normalized key header to retrieve the user configured value and assign it to the host. + if host := req.Header.Get("Host"); host != "" { + req.Host = host + } + + resp, err := conn.HTTP.Do(req) + if err != nil { + return 0, nil, err + } + defer closing(resp.Body, conn.log) + + status := resp.StatusCode + obj, err := ioutil.ReadAll(resp.Body) + if err != nil { + return status, nil, err + } + + if status >= 300 { + // add the response body with the error returned by Elasticsearch + err = fmt.Errorf("%v: %s", resp.Status, obj) + } + + return status, obj, err +} + +func closing(c io.Closer, logger *logp.Logger) { + err := c.Close() + if err != nil { + logger.Warn("Close failed with: %v", err) + } +} diff --git a/libbeat/esleg/eslegclient/connection_integration_test.go b/libbeat/esleg/eslegclient/connection_integration_test.go new file mode 100644 index 00000000000..225edd6f36c --- /dev/null +++ b/libbeat/esleg/eslegclient/connection_integration_test.go @@ -0,0 +1,177 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package eslegclient + +import ( + "context" + "io/ioutil" + "math/rand" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" + "github.com/elastic/beats/v7/libbeat/outputs" +) + +func TestConnect(t *testing.T) { + conn := getTestingElasticsearch(t) + err := conn.Connect() + assert.NoError(t, err) +} + +func TestConnectWithProxy(t *testing.T) { + wrongPort, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + go func() { + c, err := wrongPort.Accept() + if err == nil { + // Provoke an early-EOF error on client + c.Close() + } + }() + defer wrongPort.Close() + + proxy := startTestProxy(t, eslegtest.GetURL()) + defer proxy.Close() + + // Use connectTestEs instead of getTestingElasticsearch to make use of makeES + client, err := connectTestEs(t, map[string]interface{}{ + "hosts": "http://" + wrongPort.Addr().String(), + "timeout": 5, // seconds + }) + require.NoError(t, err) + assert.Error(t, client.Connect(), "it should fail without proxy") + + client, err = connectTestEs(t, map[string]interface{}{ + "hosts": "http://" + wrongPort.Addr().String(), + "proxy_url": proxy.URL, + "timeout": 5, // seconds + }) + require.NoError(t, err) + assert.NoError(t, client.Connect()) +} + +func connectTestEs(t *testing.T, cfg interface{}) (*Connection, error) { + config, err := common.NewConfigFrom(map[string]interface{}{ + "username": eslegtest.GetUser(), + "password": eslegtest.GetPass(), + }) + require.NoError(t, err) + + tmp, err := common.NewConfigFrom(cfg) + require.NoError(t, err) + + err = config.Merge(tmp) + require.NoError(t, err) + + hosts, err := config.String("hosts", -1) + require.NoError(t, err) + + username, err := config.String("username", -1) + require.NoError(t, err) + + password, err := config.String("password", -1) + require.NoError(t, err) + + timeout, err := config.Int("timeout", -1) + require.NoError(t, err) + + var proxy string + if config.HasField("proxy_url") { + proxy, err = config.String("proxy_url", -1) + require.NoError(t, err) + } + + s := ConnectionSettings{ + URL: hosts, + Username: username, + Password: password, + Timeout: time.Duration(timeout) * time.Second, + CompressionLevel: 3, + } + + if proxy != "" { + p, err := url.Parse(proxy) + require.NoError(t, err) + s.Proxy = p + } + + return NewConnection(s) +} + +// getTestingElasticsearch creates a test client. +func getTestingElasticsearch(t eslegtest.TestLogger) *Connection { + conn, err := NewConnection(ConnectionSettings{ + URL: eslegtest.GetURL(), + Username: eslegtest.GetUser(), + Password: eslegtest.GetPass(), + Timeout: 60 * time.Second, + CompressionLevel: 3, + }) + eslegtest.InitConnection(t, conn, err) + return conn +} + +func randomClient(grp outputs.Group) outputs.NetworkClient { + L := len(grp.Clients) + if L == 0 { + panic("no elasticsearch client") + } + + client := grp.Clients[rand.Intn(L)] + return client.(outputs.NetworkClient) +} + +// startTestProxy starts a proxy that redirects all connections to the specified URL +func startTestProxy(t *testing.T, redirectURL string) *httptest.Server { + t.Helper() + + realURL, err := url.Parse(redirectURL) + require.NoError(t, err) + + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + req := r.Clone(context.Background()) + req.RequestURI = "" + req.URL.Scheme = realURL.Scheme + req.URL.Host = realURL.Host + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + for _, header := range []string{"Content-Encoding", "Content-Type"} { + w.Header().Set(header, resp.Header.Get(header)) + } + w.WriteHeader(resp.StatusCode) + w.Write(body) + })) + return proxy +} diff --git a/libbeat/outputs/elasticsearch/enc.go b/libbeat/esleg/eslegclient/enc.go similarity index 95% rename from libbeat/outputs/elasticsearch/enc.go rename to libbeat/esleg/eslegclient/enc.go index 8d2497e5182..3116dc2f537 100644 --- a/libbeat/outputs/elasticsearch/enc.go +++ b/libbeat/esleg/eslegclient/enc.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package elasticsearch +package eslegclient import ( "bytes" @@ -31,20 +31,20 @@ import ( "github.com/elastic/go-structform/json" ) -type bodyEncoder interface { +type BodyEncoder interface { bulkBodyEncoder Reader() io.Reader Marshal(doc interface{}) error } type bulkBodyEncoder interface { - bulkWriter + BulkWriter AddHeader(*http.Header) Reset() } -type bulkWriter interface { +type BulkWriter interface { Add(meta, obj interface{}) error AddRaw(raw interface{}) error } @@ -69,7 +69,7 @@ type event struct { Fields common.MapStr `struct:",inline"` } -func newJSONEncoder(buf *bytes.Buffer, escapeHTML bool) *jsonEncoder { +func NewJSONEncoder(buf *bytes.Buffer, escapeHTML bool) *jsonEncoder { if buf == nil { buf = bytes.NewBuffer(nil) } @@ -142,7 +142,7 @@ func (b *jsonEncoder) Add(meta, obj interface{}) error { return nil } -func newGzipEncoder(level int, buf *bytes.Buffer, escapeHTML bool) (*gzipEncoder, error) { +func NewGzipEncoder(level int, buf *bytes.Buffer, escapeHTML bool) (*gzipEncoder, error) { if buf == nil { buf = bytes.NewBuffer(nil) } diff --git a/libbeat/outputs/elasticsearch/enc_test.go b/libbeat/esleg/eslegclient/enc_test.go similarity index 95% rename from libbeat/outputs/elasticsearch/enc_test.go rename to libbeat/esleg/eslegclient/enc_test.go index 0ccea8cd0f4..32b2f35e1f3 100644 --- a/libbeat/outputs/elasticsearch/enc_test.go +++ b/libbeat/esleg/eslegclient/enc_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package elasticsearch +package eslegclient import ( "testing" @@ -29,7 +29,7 @@ import ( ) func TestJSONEncoderMarshalBeatEvent(t *testing.T) { - encoder := newJSONEncoder(nil, true) + encoder := NewJSONEncoder(nil, true) event := beat.Event{ Timestamp: time.Date(2017, time.November, 7, 12, 0, 0, 0, time.UTC), Fields: common.MapStr{ @@ -46,7 +46,7 @@ func TestJSONEncoderMarshalBeatEvent(t *testing.T) { } func TestJSONEncoderMarshalMonitoringEvent(t *testing.T) { - encoder := newJSONEncoder(nil, true) + encoder := NewJSONEncoder(nil, true) event := report.Event{ Timestamp: time.Date(2017, time.November, 7, 12, 0, 0, 0, time.UTC), Fields: common.MapStr{ diff --git a/libbeat/outputs/elasticsearch/estest/estest.go b/libbeat/esleg/eslegclient/errors.go similarity index 53% rename from libbeat/outputs/elasticsearch/estest/estest.go rename to libbeat/esleg/eslegclient/errors.go index 3aafc7da0d7..c3636c71477 100644 --- a/libbeat/outputs/elasticsearch/estest/estest.go +++ b/libbeat/esleg/eslegclient/errors.go @@ -15,26 +15,17 @@ // specific language governing permissions and limitations // under the License. -package estest +package eslegclient -import ( - "time" +import "errors" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch/internal" - "github.com/elastic/beats/v7/libbeat/outputs/outil" -) +var ( + // ErrNotConnected indicates failure due to client having no valid connection + ErrNotConnected = errors.New("not connected") + + // ErrJSONEncodeFailed indicates encoding failures + ErrJSONEncodeFailed = errors.New("json encode failed") -// GetTestingElasticsearch creates a test client. -func GetTestingElasticsearch(t internal.TestLogger) *elasticsearch.Client { - client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ - URL: internal.GetURL(), - Index: outil.MakeSelector(), - Username: internal.GetUser(), - Password: internal.GetPass(), - Timeout: 60 * time.Second, - CompressionLevel: 3, - }, nil) - internal.InitClient(t, client, err) - return client -} + // ErrResponseRead indicates error parsing Elasticsearch response + ErrResponseRead = errors.New("bulk item status parse failed") +) diff --git a/libbeat/outputs/elasticsearch/url.go b/libbeat/esleg/eslegclient/url.go similarity index 82% rename from libbeat/outputs/elasticsearch/url.go rename to libbeat/esleg/eslegclient/url.go index 1c83419aefe..89b03c5bdb8 100644 --- a/libbeat/outputs/elasticsearch/url.go +++ b/libbeat/esleg/eslegclient/url.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package elasticsearch +package eslegclient import ( "fmt" @@ -73,19 +73,3 @@ func makePath(index string, docType string, id string) (string, error) { } return path, nil } - -// TODO: make this reusable. Same definition in elasticsearch monitoring module -func parseProxyURL(raw string) (*url.URL, error) { - if raw == "" { - return nil, nil - } - - url, err := url.Parse(raw) - if err == nil && strings.HasPrefix(url.Scheme, "http") { - return url, err - } - - // Proxy was bogus. Try prepending "http://" to it and - // see if that parses correctly. - return url.Parse("http://" + raw) -} diff --git a/libbeat/outputs/elasticsearch/url_test.go b/libbeat/esleg/eslegclient/url_test.go similarity index 64% rename from libbeat/outputs/elasticsearch/url_test.go rename to libbeat/esleg/eslegclient/url_test.go index 0b5ddcdf073..8e444a660e4 100644 --- a/libbeat/outputs/elasticsearch/url_test.go +++ b/libbeat/esleg/eslegclient/url_test.go @@ -17,9 +17,13 @@ // +build !integration -package elasticsearch +package eslegclient -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestUrlEncode(t *testing.T) { params := map[string]string{ @@ -75,3 +79,49 @@ func TestMakePath(t *testing.T) { t.Errorf("Wrong path created: %s", path) } } + +func TestAddToURL(t *testing.T) { + type Test struct { + url string + path string + pipeline string + params map[string]string + expected string + } + tests := []Test{ + { + url: "localhost:9200", + path: "/path", + pipeline: "", + params: make(map[string]string), + expected: "localhost:9200/path", + }, + { + url: "localhost:9200/", + path: "/path", + pipeline: "", + params: make(map[string]string), + expected: "localhost:9200/path", + }, + { + url: "localhost:9200", + path: "/path", + pipeline: "pipeline_1", + params: make(map[string]string), + expected: "localhost:9200/path?pipeline=pipeline_1", + }, + { + url: "localhost:9200/", + path: "/path", + pipeline: "", + params: map[string]string{ + "param": "value", + }, + expected: "localhost:9200/path?param=value", + }, + } + for _, test := range tests { + url := addToURL(test.url, test.path, test.pipeline, test.params) + assert.Equal(t, url, test.expected) + } +} diff --git a/libbeat/outputs/elasticsearch/internal/testing.go b/libbeat/esleg/eslegtest/util.go similarity index 80% rename from libbeat/outputs/elasticsearch/internal/testing.go rename to libbeat/esleg/eslegtest/util.go index b0702ec210a..8da334dc3a4 100644 --- a/libbeat/outputs/elasticsearch/internal/testing.go +++ b/libbeat/esleg/eslegtest/util.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package internal +package eslegtest import ( "fmt" @@ -40,12 +40,12 @@ type Connectable interface { Connect() error } -// InitClient initializes a new client if the no error value from creating the -// client instance is reported. +// InitConnection initializes a new connection if the no error value from creating the +// connection instance is reported. // The test logger will be used if an error is found. -func InitClient(t TestLogger, client Connectable, err error) { +func InitConnection(t TestLogger, conn Connectable, err error) { if err == nil { - err = client.Connect() + err = conn.Connect() } if err != nil { @@ -54,26 +54,30 @@ func InitClient(t TestLogger, client Connectable, err error) { } } +// GetURL return the Elasticsearch testing URL. +func GetURL() string { + return fmt.Sprintf("http://%v:%v", GetEsHost(), getEsPort()) +} + // GetEsHost returns the Elasticsearch testing host. func GetEsHost() string { return getEnv("ES_HOST", ElasticsearchDefaultHost) } -// GetEsPort returns the Elasticsearch testing port. -func GetEsPort() string { +// getEsPort returns the Elasticsearch testing port. +func getEsPort() string { return getEnv("ES_PORT", ElasticsearchDefaultPort) } -// GetURL return the Elasticsearch testing URL. -func GetURL() string { - return fmt.Sprintf("http://%v:%v", GetEsHost(), GetEsPort()) -} - // GetUser returns the Elasticsearch testing user. -func GetUser() string { return getEnv("ES_USER", "") } +func GetUser() string { + return getEnv("ES_USER", "") +} // GetPass returns the Elasticsearch testing user's password. -func GetPass() string { return getEnv("ES_PASS", "") } +func GetPass() string { + return getEnv("ES_PASS", "") +} func getEnv(name, def string) string { if v := os.Getenv(name); len(v) > 0 { diff --git a/libbeat/idxmgmt/ilm/client_handler_integration_test.go b/libbeat/idxmgmt/ilm/client_handler_integration_test.go index 2d9c2ca721e..936eb35dcd8 100644 --- a/libbeat/idxmgmt/ilm/client_handler_integration_test.go +++ b/libbeat/idxmgmt/ilm/client_handler_integration_test.go @@ -31,9 +31,8 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/idxmgmt/ilm" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/outil" "github.com/elastic/beats/v7/libbeat/version" ) @@ -179,14 +178,13 @@ func newESClientHandler(t *testing.T) ilm.ClientHandler { } func newRawESClient(t *testing.T) ilm.ESClient { - client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ + client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: getURL(), - Index: outil.MakeSelector(), Username: getUser(), - Password: getUser(), + Password: getPass(), Timeout: 60 * time.Second, CompressionLevel: 3, - }, nil) + }) if err != nil { t.Fatal(err) } diff --git a/libbeat/ml-importer/importer_integration_test.go b/libbeat/ml-importer/importer_integration_test.go index 17cc7d02190..8b913f3c051 100644 --- a/libbeat/ml-importer/importer_integration_test.go +++ b/libbeat/ml-importer/importer_integration_test.go @@ -27,8 +27,9 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" + "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch/estest" ) const sampleJob = ` @@ -104,7 +105,7 @@ const sampleDatafeed = ` func TestImportJobs(t *testing.T) { logp.TestingSetup() - client := estest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) haveXpack, err := HaveXpackML(client) assert.NoError(t, err) @@ -194,3 +195,24 @@ func TestImportJobs(t *testing.T) { err = ImportMachineLearningJob(client, &mlconfig) assert.NoError(t, err) } + +func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { + conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ + URL: eslegtest.GetURL(), + Timeout: 0, + }) + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + conn.Encoder = eslegclient.NewJSONEncoder(nil, false) + + err = conn.Connect() + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + return conn +} diff --git a/libbeat/monitoring/report/elasticsearch/client.go b/libbeat/monitoring/report/elasticsearch/client.go index 42517219a90..9e8469ab547 100644 --- a/libbeat/monitoring/report/elasticsearch/client.go +++ b/libbeat/monitoring/report/elasticsearch/client.go @@ -26,9 +26,9 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring/report" - esout "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" "github.com/elastic/beats/v7/libbeat/publisher" "github.com/elastic/beats/v7/libbeat/testing" ) @@ -36,22 +36,24 @@ import ( var createDocPrivAvailableESVersion = common.MustNewVersion("7.5.0") type publishClient struct { - log *logp.Logger - es *esout.Client + es *eslegclient.Connection params map[string]string format report.Format + + log *logp.Logger } func newPublishClient( - es *esout.Client, + es *eslegclient.Connection, params map[string]string, format report.Format, ) (*publishClient, error) { p := &publishClient{ - log: logp.NewLogger(selector), es: es, params: params, format: format, + + log: logp.NewLogger(logSelector), } return p, nil } @@ -161,7 +163,7 @@ func (c *publishClient) Test(d testing.Driver) { } func (c *publishClient) String() string { - return "publish(" + c.es.String() + ")" + return "monitoring(" + c.es.URL + ")" } func (c *publishClient) publishXPackBulk(params map[string]string, event publisher.Event, typ string) error { @@ -231,8 +233,7 @@ func (c *publishClient) publishBulk(event publisher.Event, typ string) error { // Currently one request per event is sent. Reason is that each event can contain different // interval params and X-Pack requires to send the interval param. - // FIXME: index name (first param below) - result, err := c.es.BulkWith(getMonitoringIndexName(), "", nil, nil, bulk[:]) + _, result, err := c.es.Bulk(getMonitoringIndexName(), "", nil, bulk[:]) if err != nil { return err } @@ -247,25 +248,38 @@ func getMonitoringIndexName() string { return fmt.Sprintf(".monitoring-beats-%v-%s", version, date) } -func logBulkFailures(log *logp.Logger, result esout.BulkResult, events []report.Event) { - reader := esout.NewJSONReader(result) - err := esout.BulkReadToItems(reader) - if err != nil { - log.Errorf("failed to parse monitoring bulk items: %+v", err) +func logBulkFailures(log *logp.Logger, result eslegclient.BulkResult, events []report.Event) { + var response struct { + Items []map[string]map[string]interface{} `json:"items"` + } + + if err := json.Unmarshal(result, &response); err != nil { + log.Errorf("failed to parse monitoring bulk items: %v", err) return } for i := range events { - status, msg, err := esout.BulkReadItemStatus(log, reader) - if err != nil { - log.Errorf("failed to parse monitoring bulk item status: %+v", err) - return - } - switch { - case status < 300, status == http.StatusConflict: - continue - default: - log.Warnf("monitoring bulk item insert failed (i=%v, status=%v): %s", i, status, msg) + for _, innerItem := range response.Items[i] { + var status int + if s, exists := innerItem["status"]; exists { + if v, ok := s.(int); ok { + status = v + } + } + + var errorMsg string + if e, exists := innerItem["error"]; exists { + if v, ok := e.(string); ok { + errorMsg = v + } + } + + switch { + case status < 300, status == http.StatusConflict: + continue + default: + log.Warnf("monitoring bulk item insert failed (i=%v, status=%v): %s", i, status, errorMsg) + } } } } diff --git a/libbeat/monitoring/report/elasticsearch/elasticsearch.go b/libbeat/monitoring/report/elasticsearch/elasticsearch.go index 4484ceef933..4bab9f3117d 100644 --- a/libbeat/monitoring/report/elasticsearch/elasticsearch.go +++ b/libbeat/monitoring/report/elasticsearch/elasticsearch.go @@ -24,18 +24,16 @@ import ( "math/rand" "net/url" "strconv" - "strings" "time" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/monitoring/report" "github.com/elastic/beats/v7/libbeat/outputs" - esout "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/outil" "github.com/elastic/beats/v7/libbeat/publisher/pipeline" "github.com/elastic/beats/v7/libbeat/publisher/processing" "github.com/elastic/beats/v7/libbeat/publisher/queue" @@ -59,7 +57,7 @@ type reporter struct { out []outputs.NetworkClient } -const selector = "monitoring" +const logSelector = "monitoring" var errNoMonitoring = errors.New("xpack monitoring not available") @@ -112,7 +110,7 @@ func defaultConfig(settings report.Settings) config { } func makeReporter(beat beat.Info, settings report.Settings, cfg *common.Config) (report.Reporter, error) { - log := logp.NewLogger(selector) + log := logp.NewLogger(logSelector) config := defaultConfig(settings) if err := cfg.Unpack(&config); err != nil { return nil, err @@ -131,7 +129,7 @@ func makeReporter(beat beat.Info, settings report.Settings, cfg *common.Config) windowSize = 1 } - proxyURL, err := parseProxyURL(config.ProxyURL) + proxyURL, err := common.ParseURL(config.ProxyURL) if err != nil { return nil, err } @@ -337,7 +335,7 @@ func makeClient( return nil, err } - esClient, err := esout.NewClient(esout.ClientSettings{ + esClient, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: url, Proxy: proxyURL, TLS: tlsConfig, @@ -346,11 +344,9 @@ func makeClient( APIKey: config.APIKey, Parameters: params, Headers: config.Headers, - Index: outil.MakeSelector(outil.ConstSelectorExpr("_xpack")), - Pipeline: nil, Timeout: config.Timeout, CompressionLevel: config.CompressionLevel, - }, nil) + }) if err != nil { return nil, err } @@ -368,22 +364,6 @@ func closing(log *logp.Logger, c io.Closer) { } } -// TODO: make this reusable. Same definition in elasticsearch monitoring module -func parseProxyURL(raw string) (*url.URL, error) { - if raw == "" { - return nil, nil - } - - url, err := url.Parse(raw) - if err == nil && strings.HasPrefix(url.Scheme, "http") { - return url, err - } - - // Proxy was bogus. Try prepending "http://" to it and - // see if that parses correctly. - return url.Parse("http://" + raw) -} - func makeMeta(beat beat.Info) common.MapStr { return common.MapStr{ "type": beat.Beat, diff --git a/libbeat/outputs/elasticsearch/bulk.go b/libbeat/outputs/elasticsearch/bulk.go new file mode 100644 index 00000000000..c771184afb8 --- /dev/null +++ b/libbeat/outputs/elasticsearch/bulk.go @@ -0,0 +1,155 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package elasticsearch + +import ( + "bytes" + "errors" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +var ( + errExpectedItemsArray = errors.New("expected items array") + errExpectedItemObject = errors.New("expected item response object") + errExpectedStatusCode = errors.New("expected item status code") + errUnexpectedEmptyObject = errors.New("empty object") + errExpectedObjectEnd = errors.New("expected end of object") + + nameItems = []byte("items") + nameStatus = []byte("status") + nameError = []byte("error") +) + +// bulkReadToItems reads the bulk response up to (but not including) items +func bulkReadToItems(reader *jsonReader) error { + if err := reader.ExpectDict(); err != nil { + return errExpectedObject + } + + // find 'items' field in response + for { + kind, name, err := reader.nextFieldName() + if err != nil { + return err + } + + if kind == dictEnd { + return errExpectedItemsArray + } + + // found items array -> continue + if bytes.Equal(name, nameItems) { + break + } + + reader.ignoreNext() + } + + // check items field is an array + if err := reader.ExpectArray(); err != nil { + return errExpectedItemsArray + } + + return nil +} + +// bulkReadItemStatus reads the status and error fields from the bulk item +func bulkReadItemStatus(logger *logp.Logger, reader *jsonReader) (int, []byte, error) { + // skip outer dictionary + if err := reader.ExpectDict(); err != nil { + return 0, nil, errExpectedItemObject + } + + // find first field in outer dictionary (e.g. 'create') + kind, _, err := reader.nextFieldName() + if err != nil { + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + if kind == dictEnd { + err = errUnexpectedEmptyObject + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + // parse actual item response code and error message + status, msg, err := itemStatusInner(reader, logger) + if err != nil { + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + // close dictionary. Expect outer dictionary to have only one element + kind, _, err = reader.step() + if err != nil { + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + if kind != dictEnd { + err = errExpectedObjectEnd + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + return status, msg, nil +} + +func itemStatusInner(reader *jsonReader, logger *logp.Logger) (int, []byte, error) { + if err := reader.ExpectDict(); err != nil { + return 0, nil, errExpectedItemObject + } + + status := -1 + var msg []byte + for { + kind, name, err := reader.nextFieldName() + if err != nil { + logger.Errorf("Failed to parse bulk response item: %s", err) + } + if kind == dictEnd { + break + } + + switch { + case bytes.Equal(name, nameStatus): // name == "status" + status, err = reader.nextInt() + if err != nil { + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + case bytes.Equal(name, nameError): // name == "error" + msg, err = reader.ignoreNext() // collect raw string for "error" field + if err != nil { + return 0, nil, err + } + + default: // ignore unknown fields + _, err = reader.ignoreNext() + if err != nil { + return 0, nil, err + } + } + } + + if status < 0 { + return 0, nil, errExpectedStatusCode + } + return status, msg, nil +} diff --git a/libbeat/outputs/elasticsearch/bulk_test.go b/libbeat/outputs/elasticsearch/bulk_test.go new file mode 100644 index 00000000000..49e84128a47 --- /dev/null +++ b/libbeat/outputs/elasticsearch/bulk_test.go @@ -0,0 +1,131 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package elasticsearch + +import ( + "testing" + + "github.com/elastic/beats/v7/libbeat/logp" + + "github.com/stretchr/testify/assert" +) + +func TestBulkReadToItems(t *testing.T) { + response := []byte(`{ + "errors": false, + "items": [ + {"create": {"status": 200}}, + {"create": {"status": 300}}, + {"create": {"status": 400}} + ]}`) + + reader := newJSONReader(response) + + err := bulkReadToItems(reader) + assert.NoError(t, err) + + for status := 200; status <= 400; status += 100 { + err = reader.ExpectDict() + assert.NoError(t, err) + + kind, raw, err := reader.nextFieldName() + assert.NoError(t, err) + assert.Equal(t, mapKeyEntity, kind) + assert.Equal(t, []byte("create"), raw) + + err = reader.ExpectDict() + assert.NoError(t, err) + + kind, raw, err = reader.nextFieldName() + assert.NoError(t, err) + assert.Equal(t, mapKeyEntity, kind) + assert.Equal(t, []byte("status"), raw) + + code, err := reader.nextInt() + assert.NoError(t, err) + assert.Equal(t, status, code) + + _, _, err = reader.endDict() + assert.NoError(t, err) + + _, _, err = reader.endDict() + assert.NoError(t, err) + } +} + +func TestBulkReadItemStatus(t *testing.T) { + response := []byte(`{"create": {"status": 200}}`) + + reader := newJSONReader(response) + code, _, err := bulkReadItemStatus(logp.L(), reader) + assert.NoError(t, err) + assert.Equal(t, 200, code) +} + +func TestESNoErrorStatus(t *testing.T) { + response := []byte(`{"create": {"status": 200}}`) + code, msg, err := readStatusItem(response) + + assert.Nil(t, err) + assert.Equal(t, 200, code) + assert.Equal(t, "", msg) +} + +func TestES1StyleErrorStatus(t *testing.T) { + response := []byte(`{"create": {"status": 400, "error": "test error"}}`) + code, msg, err := readStatusItem(response) + + assert.Nil(t, err) + assert.Equal(t, 400, code) + assert.Equal(t, `"test error"`, msg) +} + +func TestES2StyleErrorStatus(t *testing.T) { + response := []byte(`{"create": {"status": 400, "error": {"reason": "test_error"}}}`) + code, msg, err := readStatusItem(response) + + assert.Nil(t, err) + assert.Equal(t, 400, code) + assert.Equal(t, `{"reason": "test_error"}`, msg) +} + +func TestES2StyleExtendedErrorStatus(t *testing.T) { + response := []byte(` + { + "create": { + "status": 400, + "error": { + "reason": "test_error", + "transient": false, + "extra": null + } + } + }`) + code, _, err := readStatusItem(response) + + assert.Nil(t, err) + assert.Equal(t, 400, code) +} + +func readStatusItem(in []byte) (int, string, error) { + reader := newJSONReader(in) + code, msg, err := bulkReadItemStatus(logp.L(), reader) + return code, string(msg), err +} diff --git a/libbeat/outputs/elasticsearch/callbacks.go b/libbeat/outputs/elasticsearch/callbacks.go new file mode 100644 index 00000000000..dcbbd971adb --- /dev/null +++ b/libbeat/outputs/elasticsearch/callbacks.go @@ -0,0 +1,111 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package elasticsearch + +import ( + "sync" + + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" + + "github.com/gofrs/uuid" +) + +// ConnectCallback defines the type for the function to be called when the Elasticsearch client successfully connects to the cluster +type ConnectCallback func(*eslegclient.Connection) error + +// Callbacks must not depend on the result of a previous one, +// because the ordering is not fixed. +type callbacksRegistry struct { + callbacks map[uuid.UUID]ConnectCallback + mutex sync.Mutex +} + +// XXX: it would be fantastic to do this without a package global +var connectCallbackRegistry = newCallbacksRegistry() + +// NOTE(ph): We need to refactor this, right now this is the only way to ensure that every calls +// to an ES cluster executes a callback. +var globalCallbackRegistry = newCallbacksRegistry() + +func newCallbacksRegistry() callbacksRegistry { + return callbacksRegistry{ + callbacks: make(map[uuid.UUID]ConnectCallback), + } +} + +// RegisterGlobalCallback register a global callbacks. +func RegisterGlobalCallback(callback ConnectCallback) (uuid.UUID, error) { + globalCallbackRegistry.mutex.Lock() + defer globalCallbackRegistry.mutex.Unlock() + + // find the next unique key + var key uuid.UUID + var err error + exists := true + for exists { + key, err = uuid.NewV4() + if err != nil { + return uuid.Nil, err + } + _, exists = globalCallbackRegistry.callbacks[key] + } + + globalCallbackRegistry.callbacks[key] = callback + return key, nil +} + +// RegisterConnectCallback registers a callback for the elasticsearch output +// The callback is called each time the client connects to elasticsearch. +// It returns the key of the newly added callback, so it can be deregistered later. +func RegisterConnectCallback(callback ConnectCallback) (uuid.UUID, error) { + connectCallbackRegistry.mutex.Lock() + defer connectCallbackRegistry.mutex.Unlock() + + // find the next unique key + var key uuid.UUID + var err error + exists := true + for exists { + key, err = uuid.NewV4() + if err != nil { + return uuid.Nil, err + } + _, exists = connectCallbackRegistry.callbacks[key] + } + + connectCallbackRegistry.callbacks[key] = callback + return key, nil +} + +// DeregisterGlobalCallback deregisters a callback for the elasticsearch output +// specified by its key. If a callback does not exist, nothing happens. +func DeregisterGlobalCallback(key uuid.UUID) { + globalCallbackRegistry.mutex.Lock() + defer globalCallbackRegistry.mutex.Unlock() + + delete(globalCallbackRegistry.callbacks, key) +} + +// DeregisterConnectCallback deregisters a callback for the elasticsearch output +// specified by its key. If a callback does not exist, nothing happens. +func DeregisterConnectCallback(key uuid.UUID) { + connectCallbackRegistry.mutex.Lock() + defer connectCallbackRegistry.mutex.Unlock() + + delete(connectCallbackRegistry.callbacks, key) +} diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 40cd8f46836..4dfbd3e0b9b 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -18,22 +18,15 @@ package elasticsearch import ( - "bytes" "encoding/base64" - "encoding/json" + "errors" "fmt" - "io" - "io/ioutil" "net/http" - "net/url" "time" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/transport" - "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" "github.com/elastic/beats/v7/libbeat/outputs/outil" @@ -43,78 +36,22 @@ import ( // Client is an elasticsearch client. type Client struct { - Connection - tlsConfig *tlscommon.TLSConfig + conn eslegclient.Connection index outputs.IndexSelector pipeline *outil.Selector - params map[string]string - timeout time.Duration - - // buffered bulk requests - bulkRequ *bulkRequest - - // buffered json response reader - json JSONReader - - // additional configs - compressionLevel int - proxyURL *url.URL observer outputs.Observer -} -// ClientSettings contains the settings for a client. -type ClientSettings struct { - URL string - Proxy *url.URL - ProxyDisable bool - TLS *tlscommon.TLSConfig - Username, Password string - APIKey string - EscapeHTML bool - Parameters map[string]string - Headers map[string]string - Index outputs.IndexSelector - Pipeline *outil.Selector - Timeout time.Duration - CompressionLevel int - Observer outputs.Observer -} - -// ConnectCallback defines the type for the function to be called when the Elasticsearch client successfully connects to the cluster -type ConnectCallback func(client *Client) error - -// Connection manages the connection for a given client. -type Connection struct { log *logp.Logger - - URL string - Username string - Password string - APIKey string - Headers map[string]string - - http *http.Client - onConnectCallback func() error - - encoder bodyEncoder - version common.Version } -type bulkIndexAction struct { - Index bulkEventMeta `json:"index" struct:"index"` -} - -type bulkCreateAction struct { - Create bulkEventMeta `json:"create" struct:"create"` -} - -type bulkEventMeta struct { - Index string `json:"_index" struct:"_index"` - DocType string `json:"_type,omitempty" struct:"_type,omitempty"` - Pipeline string `json:"pipeline,omitempty" struct:"pipeline,omitempty"` - ID string `json:"_id,omitempty" struct:"_id,omitempty"` +// ClientSettings contains the settings for a client. +type ClientSettings struct { + eslegclient.ConnectionSettings + Index outputs.IndexSelector + Pipeline *outil.Selector + Observer outputs.Observer } type bulkResultStats struct { @@ -125,21 +62,6 @@ type bulkResultStats struct { tooMany int // number of events receiving HTTP 429 Too Many Requests } -var ( - nameItems = []byte("items") - nameStatus = []byte("status") - nameError = []byte("error") -) - -var ( - errExpectedItemsArray = errors.New("expected items array") - errExpectedItemObject = errors.New("expected item response object") - errExpectedStatusCode = errors.New("expected item status code") - errUnexpectedEmptyObject = errors.New("empty object") - errExpectedObjectEnd = errors.New("expected end of object") - errTempBulkFailure = errors.New("temporary bulk send failure") -) - const ( defaultEventType = "doc" ) @@ -149,104 +71,35 @@ func NewClient( s ClientSettings, onConnect *callbacksRegistry, ) (*Client, error) { - var proxy func(*http.Request) (*url.URL, error) - if !s.ProxyDisable { - proxy = http.ProxyFromEnvironment - if s.Proxy != nil { - proxy = http.ProxyURL(s.Proxy) - } - } - pipeline := s.Pipeline if pipeline != nil && pipeline.IsEmpty() { pipeline = nil } - u, err := url.Parse(s.URL) - if err != nil { - return nil, fmt.Errorf("failed to parse elasticsearch URL: %v", err) - } - if u.User != nil { - s.Username = u.User.Username() - s.Password, _ = u.User.Password() - u.User = nil - - // Re-write URL without credentials. - s.URL = u.String() - } - - log := logp.NewLogger(logSelector) - log.Infof("Elasticsearch url: %s", s.URL) - - // TODO: add socks5 proxy support - var dialer, tlsDialer transport.Dialer - - dialer = transport.NetDialer(s.Timeout) - tlsDialer, err = transport.TLSDialer(dialer, s.TLS, s.Timeout) - if err != nil { - return nil, err - } - - if st := s.Observer; st != nil { - dialer = transport.StatsDialer(dialer, st) - tlsDialer = transport.StatsDialer(tlsDialer, st) - } - - params := s.Parameters - bulkRequ, err := newBulkRequest(s.URL, "", "", params, nil) + conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ + URL: s.URL, + Username: s.Username, + Password: s.Password, + APIKey: base64.StdEncoding.EncodeToString([]byte(s.APIKey)), + Headers: s.Headers, + TLS: s.TLS, + Proxy: s.Proxy, + ProxyDisable: s.ProxyDisable, + Parameters: s.Parameters, + CompressionLevel: s.CompressionLevel, + EscapeHTML: s.EscapeHTML, + Timeout: s.Timeout, + }) if err != nil { return nil, err } - var encoder bodyEncoder - compression := s.CompressionLevel - if compression == 0 { - encoder = newJSONEncoder(nil, s.EscapeHTML) - } else { - encoder, err = newGzipEncoder(compression, nil, s.EscapeHTML) - if err != nil { - return nil, err - } - } - - client := &Client{ - Connection: Connection{ - log: log, - URL: s.URL, - Username: s.Username, - Password: s.Password, - APIKey: base64.StdEncoding.EncodeToString([]byte(s.APIKey)), - Headers: s.Headers, - http: &http.Client{ - Transport: &http.Transport{ - Dial: dialer.Dial, - DialTLS: tlsDialer.Dial, - TLSClientConfig: s.TLS.ToConfig(), - Proxy: proxy, - }, - Timeout: s.Timeout, - }, - encoder: encoder, - }, - tlsConfig: s.TLS, - index: s.Index, - pipeline: pipeline, - params: params, - timeout: s.Timeout, - - bulkRequ: bulkRequ, - - compressionLevel: compression, - proxyURL: s.Proxy, - observer: s.Observer, - } - - client.Connection.onConnectCallback = func() error { + conn.OnConnectCallback = func() error { globalCallbackRegistry.mutex.Lock() defer globalCallbackRegistry.mutex.Unlock() for _, callback := range globalCallbackRegistry.callbacks { - err := callback(client) + err := callback(conn) if err != nil { return err } @@ -257,7 +110,7 @@ func NewClient( defer onConnect.mutex.Unlock() for _, callback := range onConnect.callbacks { - err := callback(client) + err := callback(conn) if err != nil { return err } @@ -266,6 +119,16 @@ func NewClient( return nil } + client := &Client{ + conn: *conn, + index: s.Index, + pipeline: pipeline, + + observer: s.Observer, + + log: logp.NewLogger("elasticsearch"), + } + return client, nil } @@ -278,22 +141,27 @@ func (client *Client) Clone() *Client { c, _ := NewClient( ClientSettings{ - URL: client.URL, + ConnectionSettings: eslegclient.ConnectionSettings{ + URL: client.conn.URL, + Proxy: client.conn.Proxy, + // Without the following nil check on proxyURL, a nil Proxy field will try + // reloading proxy settings from the environment instead of leaving them + // empty. + ProxyDisable: client.conn.Proxy == nil, + TLS: client.conn.TLS, + Username: client.conn.Username, + Password: client.conn.Password, + APIKey: client.conn.APIKey, + Parameters: nil, // XXX: do not pass params? + Headers: client.conn.Headers, + Timeout: client.conn.HTTP.Timeout, + CompressionLevel: client.conn.CompressionLevel, + OnConnectCallback: nil, + Observer: nil, + EscapeHTML: false, + }, Index: client.index, Pipeline: client.pipeline, - Proxy: client.proxyURL, - // Without the following nil check on proxyURL, a nil Proxy field will try - // reloading proxy settings from the environment instead of leaving them - // empty. - ProxyDisable: client.proxyURL == nil, - TLS: client.tlsConfig, - Username: client.Username, - Password: client.Password, - APIKey: client.APIKey, - Parameters: nil, // XXX: do not pass params? - Headers: client.Headers, - Timeout: client.http.Timeout, - CompressionLevel: client.compressionLevel, }, nil, // XXX: do not pass connection callback? ) @@ -328,19 +196,10 @@ func (client *Client) publishEvents( return nil, nil } - body := client.encoder - body.Reset() - // encode events into bulk request buffer, dropping failed elements from // events slice - - eventType := "" - if client.GetVersion().Major < 7 { - eventType = defaultEventType - } - origCount := len(data) - data = bulkEncodePublishRequest(client.Connection.log, client.GetVersion(), body, client.index, client.pipeline, eventType, data) + data, bulkItems := bulkEncodePublishRequest(client.log, client.conn.GetVersion(), client.index, client.pipeline, data) newCount := len(data) if st != nil && origCount > newCount { st.Dropped(origCount - newCount) @@ -349,15 +208,13 @@ func (client *Client) publishEvents( return nil, nil } - requ := client.bulkRequ - requ.Reset(body) - status, result, sendErr := client.sendBulkRequest(requ) + status, result, sendErr := client.conn.Bulk("", "", nil, bulkItems) if sendErr != nil { - client.Connection.log.Error("Failed to perform any bulk index operations: %+v", sendErr) + client.log.Errorf("Failed to perform any bulk index operations: %s", sendErr) return data, sendErr } - client.Connection.log.Debugf("PublishEvents: %d events have been published to elasticsearch in %v.", + client.log.Debugf("PublishEvents: %d events have been published to elasticsearch in %v.", len(data), time.Now().Sub(begin)) @@ -368,8 +225,7 @@ func (client *Client) publishEvents( failedEvents = data stats.fails = len(failedEvents) } else { - client.json.init(result) - failedEvents, stats = bulkCollectPublishFails(client.Connection.log, &client.json, data) + failedEvents, stats = bulkCollectPublishFails(client.log, result, data) } failed := len(failedEvents) @@ -387,40 +243,36 @@ func (client *Client) publishEvents( if failed > 0 { if sendErr == nil { - sendErr = errTempBulkFailure + sendErr = eslegclient.ErrTempBulkFailure } return failedEvents, sendErr } return nil, nil } -// fillBulkRequest encodes all bulk requests and returns slice of events -// successfully added to bulk request. +// bulkEncodePublishRequest encodes all bulk requests and returns slice of events +// successfully added to the list of bulk items and the list of bulk items. func bulkEncodePublishRequest( log *logp.Logger, version common.Version, - body bulkWriter, index outputs.IndexSelector, pipeline *outil.Selector, - eventType string, data []publisher.Event, -) []publisher.Event { +) ([]publisher.Event, []interface{}) { + okEvents := data[:0] + bulkItems := []interface{}{} for i := range data { event := &data[i].Content - meta, err := createEventBulkMeta(log, version, index, pipeline, eventType, event) + meta, err := createEventBulkMeta(log, version, index, pipeline, event) if err != nil { log.Errorf("Failed to encode event meta data: %+v", err) continue } - if err := body.Add(meta, event); err != nil { - log.Errorf("Failed to encode event: %+v", err) - log.Debugf("Failed event: %v", event) - continue - } + bulkItems = append(bulkItems, meta, event) okEvents = append(okEvents, data[i]) } - return okEvents + return okEvents, bulkItems } func createEventBulkMeta( @@ -428,9 +280,13 @@ func createEventBulkMeta( version common.Version, indexSel outputs.IndexSelector, pipelineSel *outil.Selector, - eventType string, event *beat.Event, ) (interface{}, error) { + eventType := "" + if version.Major < 7 { + eventType = defaultEventType + } + pipeline, err := getPipeline(event, pipelineSel) if err != nil { err := fmt.Errorf("failed to select pipeline: %v", err) @@ -454,7 +310,7 @@ func createEventBulkMeta( } } - meta := bulkEventMeta{ + meta := eslegclient.BulkMeta{ Index: index, DocType: eventType, Pipeline: pipeline, @@ -462,9 +318,9 @@ func createEventBulkMeta( } if id != "" || version.Major > 7 || (version.Major == 7 && version.Minor >= 5) { - return bulkCreateAction{meta}, nil + return eslegclient.BulkCreateAction{Create: meta}, nil } - return bulkIndexAction{meta}, nil + return eslegclient.BulkIndexAction{Index: meta}, nil } func getPipeline(event *beat.Event, pipelineSel *outil.Selector) (string, error) { @@ -489,11 +345,12 @@ func getPipeline(event *beat.Event, pipelineSel *outil.Selector) (string, error) // the event will be dropped. func bulkCollectPublishFails( log *logp.Logger, - reader *JSONReader, + result eslegclient.BulkResult, data []publisher.Event, ) ([]publisher.Event, bulkResultStats) { - if err := BulkReadToItems(reader); err != nil { - log.Errorf("failed to parse bulk response: %+v", err) + reader := newJSONReader(result) + if err := bulkReadToItems(reader); err != nil { + log.Errorf("failed to parse bulk response: %v", err.Error()) return nil, bulkResultStats{} } @@ -501,7 +358,7 @@ func bulkCollectPublishFails( failed := data[:0] stats := bulkResultStats{} for i := 0; i < count; i++ { - status, msg, err := BulkReadItemStatus(log, reader) + status, msg, err := bulkReadItemStatus(log, reader) if err != nil { log.Error(err) return nil, bulkResultStats{} @@ -538,336 +395,18 @@ func bulkCollectPublishFails( return failed, stats } -// BulkReadToItems reads the bulk response up to (but not including) items -func BulkReadToItems(reader *JSONReader) error { - if err := reader.ExpectDict(); err != nil { - return errExpectedObject - } - - // find 'items' field in response - for { - kind, name, err := reader.nextFieldName() - if err != nil { - return err - } - - if kind == dictEnd { - return errExpectedItemsArray - } - - // found items array -> continue - if bytes.Equal(name, nameItems) { - break - } - - reader.ignoreNext() - } - - // check items field is an array - if err := reader.ExpectArray(); err != nil { - return errExpectedItemsArray - } - - return nil -} - -// BulkReadItemStatus reads the status and error fields from the bulk item -func BulkReadItemStatus(log *logp.Logger, reader *JSONReader) (int, []byte, error) { - // skip outer dictionary - if err := reader.ExpectDict(); err != nil { - return 0, nil, errExpectedItemObject - } - - // find first field in outer dictionary (e.g. 'create') - kind, _, err := reader.nextFieldName() - parserErr := func(err error) error { - return errors.Wrapf(err, "Failed to parse bulk response item") - } - if err != nil { - return 0, nil, parserErr(err) - } - if kind == dictEnd { - err = errUnexpectedEmptyObject - return 0, nil, parserErr(err) - } - - // parse actual item response code and error message - status, msg, err := itemStatusInner(log, reader) - if err != nil { - return 0, nil, parserErr(err) - } - - // close dictionary. Expect outer dictionary to have only one element - kind, _, err = reader.step() - if err != nil { - return 0, nil, parserErr(err) - } - if kind != dictEnd { - err = errExpectedObjectEnd - return 0, nil, parserErr(err) - } - - return status, msg, nil -} - -func itemStatusInner(log *logp.Logger, reader *JSONReader) (int, []byte, error) { - if err := reader.ExpectDict(); err != nil { - return 0, nil, errExpectedItemObject - } - - status := -1 - var msg []byte - for { - kind, name, err := reader.nextFieldName() - if err != nil { - log.Errorf("Failed to parse bulk response item: %+v", err) - } - if kind == dictEnd { - break - } - - switch { - case bytes.Equal(name, nameStatus): // name == "status" - status, err = reader.nextInt() - if err != nil { - return 0, nil, err - } - - case bytes.Equal(name, nameError): // name == "error" - msg, err = reader.ignoreNext() // collect raw string for "error" field - if err != nil { - return 0, nil, err - } - - default: // ignore unknown fields - _, err = reader.ignoreNext() - if err != nil { - return 0, nil, err - } - } - } - - if status < 0 { - return 0, nil, errExpectedStatusCode - } - return status, msg, nil -} - -// LoadJSON creates a PUT request based on a JSON document. -func (client *Client) LoadJSON(path string, json map[string]interface{}) ([]byte, error) { - status, body, err := client.Request("PUT", path, "", nil, json) - if err != nil { - return body, fmt.Errorf("couldn't load json. Error: %s", err) - } - if status > 300 { - return body, fmt.Errorf("couldn't load json. Status: %v", status) - } - - return body, nil -} - -// GetVersion returns the elasticsearch version the client is connected to. -func (client *Client) GetVersion() common.Version { - return client.Connection.version -} - -func (client *Client) Test(d testing.Driver) { - d.Run("elasticsearch: "+client.URL, func(d testing.Driver) { - u, err := url.Parse(client.URL) - d.Fatal("parse url", err) - - address := u.Host - - d.Run("connection", func(d testing.Driver) { - netDialer := transport.TestNetDialer(d, client.timeout) - _, err = netDialer.Dial("tcp", address) - d.Fatal("dial up", err) - }) - - if u.Scheme != "https" { - d.Warn("TLS", "secure connection disabled") - } else { - d.Run("TLS", func(d testing.Driver) { - netDialer := transport.NetDialer(client.timeout) - tlsDialer, err := transport.TestTLSDialer(d, netDialer, client.tlsConfig, client.timeout) - _, err = tlsDialer.Dial("tcp", address) - d.Fatal("dial up", err) - }) - } - - err = client.Connect() - d.Fatal("talk to server", err) - d.Info("version", client.version.String()) - }) -} - -func (client *Client) String() string { - return "elasticsearch(" + client.Connection.URL + ")" -} - -// Connect connects the client. It runs a GET request against the root URL of -// the configured host, updates the known Elasticsearch version and calls -// globally configured handlers. func (client *Client) Connect() error { - return client.Connection.Connect() -} - -// Connect connects the client. It runs a GET request against the root URL of -// the configured host, updates the known Elasticsearch version and calls -// globally configured handlers. -func (conn *Connection) Connect() error { - versionString, err := conn.Ping() - if err != nil { - return err - } - - if version, err := common.NewVersion(versionString); err != nil { - conn.log.Errorf("Invalid version from Elasticsearch: %s", versionString) - conn.version = common.Version{} - } else { - conn.version = *version - } - - err = conn.onConnectCallback() - if err != nil { - return fmt.Errorf("Connection marked as failed because the onConnect callback failed: %v", err) - } - return nil -} - -// Ping sends a GET request to the Elasticsearch. -func (conn *Connection) Ping() (string, error) { - conn.log.Debugf("ES Ping(url=%v)", conn.URL) - - status, body, err := conn.execRequest("GET", conn.URL, nil) - if err != nil { - conn.log.Debugf("Ping request failed with: %+v", err) - return "", err - } - - if status >= 300 { - return "", fmt.Errorf("Non 2xx response code: %d", status) - } - - var response struct { - Version struct { - Number string - } - } - - err = json.Unmarshal(body, &response) - if err != nil { - return "", fmt.Errorf("Failed to parse JSON response: %v", err) - } - - conn.log.Debugf("Ping status code: %v", status) - conn.log.Infof("Attempting to connect to Elasticsearch version %s", response.Version.Number) - return response.Version.Number, nil -} - -// Close closes a connection. -func (conn *Connection) Close() error { - return nil -} - -// Request sends a request via the connection. -func (conn *Connection) Request( - method, path string, - pipeline string, - params map[string]string, - body interface{}, -) (int, []byte, error) { - - url := addToURL(conn.URL, path, pipeline, params) - conn.log.Debugf("%s %s %s %v", method, url, pipeline, body) - - return conn.RequestURL(method, url, body) -} - -// RequestURL sends a request with the connection object to an alternative url -func (conn *Connection) RequestURL( - method, url string, - body interface{}, -) (int, []byte, error) { - - if body == nil { - return conn.execRequest(method, url, nil) - } - - if err := conn.encoder.Marshal(body); err != nil { - conn.log.Warnf("Failed to json encode body (%+v): %#v", err, body) - return 0, nil, ErrJSONEncodeFailed - } - return conn.execRequest(method, url, conn.encoder.Reader()) -} - -func (conn *Connection) execRequest( - method, url string, - body io.Reader, -) (int, []byte, error) { - req, err := http.NewRequest(method, url, body) - if err != nil { - conn.log.Warnf("Failed to create request %+v", err) - return 0, nil, err - } - if body != nil { - conn.encoder.AddHeader(&req.Header) - } - return conn.execHTTPRequest(req) + return client.conn.Connect() } -func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) { - req.Header.Add("Accept", "application/json") - - if conn.Username != "" || conn.Password != "" { - req.SetBasicAuth(conn.Username, conn.Password) - } - - if conn.APIKey != "" { - req.Header.Add("Authorization", "ApiKey "+conn.APIKey) - } - - for name, value := range conn.Headers { - req.Header.Add(name, value) - } - - // The stlib will override the value in the header based on the configured `Host` - // on the request which default to the current machine. - // - // We use the normalized key header to retrieve the user configured value and assign it to the host. - if host := req.Header.Get("Host"); host != "" { - req.Host = host - } - - resp, err := conn.http.Do(req) - if err != nil { - return 0, nil, err - } - defer closing(conn.log, resp.Body) - - status := resp.StatusCode - obj, err := ioutil.ReadAll(resp.Body) - if err != nil { - return status, nil, err - } - - if status >= 300 { - // add the response body with the error returned by Elasticsearch - err = fmt.Errorf("%v: %s", resp.Status, obj) - } - - return status, obj, err +func (client *Client) Close() error { + return client.conn.Close() } -// GetVersion returns the elasticsearch version the client is connected to. -// The version is read and updated on 'Connect'. -func (conn *Connection) GetVersion() common.Version { - return conn.version +func (client *Client) String() string { + return "elasticsearch(" + client.conn.URL + ")" } -func closing(log *logp.Logger, c io.Closer) { - err := c.Close() - if err != nil { - log.Warnf("Close failed with: %+v", err) - } +func (client *Client) Test(d testing.Driver) { + client.conn.Test(d) } diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index 98c9addc081..1e01b757da0 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -23,7 +23,6 @@ import ( "context" "io/ioutil" "math/rand" - "net" "net/http" "net/http/httptest" "net/url" @@ -35,50 +34,13 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch/internal" "github.com/elastic/beats/v7/libbeat/outputs/outest" - "github.com/elastic/beats/v7/libbeat/outputs/outil" ) -func TestClientConnect(t *testing.T) { - client := getTestingElasticsearch(t) - err := client.Connect() - assert.NoError(t, err) -} - -func TestClientConnectWithProxy(t *testing.T) { - wrongPort, err := net.Listen("tcp", "localhost:0") - require.NoError(t, err) - go func() { - c, err := wrongPort.Accept() - if err == nil { - // Provoke an early-EOF error on client - c.Close() - } - }() - defer wrongPort.Close() - - proxy := startTestProxy(t, internal.GetURL()) - defer proxy.Close() - - // Use connectTestEs instead of getTestingElasticsearch to make use of makeES - _, client := connectTestEs(t, map[string]interface{}{ - "hosts": "http://" + wrongPort.Addr().String(), - "timeout": 5, // seconds - }) - assert.Error(t, client.Connect(), "it should fail without proxy") - - _, client = connectTestEs(t, map[string]interface{}{ - "hosts": "http://" + wrongPort.Addr().String(), - "proxy_url": proxy.URL, - "timeout": 5, // seconds - }) - assert.NoError(t, client.Connect()) -} - func TestClientPublishEvent(t *testing.T) { index := "beat-int-pub-single-event" output, client := connectTestEs(t, map[string]interface{}{ @@ -86,7 +48,7 @@ func TestClientPublishEvent(t *testing.T) { }) // drop old index preparing test - client.Delete(index, "", "", nil) + client.conn.Delete(index, "", "", nil) batch := outest.NewBatch(beat.Event{ Timestamp: time.Now(), @@ -101,12 +63,12 @@ func TestClientPublishEvent(t *testing.T) { t.Fatal(err) } - _, _, err = client.Refresh(index) + _, _, err = client.conn.Refresh(index) if err != nil { t.Fatal(err) } - _, resp, err := client.CountSearchURI(index, "", nil) + _, resp, err := client.conn.CountSearchURI(index, "", nil) if err != nil { t.Fatal(err) } @@ -126,10 +88,10 @@ func TestClientPublishEventWithPipeline(t *testing.T) { "index": index, "pipeline": "%{[pipeline]}", }) - client.Delete(index, "", "", nil) + client.conn.Delete(index, "", "", nil) // Check version - if client.Connection.version.Major < 5 { + if client.conn.GetVersion().Major < 5 { t.Skip("Skipping tests as pipeline not available in <5.x releases") } @@ -141,7 +103,7 @@ func TestClientPublishEventWithPipeline(t *testing.T) { } getCount := func(query string) int { - _, resp, err := client.CountSearchURI(index, "", map[string]string{ + _, resp, err := client.conn.CountSearchURI(index, "", map[string]string{ "q": query, }) if err != nil { @@ -162,8 +124,8 @@ func TestClientPublishEventWithPipeline(t *testing.T) { }, } - client.DeletePipeline(pipeline, nil) - _, resp, err := client.CreatePipeline(pipeline, nil, pipelineBody) + client.conn.DeletePipeline(pipeline, nil) + _, resp, err := client.conn.CreatePipeline(pipeline, nil, pipelineBody) if err != nil { t.Fatal(err) } @@ -187,7 +149,7 @@ func TestClientPublishEventWithPipeline(t *testing.T) { "testfield": 0, }}) - _, _, err = client.Refresh(index) + _, _, err = client.conn.Refresh(index) if err != nil { t.Fatal(err) } @@ -208,9 +170,9 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { "index": index, "pipeline": "%{[pipeline]}", }) - client.Delete(index, "", "", nil) + client.conn.Delete(index, "", "", nil) - if client.Connection.version.Major < 5 { + if client.conn.GetVersion().Major < 5 { t.Skip("Skipping tests as pipeline not available in <5.x releases") } @@ -222,7 +184,7 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { } getCount := func(query string) int { - _, resp, err := client.CountSearchURI(index, "", map[string]string{ + _, resp, err := client.conn.CountSearchURI(index, "", map[string]string{ "q": query, }) if err != nil { @@ -243,8 +205,8 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { }, } - client.DeletePipeline(pipeline, nil) - _, resp, err := client.CreatePipeline(pipeline, nil, pipelineBody) + client.conn.DeletePipeline(pipeline, nil) + _, resp, err := client.conn.CreatePipeline(pipeline, nil, pipelineBody) if err != nil { t.Fatal(err) } @@ -270,7 +232,7 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { }}, ) - _, _, err = client.Refresh(index) + _, _, err = client.conn.Refresh(index) if err != nil { t.Fatal(err) } @@ -281,9 +243,9 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { func connectTestEs(t *testing.T, cfg interface{}) (outputs.Client, *Client) { config, err := common.NewConfigFrom(map[string]interface{}{ - "hosts": internal.GetEsHost(), - "username": internal.GetUser(), - "password": internal.GetPass(), + "hosts": eslegtest.GetEsHost(), + "username": eslegtest.GetUser(), + "password": eslegtest.GetPass(), "template.enabled": false, }) if err != nil { @@ -319,20 +281,6 @@ func connectTestEs(t *testing.T, cfg interface{}) (outputs.Client, *Client) { return client, client } -// getTestingElasticsearch creates a test client. -func getTestingElasticsearch(t internal.TestLogger) *Client { - client, err := NewClient(ClientSettings{ - URL: internal.GetURL(), - Index: outil.MakeSelector(), - Username: internal.GetUser(), - Password: internal.GetPass(), - Timeout: 60 * time.Second, - CompressionLevel: 3, - }, nil) - internal.InitClient(t, client, err) - return client -} - func randomClient(grp outputs.Group) outputs.NetworkClient { L := len(grp.Clients) if L == 0 { diff --git a/libbeat/outputs/elasticsearch/client_proxy_test.go b/libbeat/outputs/elasticsearch/client_proxy_test.go index 4d57f87fb78..b6751860e0a 100644 --- a/libbeat/outputs/elasticsearch/client_proxy_test.go +++ b/libbeat/outputs/elasticsearch/client_proxy_test.go @@ -34,6 +34,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/outputs/outil" ) @@ -184,10 +185,12 @@ func doClientPing(t *testing.T) { // if TEST_PROXY_DISABLE is nonempty, set ClientSettings.ProxyDisable. proxyDisable := os.Getenv("TEST_PROXY_DISABLE") clientSettings := ClientSettings{ - URL: serverURL, - Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), - Headers: map[string]string{headerTestField: headerTestValue}, - ProxyDisable: proxyDisable != "", + ConnectionSettings: eslegclient.ConnectionSettings{ + URL: serverURL, + Headers: map[string]string{headerTestField: headerTestValue}, + ProxyDisable: proxyDisable != "", + }, + Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), } if proxy != "" { proxyURL, err := url.Parse(proxy) @@ -200,7 +203,7 @@ func doClientPing(t *testing.T) { // This ping won't succeed; we aren't testing end-to-end communication // (which would require a lot more setup work), we just want to make sure // the client is pointed at the right server or proxy. - client.Ping() + client.Connect() } // serverState contains the state of the http listeners for proxy tests, diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index 9cabf8f9a6f..d69849dabab 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -32,6 +32,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs/outest" @@ -40,57 +41,6 @@ import ( "github.com/elastic/beats/v7/libbeat/version" ) -func readStatusItem(in []byte) (int, string, error) { - reader := NewJSONReader(in) - code, msg, err := BulkReadItemStatus(logp.L(), reader) - return code, string(msg), err -} - -func TestESNoErrorStatus(t *testing.T) { - response := []byte(`{"create": {"status": 200}}`) - code, msg, err := readStatusItem(response) - - assert.Nil(t, err) - assert.Equal(t, 200, code) - assert.Equal(t, "", msg) -} - -func TestES1StyleErrorStatus(t *testing.T) { - response := []byte(`{"create": {"status": 400, "error": "test error"}}`) - code, msg, err := readStatusItem(response) - - assert.Nil(t, err) - assert.Equal(t, 400, code) - assert.Equal(t, `"test error"`, msg) -} - -func TestES2StyleErrorStatus(t *testing.T) { - response := []byte(`{"create": {"status": 400, "error": {"reason": "test_error"}}}`) - code, msg, err := readStatusItem(response) - - assert.Nil(t, err) - assert.Equal(t, 400, code) - assert.Equal(t, `{"reason": "test_error"}`, msg) -} - -func TestES2StyleExtendedErrorStatus(t *testing.T) { - response := []byte(` - { - "create": { - "status": 400, - "error": { - "reason": "test_error", - "transient": false, - "extra": null - } - } - }`) - code, _, err := readStatusItem(response) - - assert.Nil(t, err) - assert.Equal(t, 400, code) -} - func TestCollectPublishFailsNone(t *testing.T) { N := 100 item := `{"create": {"status": 200}},` @@ -102,8 +52,7 @@ func TestCollectPublishFailsNone(t *testing.T) { events[i] = publisher.Event{Content: beat.Event{Fields: event}} } - reader := NewJSONReader(response) - res, _ := bulkCollectPublishFails(logp.L(), reader, events) + res, _ := bulkCollectPublishFails(logp.L(), response, events) assert.Equal(t, 0, len(res)) } @@ -120,8 +69,7 @@ func TestCollectPublishFailMiddle(t *testing.T) { eventFail := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 2}}} events := []publisher.Event{event, eventFail, event} - reader := NewJSONReader(response) - res, stats := bulkCollectPublishFails(logp.L(), reader, events) + res, stats := bulkCollectPublishFails(logp.L(), response, events) assert.Equal(t, 1, len(res)) if len(res) == 1 { assert.Equal(t, eventFail, res[0]) @@ -141,8 +89,7 @@ func TestCollectPublishFailAll(t *testing.T) { event := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 2}}} events := []publisher.Event{event, event, event} - reader := NewJSONReader(response) - res, stats := bulkCollectPublishFails(logp.L(), reader, events) + res, stats := bulkCollectPublishFails(logp.L(), response, events) assert.Equal(t, 3, len(res)) assert.Equal(t, events, res) assert.Equal(t, stats, bulkResultStats{fails: 3, tooMany: 3}) @@ -183,8 +130,7 @@ func TestCollectPipelinePublishFail(t *testing.T) { event := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 2}}} events := []publisher.Event{event} - reader := NewJSONReader(response) - res, _ := bulkCollectPublishFails(logp.L(), reader, events) + res, _ := bulkCollectPublishFails(logp.L(), response, events) assert.Equal(t, 1, len(res)) assert.Equal(t, events, res) } @@ -201,10 +147,8 @@ func BenchmarkCollectPublishFailsNone(b *testing.B) { event := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 1}}} events := []publisher.Event{event, event, event} - reader := NewJSONReader(nil) for i := 0; i < b.N; i++ { - reader.init(response) - res, _ := bulkCollectPublishFails(logp.L(), reader, events) + res, _ := bulkCollectPublishFails(logp.L(), response, events) if len(res) != 0 { b.Fail() } @@ -224,10 +168,8 @@ func BenchmarkCollectPublishFailMiddle(b *testing.B) { eventFail := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 2}}} events := []publisher.Event{event, eventFail, event} - reader := NewJSONReader(nil) for i := 0; i < b.N; i++ { - reader.init(response) - res, _ := bulkCollectPublishFails(logp.L(), reader, events) + res, _ := bulkCollectPublishFails(logp.L(), response, events) if len(res) != 1 { b.Fail() } @@ -246,10 +188,8 @@ func BenchmarkCollectPublishFailAll(b *testing.B) { event := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 2}}} events := []publisher.Event{event, event, event} - reader := NewJSONReader(nil) for i := 0; i < b.N; i++ { - reader.init(response) - res, _ := bulkCollectPublishFails(logp.L(), reader, events) + res, _ := bulkCollectPublishFails(logp.L(), response, events) if len(res) != 3 { b.Fail() } @@ -266,24 +206,32 @@ func TestClientWithHeaders(t *testing.T) { // Request.Host field and removed from the Header map. assert.Equal(t, "myhost.local", r.Host) - bulkResponse := `{"items":[{"index":{}},{"index":{}},{"index":{}}]}` - fmt.Fprintln(w, bulkResponse) + var response string + if r.URL.Path == "/" { + response = `{ "version": { "number": "7.6.0" } }` + } else { + response = `{"items":[{"index":{}},{"index":{}},{"index":{}}]}` + + } + fmt.Fprintln(w, response) requestCount++ })) defer ts.Close() client, err := NewClient(ClientSettings{ - URL: ts.URL, - Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), - Headers: map[string]string{ - "host": "myhost.local", - "X-Test": "testing value", + ConnectionSettings: eslegclient.ConnectionSettings{ + URL: ts.URL, + Headers: map[string]string{ + "host": "myhost.local", + "X-Test": "testing value", + }, }, + Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), }, nil) assert.NoError(t, err) // simple ping - client.Ping() + client.Connect() assert.Equal(t, 1, requestCount) // bulk request @@ -299,68 +247,25 @@ func TestClientWithHeaders(t *testing.T) { assert.Equal(t, 2, requestCount) } -func TestAddToURL(t *testing.T) { - type Test struct { - url string - path string - pipeline string - params map[string]string - expected string - } - tests := []Test{ - { - url: "localhost:9200", - path: "/path", - pipeline: "", - params: make(map[string]string), - expected: "localhost:9200/path", - }, - { - url: "localhost:9200/", - path: "/path", - pipeline: "", - params: make(map[string]string), - expected: "localhost:9200/path", - }, - { - url: "localhost:9200", - path: "/path", - pipeline: "pipeline_1", - params: make(map[string]string), - expected: "localhost:9200/path?pipeline=pipeline_1", - }, - { - url: "localhost:9200/", - path: "/path", - pipeline: "", - params: map[string]string{ - "param": "value", - }, - expected: "localhost:9200/path?param=value", - }, - } - for _, test := range tests { - url := addToURL(test.url, test.path, test.pipeline, test.params) - assert.Equal(t, url, test.expected) - } -} - -type testBulkRecorder struct { - data []interface{} - inAction bool -} - func TestBulkEncodeEvents(t *testing.T) { cases := map[string]struct { + version string docType string config common.MapStr events []common.MapStr }{ - "Beats 7.x event": { + "6.x": { + version: "6.8.0", docType: "doc", config: common.MapStr{}, events: []common.MapStr{{"message": "test"}}, }, + "latest": { + version: version.GetDefaultVersion(), + docType: "", + config: common.MapStr{}, + events: []common.MapStr{{"message": "test"}}, + }, } for name, test := range cases { @@ -369,7 +274,7 @@ func TestBulkEncodeEvents(t *testing.T) { cfg := common.MustNewConfigFrom(test.config) info := beat.Info{ IndexPrefix: "test", - Version: version.GetDefaultVersion(), + Version: test.version, } im, err := idxmgmt.DefaultSupport(nil, info, common.NewConfig()) @@ -388,19 +293,17 @@ func TestBulkEncodeEvents(t *testing.T) { } } - recorder := &testBulkRecorder{} - - encoded := bulkEncodePublishRequest(logp.L(), common.Version{Major: 7, Minor: 5}, recorder, index, pipeline, test.docType, events) + encoded, bulkItems := bulkEncodePublishRequest(logp.L(), *common.MustNewVersion(test.version), index, pipeline, events) assert.Equal(t, len(events), len(encoded), "all events should have been encoded") - assert.False(t, recorder.inAction, "incomplete bulk") + assert.Equal(t, 2*len(events), len(bulkItems), "incomplete bulk") // check meta-data for each event - for i := 0; i < len(recorder.data); i += 2 { - var meta bulkEventMeta - switch v := recorder.data[i].(type) { - case bulkCreateAction: + for i := 0; i < len(bulkItems); i += 2 { + var meta eslegclient.BulkMeta + switch v := bulkItems[i].(type) { + case eslegclient.BulkCreateAction: meta = v.Create - case bulkIndexAction: + case eslegclient.BulkIndexAction: meta = v.Index default: panic("unknown type") @@ -415,21 +318,6 @@ func TestBulkEncodeEvents(t *testing.T) { } } -func (r *testBulkRecorder) Add(meta, obj interface{}) error { - if r.inAction { - panic("can not add a new action if other action is active") - } - - r.data = append(r.data, meta, obj) - return nil -} - -func (r *testBulkRecorder) AddRaw(raw interface{}) error { - r.data = append(r.data) - r.inAction = !r.inAction - return nil -} - func TestClientWithAPIKey(t *testing.T) { var headers http.Header @@ -440,63 +328,13 @@ func TestClientWithAPIKey(t *testing.T) { defer ts.Close() client, err := NewClient(ClientSettings{ - URL: ts.URL, - APIKey: "hyokHG4BfWk5viKZ172X:o45JUkyuS--yiSAuuxl8Uw", + ConnectionSettings: eslegclient.ConnectionSettings{ + URL: ts.URL, + APIKey: "hyokHG4BfWk5viKZ172X:o45JUkyuS--yiSAuuxl8Uw", + }, }, nil) assert.NoError(t, err) - client.Ping() + client.Connect() assert.Equal(t, "ApiKey aHlva0hHNEJmV2s1dmlLWjE3Mlg6bzQ1SlVreXVTLS15aVNBdXV4bDhVdw==", headers.Get("Authorization")) } - -func TestBulkReadToItems(t *testing.T) { - response := []byte(`{ - "errors": false, - "items": [ - {"create": {"status": 200}}, - {"create": {"status": 300}}, - {"create": {"status": 400}} - ]}`) - - reader := NewJSONReader(response) - - err := BulkReadToItems(reader) - assert.NoError(t, err) - - for status := 200; status <= 400; status += 100 { - err = reader.ExpectDict() - assert.NoError(t, err) - - kind, raw, err := reader.nextFieldName() - assert.NoError(t, err) - assert.Equal(t, mapKeyEntity, kind) - assert.Equal(t, []byte("create"), raw) - - err = reader.ExpectDict() - assert.NoError(t, err) - - kind, raw, err = reader.nextFieldName() - assert.NoError(t, err) - assert.Equal(t, mapKeyEntity, kind) - assert.Equal(t, []byte("status"), raw) - - code, err := reader.nextInt() - assert.NoError(t, err) - assert.Equal(t, status, code) - - _, _, err = reader.endDict() - assert.NoError(t, err) - - _, _, err = reader.endDict() - assert.NoError(t, err) - } -} - -func TestBulkReadItemStatus(t *testing.T) { - response := []byte(`{"create": {"status": 200}}`) - - reader := NewJSONReader(response) - code, _, err := BulkReadItemStatus(logp.L(), reader) - assert.NoError(t, err) - assert.Equal(t, 200, code) -} diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index 4cbf449b6ec..499bba2eeff 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) @@ -78,7 +79,7 @@ var ( func (c *elasticsearchConfig) Validate() error { if c.ProxyURL != "" && !c.ProxyDisable { - if _, err := parseProxyURL(c.ProxyURL); err != nil { + if _, err := common.ParseURL(c.ProxyURL); err != nil { return err } } diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index b80fd832d1f..b6c3bd797a9 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -18,16 +18,12 @@ package elasticsearch import ( - "errors" - "fmt" "net/url" - "sync" - - "github.com/gofrs/uuid" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" "github.com/elastic/beats/v7/libbeat/outputs/outil" @@ -37,101 +33,8 @@ func init() { outputs.RegisterType("elasticsearch", makeES) } -var ( - // ErrNotConnected indicates failure due to client having no valid connection - ErrNotConnected = errors.New("not connected") - - // ErrJSONEncodeFailed indicates encoding failures - ErrJSONEncodeFailed = errors.New("json encode failed") - - // ErrResponseRead indicates error parsing Elasticsearch response - ErrResponseRead = errors.New("bulk item status parse failed") -) - const logSelector = "elasticsearch" -// Callbacks must not depend on the result of a previous one, -// because the ordering is not fixed. -type callbacksRegistry struct { - callbacks map[uuid.UUID]ConnectCallback - mutex sync.Mutex -} - -// XXX: it would be fantastic to do this without a package global -var connectCallbackRegistry = newCallbacksRegistry() - -// NOTE(ph): We need to refactor this, right now this is the only way to ensure that every calls -// to an ES cluster executes a callback. -var globalCallbackRegistry = newCallbacksRegistry() - -// RegisterGlobalCallback register a global callbacks. -func RegisterGlobalCallback(callback ConnectCallback) (uuid.UUID, error) { - globalCallbackRegistry.mutex.Lock() - defer globalCallbackRegistry.mutex.Unlock() - - // find the next unique key - var key uuid.UUID - var err error - exists := true - for exists { - key, err = uuid.NewV4() - if err != nil { - return uuid.Nil, err - } - _, exists = globalCallbackRegistry.callbacks[key] - } - - globalCallbackRegistry.callbacks[key] = callback - return key, nil -} - -func newCallbacksRegistry() callbacksRegistry { - return callbacksRegistry{ - callbacks: make(map[uuid.UUID]ConnectCallback), - } -} - -// RegisterConnectCallback registers a callback for the elasticsearch output -// The callback is called each time the client connects to elasticsearch. -// It returns the key of the newly added callback, so it can be deregistered later. -func RegisterConnectCallback(callback ConnectCallback) (uuid.UUID, error) { - connectCallbackRegistry.mutex.Lock() - defer connectCallbackRegistry.mutex.Unlock() - - // find the next unique key - var key uuid.UUID - var err error - exists := true - for exists { - key, err = uuid.NewV4() - if err != nil { - return uuid.Nil, err - } - _, exists = connectCallbackRegistry.callbacks[key] - } - - connectCallbackRegistry.callbacks[key] = callback - return key, nil -} - -// DeregisterConnectCallback deregisters a callback for the elasticsearch output -// specified by its key. If a callback does not exist, nothing happens. -func DeregisterConnectCallback(key uuid.UUID) { - connectCallbackRegistry.mutex.Lock() - defer connectCallbackRegistry.mutex.Unlock() - - delete(connectCallbackRegistry.callbacks, key) -} - -// DeregisterGlobalCallback deregisters a callback for the elasticsearch output -// specified by its key. If a callback does not exist, nothing happens. -func DeregisterGlobalCallback(key uuid.UUID) { - globalCallbackRegistry.mutex.Lock() - defer globalCallbackRegistry.mutex.Unlock() - - delete(globalCallbackRegistry.callbacks, key) -} - func makeES( im outputs.IndexManager, beat beat.Info, @@ -165,7 +68,7 @@ func makeES( var proxyURL *url.URL if !config.ProxyDisable { - proxyURL, err = parseProxyURL(config.ProxyURL) + proxyURL, err = common.ParseURL(config.ProxyURL) if err != nil { return outputs.Fail(err) } @@ -189,21 +92,24 @@ func makeES( var client outputs.NetworkClient client, err = NewClient(ClientSettings{ - URL: esURL, - Index: index, - Pipeline: pipeline, - Proxy: proxyURL, - ProxyDisable: config.ProxyDisable, - TLS: tlsConfig, - Username: config.Username, - Password: config.Password, - APIKey: config.APIKey, - Parameters: params, - Headers: config.Headers, - Timeout: config.Timeout, - CompressionLevel: config.CompressionLevel, - Observer: observer, - EscapeHTML: config.EscapeHTML, + ConnectionSettings: eslegclient.ConnectionSettings{ + URL: esURL, + Proxy: proxyURL, + ProxyDisable: config.ProxyDisable, + TLS: tlsConfig, + Username: config.Username, + Password: config.Password, + APIKey: config.APIKey, + Parameters: params, + Headers: config.Headers, + Timeout: config.Timeout, + CompressionLevel: config.CompressionLevel, + Observer: observer, + EscapeHTML: config.EscapeHTML, + }, + Index: index, + Pipeline: pipeline, + Observer: observer, }, &connectCallbackRegistry) if err != nil { return outputs.Fail(err) @@ -242,97 +148,3 @@ func buildSelectors( return index, pipeline, err } - -// NewConnectedClient creates a new Elasticsearch client based on the given config. -// It uses the NewElasticsearchClients to create a list of clients then returns -// the first from the list that successfully connects. -func NewConnectedClient(cfg *common.Config) (*Client, error) { - clients, err := NewElasticsearchClients(cfg) - if err != nil { - return nil, err - } - - errors := []string{} - - for _, client := range clients { - err = client.Connect() - if err != nil { - client.Connection.log.Errorf("Error connecting to Elasticsearch at %v: %+v", client.Connection.URL, err) - err = fmt.Errorf("Error connection to Elasticsearch %v: %v", client.Connection.URL, err) - errors = append(errors, err.Error()) - continue - } - return &client, nil - } - return nil, fmt.Errorf("Couldn't connect to any of the configured Elasticsearch hosts. Errors: %v", errors) -} - -// NewElasticsearchClients returns a list of Elasticsearch clients based on the given -// configuration. It accepts the same configuration parameters as the output, -// except for the output specific configuration options (index, pipeline, -// template) .If multiple hosts are defined in the configuration, a client is returned -// for each of them. -func NewElasticsearchClients(cfg *common.Config) ([]Client, error) { - hosts, err := outputs.ReadHostList(cfg) - if err != nil { - return nil, err - } - - config := defaultConfig - if err = cfg.Unpack(&config); err != nil { - return nil, err - } - - tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS) - if err != nil { - return nil, err - } - - log := logp.NewLogger(logSelector) - var proxyURL *url.URL - if !config.ProxyDisable { - proxyURL, err = parseProxyURL(config.ProxyURL) - if err != nil { - return nil, err - } - if proxyURL != nil { - log.Infof("Using proxy URL: %s", proxyURL) - } - } - - params := config.Params - if len(params) == 0 { - params = nil - } - - clients := []Client{} - for _, host := range hosts { - esURL, err := common.MakeURL(config.Protocol, config.Path, host, 9200) - if err != nil { - log.Errorf("Invalid host param set: %s, Error: %+v", host, err) - return nil, err - } - - client, err := NewClient(ClientSettings{ - URL: esURL, - Proxy: proxyURL, - ProxyDisable: config.ProxyDisable, - TLS: tlsConfig, - Username: config.Username, - Password: config.Password, - APIKey: config.APIKey, - Parameters: params, - Headers: config.Headers, - Timeout: config.Timeout, - CompressionLevel: config.CompressionLevel, - }, nil) - if err != nil { - return clients, err - } - clients = append(clients, *client) - } - if len(clients) == 0 { - return clients, fmt.Errorf("No hosts defined in the Elasticsearch output") - } - return clients, nil -} diff --git a/libbeat/outputs/elasticsearch/elasticsearch_test.go b/libbeat/outputs/elasticsearch/elasticsearch_test.go index 0fd3ae91db3..60268b59602 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch_test.go +++ b/libbeat/outputs/elasticsearch/elasticsearch_test.go @@ -20,12 +20,14 @@ package elasticsearch import ( "fmt" "testing" + + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" ) func TestConnectCallbacksManagement(t *testing.T) { - f0 := func(client *Client) error { fmt.Println("i am function #0"); return nil } - f1 := func(client *Client) error { fmt.Println("i am function #1"); return nil } - f2 := func(client *Client) error { fmt.Println("i am function #2"); return nil } + f0 := func(client *eslegclient.Connection) error { fmt.Println("i am function #0"); return nil } + f1 := func(client *eslegclient.Connection) error { fmt.Println("i am function #1"); return nil } + f2 := func(client *eslegclient.Connection) error { fmt.Println("i am function #2"); return nil } _, err := RegisterConnectCallback(f0) if err != nil { @@ -48,9 +50,9 @@ func TestConnectCallbacksManagement(t *testing.T) { } func TestGlobalConnectCallbacksManagement(t *testing.T) { - f0 := func(client *Client) error { fmt.Println("i am function #0"); return nil } - f1 := func(client *Client) error { fmt.Println("i am function #1"); return nil } - f2 := func(client *Client) error { fmt.Println("i am function #2"); return nil } + f0 := func(client *eslegclient.Connection) error { fmt.Println("i am function #0"); return nil } + f1 := func(client *eslegclient.Connection) error { fmt.Println("i am function #1"); return nil } + f2 := func(client *eslegclient.Connection) error { fmt.Println("i am function #2"); return nil } _, err := RegisterGlobalCallback(f0) if err != nil { diff --git a/libbeat/outputs/elasticsearch/json_read.go b/libbeat/outputs/elasticsearch/json_read.go index 8df87e5cb43..eb7d3c696ee 100644 --- a/libbeat/outputs/elasticsearch/json_read.go +++ b/libbeat/outputs/elasticsearch/json_read.go @@ -30,7 +30,7 @@ import ( // // Due to parser simply stepping through the input buffer, almost no additional // allocations are required. -type JSONReader struct { +type jsonReader struct { streambuf.Buffer // parser state machine @@ -133,14 +133,13 @@ func (s state) String() string { return "unknown" } -// NewJSONReader returns a new JSONReader initialized with in -func NewJSONReader(in []byte) *JSONReader { - r := &JSONReader{} +func newJSONReader(in []byte) *jsonReader { + r := &jsonReader{} r.init(in) return r } -func (r *JSONReader) init(in []byte) { +func (r *jsonReader) init(in []byte) { r.Buffer.Init(in, true) r.currentState = startState r.states = r.statesBuf[:0] @@ -148,18 +147,18 @@ func (r *JSONReader) init(in []byte) { var whitespace = []byte(" \t\r\n") -func (r *JSONReader) skipWS() { +func (r *jsonReader) skipWS() { r.IgnoreSymbols(whitespace) } -func (r *JSONReader) pushState(next state) { +func (r *jsonReader) pushState(next state) { if r.currentState != failedState { r.states = append(r.states, r.currentState) } r.currentState = next } -func (r *JSONReader) popState() { +func (r *jsonReader) popState() { if len(r.states) == 0 { r.currentState = failedState } else { @@ -170,7 +169,7 @@ func (r *JSONReader) popState() { } // ExpectDict checks if the next entity is a json object -func (r *JSONReader) ExpectDict() error { +func (r *jsonReader) ExpectDict() error { e, _, err := r.step() if err != nil { @@ -185,7 +184,7 @@ func (r *JSONReader) ExpectDict() error { } // ExpectArray checks if the next entity is a json array -func (r *JSONReader) ExpectArray() error { +func (r *jsonReader) ExpectArray() error { e, _, err := r.step() if err != nil { return err @@ -198,7 +197,7 @@ func (r *JSONReader) ExpectArray() error { return nil } -func (r *JSONReader) nextFieldName() (entity, []byte, error) { +func (r *jsonReader) nextFieldName() (entity, []byte, error) { e, raw, err := r.step() if err != nil { return e, raw, err @@ -211,7 +210,7 @@ func (r *JSONReader) nextFieldName() (entity, []byte, error) { return e, raw, err } -func (r *JSONReader) nextInt() (int, error) { +func (r *jsonReader) nextInt() (int, error) { e, raw, err := r.step() if err != nil { return 0, err @@ -227,7 +226,7 @@ func (r *JSONReader) nextInt() (int, error) { } // ignore type of next element and return raw content. -func (r *JSONReader) ignoreNext() (raw []byte, err error) { +func (r *jsonReader) ignoreNext() (raw []byte, err error) { r.skipWS() snapshot := r.Snapshot() @@ -256,7 +255,7 @@ func (r *JSONReader) ignoreNext() (raw []byte, err error) { return bytes, nil } -func ignoreKind(r *JSONReader, kind entity) error { +func ignoreKind(r *jsonReader, kind entity) error { for { e, _, err := r.step() if err != nil { @@ -279,7 +278,7 @@ func ignoreKind(r *JSONReader, kind entity) error { } // step continues the JSON parser state machine until next entity has been parsed. -func (r *JSONReader) step() (entity, []byte, error) { +func (r *jsonReader) step() (entity, []byte, error) { r.skipWS() switch r.currentState { case failedState: @@ -301,11 +300,11 @@ func (r *JSONReader) step() (entity, []byte, error) { } } -func (r *JSONReader) stepFailing() (entity, []byte, error) { +func (r *jsonReader) stepFailing() (entity, []byte, error) { return failEntity, nil, r.Err() } -func (r *JSONReader) stepStart() (entity, []byte, error) { +func (r *jsonReader) stepStart() (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(err) @@ -314,11 +313,11 @@ func (r *JSONReader) stepStart() (entity, []byte, error) { return r.tryStepPrimitive(c) } -func (r *JSONReader) stepArray() (entity, []byte, error) { +func (r *jsonReader) stepArray() (entity, []byte, error) { return r.doStepArray(true) } -func (r *JSONReader) stepArrayNext() (entity, []byte, error) { +func (r *jsonReader) stepArrayNext() (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(errFailing) @@ -337,7 +336,7 @@ func (r *JSONReader) stepArrayNext() (entity, []byte, error) { } } -func (r *JSONReader) doStepArray(allowArrayEnd bool) (entity, []byte, error) { +func (r *jsonReader) doStepArray(allowArrayEnd bool) (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(err) @@ -354,11 +353,11 @@ func (r *JSONReader) doStepArray(allowArrayEnd bool) (entity, []byte, error) { return r.tryStepPrimitive(c) } -func (r *JSONReader) stepDict() (entity, []byte, error) { +func (r *jsonReader) stepDict() (entity, []byte, error) { return r.doStepDict(true) } -func (r *JSONReader) doStepDict(allowEnd bool) (entity, []byte, error) { +func (r *jsonReader) doStepDict(allowEnd bool) (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(err) @@ -378,7 +377,7 @@ func (r *JSONReader) doStepDict(allowEnd bool) (entity, []byte, error) { } } -func (r *JSONReader) stepDictValue() (entity, []byte, error) { +func (r *jsonReader) stepDictValue() (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(err) @@ -388,7 +387,7 @@ func (r *JSONReader) stepDictValue() (entity, []byte, error) { return r.tryStepPrimitive(c) } -func (r *JSONReader) stepDictValueEnd() (entity, []byte, error) { +func (r *jsonReader) stepDictValueEnd() (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(err) @@ -407,7 +406,7 @@ func (r *JSONReader) stepDictValueEnd() (entity, []byte, error) { } } -func (r *JSONReader) tryStepPrimitive(c byte) (entity, []byte, error) { +func (r *jsonReader) tryStepPrimitive(c byte) (entity, []byte, error) { switch c { case '{': // start dictionary return r.startDict() @@ -435,19 +434,19 @@ func (r *JSONReader) tryStepPrimitive(c byte) (entity, []byte, error) { } } -func (r *JSONReader) stepNull() (entity, []byte, error) { +func (r *jsonReader) stepNull() (entity, []byte, error) { return stepSymbol(r, nullValue, nullSymbol, errExpectedNull) } -func (r *JSONReader) stepTrue() (entity, []byte, error) { +func (r *jsonReader) stepTrue() (entity, []byte, error) { return stepSymbol(r, trueValue, trueSymbol, errExpectedTrue) } -func (r *JSONReader) stepFalse() (entity, []byte, error) { +func (r *jsonReader) stepFalse() (entity, []byte, error) { return stepSymbol(r, falseValue, falseSymbol, errExpectedFalse) } -func stepSymbol(r *JSONReader, e entity, symb []byte, fail error) (entity, []byte, error) { +func stepSymbol(r *jsonReader, e entity, symb []byte, fail error) (entity, []byte, error) { ok, err := r.MatchASCII(symb) if err != nil { return failEntity, nil, err @@ -460,7 +459,7 @@ func stepSymbol(r *JSONReader, e entity, symb []byte, fail error) (entity, []byt return e, nil, nil } -func (r *JSONReader) stepMapKey() (entity, []byte, error) { +func (r *jsonReader) stepMapKey() (entity, []byte, error) { e, key, err := r.stepString() if err != nil { return e, key, err @@ -482,7 +481,7 @@ func (r *JSONReader) stepMapKey() (entity, []byte, error) { return mapKeyEntity, key, nil } -func (r *JSONReader) stepString() (entity, []byte, error) { +func (r *jsonReader) stepString() (entity, []byte, error) { start := 1 for { idxQuote := r.IndexByteFrom(start, '"') @@ -502,36 +501,36 @@ func (r *JSONReader) stepString() (entity, []byte, error) { } } -func (r *JSONReader) startDict() (entity, []byte, error) { +func (r *jsonReader) startDict() (entity, []byte, error) { r.Advance(1) r.pushState(dictState) return dictStart, nil, nil } -func (r *JSONReader) endDict() (entity, []byte, error) { +func (r *jsonReader) endDict() (entity, []byte, error) { r.Advance(1) r.popState() return dictEnd, nil, nil } -func (r *JSONReader) startArray() (entity, []byte, error) { +func (r *jsonReader) startArray() (entity, []byte, error) { r.Advance(1) r.pushState(arrState) return arrStart, nil, nil } -func (r *JSONReader) endArray() (entity, []byte, error) { +func (r *jsonReader) endArray() (entity, []byte, error) { r.Advance(1) r.popState() return arrEnd, nil, nil } -func (r *JSONReader) failWith(err error) (entity, []byte, error) { +func (r *jsonReader) failWith(err error) (entity, []byte, error) { r.currentState = failedState return failEntity, nil, r.SetError(err) } -func (r *JSONReader) stepNumber() (entity, []byte, error) { +func (r *jsonReader) stepNumber() (entity, []byte, error) { snapshot := r.Snapshot() lenBefore := r.Len() isDouble := false diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 2b5a0dbfc9b..6dfebbabcec 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -32,9 +32,10 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/fmtstr" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/outputs" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" + _ "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" "github.com/elastic/beats/v7/libbeat/outputs/outest" "github.com/elastic/beats/v7/libbeat/outputs/outil" ) @@ -49,7 +50,7 @@ const ( ) type esConnection struct { - *elasticsearch.Client + *eslegclient.Connection t *testing.T index string } @@ -100,13 +101,12 @@ func esConnect(t *testing.T, index string) *esConnection { username := os.Getenv("ES_USER") password := os.Getenv("ES_PASS") - client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ + client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: host, - Index: indexSel, Username: username, Password: password, Timeout: 60 * time.Second, - }, nil) + }) if err != nil { t.Fatal(err) } @@ -126,7 +126,7 @@ func esConnect(t *testing.T, index string) *esConnection { es := &esConnection{} es.t = t - es.Client = client + es.Connection = client es.index = index return es } diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 958b311057f..1a53cc75073 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -34,8 +34,8 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch/estest" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" + "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" "github.com/elastic/beats/v7/libbeat/version" ) @@ -60,7 +60,7 @@ func newTestSetup(t *testing.T, cfg TemplateConfig) *testSetup { if cfg.Name == "" { cfg.Name = fmt.Sprintf("load-test-%+v", rand.Int()) } - client := estest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if err := client.Connect(); err != nil { t.Fatal(err) } @@ -290,7 +290,7 @@ func TestTemplateWithData(t *testing.T) { setup := newTestSetup(t, TemplateConfig{Enabled: true}) require.NoError(t, setup.loadFromFile([]string{"testdata", "fields.yml"})) require.True(t, setup.loader.templateExists(setup.config.Name)) - esClient := setup.client.(*elasticsearch.Client) + esClient := setup.client.(*eslegclient.Connection) for _, test := range dataTests { _, _, err := esClient.Index(setup.config.Name, "_doc", "", nil, test.data) if test.error { @@ -350,3 +350,24 @@ func path(t *testing.T, fileElems []string) string { require.NoError(t, err) return fieldsPath } + +func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { + conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ + URL: eslegtest.GetURL(), + Timeout: 0, + }) + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + conn.Encoder = eslegclient.NewJSONEncoder(nil, false) + + err = conn.Connect() + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + return conn +} diff --git a/x-pack/libbeat/licenser/elastic_fetcher.go b/x-pack/libbeat/licenser/elastic_fetcher.go index 1edde4e7992..6ffa5f6fa37 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher.go +++ b/x-pack/libbeat/licenser/elastic_fetcher.go @@ -14,8 +14,8 @@ import ( "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" ) const xPackURL = "/_license" @@ -146,7 +146,7 @@ func (f *ElasticFetcher) parseJSON(b []byte) (*License, error) { // esClientMux is taking care of round robin request over an array of elasticsearch client, note that // calling request is not threadsafe. type esClientMux struct { - clients []elasticsearch.Client + clients []eslegclient.Connection idx int } @@ -177,12 +177,12 @@ func (mux *esClientMux) Request( // newESClientMux takes a list of clients and randomize where we start and the list of host we are // querying. -func newESClientMux(clients []elasticsearch.Client) *esClientMux { +func newESClientMux(clients []eslegclient.Connection) *esClientMux { // randomize where we start idx := rand.Intn(len(clients)) // randomize the list of round robin hosts. - tmp := make([]elasticsearch.Client, len(clients)) + tmp := make([]eslegclient.Connection, len(clients)) copy(tmp, clients) rand.Shuffle(len(tmp), func(i, j int) { tmp[i], tmp[j] = tmp[j], tmp[i] diff --git a/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go b/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go index 63b38fdb962..8cf182ad717 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go +++ b/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go @@ -13,8 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common/cli" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/outil" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" ) const ( @@ -22,16 +21,15 @@ const ( elasticsearchPort = "9200" ) -func getTestClient() *elasticsearch.Client { +func getTestClient() *eslegclient.Connection { host := "http://" + cli.GetEnvOr("ES_HOST", elasticsearchHost) + ":" + cli.GetEnvOr("ES_POST", elasticsearchPort) - client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ + client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: host, - Index: outil.MakeSelector(), Username: "myelastic", // NOTE: I will refactor this in a followup PR Password: "changeme", - Timeout: 60 * time.Second, CompressionLevel: 3, - }, nil) + Timeout: 60 * time.Second, + }) if err != nil { panic(err) diff --git a/x-pack/libbeat/licenser/elastic_fetcher_test.go b/x-pack/libbeat/licenser/elastic_fetcher_test.go index 87b92d6bcca..660df43fbba 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher_test.go +++ b/x-pack/libbeat/licenser/elastic_fetcher_test.go @@ -14,18 +14,21 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" + "github.com/stretchr/testify/assert" ) -func newServerClientPair(t *testing.T, handler http.HandlerFunc) (*httptest.Server, *elasticsearch.Client) { +func newServerClientPair(t *testing.T, handler http.HandlerFunc) (*httptest.Server, *eslegclient.Connection) { mux := http.NewServeMux() mux.Handle("/_license/", http.HandlerFunc(handler)) server := httptest.NewServer(mux) - client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{URL: server.URL}, nil) + client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ + URL: server.URL, + Timeout: 90 * time.Second, + }) if err != nil { t.Fatalf("could not create the elasticsearch client, error: %s", err) } diff --git a/x-pack/libbeat/licenser/es_callback.go b/x-pack/libbeat/licenser/es_callback.go index 5c15c3eaab4..0ca99db8f5f 100644 --- a/x-pack/libbeat/licenser/es_callback.go +++ b/x-pack/libbeat/licenser/es_callback.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" ) @@ -21,7 +22,7 @@ const licenseDebugK = "license" func Enforce(name string, checks ...CheckFunc) { name = strings.Title(name) - cb := func(client *elasticsearch.Client) error { + cb := func(client *eslegclient.Connection) error { // Logger created earlier than this place are at risk of discarding any log statement. log := logp.NewLogger(licenseDebugK)