From 5cffba87143ffcb9b83755f263a0a3228d1278d9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Feb 2020 11:56:55 -0800 Subject: [PATCH 01/91] Basic extraction of ES client code from ES output code --- go.mod | 1 + go.sum | 3 + .../elasticsearch => esclientleg}/api.go | 2 +- .../api_integration_test.go | 2 +- .../api_mock_test.go | 2 +- .../elasticsearch => esclientleg}/api_test.go | 8 +- .../elasticsearch => esclientleg}/bulkapi.go | 38 +-- .../bulkapi_integration_test.go | 2 +- .../bulkapi_mock_test.go | 2 +- .../elasticsearch => esclientleg}/enc.go | 12 +- .../elasticsearch => esclientleg}/enc_test.go | 6 +- libbeat/esclientleg/errors.go | 31 +++ libbeat/esclientleg/esclientleg.go | 209 ++++++++++++++++ .../elasticsearch => esclientleg}/url.go | 4 +- .../elasticsearch => esclientleg}/url_test.go | 2 +- .../monitoring/report/elasticsearch/client.go | 4 +- libbeat/outputs/elasticsearch/client.go | 226 ++---------------- libbeat/outputs/elasticsearch/client_test.go | 2 +- libbeat/outputs/elasticsearch/config.go | 3 +- .../outputs/elasticsearch/elasticsearch.go | 17 +- 20 files changed, 312 insertions(+), 264 deletions(-) rename libbeat/{outputs/elasticsearch => esclientleg}/api.go (99%) rename libbeat/{outputs/elasticsearch => esclientleg}/api_integration_test.go (99%) rename libbeat/{outputs/elasticsearch => esclientleg}/api_mock_test.go (99%) rename libbeat/{outputs/elasticsearch => esclientleg}/api_test.go (95%) rename libbeat/{outputs/elasticsearch => esclientleg}/bulkapi.go (87%) rename libbeat/{outputs/elasticsearch => esclientleg}/bulkapi_integration_test.go (99%) rename libbeat/{outputs/elasticsearch => esclientleg}/bulkapi_mock_test.go (99%) rename libbeat/{outputs/elasticsearch => esclientleg}/enc.go (95%) rename libbeat/{outputs/elasticsearch => esclientleg}/enc_test.go (95%) create mode 100644 libbeat/esclientleg/errors.go create mode 100644 libbeat/esclientleg/esclientleg.go rename libbeat/{outputs/elasticsearch => esclientleg}/url.go (96%) rename libbeat/{outputs/elasticsearch => esclientleg}/url_test.go (98%) diff --git a/go.mod b/go.mod index d49fb8a0ebe..5332f700520 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 + github.com/elastic/beats v7.6.0+incompatible github.com/elastic/ecs v1.4.0 github.com/elastic/go-libaudit v0.4.0 github.com/elastic/go-licenser v0.2.1 diff --git a/go.sum b/go.sum index d95286d6c56..8bf26d36cfc 100644 --- a/go.sum +++ b/go.sum @@ -209,6 +209,9 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/elastic/beats v7.6.0+incompatible h1:vUcDbZ/qsvox6pC/t/tSdSItlt1/pKbsKaHEVS4GWss= +github.com/elastic/beats v7.6.0+incompatible/go.mod h1:7cX7zGsOwJ01FLkZs9Tg5nBdnQi6XB3hYAyWekpKgeY= +github.com/elastic/beats v7.6.1+incompatible h1:4iPVpFr8BSJW2fUVi+/tYXQ4v/IYVx0k2PPLTg8cfks= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/ecs v1.4.0 h1:BGIUwWJhThRO2IQxzm7ekV9TMUGwZoYyevT5/1xmMf0= diff --git a/libbeat/outputs/elasticsearch/api.go b/libbeat/esclientleg/api.go similarity index 99% rename from libbeat/outputs/elasticsearch/api.go rename to libbeat/esclientleg/api.go index d267fb6a98a..15becfabe21 100644 --- a/libbeat/outputs/elasticsearch/api.go +++ b/libbeat/esclientleg/api.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package elasticsearch +package esclientleg import ( "encoding/json" diff --git a/libbeat/outputs/elasticsearch/api_integration_test.go b/libbeat/esclientleg/api_integration_test.go similarity index 99% rename from libbeat/outputs/elasticsearch/api_integration_test.go rename to libbeat/esclientleg/api_integration_test.go index 41ec6c0634b..f0d4943f6b9 100644 --- a/libbeat/outputs/elasticsearch/api_integration_test.go +++ b/libbeat/esclientleg/api_integration_test.go @@ -17,7 +17,7 @@ // +build integration -package elasticsearch +package esclientleg import ( "encoding/json" diff --git a/libbeat/outputs/elasticsearch/api_mock_test.go b/libbeat/esclientleg/api_mock_test.go similarity index 99% rename from libbeat/outputs/elasticsearch/api_mock_test.go rename to libbeat/esclientleg/api_mock_test.go index 65e68754833..406812c7810 100644 --- a/libbeat/outputs/elasticsearch/api_mock_test.go +++ b/libbeat/esclientleg/api_mock_test.go @@ -17,7 +17,7 @@ // +build !integration -package elasticsearch +package esclientleg import ( "encoding/json" diff --git a/libbeat/outputs/elasticsearch/api_test.go b/libbeat/esclientleg/api_test.go similarity index 95% rename from libbeat/outputs/elasticsearch/api_test.go rename to libbeat/esclientleg/api_test.go index 3706dd64511..2a114b06bc6 100644 --- a/libbeat/outputs/elasticsearch/api_test.go +++ b/libbeat/esclientleg/api_test.go @@ -16,13 +16,15 @@ // under the License. // Need for unit and integration tests -package elasticsearch +package esclientleg import ( "encoding/json" "testing" "time" + "github.com/elastic/beats/libbeat/outputs/elasticsearch" + "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/logp" @@ -172,8 +174,8 @@ func TestReadSearchResult_invalid(t *testing.T) { assert.Error(t, err) } -func newTestClient(url string) *Client { - client, err := NewClient(ClientSettings{ +func newTestClient(url string) *elasticsearch.Client { + client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ URL: url, Index: outil.MakeSelector(), Timeout: 60 * time.Second, diff --git a/libbeat/outputs/elasticsearch/bulkapi.go b/libbeat/esclientleg/bulkapi.go similarity index 87% rename from libbeat/outputs/elasticsearch/bulkapi.go rename to libbeat/esclientleg/bulkapi.go index 325b54e9bef..d954d1d4850 100644 --- a/libbeat/outputs/elasticsearch/bulkapi.go +++ b/libbeat/esclientleg/bulkapi.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package elasticsearch +package esclientleg import ( "bytes" @@ -32,7 +32,7 @@ import ( // MetaBuilder creates meta data for bulk requests type MetaBuilder func(interface{}) interface{} -type bulkRequest struct { +type BulkRequest struct { requ *http.Request } @@ -62,18 +62,18 @@ func (conn *Connection) BulkWith( return nil, nil } - enc := conn.encoder + enc := conn.Encoder enc.Reset() if err := bulkEncode(conn.log, enc, metaBuilder, body); err != nil { return nil, err } - requ, err := newBulkRequest(conn.URL, index, docType, params, enc) + requ, err := NewBulkRequest(conn.URL, index, docType, params, enc) if err != nil { return nil, err } - _, result, err := conn.sendBulkRequest(requ) + _, result, err := conn.SendBulkRequest(requ) if err != nil { return nil, err } @@ -91,7 +91,7 @@ func (conn *Connection) SendMonitoringBulk( return nil, nil } - enc := conn.encoder + enc := conn.Encoder enc.Reset() if err := bulkEncode(conn.log, enc, nil, body); err != nil { return nil, err @@ -103,24 +103,24 @@ func (conn *Connection) SendMonitoringBulk( } } - requ, err := newMonitoringBulkRequest(conn.version, conn.URL, params, enc) + requ, err := newMonitoringBulkRequest(conn.GetVersion(), conn.URL, params, enc) if err != nil { return nil, err } - _, result, err := conn.sendBulkRequest(requ) + _, result, err := conn.SendBulkRequest(requ) if err != nil { return nil, err } return result, nil } -func newBulkRequest( +func NewBulkRequest( urlStr string, index, docType string, params map[string]string, - body bodyEncoder, -) (*bulkRequest, error) { + body BodyEncoder, +) (*BulkRequest, error) { path, err := makePath(index, docType, "_bulk") if err != nil { return nil, err @@ -133,8 +133,8 @@ func newMonitoringBulkRequest( esVersion common.Version, urlStr string, params map[string]string, - body bodyEncoder, -) (*bulkRequest, error) { + body BodyEncoder, +) (*BulkRequest, error) { var path string var err error if esVersion.Major < 7 { @@ -154,8 +154,8 @@ func newBulkRequestWithPath( urlStr string, path string, params map[string]string, - body bodyEncoder, -) (*bulkRequest, error) { + body BodyEncoder, +) (*BulkRequest, error) { url := addToURL(urlStr, path, "", params) var reader io.Reader @@ -172,12 +172,12 @@ func newBulkRequestWithPath( body.AddHeader(&requ.Header) } - return &bulkRequest{ + return &BulkRequest{ requ: requ, }, nil } -func (r *bulkRequest) Reset(body bodyEncoder) { +func (r *BulkRequest) Reset(body BodyEncoder) { bdy := body.Reader() rc, ok := bdy.(io.ReadCloser) @@ -200,12 +200,12 @@ func (r *bulkRequest) Reset(body bodyEncoder) { body.AddHeader(&r.requ.Header) } -func (conn *Connection) sendBulkRequest(requ *bulkRequest) (int, BulkResult, error) { +func (conn *Connection) SendBulkRequest(requ *BulkRequest) (int, BulkResult, error) { status, resp, err := conn.execHTTPRequest(requ.requ) return status, BulkResult(resp), err } -func bulkEncode(log *logp.Logger, out bulkWriter, metaBuilder MetaBuilder, body []interface{}) error { +func bulkEncode(log *logp.Logger, out BulkWriter, metaBuilder MetaBuilder, body []interface{}) error { if metaBuilder == nil { for _, obj := range body { if err := out.AddRaw(obj); err != nil { diff --git a/libbeat/outputs/elasticsearch/bulkapi_integration_test.go b/libbeat/esclientleg/bulkapi_integration_test.go similarity index 99% rename from libbeat/outputs/elasticsearch/bulkapi_integration_test.go rename to libbeat/esclientleg/bulkapi_integration_test.go index 54e2201b398..7f6c16cc8b9 100644 --- a/libbeat/outputs/elasticsearch/bulkapi_integration_test.go +++ b/libbeat/esclientleg/bulkapi_integration_test.go @@ -17,7 +17,7 @@ // +build integration -package elasticsearch +package esclientleg import ( "fmt" diff --git a/libbeat/outputs/elasticsearch/bulkapi_mock_test.go b/libbeat/esclientleg/bulkapi_mock_test.go similarity index 99% rename from libbeat/outputs/elasticsearch/bulkapi_mock_test.go rename to libbeat/esclientleg/bulkapi_mock_test.go index a87d0d1046c..06abdc90c4a 100644 --- a/libbeat/outputs/elasticsearch/bulkapi_mock_test.go +++ b/libbeat/esclientleg/bulkapi_mock_test.go @@ -17,7 +17,7 @@ // +build !integration -package elasticsearch +package esclientleg import ( "fmt" diff --git a/libbeat/outputs/elasticsearch/enc.go b/libbeat/esclientleg/enc.go similarity index 95% rename from libbeat/outputs/elasticsearch/enc.go rename to libbeat/esclientleg/enc.go index 8d2497e5182..79eab7af9fd 100644 --- a/libbeat/outputs/elasticsearch/enc.go +++ b/libbeat/esclientleg/enc.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package elasticsearch +package esclientleg import ( "bytes" @@ -31,20 +31,20 @@ import ( "github.com/elastic/go-structform/json" ) -type bodyEncoder interface { +type BodyEncoder interface { bulkBodyEncoder Reader() io.Reader Marshal(doc interface{}) error } type bulkBodyEncoder interface { - bulkWriter + BulkWriter AddHeader(*http.Header) Reset() } -type bulkWriter interface { +type BulkWriter interface { Add(meta, obj interface{}) error AddRaw(raw interface{}) error } @@ -69,7 +69,7 @@ type event struct { Fields common.MapStr `struct:",inline"` } -func newJSONEncoder(buf *bytes.Buffer, escapeHTML bool) *jsonEncoder { +func NewJSONEncoder(buf *bytes.Buffer, escapeHTML bool) *jsonEncoder { if buf == nil { buf = bytes.NewBuffer(nil) } @@ -142,7 +142,7 @@ func (b *jsonEncoder) Add(meta, obj interface{}) error { return nil } -func newGzipEncoder(level int, buf *bytes.Buffer, escapeHTML bool) (*gzipEncoder, error) { +func NewGzipEncoder(level int, buf *bytes.Buffer, escapeHTML bool) (*gzipEncoder, error) { if buf == nil { buf = bytes.NewBuffer(nil) } diff --git a/libbeat/outputs/elasticsearch/enc_test.go b/libbeat/esclientleg/enc_test.go similarity index 95% rename from libbeat/outputs/elasticsearch/enc_test.go rename to libbeat/esclientleg/enc_test.go index 0ccea8cd0f4..c3997fd5b9b 100644 --- a/libbeat/outputs/elasticsearch/enc_test.go +++ b/libbeat/esclientleg/enc_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package elasticsearch +package esclientleg import ( "testing" @@ -29,7 +29,7 @@ import ( ) func TestJSONEncoderMarshalBeatEvent(t *testing.T) { - encoder := newJSONEncoder(nil, true) + encoder := NewJSONEncoder(nil, true) event := beat.Event{ Timestamp: time.Date(2017, time.November, 7, 12, 0, 0, 0, time.UTC), Fields: common.MapStr{ @@ -46,7 +46,7 @@ func TestJSONEncoderMarshalBeatEvent(t *testing.T) { } func TestJSONEncoderMarshalMonitoringEvent(t *testing.T) { - encoder := newJSONEncoder(nil, true) + encoder := NewJSONEncoder(nil, true) event := report.Event{ Timestamp: time.Date(2017, time.November, 7, 12, 0, 0, 0, time.UTC), Fields: common.MapStr{ diff --git a/libbeat/esclientleg/errors.go b/libbeat/esclientleg/errors.go new file mode 100644 index 00000000000..f74599f8f01 --- /dev/null +++ b/libbeat/esclientleg/errors.go @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package esclientleg + +import "errors" + +var ( + // ErrNotConnected indicates failure due to client having no valid connection + ErrNotConnected = errors.New("not connected") + + // ErrJSONEncodeFailed indicates encoding failures + ErrJSONEncodeFailed = errors.New("json encode failed") + + // ErrResponseRead indicates error parsing Elasticsearch response + ErrResponseRead = errors.New("bulk item status parse failed") +) diff --git a/libbeat/esclientleg/esclientleg.go b/libbeat/esclientleg/esclientleg.go new file mode 100644 index 00000000000..38c06b4d321 --- /dev/null +++ b/libbeat/esclientleg/esclientleg.go @@ -0,0 +1,209 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package esclientleg + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + + "github.com/elastic/beats/libbeat/logp" + + "github.com/elastic/beats/libbeat/common" +) + +var ( + debugf = logp.MakeDebug("esclientleg") +) + +// Connection manages the connection for a given client. +type Connection struct { + URL string + Username string + Password string + APIKey string + Headers map[string]string + + HTTP *http.Client + OnConnectCallback func() error + + Encoder BodyEncoder + version common.Version +} + +// Connect connects the client. It runs a GET request against the root URL of +// the configured host, updates the known Elasticsearch version and calls +// globally configured handlers. +func (conn *Connection) Connect() error { + versionString, err := conn.Ping() + if err != nil { + return err + } + + if version, err := common.NewVersion(versionString); err != nil { + logp.Err("Invalid version from Elasticsearch: %v", versionString) + conn.version = common.Version{} + } else { + conn.version = *version + } + + err = conn.OnConnectCallback() + if err != nil { + return fmt.Errorf("Connection marked as failed because the onConnect callback failed: %v", err) + } + return nil +} + +// Ping sends a GET request to the Elasticsearch. +func (conn *Connection) Ping() (string, error) { + debugf("ES Ping(url=%v)", conn.URL) + + status, body, err := conn.execRequest("GET", conn.URL, nil) + if err != nil { + debugf("Ping request failed with: %v", err) + return "", err + } + + if status >= 300 { + return "", fmt.Errorf("Non 2xx response code: %d", status) + } + + var response struct { + Version struct { + Number string + } + } + + err = json.Unmarshal(body, &response) + if err != nil { + return "", fmt.Errorf("Failed to parse JSON response: %v", err) + } + + debugf("Ping status code: %v", status) + logp.Info("Attempting to connect to Elasticsearch version %s", response.Version.Number) + return response.Version.Number, nil +} + +// Close closes a connection. +func (conn *Connection) Close() error { + return nil +} + +// Request sends a request via the connection. +func (conn *Connection) Request( + method, path string, + pipeline string, + params map[string]string, + body interface{}, +) (int, []byte, error) { + + url := addToURL(conn.URL, path, pipeline, params) + debugf("%s %s %s %v", method, url, pipeline, body) + + return conn.RequestURL(method, url, body) +} + +// RequestURL sends a request with the connection object to an alternative url +func (conn *Connection) RequestURL( + method, url string, + body interface{}, +) (int, []byte, error) { + + if body == nil { + return conn.execRequest(method, url, nil) + } + + if err := conn.Encoder.Marshal(body); err != nil { + logp.Warn("Failed to json encode body (%v): %#v", err, body) + return 0, nil, ErrJSONEncodeFailed + } + return conn.execRequest(method, url, conn.Encoder.Reader()) +} + +func (conn *Connection) execRequest( + method, url string, + body io.Reader, +) (int, []byte, error) { + req, err := http.NewRequest(method, url, body) + if err != nil { + logp.Warn("Failed to create request %+v", err) + return 0, nil, err + } + if body != nil { + conn.Encoder.AddHeader(&req.Header) + } + return conn.execHTTPRequest(req) +} + +func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) { + req.Header.Add("Accept", "application/json") + + if conn.Username != "" || conn.Password != "" { + req.SetBasicAuth(conn.Username, conn.Password) + } + + if conn.APIKey != "" { + req.Header.Add("Authorization", "ApiKey "+conn.APIKey) + } + + for name, value := range conn.Headers { + req.Header.Add(name, value) + } + + // The stlib will override the value in the header based on the configured `Host` + // on the request which default to the current machine. + // + // We use the normalized key header to retrieve the user configured value and assign it to the host. + if host := req.Header.Get("Host"); host != "" { + req.Host = host + } + + resp, err := conn.HTTP.Do(req) + if err != nil { + return 0, nil, err + } + defer closing(resp.Body) + + status := resp.StatusCode + obj, err := ioutil.ReadAll(resp.Body) + if err != nil { + return status, nil, err + } + + if status >= 300 { + // add the response body with the error returned by Elasticsearch + err = fmt.Errorf("%v: %s", resp.Status, obj) + } + + return status, obj, err +} + +// GetVersion returns the elasticsearch version the client is connected to. +// The version is read and updated on 'Connect'. +func (conn *Connection) GetVersion() common.Version { + return conn.version +} + +func closing(c io.Closer) { + err := c.Close() + if err != nil { + logp.Warn("Close failed with: %v", err) + } +} diff --git a/libbeat/outputs/elasticsearch/url.go b/libbeat/esclientleg/url.go similarity index 96% rename from libbeat/outputs/elasticsearch/url.go rename to libbeat/esclientleg/url.go index 1c83419aefe..bdea7ca4c6e 100644 --- a/libbeat/outputs/elasticsearch/url.go +++ b/libbeat/esclientleg/url.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package elasticsearch +package esclientleg import ( "fmt" @@ -75,7 +75,7 @@ func makePath(index string, docType string, id string) (string, error) { } // TODO: make this reusable. Same definition in elasticsearch monitoring module -func parseProxyURL(raw string) (*url.URL, error) { +func ParseProxyURL(raw string) (*url.URL, error) { if raw == "" { return nil, nil } diff --git a/libbeat/outputs/elasticsearch/url_test.go b/libbeat/esclientleg/url_test.go similarity index 98% rename from libbeat/outputs/elasticsearch/url_test.go rename to libbeat/esclientleg/url_test.go index 0b5ddcdf073..566e5e09c23 100644 --- a/libbeat/outputs/elasticsearch/url_test.go +++ b/libbeat/esclientleg/url_test.go @@ -17,7 +17,7 @@ // +build !integration -package elasticsearch +package esclientleg import "testing" diff --git a/libbeat/monitoring/report/elasticsearch/client.go b/libbeat/monitoring/report/elasticsearch/client.go index 42517219a90..43c16a2e35b 100644 --- a/libbeat/monitoring/report/elasticsearch/client.go +++ b/libbeat/monitoring/report/elasticsearch/client.go @@ -23,6 +23,8 @@ import ( "net/http" "time" + "github.com/elastic/beats/libbeat/esclientleg" + "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" @@ -247,7 +249,7 @@ func getMonitoringIndexName() string { return fmt.Sprintf(".monitoring-beats-%v-%s", version, date) } -func logBulkFailures(log *logp.Logger, result esout.BulkResult, events []report.Event) { +func logBulkFailures(log *logp.Logger, result esclientleg.BulkResult, events []report.Event) { reader := esout.NewJSONReader(result) err := esout.BulkReadToItems(reader) if err != nil { diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 40cd8f46836..bf8d8dbafe9 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -20,10 +20,8 @@ package elasticsearch import ( "bytes" "encoding/base64" - "encoding/json" + "errors" "fmt" - "io" - "io/ioutil" "net/http" "net/url" "time" @@ -34,6 +32,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" "github.com/elastic/beats/v7/libbeat/outputs/outil" @@ -43,7 +42,7 @@ import ( // Client is an elasticsearch client. type Client struct { - Connection + esclientleg.Connection tlsConfig *tlscommon.TLSConfig index outputs.IndexSelector @@ -52,7 +51,7 @@ type Client struct { timeout time.Duration // buffered bulk requests - bulkRequ *bulkRequest + bulkRequ *esclientleg.BulkRequest // buffered json response reader json JSONReader @@ -85,23 +84,6 @@ type ClientSettings struct { // ConnectCallback defines the type for the function to be called when the Elasticsearch client successfully connects to the cluster type ConnectCallback func(client *Client) error -// Connection manages the connection for a given client. -type Connection struct { - log *logp.Logger - - URL string - Username string - Password string - APIKey string - Headers map[string]string - - http *http.Client - onConnectCallback func() error - - encoder bodyEncoder - version common.Version -} - type bulkIndexAction struct { Index bulkEventMeta `json:"index" struct:"index"` } @@ -193,31 +175,30 @@ func NewClient( } params := s.Parameters - bulkRequ, err := newBulkRequest(s.URL, "", "", params, nil) + bulkRequ, err := esclientleg.NewBulkRequest(s.URL, "", "", params, nil) if err != nil { return nil, err } - var encoder bodyEncoder + var encoder esclientleg.BodyEncoder compression := s.CompressionLevel if compression == 0 { - encoder = newJSONEncoder(nil, s.EscapeHTML) + encoder = esclientleg.NewJSONEncoder(nil, s.EscapeHTML) } else { - encoder, err = newGzipEncoder(compression, nil, s.EscapeHTML) + encoder, err = esclientleg.NewGzipEncoder(compression, nil, s.EscapeHTML) if err != nil { return nil, err } } client := &Client{ - Connection: Connection{ - log: log, + Connection: esclientleg.Connection{ URL: s.URL, Username: s.Username, Password: s.Password, APIKey: base64.StdEncoding.EncodeToString([]byte(s.APIKey)), Headers: s.Headers, - http: &http.Client{ + HTTP: &http.Client{ Transport: &http.Transport{ Dial: dialer.Dial, DialTLS: tlsDialer.Dial, @@ -226,7 +207,7 @@ func NewClient( }, Timeout: s.Timeout, }, - encoder: encoder, + Encoder: encoder, }, tlsConfig: s.TLS, index: s.Index, @@ -241,7 +222,7 @@ func NewClient( observer: s.Observer, } - client.Connection.onConnectCallback = func() error { + client.Connection.OnConnectCallback = func() error { globalCallbackRegistry.mutex.Lock() defer globalCallbackRegistry.mutex.Unlock() @@ -292,7 +273,7 @@ func (client *Client) Clone() *Client { APIKey: client.APIKey, Parameters: nil, // XXX: do not pass params? Headers: client.Headers, - Timeout: client.http.Timeout, + Timeout: client.HTTP.Timeout, CompressionLevel: client.compressionLevel, }, nil, // XXX: do not pass connection callback? @@ -328,7 +309,7 @@ func (client *Client) publishEvents( return nil, nil } - body := client.encoder + body := client.Encoder body.Reset() // encode events into bulk request buffer, dropping failed elements from @@ -351,7 +332,7 @@ func (client *Client) publishEvents( requ := client.bulkRequ requ.Reset(body) - status, result, sendErr := client.sendBulkRequest(requ) + status, result, sendErr := client.SendBulkRequest(requ) if sendErr != nil { client.Connection.log.Error("Failed to perform any bulk index operations: %+v", sendErr) return data, sendErr @@ -399,7 +380,7 @@ func (client *Client) publishEvents( func bulkEncodePublishRequest( log *logp.Logger, version common.Version, - body bulkWriter, + body esclientleg.BulkWriter, index outputs.IndexSelector, pipeline *outil.Selector, eventType string, @@ -666,11 +647,6 @@ func (client *Client) LoadJSON(path string, json map[string]interface{}) ([]byte return body, nil } -// GetVersion returns the elasticsearch version the client is connected to. -func (client *Client) GetVersion() common.Version { - return client.Connection.version -} - func (client *Client) Test(d testing.Driver) { d.Run("elasticsearch: "+client.URL, func(d testing.Driver) { u, err := url.Parse(client.URL) @@ -697,177 +673,11 @@ func (client *Client) Test(d testing.Driver) { err = client.Connect() d.Fatal("talk to server", err) - d.Info("version", client.version.String()) + version := client.GetVersion() + d.Info("version", version.String()) }) } func (client *Client) String() string { return "elasticsearch(" + client.Connection.URL + ")" } - -// Connect connects the client. It runs a GET request against the root URL of -// the configured host, updates the known Elasticsearch version and calls -// globally configured handlers. -func (client *Client) Connect() error { - return client.Connection.Connect() -} - -// Connect connects the client. It runs a GET request against the root URL of -// the configured host, updates the known Elasticsearch version and calls -// globally configured handlers. -func (conn *Connection) Connect() error { - versionString, err := conn.Ping() - if err != nil { - return err - } - - if version, err := common.NewVersion(versionString); err != nil { - conn.log.Errorf("Invalid version from Elasticsearch: %s", versionString) - conn.version = common.Version{} - } else { - conn.version = *version - } - - err = conn.onConnectCallback() - if err != nil { - return fmt.Errorf("Connection marked as failed because the onConnect callback failed: %v", err) - } - return nil -} - -// Ping sends a GET request to the Elasticsearch. -func (conn *Connection) Ping() (string, error) { - conn.log.Debugf("ES Ping(url=%v)", conn.URL) - - status, body, err := conn.execRequest("GET", conn.URL, nil) - if err != nil { - conn.log.Debugf("Ping request failed with: %+v", err) - return "", err - } - - if status >= 300 { - return "", fmt.Errorf("Non 2xx response code: %d", status) - } - - var response struct { - Version struct { - Number string - } - } - - err = json.Unmarshal(body, &response) - if err != nil { - return "", fmt.Errorf("Failed to parse JSON response: %v", err) - } - - conn.log.Debugf("Ping status code: %v", status) - conn.log.Infof("Attempting to connect to Elasticsearch version %s", response.Version.Number) - return response.Version.Number, nil -} - -// Close closes a connection. -func (conn *Connection) Close() error { - return nil -} - -// Request sends a request via the connection. -func (conn *Connection) Request( - method, path string, - pipeline string, - params map[string]string, - body interface{}, -) (int, []byte, error) { - - url := addToURL(conn.URL, path, pipeline, params) - conn.log.Debugf("%s %s %s %v", method, url, pipeline, body) - - return conn.RequestURL(method, url, body) -} - -// RequestURL sends a request with the connection object to an alternative url -func (conn *Connection) RequestURL( - method, url string, - body interface{}, -) (int, []byte, error) { - - if body == nil { - return conn.execRequest(method, url, nil) - } - - if err := conn.encoder.Marshal(body); err != nil { - conn.log.Warnf("Failed to json encode body (%+v): %#v", err, body) - return 0, nil, ErrJSONEncodeFailed - } - return conn.execRequest(method, url, conn.encoder.Reader()) -} - -func (conn *Connection) execRequest( - method, url string, - body io.Reader, -) (int, []byte, error) { - req, err := http.NewRequest(method, url, body) - if err != nil { - conn.log.Warnf("Failed to create request %+v", err) - return 0, nil, err - } - if body != nil { - conn.encoder.AddHeader(&req.Header) - } - return conn.execHTTPRequest(req) -} - -func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) { - req.Header.Add("Accept", "application/json") - - if conn.Username != "" || conn.Password != "" { - req.SetBasicAuth(conn.Username, conn.Password) - } - - if conn.APIKey != "" { - req.Header.Add("Authorization", "ApiKey "+conn.APIKey) - } - - for name, value := range conn.Headers { - req.Header.Add(name, value) - } - - // The stlib will override the value in the header based on the configured `Host` - // on the request which default to the current machine. - // - // We use the normalized key header to retrieve the user configured value and assign it to the host. - if host := req.Header.Get("Host"); host != "" { - req.Host = host - } - - resp, err := conn.http.Do(req) - if err != nil { - return 0, nil, err - } - defer closing(conn.log, resp.Body) - - status := resp.StatusCode - obj, err := ioutil.ReadAll(resp.Body) - if err != nil { - return status, nil, err - } - - if status >= 300 { - // add the response body with the error returned by Elasticsearch - err = fmt.Errorf("%v: %s", resp.Status, obj) - } - - return status, obj, err -} - -// GetVersion returns the elasticsearch version the client is connected to. -// The version is read and updated on 'Connect'. -func (conn *Connection) GetVersion() common.Version { - return conn.version -} - -func closing(log *logp.Logger, c io.Closer) { - err := c.Close() - if err != nil { - log.Warnf("Close failed with: %+v", err) - } -} diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index 9cabf8f9a6f..2bd3f7e7f46 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -340,7 +340,7 @@ func TestAddToURL(t *testing.T) { }, } for _, test := range tests { - url := addToURL(test.url, test.path, test.pipeline, test.params) + url := esclientleg.addToURL(test.url, test.path, test.pipeline, test.params) assert.Equal(t, url, test.expected) } } diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index 4cbf449b6ec..342b3129799 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -22,6 +22,7 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/esclientleg" ) type elasticsearchConfig struct { @@ -78,7 +79,7 @@ var ( func (c *elasticsearchConfig) Validate() error { if c.ProxyURL != "" && !c.ProxyDisable { - if _, err := parseProxyURL(c.ProxyURL); err != nil { + if _, err := esclientleg.ParseProxyURL(c.ProxyURL); err != nil { return err } } diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index b80fd832d1f..441ca769937 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -18,7 +18,6 @@ package elasticsearch import ( - "errors" "fmt" "net/url" "sync" @@ -28,6 +27,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" "github.com/elastic/beats/v7/libbeat/outputs/outil" @@ -37,17 +37,6 @@ func init() { outputs.RegisterType("elasticsearch", makeES) } -var ( - // ErrNotConnected indicates failure due to client having no valid connection - ErrNotConnected = errors.New("not connected") - - // ErrJSONEncodeFailed indicates encoding failures - ErrJSONEncodeFailed = errors.New("json encode failed") - - // ErrResponseRead indicates error parsing Elasticsearch response - ErrResponseRead = errors.New("bulk item status parse failed") -) - const logSelector = "elasticsearch" // Callbacks must not depend on the result of a previous one, @@ -165,7 +154,7 @@ func makeES( var proxyURL *url.URL if !config.ProxyDisable { - proxyURL, err = parseProxyURL(config.ProxyURL) + proxyURL, err = esclientleg.ParseProxyURL(config.ProxyURL) if err != nil { return outputs.Fail(err) } @@ -291,7 +280,7 @@ func NewElasticsearchClients(cfg *common.Config) ([]Client, error) { log := logp.NewLogger(logSelector) var proxyURL *url.URL if !config.ProxyDisable { - proxyURL, err = parseProxyURL(config.ProxyURL) + proxyURL, err = esclientleg.ParseProxyURL(config.ProxyURL) if err != nil { return nil, err } From 0b1603e68469a6b6234a20f68fe17e0579c53d4f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Feb 2020 16:26:22 -0800 Subject: [PATCH 02/91] Move test --- .../{esclientleg.go => connection.go} | 0 libbeat/esclientleg/url_test.go | 52 ++++++++++++++++++- libbeat/outputs/elasticsearch/client_test.go | 46 ---------------- 3 files changed, 51 insertions(+), 47 deletions(-) rename libbeat/esclientleg/{esclientleg.go => connection.go} (100%) diff --git a/libbeat/esclientleg/esclientleg.go b/libbeat/esclientleg/connection.go similarity index 100% rename from libbeat/esclientleg/esclientleg.go rename to libbeat/esclientleg/connection.go diff --git a/libbeat/esclientleg/url_test.go b/libbeat/esclientleg/url_test.go index 566e5e09c23..7893aac62f0 100644 --- a/libbeat/esclientleg/url_test.go +++ b/libbeat/esclientleg/url_test.go @@ -19,7 +19,11 @@ package esclientleg -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestUrlEncode(t *testing.T) { params := map[string]string{ @@ -75,3 +79,49 @@ func TestMakePath(t *testing.T) { t.Errorf("Wrong path created: %s", path) } } + +func TestAddToURL(t *testing.T) { + type Test struct { + url string + path string + pipeline string + params map[string]string + expected string + } + tests := []Test{ + { + url: "localhost:9200", + path: "/path", + pipeline: "", + params: make(map[string]string), + expected: "localhost:9200/path", + }, + { + url: "localhost:9200/", + path: "/path", + pipeline: "", + params: make(map[string]string), + expected: "localhost:9200/path", + }, + { + url: "localhost:9200", + path: "/path", + pipeline: "pipeline_1", + params: make(map[string]string), + expected: "localhost:9200/path?pipeline=pipeline_1", + }, + { + url: "localhost:9200/", + path: "/path", + pipeline: "", + params: map[string]string{ + "param": "value", + }, + expected: "localhost:9200/path?param=value", + }, + } + for _, test := range tests { + url := addToURL(test.url, test.path, test.pipeline, test.params) + assert.Equal(t, url, test.expected) + } +} diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index 2bd3f7e7f46..c9a83e5f57e 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -299,52 +299,6 @@ func TestClientWithHeaders(t *testing.T) { assert.Equal(t, 2, requestCount) } -func TestAddToURL(t *testing.T) { - type Test struct { - url string - path string - pipeline string - params map[string]string - expected string - } - tests := []Test{ - { - url: "localhost:9200", - path: "/path", - pipeline: "", - params: make(map[string]string), - expected: "localhost:9200/path", - }, - { - url: "localhost:9200/", - path: "/path", - pipeline: "", - params: make(map[string]string), - expected: "localhost:9200/path", - }, - { - url: "localhost:9200", - path: "/path", - pipeline: "pipeline_1", - params: make(map[string]string), - expected: "localhost:9200/path?pipeline=pipeline_1", - }, - { - url: "localhost:9200/", - path: "/path", - pipeline: "", - params: map[string]string{ - "param": "value", - }, - expected: "localhost:9200/path?param=value", - }, - } - for _, test := range tests { - url := esclientleg.addToURL(test.url, test.path, test.pipeline, test.params) - assert.Equal(t, url, test.expected) - } -} - type testBulkRecorder struct { data []interface{} inAction bool From 6d544b9189a5b2f2d3469ed9167c9e2db79c0767 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Feb 2020 16:33:32 -0800 Subject: [PATCH 03/91] Removing duplicate function in monitoring reporter --- libbeat/esclientleg/url.go | 1 - .../report/elasticsearch/elasticsearch.go | 20 ++----------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/libbeat/esclientleg/url.go b/libbeat/esclientleg/url.go index bdea7ca4c6e..e5bbed2cbfb 100644 --- a/libbeat/esclientleg/url.go +++ b/libbeat/esclientleg/url.go @@ -74,7 +74,6 @@ func makePath(index string, docType string, id string) (string, error) { return path, nil } -// TODO: make this reusable. Same definition in elasticsearch monitoring module func ParseProxyURL(raw string) (*url.URL, error) { if raw == "" { return nil, nil diff --git a/libbeat/monitoring/report/elasticsearch/elasticsearch.go b/libbeat/monitoring/report/elasticsearch/elasticsearch.go index 4484ceef933..d4f767690e5 100644 --- a/libbeat/monitoring/report/elasticsearch/elasticsearch.go +++ b/libbeat/monitoring/report/elasticsearch/elasticsearch.go @@ -24,12 +24,12 @@ import ( "math/rand" "net/url" "strconv" - "strings" "time" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/monitoring/report" @@ -131,7 +131,7 @@ func makeReporter(beat beat.Info, settings report.Settings, cfg *common.Config) windowSize = 1 } - proxyURL, err := parseProxyURL(config.ProxyURL) + proxyURL, err := esclientleg.ParseProxyURL(config.ProxyURL) if err != nil { return nil, err } @@ -368,22 +368,6 @@ func closing(log *logp.Logger, c io.Closer) { } } -// TODO: make this reusable. Same definition in elasticsearch monitoring module -func parseProxyURL(raw string) (*url.URL, error) { - if raw == "" { - return nil, nil - } - - url, err := url.Parse(raw) - if err == nil && strings.HasPrefix(url.Scheme, "http") { - return url, err - } - - // Proxy was bogus. Try prepending "http://" to it and - // see if that parses correctly. - return url.Parse("http://" + raw) -} - func makeMeta(beat beat.Info) common.MapStr { return common.MapStr{ "type": beat.Beat, From 84dff7af6cda8e1467a2ab3210cf393c4f8c4540 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Feb 2020 16:42:54 -0800 Subject: [PATCH 04/91] Break import cycle --- libbeat/esclientleg/api_mock_test.go | 6 +++--- libbeat/esclientleg/api_test.go | 26 +++++++++++------------- libbeat/esclientleg/bulkapi_mock_test.go | 6 +++--- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/libbeat/esclientleg/api_mock_test.go b/libbeat/esclientleg/api_mock_test.go index 406812c7810..ea44f249ead 100644 --- a/libbeat/esclientleg/api_mock_test.go +++ b/libbeat/esclientleg/api_mock_test.go @@ -63,7 +63,7 @@ func TestOneHostSuccessResp(t *testing.T) { server := ElasticsearchMock(200, expectedResp) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) params := map[string]string{ "refresh": "true", @@ -89,7 +89,7 @@ func TestOneHost500Resp(t *testing.T) { server := ElasticsearchMock(http.StatusInternalServerError, []byte("Something wrong happened")) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) err := client.Connect() if err != nil { t.Fatalf("Failed to connect: %v", err) @@ -121,7 +121,7 @@ func TestOneHost503Resp(t *testing.T) { server := ElasticsearchMock(503, []byte("Something wrong happened")) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) params := map[string]string{ "refresh": "true", diff --git a/libbeat/esclientleg/api_test.go b/libbeat/esclientleg/api_test.go index 2a114b06bc6..a5d7da4017b 100644 --- a/libbeat/esclientleg/api_test.go +++ b/libbeat/esclientleg/api_test.go @@ -20,15 +20,13 @@ package esclientleg import ( "encoding/json" + "net/http" "testing" - "time" - - "github.com/elastic/beats/libbeat/outputs/elasticsearch" "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/libbeat/outputs/outil" + "github.com/elastic/beats/v7/libbeat/outputs/transport" ) func GetValidQueryResult() QueryResult { @@ -174,17 +172,17 @@ func TestReadSearchResult_invalid(t *testing.T) { assert.Error(t, err) } -func newTestClient(url string) *elasticsearch.Client { - client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ - URL: url, - Index: outil.MakeSelector(), - Timeout: 60 * time.Second, - CompressionLevel: 3, - }, nil) - if err != nil { - panic(err) +func newTestConnection(url string) Connection { + return Connection{ + URL: url, + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(0).Dial, + }, + Timeout: 0, + }, + Encoder: NewJSONEncoder(nil, false), } - return client } func (r QueryResult) String() string { diff --git a/libbeat/esclientleg/bulkapi_mock_test.go b/libbeat/esclientleg/bulkapi_mock_test.go index 06abdc90c4a..cad04de0df8 100644 --- a/libbeat/esclientleg/bulkapi_mock_test.go +++ b/libbeat/esclientleg/bulkapi_mock_test.go @@ -55,7 +55,7 @@ func TestOneHostSuccessResp_Bulk(t *testing.T) { server := ElasticsearchMock(200, expectedResp) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) params := map[string]string{ "refresh": "true", @@ -91,7 +91,7 @@ func TestOneHost500Resp_Bulk(t *testing.T) { server := ElasticsearchMock(http.StatusInternalServerError, []byte("Something wrong happened")) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) params := map[string]string{ "refresh": "true", @@ -131,7 +131,7 @@ func TestOneHost503Resp_Bulk(t *testing.T) { server := ElasticsearchMock(503, []byte("Something wrong happened")) - client := newTestClient(server.URL) + client := newTestConnection(server.URL) params := map[string]string{ "refresh": "true", From 2f9b881aeb81a8819c246a61f4c0e076cff969a4 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Feb 2020 17:46:06 -0800 Subject: [PATCH 05/91] Guard onConnect callback execution --- libbeat/esclientleg/connection.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libbeat/esclientleg/connection.go b/libbeat/esclientleg/connection.go index 38c06b4d321..3639d243f1b 100644 --- a/libbeat/esclientleg/connection.go +++ b/libbeat/esclientleg/connection.go @@ -64,10 +64,13 @@ func (conn *Connection) Connect() error { conn.version = *version } - err = conn.OnConnectCallback() - if err != nil { - return fmt.Errorf("Connection marked as failed because the onConnect callback failed: %v", err) + if conn.OnConnectCallback != nil { + err = conn.OnConnectCallback() + if err != nil { + return fmt.Errorf("Connection marked as failed because the onConnect callback failed: %v", err) + } } + return nil } From 5c2e5351cb4e9004956fb89c2955575ab4593b17 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 04:28:27 -0800 Subject: [PATCH 06/91] Replace use of field with getter --- libbeat/outputs/elasticsearch/client_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index 98c9addc081..8daff6e9291 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -129,7 +129,7 @@ func TestClientPublishEventWithPipeline(t *testing.T) { client.Delete(index, "", "", nil) // Check version - if client.Connection.version.Major < 5 { + if client.Connection.GetVersion().Major < 5 { t.Skip("Skipping tests as pipeline not available in <5.x releases") } @@ -210,7 +210,7 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { }) client.Delete(index, "", "", nil) - if client.Connection.version.Major < 5 { + if client.Connection.GetVersion().Major < 5 { t.Skip("Skipping tests as pipeline not available in <5.x releases") } From 805469b45f2876d9211a7a8f6bcf4499d0fc54a9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 05:06:00 -0800 Subject: [PATCH 07/91] Moving common bulk API response processing into esclientleg --- libbeat/esclientleg/bulkapi.go | 147 +++++++++++++++ libbeat/esclientleg/bulkapi_test.go | 127 +++++++++++++ .../json_read.go | 2 +- .../monitoring/report/elasticsearch/client.go | 9 +- libbeat/outputs/elasticsearch/client.go | 171 ++---------------- libbeat/outputs/elasticsearch/client_test.go | 127 +------------ 6 files changed, 297 insertions(+), 286 deletions(-) create mode 100644 libbeat/esclientleg/bulkapi_test.go rename libbeat/{outputs/elasticsearch => esclientleg}/json_read.go (99%) diff --git a/libbeat/esclientleg/bulkapi.go b/libbeat/esclientleg/bulkapi.go index d954d1d4850..d4d39b07366 100644 --- a/libbeat/esclientleg/bulkapi.go +++ b/libbeat/esclientleg/bulkapi.go @@ -20,6 +20,7 @@ package esclientleg import ( "bytes" "encoding/json" + "errors" "io" "io/ioutil" "net/http" @@ -29,6 +30,34 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) +var ( + errExpectedItemsArray = errors.New("expected items array") + errExpectedItemObject = errors.New("expected item response object") + errExpectedStatusCode = errors.New("expected item status code") + errUnexpectedEmptyObject = errors.New("empty object") + errExpectedObjectEnd = errors.New("expected end of object") + ErrTempBulkFailure = errors.New("temporary bulk send failure") + + nameItems = []byte("items") + nameStatus = []byte("status") + nameError = []byte("error") +) + +type BulkIndexAction struct { + Index BulkMeta `json:"index" struct:"index"` +} + +type BulkCreateAction struct { + Create BulkMeta `json:"create" struct:"create"` +} + +type BulkMeta struct { + Index string `json:"_index" struct:"_index"` + DocType string `json:"_type,omitempty" struct:"_type,omitempty"` + Pipeline string `json:"pipeline,omitempty" struct:"pipeline,omitempty"` + ID string `json:"_id,omitempty" struct:"_id,omitempty"` +} + // MetaBuilder creates meta data for bulk requests type MetaBuilder func(interface{}) interface{} @@ -205,6 +234,124 @@ func (conn *Connection) SendBulkRequest(requ *BulkRequest) (int, BulkResult, err return status, BulkResult(resp), err } +// BulkReadToItems reads the bulk response up to (but not including) items +func BulkReadToItems(reader *JSONReader) error { + if err := reader.ExpectDict(); err != nil { + return errExpectedObject + } + + // find 'items' field in response + for { + kind, name, err := reader.nextFieldName() + if err != nil { + return err + } + + if kind == dictEnd { + return errExpectedItemsArray + } + + // found items array -> continue + if bytes.Equal(name, nameItems) { + break + } + + reader.ignoreNext() + } + + // check items field is an array + if err := reader.ExpectArray(); err != nil { + return errExpectedItemsArray + } + + return nil +} + +// BulkReadItemStatus reads the status and error fields from the bulk item +func BulkReadItemStatus(log *logp.Logger, reader *JSONReader) (int, []byte, error) { + // skip outer dictionary + if err := reader.ExpectDict(); err != nil { + return 0, nil, errExpectedItemObject + } + + // find first field in outer dictionary (e.g. 'create') + kind, _, err := reader.nextFieldName() + if err != nil { + log.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + if kind == dictEnd { + err = errUnexpectedEmptyObject + log.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + // parse actual item response code and error message + status, msg, err := itemStatusInner(reader) + if err != nil { + log.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + // close dictionary. Expect outer dictionary to have only one element + kind, _, err = reader.step() + if err != nil { + log.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + if kind != dictEnd { + err = errExpectedObjectEnd + log.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + return status, msg, nil +} + +func itemStatusInner(reader *JSONReader) (int, []byte, error) { + if err := reader.ExpectDict(); err != nil { + return 0, nil, errExpectedItemObject + } + + status := -1 + var msg []byte + for { + kind, name, err := reader.nextFieldName() + if err != nil { + logp.Err("Failed to parse bulk response item: %s", err) + } + if kind == dictEnd { + break + } + + switch { + case bytes.Equal(name, nameStatus): // name == "status" + status, err = reader.nextInt() + if err != nil { + logp.Err("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + case bytes.Equal(name, nameError): // name == "error" + msg, err = reader.ignoreNext() // collect raw string for "error" field + if err != nil { + return 0, nil, err + } + + default: // ignore unknown fields + _, err = reader.ignoreNext() + if err != nil { + return 0, nil, err + } + } + } + + if status < 0 { + return 0, nil, errExpectedStatusCode + } + return status, msg, nil +} + func bulkEncode(log *logp.Logger, out BulkWriter, metaBuilder MetaBuilder, body []interface{}) error { if metaBuilder == nil { for _, obj := range body { diff --git a/libbeat/esclientleg/bulkapi_test.go b/libbeat/esclientleg/bulkapi_test.go new file mode 100644 index 00000000000..d55abd18e14 --- /dev/null +++ b/libbeat/esclientleg/bulkapi_test.go @@ -0,0 +1,127 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package esclientleg + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBulkReadToItems(t *testing.T) { + response := []byte(`{ + "errors": false, + "items": [ + {"create": {"status": 200}}, + {"create": {"status": 300}}, + {"create": {"status": 400}} + ]}`) + + reader := NewJSONReader(response) + + err := BulkReadToItems(reader) + assert.NoError(t, err) + + for status := 200; status <= 400; status += 100 { + err = reader.ExpectDict() + assert.NoError(t, err) + + kind, raw, err := reader.nextFieldName() + assert.NoError(t, err) + assert.Equal(t, mapKeyEntity, kind) + assert.Equal(t, []byte("create"), raw) + + err = reader.ExpectDict() + assert.NoError(t, err) + + kind, raw, err = reader.nextFieldName() + assert.NoError(t, err) + assert.Equal(t, mapKeyEntity, kind) + assert.Equal(t, []byte("status"), raw) + + code, err := reader.nextInt() + assert.NoError(t, err) + assert.Equal(t, status, code) + + _, _, err = reader.endDict() + assert.NoError(t, err) + + _, _, err = reader.endDict() + assert.NoError(t, err) + } +} + +func TestBulkReadItemStatus(t *testing.T) { + response := []byte(`{"create": {"status": 200}}`) + + reader := NewJSONReader(response) + code, _, err := BulkReadItemStatus(reader) + assert.NoError(t, err) + assert.Equal(t, 200, code) +} + +func TestESNoErrorStatus(t *testing.T) { + response := []byte(`{"create": {"status": 200}}`) + code, msg, err := readStatusItem(response) + + assert.Nil(t, err) + assert.Equal(t, 200, code) + assert.Equal(t, "", msg) +} + +func TestES1StyleErrorStatus(t *testing.T) { + response := []byte(`{"create": {"status": 400, "error": "test error"}}`) + code, msg, err := readStatusItem(response) + + assert.Nil(t, err) + assert.Equal(t, 400, code) + assert.Equal(t, `"test error"`, msg) +} + +func TestES2StyleErrorStatus(t *testing.T) { + response := []byte(`{"create": {"status": 400, "error": {"reason": "test_error"}}}`) + code, msg, err := readStatusItem(response) + + assert.Nil(t, err) + assert.Equal(t, 400, code) + assert.Equal(t, `{"reason": "test_error"}`, msg) +} + +func TestES2StyleExtendedErrorStatus(t *testing.T) { + response := []byte(` + { + "create": { + "status": 400, + "error": { + "reason": "test_error", + "transient": false, + "extra": null + } + } + }`) + code, _, err := readStatusItem(response) + + assert.Nil(t, err) + assert.Equal(t, 400, code) +} + +func readStatusItem(in []byte) (int, string, error) { + reader := NewJSONReader(in) + code, msg, err := BulkReadItemStatus(reader) + return code, string(msg), err +} diff --git a/libbeat/outputs/elasticsearch/json_read.go b/libbeat/esclientleg/json_read.go similarity index 99% rename from libbeat/outputs/elasticsearch/json_read.go rename to libbeat/esclientleg/json_read.go index 8df87e5cb43..94ca887b092 100644 --- a/libbeat/outputs/elasticsearch/json_read.go +++ b/libbeat/esclientleg/json_read.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package elasticsearch +package esclientleg import ( "errors" diff --git a/libbeat/monitoring/report/elasticsearch/client.go b/libbeat/monitoring/report/elasticsearch/client.go index 43c16a2e35b..36ec6b28176 100644 --- a/libbeat/monitoring/report/elasticsearch/client.go +++ b/libbeat/monitoring/report/elasticsearch/client.go @@ -23,11 +23,10 @@ import ( "net/http" "time" - "github.com/elastic/beats/libbeat/esclientleg" - "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring/report" esout "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" @@ -250,15 +249,15 @@ func getMonitoringIndexName() string { } func logBulkFailures(log *logp.Logger, result esclientleg.BulkResult, events []report.Event) { - reader := esout.NewJSONReader(result) - err := esout.BulkReadToItems(reader) + reader := esclientleg.NewJSONReader(result) + err := esclientleg.BulkReadToItems(reader) if err != nil { log.Errorf("failed to parse monitoring bulk items: %+v", err) return } for i := range events { - status, msg, err := esout.BulkReadItemStatus(log, reader) + status, msg, err := esclientleg.BulkReadItemStatus(log, reader) if err != nil { log.Errorf("failed to parse monitoring bulk item status: %+v", err) return diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index bf8d8dbafe9..2dd9cf1e5f6 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -18,7 +18,6 @@ package elasticsearch import ( - "bytes" "encoding/base64" "errors" "fmt" @@ -53,9 +52,6 @@ type Client struct { // buffered bulk requests bulkRequ *esclientleg.BulkRequest - // buffered json response reader - json JSONReader - // additional configs compressionLevel int proxyURL *url.URL @@ -84,21 +80,6 @@ type ClientSettings struct { // ConnectCallback defines the type for the function to be called when the Elasticsearch client successfully connects to the cluster type ConnectCallback func(client *Client) error -type bulkIndexAction struct { - Index bulkEventMeta `json:"index" struct:"index"` -} - -type bulkCreateAction struct { - Create bulkEventMeta `json:"create" struct:"create"` -} - -type bulkEventMeta struct { - Index string `json:"_index" struct:"_index"` - DocType string `json:"_type,omitempty" struct:"_type,omitempty"` - Pipeline string `json:"pipeline,omitempty" struct:"pipeline,omitempty"` - ID string `json:"_id,omitempty" struct:"_id,omitempty"` -} - type bulkResultStats struct { acked int // number of events ACKed by Elasticsearch duplicates int // number of events failed with `create` due to ID already being indexed @@ -107,21 +88,6 @@ type bulkResultStats struct { tooMany int // number of events receiving HTTP 429 Too Many Requests } -var ( - nameItems = []byte("items") - nameStatus = []byte("status") - nameError = []byte("error") -) - -var ( - errExpectedItemsArray = errors.New("expected items array") - errExpectedItemObject = errors.New("expected item response object") - errExpectedStatusCode = errors.New("expected item status code") - errUnexpectedEmptyObject = errors.New("empty object") - errExpectedObjectEnd = errors.New("expected end of object") - errTempBulkFailure = errors.New("temporary bulk send failure") -) - const ( defaultEventType = "doc" ) @@ -321,7 +287,7 @@ func (client *Client) publishEvents( } origCount := len(data) - data = bulkEncodePublishRequest(client.Connection.log, client.GetVersion(), body, client.index, client.pipeline, eventType, data) + data = bulkEncodePublishRequest(client.GetVersion(), body, client.index, client.pipeline, eventType, data) newCount := len(data) if st != nil && origCount > newCount { st.Dropped(origCount - newCount) @@ -349,8 +315,7 @@ func (client *Client) publishEvents( failedEvents = data stats.fails = len(failedEvents) } else { - client.json.init(result) - failedEvents, stats = bulkCollectPublishFails(client.Connection.log, &client.json, data) + failedEvents, stats = bulkCollectPublishFails(result, data) } failed := len(failedEvents) @@ -368,7 +333,7 @@ func (client *Client) publishEvents( if failed > 0 { if sendErr == nil { - sendErr = errTempBulkFailure + sendErr = esclientleg.ErrTempBulkFailure } return failedEvents, sendErr } @@ -435,7 +400,7 @@ func createEventBulkMeta( } } - meta := bulkEventMeta{ + meta := esclientleg.BulkMeta{ Index: index, DocType: eventType, Pipeline: pipeline, @@ -443,9 +408,9 @@ func createEventBulkMeta( } if id != "" || version.Major > 7 || (version.Major == 7 && version.Minor >= 5) { - return bulkCreateAction{meta}, nil + return esclientleg.BulkCreateAction{meta}, nil } - return bulkIndexAction{meta}, nil + return esclientleg.BulkIndexAction{meta}, nil } func getPipeline(event *beat.Event, pipelineSel *outil.Selector) (string, error) { @@ -470,11 +435,12 @@ func getPipeline(event *beat.Event, pipelineSel *outil.Selector) (string, error) // the event will be dropped. func bulkCollectPublishFails( log *logp.Logger, - reader *JSONReader, + result esclientleg.BulkResult, data []publisher.Event, ) ([]publisher.Event, bulkResultStats) { - if err := BulkReadToItems(reader); err != nil { - log.Errorf("failed to parse bulk response: %+v", err) + reader := esclientleg.NewJSONReader(result) + if err := esclientleg.BulkReadToItems(reader); err != nil { + log.Errorf("failed to parse bulk response: %v", err.Error()) return nil, bulkResultStats{} } @@ -482,7 +448,7 @@ func bulkCollectPublishFails( failed := data[:0] stats := bulkResultStats{} for i := 0; i < count; i++ { - status, msg, err := BulkReadItemStatus(log, reader) + status, msg, err := esclientleg.BulkReadItemStatus(log, reader) if err != nil { log.Error(err) return nil, bulkResultStats{} @@ -519,121 +485,6 @@ func bulkCollectPublishFails( return failed, stats } -// BulkReadToItems reads the bulk response up to (but not including) items -func BulkReadToItems(reader *JSONReader) error { - if err := reader.ExpectDict(); err != nil { - return errExpectedObject - } - - // find 'items' field in response - for { - kind, name, err := reader.nextFieldName() - if err != nil { - return err - } - - if kind == dictEnd { - return errExpectedItemsArray - } - - // found items array -> continue - if bytes.Equal(name, nameItems) { - break - } - - reader.ignoreNext() - } - - // check items field is an array - if err := reader.ExpectArray(); err != nil { - return errExpectedItemsArray - } - - return nil -} - -// BulkReadItemStatus reads the status and error fields from the bulk item -func BulkReadItemStatus(log *logp.Logger, reader *JSONReader) (int, []byte, error) { - // skip outer dictionary - if err := reader.ExpectDict(); err != nil { - return 0, nil, errExpectedItemObject - } - - // find first field in outer dictionary (e.g. 'create') - kind, _, err := reader.nextFieldName() - parserErr := func(err error) error { - return errors.Wrapf(err, "Failed to parse bulk response item") - } - if err != nil { - return 0, nil, parserErr(err) - } - if kind == dictEnd { - err = errUnexpectedEmptyObject - return 0, nil, parserErr(err) - } - - // parse actual item response code and error message - status, msg, err := itemStatusInner(log, reader) - if err != nil { - return 0, nil, parserErr(err) - } - - // close dictionary. Expect outer dictionary to have only one element - kind, _, err = reader.step() - if err != nil { - return 0, nil, parserErr(err) - } - if kind != dictEnd { - err = errExpectedObjectEnd - return 0, nil, parserErr(err) - } - - return status, msg, nil -} - -func itemStatusInner(log *logp.Logger, reader *JSONReader) (int, []byte, error) { - if err := reader.ExpectDict(); err != nil { - return 0, nil, errExpectedItemObject - } - - status := -1 - var msg []byte - for { - kind, name, err := reader.nextFieldName() - if err != nil { - log.Errorf("Failed to parse bulk response item: %+v", err) - } - if kind == dictEnd { - break - } - - switch { - case bytes.Equal(name, nameStatus): // name == "status" - status, err = reader.nextInt() - if err != nil { - return 0, nil, err - } - - case bytes.Equal(name, nameError): // name == "error" - msg, err = reader.ignoreNext() // collect raw string for "error" field - if err != nil { - return 0, nil, err - } - - default: // ignore unknown fields - _, err = reader.ignoreNext() - if err != nil { - return 0, nil, err - } - } - } - - if status < 0 { - return 0, nil, errExpectedStatusCode - } - return status, msg, nil -} - // LoadJSON creates a PUT request based on a JSON document. func (client *Client) LoadJSON(path string, json map[string]interface{}) ([]byte, error) { status, body, err := client.Request("PUT", path, "", nil, json) diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index c9a83e5f57e..9af44f5c0a4 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -40,57 +40,6 @@ import ( "github.com/elastic/beats/v7/libbeat/version" ) -func readStatusItem(in []byte) (int, string, error) { - reader := NewJSONReader(in) - code, msg, err := BulkReadItemStatus(logp.L(), reader) - return code, string(msg), err -} - -func TestESNoErrorStatus(t *testing.T) { - response := []byte(`{"create": {"status": 200}}`) - code, msg, err := readStatusItem(response) - - assert.Nil(t, err) - assert.Equal(t, 200, code) - assert.Equal(t, "", msg) -} - -func TestES1StyleErrorStatus(t *testing.T) { - response := []byte(`{"create": {"status": 400, "error": "test error"}}`) - code, msg, err := readStatusItem(response) - - assert.Nil(t, err) - assert.Equal(t, 400, code) - assert.Equal(t, `"test error"`, msg) -} - -func TestES2StyleErrorStatus(t *testing.T) { - response := []byte(`{"create": {"status": 400, "error": {"reason": "test_error"}}}`) - code, msg, err := readStatusItem(response) - - assert.Nil(t, err) - assert.Equal(t, 400, code) - assert.Equal(t, `{"reason": "test_error"}`, msg) -} - -func TestES2StyleExtendedErrorStatus(t *testing.T) { - response := []byte(` - { - "create": { - "status": 400, - "error": { - "reason": "test_error", - "transient": false, - "extra": null - } - } - }`) - code, _, err := readStatusItem(response) - - assert.Nil(t, err) - assert.Equal(t, 400, code) -} - func TestCollectPublishFailsNone(t *testing.T) { N := 100 item := `{"create": {"status": 200}},` @@ -102,8 +51,7 @@ func TestCollectPublishFailsNone(t *testing.T) { events[i] = publisher.Event{Content: beat.Event{Fields: event}} } - reader := NewJSONReader(response) - res, _ := bulkCollectPublishFails(logp.L(), reader, events) + res, _ := bulkCollectPublishFails(logp.L(), response, events) assert.Equal(t, 0, len(res)) } @@ -120,8 +68,7 @@ func TestCollectPublishFailMiddle(t *testing.T) { eventFail := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 2}}} events := []publisher.Event{event, eventFail, event} - reader := NewJSONReader(response) - res, stats := bulkCollectPublishFails(logp.L(), reader, events) + res, stats := bulkCollectPublishFails(logp.L(), response, events) assert.Equal(t, 1, len(res)) if len(res) == 1 { assert.Equal(t, eventFail, res[0]) @@ -141,8 +88,7 @@ func TestCollectPublishFailAll(t *testing.T) { event := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 2}}} events := []publisher.Event{event, event, event} - reader := NewJSONReader(response) - res, stats := bulkCollectPublishFails(logp.L(), reader, events) + res, stats := bulkCollectPublishFails(logp.L(), response, events) assert.Equal(t, 3, len(res)) assert.Equal(t, events, res) assert.Equal(t, stats, bulkResultStats{fails: 3, tooMany: 3}) @@ -183,8 +129,7 @@ func TestCollectPipelinePublishFail(t *testing.T) { event := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 2}}} events := []publisher.Event{event} - reader := NewJSONReader(response) - res, _ := bulkCollectPublishFails(logp.L(), reader, events) + res, _ := bulkCollectPublishFails(logp.L(), response, events) assert.Equal(t, 1, len(res)) assert.Equal(t, events, res) } @@ -201,10 +146,8 @@ func BenchmarkCollectPublishFailsNone(b *testing.B) { event := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 1}}} events := []publisher.Event{event, event, event} - reader := NewJSONReader(nil) for i := 0; i < b.N; i++ { - reader.init(response) - res, _ := bulkCollectPublishFails(logp.L(), reader, events) + res, _ := bulkCollectPublishFails(logp.L(), response, events) if len(res) != 0 { b.Fail() } @@ -224,10 +167,8 @@ func BenchmarkCollectPublishFailMiddle(b *testing.B) { eventFail := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 2}}} events := []publisher.Event{event, eventFail, event} - reader := NewJSONReader(nil) for i := 0; i < b.N; i++ { - reader.init(response) - res, _ := bulkCollectPublishFails(logp.L(), reader, events) + res, _ := bulkCollectPublishFails(logp.L(), response, events) if len(res) != 1 { b.Fail() } @@ -246,10 +187,8 @@ func BenchmarkCollectPublishFailAll(b *testing.B) { event := publisher.Event{Content: beat.Event{Fields: common.MapStr{"field": 2}}} events := []publisher.Event{event, event, event} - reader := NewJSONReader(nil) for i := 0; i < b.N; i++ { - reader.init(response) - res, _ := bulkCollectPublishFails(logp.L(), reader, events) + res, _ := bulkCollectPublishFails(logp.L(), response, events) if len(res) != 3 { b.Fail() } @@ -402,55 +341,3 @@ func TestClientWithAPIKey(t *testing.T) { client.Ping() assert.Equal(t, "ApiKey aHlva0hHNEJmV2s1dmlLWjE3Mlg6bzQ1SlVreXVTLS15aVNBdXV4bDhVdw==", headers.Get("Authorization")) } - -func TestBulkReadToItems(t *testing.T) { - response := []byte(`{ - "errors": false, - "items": [ - {"create": {"status": 200}}, - {"create": {"status": 300}}, - {"create": {"status": 400}} - ]}`) - - reader := NewJSONReader(response) - - err := BulkReadToItems(reader) - assert.NoError(t, err) - - for status := 200; status <= 400; status += 100 { - err = reader.ExpectDict() - assert.NoError(t, err) - - kind, raw, err := reader.nextFieldName() - assert.NoError(t, err) - assert.Equal(t, mapKeyEntity, kind) - assert.Equal(t, []byte("create"), raw) - - err = reader.ExpectDict() - assert.NoError(t, err) - - kind, raw, err = reader.nextFieldName() - assert.NoError(t, err) - assert.Equal(t, mapKeyEntity, kind) - assert.Equal(t, []byte("status"), raw) - - code, err := reader.nextInt() - assert.NoError(t, err) - assert.Equal(t, status, code) - - _, _, err = reader.endDict() - assert.NoError(t, err) - - _, _, err = reader.endDict() - assert.NoError(t, err) - } -} - -func TestBulkReadItemStatus(t *testing.T) { - response := []byte(`{"create": {"status": 200}}`) - - reader := NewJSONReader(response) - code, _, err := BulkReadItemStatus(logp.L(), reader) - assert.NoError(t, err) - assert.Equal(t, 200, code) -} From cdb59c1686766d6a386c827591ec45c2612fe547 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 05:14:31 -0800 Subject: [PATCH 08/91] Moving API integration tests --- libbeat/esclientleg/api_integration_test.go | 96 ++++++++++++++++++--- 1 file changed, 83 insertions(+), 13 deletions(-) diff --git a/libbeat/esclientleg/api_integration_test.go b/libbeat/esclientleg/api_integration_test.go index f0d4943f6b9..5ab755c8586 100644 --- a/libbeat/esclientleg/api_integration_test.go +++ b/libbeat/esclientleg/api_integration_test.go @@ -29,6 +29,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/outputs/transport" +) + +const ( + // ElasticsearchDefaultHost is the default host for elasticsearch. + ElasticsearchDefaultHost = "localhost" + // ElasticsearchDefaultPort is the default port for elasticsearch. + ElasticsearchDefaultPort = "9200" +>>>>>>> Moving API integration tests ) func TestIndex(t *testing.T) { @@ -36,7 +45,7 @@ func TestIndex(t *testing.T) { index := fmt.Sprintf("beats-test-index-%d", os.Getpid()) - client := getTestingElasticsearch(t) + conn := getTestingElasticsearch(t) body := map[string]interface{}{ "user": "test", @@ -46,7 +55,7 @@ func TestIndex(t *testing.T) { params := map[string]string{ "refresh": "true", } - _, resp, err := client.Index(index, "_doc", "1", params, body) + _, resp, err := conn.Index(index, "_doc", "1", params, body) if err != nil { t.Fatalf("Index() returns error: %s", err) } @@ -59,7 +68,7 @@ func TestIndex(t *testing.T) { "match_all": map[string]interface{}{}, }, } - _, result, err := client.SearchURIWithBody(index, "", nil, map[string]interface{}{}) + _, result, err := conn.SearchURIWithBody(index, "", nil, map[string]interface{}{}) if err != nil { t.Fatalf("SearchUriWithBody() returns an error: %s", err) } @@ -70,7 +79,7 @@ func TestIndex(t *testing.T) { params = map[string]string{ "q": "user:test", } - _, result, err = client.SearchURI(index, "test", params) + _, result, err = conn.SearchURI(index, "test", params) if err != nil { t.Fatalf("SearchUri() returns an error: %s", err) } @@ -78,7 +87,7 @@ func TestIndex(t *testing.T) { t.Errorf("Wrong number of search results: %d", result.Hits.Total.Value) } - _, resp, err = client.Delete(index, "_doc", "1", nil) + _, resp, err = conn.Delete(index, "_doc", "1", nil) if err != nil { t.Errorf("Delete() returns error: %s", err) } @@ -103,17 +112,17 @@ func TestIngest(t *testing.T) { }, } - client := getTestingElasticsearch(t) - if client.Connection.version.Major < 5 { + conn := getTestingElasticsearch(t) + if conn.GetVersion().Major < 5 { t.Skip("Skipping tests as pipeline not available in <5.x releases") } - status, _, err := client.DeletePipeline(pipeline, nil) + status, _, err := conn.DeletePipeline(pipeline, nil) if err != nil && status != http.StatusNotFound { t.Fatal(err) } - exists, err := client.PipelineExists(pipeline) + exists, err := conn.PipelineExists(pipeline) if err != nil { t.Fatal(err) } @@ -121,7 +130,7 @@ func TestIngest(t *testing.T) { t.Fatalf("Test expected PipelineExists to return false for %v", pipeline) } - _, resp, err := client.CreatePipeline(pipeline, nil, pipelineBody) + _, resp, err := conn.CreatePipeline(pipeline, nil, pipelineBody) if err != nil { t.Fatal(err) } @@ -129,7 +138,7 @@ func TestIngest(t *testing.T) { t.Fatalf("Test pipeline %v not created", pipeline) } - exists, err = client.PipelineExists(pipeline) + exists, err = conn.PipelineExists(pipeline) if err != nil { t.Fatal(err) } @@ -138,7 +147,7 @@ func TestIngest(t *testing.T) { } params := map[string]string{"refresh": "true"} - _, resp, err = client.Ingest(index, "_doc", pipeline, "1", params, obj{ + _, resp, err = conn.Ingest(index, "_doc", pipeline, "1", params, obj{ "testfield": "TEST", }) if err != nil { @@ -149,7 +158,7 @@ func TestIngest(t *testing.T) { } // get _source field from indexed document - _, docBody, err := client.apiCall("GET", index, "", "_source/1", "", nil, nil) + _, docBody, err := conn.apiCall("GET", index, "", "_source/1", "", nil, nil) if err != nil { t.Fatal(err) } @@ -164,3 +173,64 @@ func TestIngest(t *testing.T) { assert.Equal(t, "test", doc.Field) } + +// FIXME: Below code is repeated in outputs/elasticsearch/internal/testing.go + +// TestLogger is used to report fatal errors to the testing framework. +type TestLogger interface { + Fatal(args ...interface{}) +} + +// Connectable defines the minimum interface required to initialize a connected +// client. +type Connectable interface { + Connect() error +} + +// getTestingElasticsearch creates a test client. +func getTestingElasticsearch(t TestLogger) *Connection { + connection := Connection{ + URL: getURL(), + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(0).Dial, + }, + Timeout: 0, + }, + Encoder: NewJSONEncoder(nil, false), + } + err := connection.Connect() + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + return connection +} + +func getURL() string { + return fmt.Sprintf("http://%v:%v", GetEsHost(), GetEsPort()) +} + +func getEsHost() string { + return getEnv("ES_HOST", ElasticsearchDefaultHost) +} + +func getEsPort() string { + return getEnv("ES_PORT", ElasticsearchDefaultPort) +} + +func getUser() string { + return getEnv("ES_USER", "") +} + +func getPass() string { + return getEnv("ES_PASS", "") +} + +func getEnv(name, def string) string { + if v := os.Getenv(name); len(v) > 0 { + return v + } + return def +} From f0946c211dc70607e228a6c17559d14740cc88b4 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 05:32:54 -0800 Subject: [PATCH 09/91] Fixing references in tests --- libbeat/outputs/elasticsearch/client_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index 9af44f5c0a4..8f0b3926486 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -27,6 +27,8 @@ import ( "testing" "time" + "github.com/elastic/beats/libbeat/esclientleg" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -289,11 +291,11 @@ func TestBulkEncodeEvents(t *testing.T) { // check meta-data for each event for i := 0; i < len(recorder.data); i += 2 { - var meta bulkEventMeta + var meta esclientleg.BulkMeta switch v := recorder.data[i].(type) { - case bulkCreateAction: + case esclientleg.BulkCreateAction: meta = v.Create - case bulkIndexAction: + case esclientleg.BulkIndexAction: meta = v.Index default: panic("unknown type") From ce11c70212dd07d0b726f599784a2fdbdc105a3c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 05:35:35 -0800 Subject: [PATCH 10/91] Adding developer CHANGELOG entry --- CHANGELOG-developer.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 54034d78934..763cc6aaeb1 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -36,6 +36,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - The `libbeat/outputs/transport` package has been moved to `libbeat/common/transport`. {pull}16734[16734] - The `libbeat/outputs/tls.go` file has been removed. All exported symbols in that file (`libbeat/outputs.*`) are now available as `libbeat/common/tlscommon.*`. {pull}16734[16734] - The newly generated Beats are using go modules to manage dependencies. {pull}16288[16288] +- Extract Elasticsearch client logic from `outputs/elasticsearch` package into new `esclientleg` package. {pull}16150[16150] ==== Bugfixes From 6c29542502bf0d68034501a169ade75dbf8f4b3b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 05:47:17 -0800 Subject: [PATCH 11/91] Move LoadJSON method --- libbeat/esclientleg/connection.go | 25 +++++++++++++++++++------ libbeat/outputs/elasticsearch/client.go | 13 ------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/libbeat/esclientleg/connection.go b/libbeat/esclientleg/connection.go index 3639d243f1b..275741807bf 100644 --- a/libbeat/esclientleg/connection.go +++ b/libbeat/esclientleg/connection.go @@ -155,6 +155,25 @@ func (conn *Connection) execRequest( return conn.execHTTPRequest(req) } +// GetVersion returns the elasticsearch version the client is connected to. +// The version is read and updated on 'Connect'. +func (conn *Connection) GetVersion() common.Version { + return conn.version +} + +// LoadJSON creates a PUT request based on a JSON document. +func (conn *Connection) LoadJSON(path string, json map[string]interface{}) ([]byte, error) { + status, body, err := conn.Request("PUT", path, "", nil, json) + if err != nil { + return body, fmt.Errorf("couldn't load json. Error: %s", err) + } + if status > 300 { + return body, fmt.Errorf("couldn't load json. Status: %v", status) + } + + return body, nil +} + func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) { req.Header.Add("Accept", "application/json") @@ -198,12 +217,6 @@ func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) return status, obj, err } -// GetVersion returns the elasticsearch version the client is connected to. -// The version is read and updated on 'Connect'. -func (conn *Connection) GetVersion() common.Version { - return conn.version -} - func closing(c io.Closer) { err := c.Close() if err != nil { diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 2dd9cf1e5f6..cd2c474281f 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -485,19 +485,6 @@ func bulkCollectPublishFails( return failed, stats } -// LoadJSON creates a PUT request based on a JSON document. -func (client *Client) LoadJSON(path string, json map[string]interface{}) ([]byte, error) { - status, body, err := client.Request("PUT", path, "", nil, json) - if err != nil { - return body, fmt.Errorf("couldn't load json. Error: %s", err) - } - if status > 300 { - return body, fmt.Errorf("couldn't load json. Status: %v", status) - } - - return body, nil -} - func (client *Client) Test(d testing.Driver) { d.Run("elasticsearch: "+client.URL, func(d testing.Driver) { u, err := url.Parse(client.URL) From b75c8c49aaaa20def85194962c1a802329030f9b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 05:52:02 -0800 Subject: [PATCH 12/91] Move callbacks to own file --- libbeat/outputs/elasticsearch/callbacks.go | 109 ++++++++++++++++++ libbeat/outputs/elasticsearch/client.go | 3 - .../outputs/elasticsearch/elasticsearch.go | 85 -------------- 3 files changed, 109 insertions(+), 88 deletions(-) create mode 100644 libbeat/outputs/elasticsearch/callbacks.go diff --git a/libbeat/outputs/elasticsearch/callbacks.go b/libbeat/outputs/elasticsearch/callbacks.go new file mode 100644 index 00000000000..76383e884f8 --- /dev/null +++ b/libbeat/outputs/elasticsearch/callbacks.go @@ -0,0 +1,109 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package elasticsearch + +import ( + "sync" + + "github.com/gofrs/uuid" +) + +// ConnectCallback defines the type for the function to be called when the Elasticsearch client successfully connects to the cluster +type ConnectCallback func(client *Client) error + +// Callbacks must not depend on the result of a previous one, +// because the ordering is not fixed. +type callbacksRegistry struct { + callbacks map[uuid.UUID]ConnectCallback + mutex sync.Mutex +} + +// XXX: it would be fantastic to do this without a package global +var connectCallbackRegistry = newCallbacksRegistry() + +// NOTE(ph): We need to refactor this, right now this is the only way to ensure that every calls +// to an ES cluster executes a callback. +var globalCallbackRegistry = newCallbacksRegistry() + +func newCallbacksRegistry() callbacksRegistry { + return callbacksRegistry{ + callbacks: make(map[uuid.UUID]ConnectCallback), + } +} + +// RegisterGlobalCallback register a global callbacks. +func RegisterGlobalCallback(callback ConnectCallback) (uuid.UUID, error) { + globalCallbackRegistry.mutex.Lock() + defer globalCallbackRegistry.mutex.Unlock() + + // find the next unique key + var key uuid.UUID + var err error + exists := true + for exists { + key, err = uuid.NewV4() + if err != nil { + return uuid.Nil, err + } + _, exists = globalCallbackRegistry.callbacks[key] + } + + globalCallbackRegistry.callbacks[key] = callback + return key, nil +} + +// RegisterConnectCallback registers a callback for the elasticsearch output +// The callback is called each time the client connects to elasticsearch. +// It returns the key of the newly added callback, so it can be deregistered later. +func RegisterConnectCallback(callback ConnectCallback) (uuid.UUID, error) { + connectCallbackRegistry.mutex.Lock() + defer connectCallbackRegistry.mutex.Unlock() + + // find the next unique key + var key uuid.UUID + var err error + exists := true + for exists { + key, err = uuid.NewV4() + if err != nil { + return uuid.Nil, err + } + _, exists = connectCallbackRegistry.callbacks[key] + } + + connectCallbackRegistry.callbacks[key] = callback + return key, nil +} + +// DeregisterGlobalCallback deregisters a callback for the elasticsearch output +// specified by its key. If a callback does not exist, nothing happens. +func DeregisterGlobalCallback(key uuid.UUID) { + globalCallbackRegistry.mutex.Lock() + defer globalCallbackRegistry.mutex.Unlock() + + delete(globalCallbackRegistry.callbacks, key) +} + +// DeregisterConnectCallback deregisters a callback for the elasticsearch output +// specified by its key. If a callback does not exist, nothing happens. +func DeregisterConnectCallback(key uuid.UUID) { + connectCallbackRegistry.mutex.Lock() + defer connectCallbackRegistry.mutex.Unlock() + + delete(connectCallbackRegistry.callbacks, key) +} diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index cd2c474281f..d55f160f776 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -77,9 +77,6 @@ type ClientSettings struct { Observer outputs.Observer } -// ConnectCallback defines the type for the function to be called when the Elasticsearch client successfully connects to the cluster -type ConnectCallback func(client *Client) error - type bulkResultStats struct { acked int // number of events ACKed by Elasticsearch duplicates int // number of events failed with `create` due to ID already being indexed diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index 441ca769937..030d7d3c1c5 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -20,9 +20,6 @@ package elasticsearch import ( "fmt" "net/url" - "sync" - - "github.com/gofrs/uuid" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" @@ -39,88 +36,6 @@ func init() { const logSelector = "elasticsearch" -// Callbacks must not depend on the result of a previous one, -// because the ordering is not fixed. -type callbacksRegistry struct { - callbacks map[uuid.UUID]ConnectCallback - mutex sync.Mutex -} - -// XXX: it would be fantastic to do this without a package global -var connectCallbackRegistry = newCallbacksRegistry() - -// NOTE(ph): We need to refactor this, right now this is the only way to ensure that every calls -// to an ES cluster executes a callback. -var globalCallbackRegistry = newCallbacksRegistry() - -// RegisterGlobalCallback register a global callbacks. -func RegisterGlobalCallback(callback ConnectCallback) (uuid.UUID, error) { - globalCallbackRegistry.mutex.Lock() - defer globalCallbackRegistry.mutex.Unlock() - - // find the next unique key - var key uuid.UUID - var err error - exists := true - for exists { - key, err = uuid.NewV4() - if err != nil { - return uuid.Nil, err - } - _, exists = globalCallbackRegistry.callbacks[key] - } - - globalCallbackRegistry.callbacks[key] = callback - return key, nil -} - -func newCallbacksRegistry() callbacksRegistry { - return callbacksRegistry{ - callbacks: make(map[uuid.UUID]ConnectCallback), - } -} - -// RegisterConnectCallback registers a callback for the elasticsearch output -// The callback is called each time the client connects to elasticsearch. -// It returns the key of the newly added callback, so it can be deregistered later. -func RegisterConnectCallback(callback ConnectCallback) (uuid.UUID, error) { - connectCallbackRegistry.mutex.Lock() - defer connectCallbackRegistry.mutex.Unlock() - - // find the next unique key - var key uuid.UUID - var err error - exists := true - for exists { - key, err = uuid.NewV4() - if err != nil { - return uuid.Nil, err - } - _, exists = connectCallbackRegistry.callbacks[key] - } - - connectCallbackRegistry.callbacks[key] = callback - return key, nil -} - -// DeregisterConnectCallback deregisters a callback for the elasticsearch output -// specified by its key. If a callback does not exist, nothing happens. -func DeregisterConnectCallback(key uuid.UUID) { - connectCallbackRegistry.mutex.Lock() - defer connectCallbackRegistry.mutex.Unlock() - - delete(connectCallbackRegistry.callbacks, key) -} - -// DeregisterGlobalCallback deregisters a callback for the elasticsearch output -// specified by its key. If a callback does not exist, nothing happens. -func DeregisterGlobalCallback(key uuid.UUID) { - globalCallbackRegistry.mutex.Lock() - defer globalCallbackRegistry.mutex.Unlock() - - delete(globalCallbackRegistry.callbacks, key) -} - func makeES( im outputs.IndexManager, beat beat.Info, From f185aaa4c570566260a317a872fddba45ac6c31a Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 05:55:25 -0800 Subject: [PATCH 13/91] Move client-related constructors into client.go file --- libbeat/outputs/elasticsearch/client.go | 93 ++++++++++++++++++ .../outputs/elasticsearch/elasticsearch.go | 95 ------------------- 2 files changed, 93 insertions(+), 95 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index d55f160f776..b08dc1977d2 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -213,6 +213,99 @@ func NewClient( return client, nil } +// NewConnectedClient creates a new Elasticsearch client based on the given config. +// It uses the NewElasticsearchClients to create a list of clients then returns +// the first from the list that successfully connects. +func NewConnectedClient(cfg *common.Config) (*Client, error) { + clients, err := NewElasticsearchClients(cfg) + if err != nil { + return nil, err + } + + errors := []string{} + + for _, client := range clients { + err = client.Connect() + if err != nil { + logp.Err("Error connecting to Elasticsearch at %v: %v", client.Connection.URL, err) + err = fmt.Errorf("Error connection to Elasticsearch %v: %v", client.Connection.URL, err) + errors = append(errors, err.Error()) + continue + } + return &client, nil + } + return nil, fmt.Errorf("Couldn't connect to any of the configured Elasticsearch hosts. Errors: %v", errors) +} + +// NewElasticsearchClients returns a list of Elasticsearch clients based on the given +// configuration. It accepts the same configuration parameters as the output, +// except for the output specific configuration options (index, pipeline, +// template) .If multiple hosts are defined in the configuration, a client is returned +// for each of them. +func NewElasticsearchClients(cfg *common.Config) ([]Client, error) { + hosts, err := outputs.ReadHostList(cfg) + if err != nil { + return nil, err + } + + config := defaultConfig + if err = cfg.Unpack(&config); err != nil { + return nil, err + } + + tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS) + if err != nil { + return nil, err + } + + var proxyURL *url.URL + if !config.ProxyDisable { + proxyURL, err = esclientleg.ParseProxyURL(config.ProxyURL) + if err != nil { + return nil, err + } + if proxyURL != nil { + logp.Info("Using proxy URL: %s", proxyURL) + } + } + + params := config.Params + if len(params) == 0 { + params = nil + } + + clients := []Client{} + for _, host := range hosts { + esURL, err := common.MakeURL(config.Protocol, config.Path, host, 9200) + if err != nil { + logp.Err("Invalid host param set: %s, Error: %v", host, err) + return nil, err + } + + client, err := NewClient(ClientSettings{ + URL: esURL, + Proxy: proxyURL, + ProxyDisable: config.ProxyDisable, + TLS: tlsConfig, + Username: config.Username, + Password: config.Password, + APIKey: config.APIKey, + Parameters: params, + Headers: config.Headers, + Timeout: config.Timeout, + CompressionLevel: config.CompressionLevel, + }, nil) + if err != nil { + return clients, err + } + clients = append(clients, *client) + } + if len(clients) == 0 { + return clients, fmt.Errorf("No hosts defined in the Elasticsearch output") + } + return clients, nil +} + // Clone clones a client. func (client *Client) Clone() *Client { // when cloning the connection callback and params are not copied. A diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index 030d7d3c1c5..f738b96433b 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -18,7 +18,6 @@ package elasticsearch import ( - "fmt" "net/url" "github.com/elastic/beats/v7/libbeat/beat" @@ -146,97 +145,3 @@ func buildSelectors( return index, pipeline, err } - -// NewConnectedClient creates a new Elasticsearch client based on the given config. -// It uses the NewElasticsearchClients to create a list of clients then returns -// the first from the list that successfully connects. -func NewConnectedClient(cfg *common.Config) (*Client, error) { - clients, err := NewElasticsearchClients(cfg) - if err != nil { - return nil, err - } - - errors := []string{} - - for _, client := range clients { - err = client.Connect() - if err != nil { - client.Connection.log.Errorf("Error connecting to Elasticsearch at %v: %+v", client.Connection.URL, err) - err = fmt.Errorf("Error connection to Elasticsearch %v: %v", client.Connection.URL, err) - errors = append(errors, err.Error()) - continue - } - return &client, nil - } - return nil, fmt.Errorf("Couldn't connect to any of the configured Elasticsearch hosts. Errors: %v", errors) -} - -// NewElasticsearchClients returns a list of Elasticsearch clients based on the given -// configuration. It accepts the same configuration parameters as the output, -// except for the output specific configuration options (index, pipeline, -// template) .If multiple hosts are defined in the configuration, a client is returned -// for each of them. -func NewElasticsearchClients(cfg *common.Config) ([]Client, error) { - hosts, err := outputs.ReadHostList(cfg) - if err != nil { - return nil, err - } - - config := defaultConfig - if err = cfg.Unpack(&config); err != nil { - return nil, err - } - - tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS) - if err != nil { - return nil, err - } - - log := logp.NewLogger(logSelector) - var proxyURL *url.URL - if !config.ProxyDisable { - proxyURL, err = esclientleg.ParseProxyURL(config.ProxyURL) - if err != nil { - return nil, err - } - if proxyURL != nil { - log.Infof("Using proxy URL: %s", proxyURL) - } - } - - params := config.Params - if len(params) == 0 { - params = nil - } - - clients := []Client{} - for _, host := range hosts { - esURL, err := common.MakeURL(config.Protocol, config.Path, host, 9200) - if err != nil { - log.Errorf("Invalid host param set: %s, Error: %+v", host, err) - return nil, err - } - - client, err := NewClient(ClientSettings{ - URL: esURL, - Proxy: proxyURL, - ProxyDisable: config.ProxyDisable, - TLS: tlsConfig, - Username: config.Username, - Password: config.Password, - APIKey: config.APIKey, - Parameters: params, - Headers: config.Headers, - Timeout: config.Timeout, - CompressionLevel: config.CompressionLevel, - }, nil) - if err != nil { - return clients, err - } - clients = append(clients, *client) - } - if len(clients) == 0 { - return clients, fmt.Errorf("No hosts defined in the Elasticsearch output") - } - return clients, nil -} From d9ea8da685916fced07cbbcf180fddedbea8ad30 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 06:05:42 -0800 Subject: [PATCH 14/91] Reducing global logging usage --- libbeat/esclientleg/bulkapi.go | 10 ++-- libbeat/esclientleg/connection.go | 42 +++++++++----- .../monitoring/report/elasticsearch/client.go | 12 ++++ .../report/elasticsearch/elasticsearch.go | 4 +- libbeat/outputs/elasticsearch/client.go | 58 +++++++++++-------- 5 files changed, 78 insertions(+), 48 deletions(-) diff --git a/libbeat/esclientleg/bulkapi.go b/libbeat/esclientleg/bulkapi.go index d4d39b07366..75e0e8f2e74 100644 --- a/libbeat/esclientleg/bulkapi.go +++ b/libbeat/esclientleg/bulkapi.go @@ -268,7 +268,7 @@ func BulkReadToItems(reader *JSONReader) error { } // BulkReadItemStatus reads the status and error fields from the bulk item -func BulkReadItemStatus(log *logp.Logger, reader *JSONReader) (int, []byte, error) { +func BulkReadItemStatus(reader *JSONReader, logger *logp.Logger) (int, []byte, error) { // skip outer dictionary if err := reader.ExpectDict(); err != nil { return 0, nil, errExpectedItemObject @@ -287,7 +287,7 @@ func BulkReadItemStatus(log *logp.Logger, reader *JSONReader) (int, []byte, erro } // parse actual item response code and error message - status, msg, err := itemStatusInner(reader) + status, msg, err := itemStatusInner(reader, logger) if err != nil { log.Errorf("Failed to parse bulk response item: %s", err) return 0, nil, err @@ -308,7 +308,7 @@ func BulkReadItemStatus(log *logp.Logger, reader *JSONReader) (int, []byte, erro return status, msg, nil } -func itemStatusInner(reader *JSONReader) (int, []byte, error) { +func itemStatusInner(reader *JSONReader, logger *logp.Logger) (int, []byte, error) { if err := reader.ExpectDict(); err != nil { return 0, nil, errExpectedItemObject } @@ -318,7 +318,7 @@ func itemStatusInner(reader *JSONReader) (int, []byte, error) { for { kind, name, err := reader.nextFieldName() if err != nil { - logp.Err("Failed to parse bulk response item: %s", err) + logger.Errorf("Failed to parse bulk response item: %s", err) } if kind == dictEnd { break @@ -328,7 +328,7 @@ func itemStatusInner(reader *JSONReader) (int, []byte, error) { case bytes.Equal(name, nameStatus): // name == "status" status, err = reader.nextInt() if err != nil { - logp.Err("Failed to parse bulk response item: %s", err) + logger.Errorf("Failed to parse bulk response item: %s", err) return 0, nil, err } diff --git a/libbeat/esclientleg/connection.go b/libbeat/esclientleg/connection.go index 275741807bf..f1d4e8d971c 100644 --- a/libbeat/esclientleg/connection.go +++ b/libbeat/esclientleg/connection.go @@ -29,12 +29,16 @@ import ( "github.com/elastic/beats/libbeat/common" ) -var ( - debugf = logp.MakeDebug("esclientleg") -) - // Connection manages the connection for a given client. type Connection struct { + ConnectionSettings + + version common.Version + log *logp.Logger +} + +// ConnectionSettings are the settings needed for a Connection +type ConnectionSettings struct { URL string Username string Password string @@ -45,7 +49,13 @@ type Connection struct { OnConnectCallback func() error Encoder BodyEncoder - version common.Version +} + +func NewConnection(settings ConnectionSettings) *Connection { + return &Connection{ + ConnectionSettings: settings, + log: logp.NewLogger("esclientleg"), + } } // Connect connects the client. It runs a GET request against the root URL of @@ -58,7 +68,7 @@ func (conn *Connection) Connect() error { } if version, err := common.NewVersion(versionString); err != nil { - logp.Err("Invalid version from Elasticsearch: %v", versionString) + conn.log.Errorf("Invalid version from Elasticsearch: %v", versionString) conn.version = common.Version{} } else { conn.version = *version @@ -76,11 +86,11 @@ func (conn *Connection) Connect() error { // Ping sends a GET request to the Elasticsearch. func (conn *Connection) Ping() (string, error) { - debugf("ES Ping(url=%v)", conn.URL) + conn.log.Debugf("ES Ping(url=%v)", conn.URL) status, body, err := conn.execRequest("GET", conn.URL, nil) if err != nil { - debugf("Ping request failed with: %v", err) + conn.log.Debugf("Ping request failed with: %v", err) return "", err } @@ -99,8 +109,8 @@ func (conn *Connection) Ping() (string, error) { return "", fmt.Errorf("Failed to parse JSON response: %v", err) } - debugf("Ping status code: %v", status) - logp.Info("Attempting to connect to Elasticsearch version %s", response.Version.Number) + conn.log.Debugf("Ping status code: %v", status) + conn.log.Infof("Attempting to connect to Elasticsearch version %s", response.Version.Number) return response.Version.Number, nil } @@ -118,7 +128,7 @@ func (conn *Connection) Request( ) (int, []byte, error) { url := addToURL(conn.URL, path, pipeline, params) - debugf("%s %s %s %v", method, url, pipeline, body) + conn.log.Debugf("%s %s %s %v", method, url, pipeline, body) return conn.RequestURL(method, url, body) } @@ -134,7 +144,7 @@ func (conn *Connection) RequestURL( } if err := conn.Encoder.Marshal(body); err != nil { - logp.Warn("Failed to json encode body (%v): %#v", err, body) + conn.log.Warnf("Failed to json encode body (%v): %#v", err, body) return 0, nil, ErrJSONEncodeFailed } return conn.execRequest(method, url, conn.Encoder.Reader()) @@ -146,7 +156,7 @@ func (conn *Connection) execRequest( ) (int, []byte, error) { req, err := http.NewRequest(method, url, body) if err != nil { - logp.Warn("Failed to create request %+v", err) + conn.log.Warnf("Failed to create request %+v", err) return 0, nil, err } if body != nil { @@ -201,7 +211,7 @@ func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) if err != nil { return 0, nil, err } - defer closing(resp.Body) + defer closing(resp.Body, conn.log) status := resp.StatusCode obj, err := ioutil.ReadAll(resp.Body) @@ -217,9 +227,9 @@ func (conn *Connection) execHTTPRequest(req *http.Request) (int, []byte, error) return status, obj, err } -func closing(c io.Closer) { +func closing(c io.Closer, logger *logp.Logger) { err := c.Close() if err != nil { - logp.Warn("Close failed with: %v", err) + logger.Warn("Close failed with: %v", err) } } diff --git a/libbeat/monitoring/report/elasticsearch/client.go b/libbeat/monitoring/report/elasticsearch/client.go index 36ec6b28176..77876816405 100644 --- a/libbeat/monitoring/report/elasticsearch/client.go +++ b/libbeat/monitoring/report/elasticsearch/client.go @@ -41,6 +41,8 @@ type publishClient struct { es *esout.Client params map[string]string format report.Format + + log *logp.Logger } func newPublishClient( @@ -53,6 +55,8 @@ func newPublishClient( es: es, params: params, format: format, + + log: logp.NewLogger(logSelector), } return p, nil } @@ -93,7 +97,11 @@ func (c *publishClient) Connect() error { return errNoMonitoring } +<<<<<<< HEAD c.log.Debug("XPack monitoring is enabled") +======= + c.log.Debug(("XPack monitoring is enabled")) +>>>>>>> Reducing global logging usage return nil } @@ -111,7 +119,11 @@ func (c *publishClient) Publish(batch publisher.Batch) error { // Extract type t, err := event.Content.Meta.GetValue("type") if err != nil { +<<<<<<< HEAD c.log.Errorf("Type not available in monitoring reported. Please report this error: %+v", err) +======= + c.log.Errorf("Type not available in monitoring reported. Please report this error: %s", err) +>>>>>>> Reducing global logging usage continue } diff --git a/libbeat/monitoring/report/elasticsearch/elasticsearch.go b/libbeat/monitoring/report/elasticsearch/elasticsearch.go index d4f767690e5..ffe8cbe795c 100644 --- a/libbeat/monitoring/report/elasticsearch/elasticsearch.go +++ b/libbeat/monitoring/report/elasticsearch/elasticsearch.go @@ -59,7 +59,7 @@ type reporter struct { out []outputs.NetworkClient } -const selector = "monitoring" +const logSelector = "monitoring" var errNoMonitoring = errors.New("xpack monitoring not available") @@ -112,7 +112,7 @@ func defaultConfig(settings report.Settings) config { } func makeReporter(beat beat.Info, settings report.Settings, cfg *common.Config) (report.Reporter, error) { - log := logp.NewLogger(selector) + log := logp.NewLogger(logSelector) config := defaultConfig(settings) if err := cfg.Unpack(&config); err != nil { return nil, err diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index b08dc1977d2..8542941504e 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -57,6 +57,8 @@ type Client struct { proxyURL *url.URL observer outputs.Observer + + log *logp.Logger } // ClientSettings contains the settings for a client. @@ -154,35 +156,39 @@ func NewClient( } } - client := &Client{ - Connection: esclientleg.Connection{ - URL: s.URL, - Username: s.Username, - Password: s.Password, - APIKey: base64.StdEncoding.EncodeToString([]byte(s.APIKey)), - Headers: s.Headers, - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: dialer.Dial, - DialTLS: tlsDialer.Dial, - TLSClientConfig: s.TLS.ToConfig(), - Proxy: proxy, - }, - Timeout: s.Timeout, + conn := esclientleg.NewConnection(esclientleg.ConnectionSettings{ + URL: s.URL, + Username: s.Username, + Password: s.Password, + APIKey: base64.StdEncoding.EncodeToString([]byte(s.APIKey)), + Headers: s.Headers, + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: dialer.Dial, + DialTLS: tlsDialer.Dial, + TLSClientConfig: s.TLS.ToConfig(), + Proxy: proxy, }, - Encoder: encoder, + Timeout: s.Timeout, }, - tlsConfig: s.TLS, - index: s.Index, - pipeline: pipeline, - params: params, - timeout: s.Timeout, + Encoder: encoder, + }) + + client := &Client{ + Connection: *conn, + tlsConfig: s.TLS, + index: s.Index, + pipeline: pipeline, + params: params, + timeout: s.Timeout, bulkRequ: bulkRequ, compressionLevel: compression, proxyURL: s.Proxy, observer: s.Observer, + + log: logp.NewLogger("elasticsearch"), } client.Connection.OnConnectCallback = func() error { @@ -377,7 +383,7 @@ func (client *Client) publishEvents( } origCount := len(data) - data = bulkEncodePublishRequest(client.GetVersion(), body, client.index, client.pipeline, eventType, data) + data = bulkEncodePublishRequest(client.GetVersion(), body, client.index, client.pipeline, eventType, data, client.log) newCount := len(data) if st != nil && origCount > newCount { st.Dropped(origCount - newCount) @@ -390,11 +396,11 @@ func (client *Client) publishEvents( requ.Reset(body) status, result, sendErr := client.SendBulkRequest(requ) if sendErr != nil { - client.Connection.log.Error("Failed to perform any bulk index operations: %+v", sendErr) + client.log.Errorf("Failed to perform any bulk index operations: %s", sendErr) return data, sendErr } - client.Connection.log.Debugf("PublishEvents: %d events have been published to elasticsearch in %v.", + client.log.Debugf("PublishEvents: %d events have been published to elasticsearch in %v.", len(data), time.Now().Sub(begin)) @@ -405,7 +411,7 @@ func (client *Client) publishEvents( failedEvents = data stats.fails = len(failedEvents) } else { - failedEvents, stats = bulkCollectPublishFails(result, data) + failedEvents, stats = bulkCollectPublishFails(result, data, client.log) } failed := len(failedEvents) @@ -440,6 +446,7 @@ func bulkEncodePublishRequest( pipeline *outil.Selector, eventType string, data []publisher.Event, + logger *logp.Logger, ) []publisher.Event { okEvents := data[:0] for i := range data { @@ -466,6 +473,7 @@ func createEventBulkMeta( pipelineSel *outil.Selector, eventType string, event *beat.Event, + logger *logp.Logger, ) (interface{}, error) { pipeline, err := getPipeline(event, pipelineSel) if err != nil { From a9c606bce4bd6ff5060c01cea697d7a4b4717382 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 06:46:54 -0800 Subject: [PATCH 15/91] Use new constructor in test --- libbeat/esclientleg/api_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libbeat/esclientleg/api_test.go b/libbeat/esclientleg/api_test.go index a5d7da4017b..658e405353d 100644 --- a/libbeat/esclientleg/api_test.go +++ b/libbeat/esclientleg/api_test.go @@ -172,8 +172,8 @@ func TestReadSearchResult_invalid(t *testing.T) { assert.Error(t, err) } -func newTestConnection(url string) Connection { - return Connection{ +func newTestConnection(url string) *Connection { + return NewConnection(ConnectionSettings{ URL: url, HTTP: &http.Client{ Transport: &http.Transport{ @@ -182,7 +182,7 @@ func newTestConnection(url string) Connection { Timeout: 0, }, Encoder: NewJSONEncoder(nil, false), - } + }) } func (r QueryResult) String() string { From cc7e93ea329175c69afb51f04afb4977903cee00 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 07:03:54 -0800 Subject: [PATCH 16/91] Passing logger in test --- libbeat/outputs/elasticsearch/client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 8542941504e..0c7616415d5 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -446,7 +446,6 @@ func bulkEncodePublishRequest( pipeline *outil.Selector, eventType string, data []publisher.Event, - logger *logp.Logger, ) []publisher.Event { okEvents := data[:0] for i := range data { From 6dc8b2b509a88b4564b3639a9f6c559ac8f0f78e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 07:18:42 -0800 Subject: [PATCH 17/91] Use logger in test --- libbeat/esclientleg/bulkapi_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libbeat/esclientleg/bulkapi_test.go b/libbeat/esclientleg/bulkapi_test.go index d55abd18e14..a08c6b99123 100644 --- a/libbeat/esclientleg/bulkapi_test.go +++ b/libbeat/esclientleg/bulkapi_test.go @@ -20,6 +20,8 @@ package esclientleg import ( "testing" + "github.com/elastic/beats/libbeat/logp" + "github.com/stretchr/testify/assert" ) @@ -70,7 +72,7 @@ func TestBulkReadItemStatus(t *testing.T) { response := []byte(`{"create": {"status": 200}}`) reader := NewJSONReader(response) - code, _, err := BulkReadItemStatus(reader) + code, _, err := BulkReadItemStatus(reader, testLogger()) assert.NoError(t, err) assert.Equal(t, 200, code) } @@ -122,6 +124,10 @@ func TestES2StyleExtendedErrorStatus(t *testing.T) { func readStatusItem(in []byte) (int, string, error) { reader := NewJSONReader(in) - code, msg, err := BulkReadItemStatus(reader) + code, msg, err := BulkReadItemStatus(reader, testLogger()) return code, string(msg), err } + +func testLogger() *logp.Logger { + return logp.NewLogger("test_esclientleg") +} From 57864df619ad3047e1ec6bdb1602d66f7bb8a809 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 07:41:21 -0800 Subject: [PATCH 18/91] Use struct fieldnames when initializing --- libbeat/outputs/elasticsearch/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 0c7616415d5..31ba9a79fc8 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -505,9 +505,9 @@ func createEventBulkMeta( } if id != "" || version.Major > 7 || (version.Major == 7 && version.Minor >= 5) { - return esclientleg.BulkCreateAction{meta}, nil + return esclientleg.BulkCreateAction{Create: meta}, nil } - return esclientleg.BulkIndexAction{meta}, nil + return esclientleg.BulkIndexAction{Index: meta}, nil } func getPipeline(event *beat.Event, pipelineSel *outil.Selector) (string, error) { From d7f02afe14aaa8cf26ed14519c0894bcee9ac92c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 10:07:27 -0800 Subject: [PATCH 19/91] Use constructor in test --- libbeat/esclientleg/api_integration_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libbeat/esclientleg/api_integration_test.go b/libbeat/esclientleg/api_integration_test.go index 5ab755c8586..e9a820485ce 100644 --- a/libbeat/esclientleg/api_integration_test.go +++ b/libbeat/esclientleg/api_integration_test.go @@ -189,7 +189,7 @@ type Connectable interface { // getTestingElasticsearch creates a test client. func getTestingElasticsearch(t TestLogger) *Connection { - connection := Connection{ + connection := NewConnection(ConnectionSettings{ URL: getURL(), HTTP: &http.Client{ Transport: &http.Transport{ @@ -198,7 +198,8 @@ func getTestingElasticsearch(t TestLogger) *Connection { Timeout: 0, }, Encoder: NewJSONEncoder(nil, false), - } + }) + err := connection.Connect() if err != nil { t.Fatal(err) From 13bbe68e18b4053767cda1d13192352e0f6e9b46 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Feb 2020 10:08:30 -0800 Subject: [PATCH 20/91] Fixing typos --- libbeat/esclientleg/api_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/esclientleg/api_integration_test.go b/libbeat/esclientleg/api_integration_test.go index e9a820485ce..c2815f84140 100644 --- a/libbeat/esclientleg/api_integration_test.go +++ b/libbeat/esclientleg/api_integration_test.go @@ -210,7 +210,7 @@ func getTestingElasticsearch(t TestLogger) *Connection { } func getURL() string { - return fmt.Sprintf("http://%v:%v", GetEsHost(), GetEsPort()) + return fmt.Sprintf("http://%v:%v", getEsHost(), getEsPort()) } func getEsHost() string { From 9969cd04ef07045856d997f1dcae9ac067097098 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 14 Feb 2020 16:39:07 -0800 Subject: [PATCH 21/91] Replace esclient.ParseProxyURL with generic function in common --- libbeat/common/url.go | 16 +++++++ libbeat/common/url_test.go | 43 +++++++++++++++++++ libbeat/esclientleg/url.go | 15 ------- .../report/elasticsearch/elasticsearch.go | 3 +- libbeat/outputs/elasticsearch/client.go | 2 +- libbeat/outputs/elasticsearch/config.go | 4 +- .../outputs/elasticsearch/elasticsearch.go | 3 +- 7 files changed, 64 insertions(+), 22 deletions(-) diff --git a/libbeat/common/url.go b/libbeat/common/url.go index 327edeef996..d4e55154dfb 100644 --- a/libbeat/common/url.go +++ b/libbeat/common/url.go @@ -83,3 +83,19 @@ func EncodeURLParams(url string, params url.Values) string { return strings.Join([]string{url, "?", params.Encode()}, "") } + +// ParseURL tries to parse a URL and return the parsed result. +func ParseURL(raw string) (*url.URL, error) { + if raw == "" { + return nil, nil + } + + url, err := url.Parse(raw) + if err == nil && strings.HasPrefix(url.Scheme, "http") { + return url, err + } + + // Proxy was bogus. Try prepending "http://" to it and + // see if that parses correctly. + return url.Parse("http://" + raw) +} diff --git a/libbeat/common/url_test.go b/libbeat/common/url_test.go index e0ea76c9348..8a6c70fb946 100644 --- a/libbeat/common/url_test.go +++ b/libbeat/common/url_test.go @@ -24,6 +24,8 @@ import ( "net/url" "testing" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" ) @@ -114,3 +116,44 @@ func TestURLParamsEncode(t *testing.T) { assert.Equal(t, output, urlWithParams) } } + +func TestParseURL(t *testing.T) { + tests := map[string]struct { + input string + expected string + errorAssertFunc require.ErrorAssertionFunc + }{ + "http": { + "http://host:1234/path", + "http://host:1234/path", + require.NoError, + }, + "https": { + "https://host:1234/path", + "https://host:1234/path", + require.NoError, + }, + "no_scheme": { + "host:1234/path", + "http://host:1234/path", + require.NoError, + }, + "invalid": { + "foobar:port", + "", + require.Error, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + url, err := ParseURL(test.input) + test.errorAssertFunc(t, err) + if test.expected != "" { + require.Equal(t, test.expected, url.String()) + } else { + require.Nil(t, url) + } + }) + } +} diff --git a/libbeat/esclientleg/url.go b/libbeat/esclientleg/url.go index e5bbed2cbfb..95f0877b720 100644 --- a/libbeat/esclientleg/url.go +++ b/libbeat/esclientleg/url.go @@ -73,18 +73,3 @@ func makePath(index string, docType string, id string) (string, error) { } return path, nil } - -func ParseProxyURL(raw string) (*url.URL, error) { - if raw == "" { - return nil, nil - } - - url, err := url.Parse(raw) - if err == nil && strings.HasPrefix(url.Scheme, "http") { - return url, err - } - - // Proxy was bogus. Try prepending "http://" to it and - // see if that parses correctly. - return url.Parse("http://" + raw) -} diff --git a/libbeat/monitoring/report/elasticsearch/elasticsearch.go b/libbeat/monitoring/report/elasticsearch/elasticsearch.go index ffe8cbe795c..b530a8c081e 100644 --- a/libbeat/monitoring/report/elasticsearch/elasticsearch.go +++ b/libbeat/monitoring/report/elasticsearch/elasticsearch.go @@ -29,7 +29,6 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/monitoring/report" @@ -131,7 +130,7 @@ func makeReporter(beat beat.Info, settings report.Settings, cfg *common.Config) windowSize = 1 } - proxyURL, err := esclientleg.ParseProxyURL(config.ProxyURL) + proxyURL, err := common.ParseURL(config.ProxyURL) if err != nil { return nil, err } diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 31ba9a79fc8..1452f3b5484 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -266,7 +266,7 @@ func NewElasticsearchClients(cfg *common.Config) ([]Client, error) { var proxyURL *url.URL if !config.ProxyDisable { - proxyURL, err = esclientleg.ParseProxyURL(config.ProxyURL) + proxyURL, err = common.ParseURL(config.ProxyURL) if err != nil { return nil, err } diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index 342b3129799..499bba2eeff 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -21,8 +21,8 @@ import ( "fmt" "time" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/libbeat/esclientleg" ) type elasticsearchConfig struct { @@ -79,7 +79,7 @@ var ( func (c *elasticsearchConfig) Validate() error { if c.ProxyURL != "" && !c.ProxyDisable { - if _, err := esclientleg.ParseProxyURL(c.ProxyURL); err != nil { + if _, err := common.ParseURL(c.ProxyURL); err != nil { return err } } diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index f738b96433b..8e46b4f3c3f 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -23,7 +23,6 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" "github.com/elastic/beats/v7/libbeat/outputs/outil" @@ -68,7 +67,7 @@ func makeES( var proxyURL *url.URL if !config.ProxyDisable { - proxyURL, err = esclientleg.ParseProxyURL(config.ProxyURL) + proxyURL, err = common.ParseURL(config.ProxyURL) if err != nil { return outputs.Fail(err) } From 8446cd00ff84ceae0d3d71296d0b3273db65ad88 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 14 Feb 2020 16:40:58 -0800 Subject: [PATCH 22/91] Imports formatting --- libbeat/common/url_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libbeat/common/url_test.go b/libbeat/common/url_test.go index 8a6c70fb946..a0becf080fe 100644 --- a/libbeat/common/url_test.go +++ b/libbeat/common/url_test.go @@ -24,9 +24,8 @@ import ( "net/url" "testing" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGetUrl(t *testing.T) { From c0693e605ce0173eb25a03bef43114fb54a59bd3 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 24 Feb 2020 09:12:01 -0800 Subject: [PATCH 23/91] Moving more fields from ES output client to esclientleg.Connection --- libbeat/esclientleg/connection.go | 31 +++++++++++++-- libbeat/outputs/elasticsearch/client.go | 51 +++++++++---------------- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/libbeat/esclientleg/connection.go b/libbeat/esclientleg/connection.go index f1d4e8d971c..2fc1d0b1e24 100644 --- a/libbeat/esclientleg/connection.go +++ b/libbeat/esclientleg/connection.go @@ -23,6 +23,9 @@ import ( "io" "io/ioutil" "net/http" + "net/url" + + "github.com/elastic/beats/libbeat/outputs/transport" "github.com/elastic/beats/libbeat/logp" @@ -33,6 +36,7 @@ import ( type Connection struct { ConnectionSettings + Encoder BodyEncoder version common.Version log *logp.Logger } @@ -40,22 +44,41 @@ type Connection struct { // ConnectionSettings are the settings needed for a Connection type ConnectionSettings struct { URL string + ProxyURL *url.URL + Username string Password string APIKey string Headers map[string]string - HTTP *http.Client + TLSConfig *transport.TLSConfig + + HTTP *http.Client + OnConnectCallback func() error - Encoder BodyEncoder + CompressionLevel int + EscapeHTML bool } -func NewConnection(settings ConnectionSettings) *Connection { +func NewConnection(settings ConnectionSettings) (*Connection, error) { + var encoder BodyEncoder + var err error + compression := settings.CompressionLevel + if compression == 0 { + encoder = NewJSONEncoder(nil, settings.EscapeHTML) + } else { + encoder, err = NewGzipEncoder(compression, nil, settings.EscapeHTML) + if err != nil { + return nil, err + } + } + return &Connection{ ConnectionSettings: settings, + Encoder: encoder, log: logp.NewLogger("esclientleg"), - } + }, nil } // Connect connects the client. It runs a GET request against the root URL of diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 1452f3b5484..c8cd4814825 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -42,7 +42,6 @@ import ( // Client is an elasticsearch client. type Client struct { esclientleg.Connection - tlsConfig *tlscommon.TLSConfig index outputs.IndexSelector pipeline *outil.Selector @@ -52,10 +51,6 @@ type Client struct { // buffered bulk requests bulkRequ *esclientleg.BulkRequest - // additional configs - compressionLevel int - proxyURL *url.URL - observer outputs.Observer log *logp.Logger @@ -145,23 +140,13 @@ func NewClient( return nil, err } - var encoder esclientleg.BodyEncoder - compression := s.CompressionLevel - if compression == 0 { - encoder = esclientleg.NewJSONEncoder(nil, s.EscapeHTML) - } else { - encoder, err = esclientleg.NewGzipEncoder(compression, nil, s.EscapeHTML) - if err != nil { - return nil, err - } - } - - conn := esclientleg.NewConnection(esclientleg.ConnectionSettings{ - URL: s.URL, - Username: s.Username, - Password: s.Password, - APIKey: base64.StdEncoding.EncodeToString([]byte(s.APIKey)), - Headers: s.Headers, + conn, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ + URL: s.URL, + Username: s.Username, + Password: s.Password, + APIKey: base64.StdEncoding.EncodeToString([]byte(s.APIKey)), + Headers: s.Headers, + TLSConfig: s.TLS, HTTP: &http.Client{ Transport: &http.Transport{ Dial: dialer.Dial, @@ -171,12 +156,16 @@ func NewClient( }, Timeout: s.Timeout, }, - Encoder: encoder, + ProxyURL: s.Proxy, + CompressionLevel: s.CompressionLevel, + EscapeHTML: s.EscapeHTML, }) + if err != nil { + return nil, err + } client := &Client{ Connection: *conn, - tlsConfig: s.TLS, index: s.Index, pipeline: pipeline, params: params, @@ -184,9 +173,7 @@ func NewClient( bulkRequ: bulkRequ, - compressionLevel: compression, - proxyURL: s.Proxy, - observer: s.Observer, + observer: s.Observer, log: logp.NewLogger("elasticsearch"), } @@ -324,19 +311,19 @@ func (client *Client) Clone() *Client { URL: client.URL, Index: client.index, Pipeline: client.pipeline, - Proxy: client.proxyURL, + Proxy: client.ProxyURL, // Without the following nil check on proxyURL, a nil Proxy field will try // reloading proxy settings from the environment instead of leaving them // empty. - ProxyDisable: client.proxyURL == nil, - TLS: client.tlsConfig, + ProxyDisable: client.ProxyURL == nil, + TLS: client.TLSConfig, Username: client.Username, Password: client.Password, APIKey: client.APIKey, Parameters: nil, // XXX: do not pass params? Headers: client.Headers, Timeout: client.HTTP.Timeout, - CompressionLevel: client.compressionLevel, + CompressionLevel: client.CompressionLevel, }, nil, // XXX: do not pass connection callback? ) @@ -600,7 +587,7 @@ func (client *Client) Test(d testing.Driver) { } else { d.Run("TLS", func(d testing.Driver) { netDialer := transport.NetDialer(client.timeout) - tlsDialer, err := transport.TestTLSDialer(d, netDialer, client.tlsConfig, client.timeout) + tlsDialer, err := transport.TestTLSDialer(d, netDialer, client.TLSConfig, client.timeout) _, err = tlsDialer.Dial("tcp", address) d.Fatal("dial up", err) }) From 5ef678404f7cafd7b9e1f34434e2b15a9e4a0020 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 24 Feb 2020 09:34:22 -0800 Subject: [PATCH 24/91] Moving more fields --- libbeat/esclientleg/connection.go | 14 +++++++++++++- libbeat/outputs/elasticsearch/client.go | 16 ++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/libbeat/esclientleg/connection.go b/libbeat/esclientleg/connection.go index 2fc1d0b1e24..53c18dd7e2e 100644 --- a/libbeat/esclientleg/connection.go +++ b/libbeat/esclientleg/connection.go @@ -37,8 +37,13 @@ type Connection struct { ConnectionSettings Encoder BodyEncoder + + // buffered bulk requests + BulkRequ *BulkRequest + version common.Version - log *logp.Logger + + log *logp.Logger } // ConnectionSettings are the settings needed for a Connection @@ -57,6 +62,7 @@ type ConnectionSettings struct { OnConnectCallback func() error + Parameters map[string]string CompressionLevel int EscapeHTML bool } @@ -74,9 +80,15 @@ func NewConnection(settings ConnectionSettings) (*Connection, error) { } } + bulkRequ, err := NewBulkRequest(settings.URL, "", "", settings.Parameters, nil) + if err != nil { + return nil, err + } + return &Connection{ ConnectionSettings: settings, Encoder: encoder, + BulkRequ: bulkRequ, log: logp.NewLogger("esclientleg"), }, nil } diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index c8cd4814825..a7820fa2bc6 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -45,12 +45,8 @@ type Client struct { index outputs.IndexSelector pipeline *outil.Selector - params map[string]string timeout time.Duration - // buffered bulk requests - bulkRequ *esclientleg.BulkRequest - observer outputs.Observer log *logp.Logger @@ -134,12 +130,6 @@ func NewClient( tlsDialer = transport.StatsDialer(tlsDialer, st) } - params := s.Parameters - bulkRequ, err := esclientleg.NewBulkRequest(s.URL, "", "", params, nil) - if err != nil { - return nil, err - } - conn, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ URL: s.URL, Username: s.Username, @@ -157,6 +147,7 @@ func NewClient( Timeout: s.Timeout, }, ProxyURL: s.Proxy, + Parameters: s.Parameters, CompressionLevel: s.CompressionLevel, EscapeHTML: s.EscapeHTML, }) @@ -168,11 +159,8 @@ func NewClient( Connection: *conn, index: s.Index, pipeline: pipeline, - params: params, timeout: s.Timeout, - bulkRequ: bulkRequ, - observer: s.Observer, log: logp.NewLogger("elasticsearch"), @@ -379,7 +367,7 @@ func (client *Client) publishEvents( return nil, nil } - requ := client.bulkRequ + requ := client.BulkRequ requ.Reset(body) status, result, sendErr := client.SendBulkRequest(requ) if sendErr != nil { From 254b5e903c8d0078af1b80f508d3625a0c60ec08 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 24 Feb 2020 09:52:51 -0800 Subject: [PATCH 25/91] Update test code --- libbeat/esclientleg/api_integration_test.go | 13 +++++++++---- libbeat/esclientleg/api_test.go | 5 +++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/libbeat/esclientleg/api_integration_test.go b/libbeat/esclientleg/api_integration_test.go index c2815f84140..e480a39c99d 100644 --- a/libbeat/esclientleg/api_integration_test.go +++ b/libbeat/esclientleg/api_integration_test.go @@ -189,7 +189,7 @@ type Connectable interface { // getTestingElasticsearch creates a test client. func getTestingElasticsearch(t TestLogger) *Connection { - connection := NewConnection(ConnectionSettings{ + conn, err := NewConnection(ConnectionSettings{ URL: getURL(), HTTP: &http.Client{ Transport: &http.Transport{ @@ -197,16 +197,21 @@ func getTestingElasticsearch(t TestLogger) *Connection { }, Timeout: 0, }, - Encoder: NewJSONEncoder(nil, false), }) + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + conn.Encoder = NewJSONEncoder(nil, false) - err := connection.Connect() + err = conn.Connect() if err != nil { t.Fatal(err) panic(err) // panic in case TestLogger did not stop test } - return connection + return conn } func getURL() string { diff --git a/libbeat/esclientleg/api_test.go b/libbeat/esclientleg/api_test.go index 658e405353d..3a59b65da32 100644 --- a/libbeat/esclientleg/api_test.go +++ b/libbeat/esclientleg/api_test.go @@ -173,7 +173,7 @@ func TestReadSearchResult_invalid(t *testing.T) { } func newTestConnection(url string) *Connection { - return NewConnection(ConnectionSettings{ + conn, _ := NewConnection(ConnectionSettings{ URL: url, HTTP: &http.Client{ Transport: &http.Transport{ @@ -181,8 +181,9 @@ func newTestConnection(url string) *Connection { }, Timeout: 0, }, - Encoder: NewJSONEncoder(nil, false), }) + conn.Encoder = NewJSONEncoder(nil, false) + return conn } func (r QueryResult) String() string { From b4ec366766a229d17c9e2f0f792703f2c0ebd071 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 25 Feb 2020 13:48:45 -0800 Subject: [PATCH 26/91] Use new TLS package --- libbeat/esclientleg/connection.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libbeat/esclientleg/connection.go b/libbeat/esclientleg/connection.go index 53c18dd7e2e..e300b28a31f 100644 --- a/libbeat/esclientleg/connection.go +++ b/libbeat/esclientleg/connection.go @@ -25,8 +25,7 @@ import ( "net/http" "net/url" - "github.com/elastic/beats/libbeat/outputs/transport" - + "github.com/elastic/beats/libbeat/common/transport/tlscommon" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/common" @@ -56,7 +55,7 @@ type ConnectionSettings struct { APIKey string Headers map[string]string - TLSConfig *transport.TLSConfig + TLSConfig *tlscommon.TLSConfig HTTP *http.Client From 3a85b3a2b0b07723783f696af65de2fb8d0f5bbf Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 25 Feb 2020 16:41:45 -0800 Subject: [PATCH 27/91] Extracting common test code into eslegtest package --- filebeat/fileset/modules_integration_test.go | 21 +-- libbeat/esclientleg/api_integration_test.go | 76 -------- .../connection_integration_test.go | 170 ++++++++++++++++++ libbeat/esclientleg/eslegtest/util.go | 119 ++++++++++++ .../elasticsearch/client_integration_test.go | 47 +---- .../outputs/elasticsearch/estest/estest.go | 40 ----- .../outputs/elasticsearch/internal/testing.go | 65 ------- libbeat/template/load_integration_test.go | 4 +- 8 files changed, 308 insertions(+), 234 deletions(-) create mode 100644 libbeat/esclientleg/connection_integration_test.go create mode 100644 libbeat/esclientleg/eslegtest/util.go delete mode 100644 libbeat/outputs/elasticsearch/estest/estest.go diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index 0afadfcbcea..b18dcd1d78b 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -24,11 +24,12 @@ import ( "path/filepath" "testing" + "github.com/elastic/beats/v7/libbeat/esclientleg" + "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch/estest" + "github.com/elastic/beats/v7/libbeat/esclientleg/eslegtest" ) func makeTestInfo(version string) beat.Info { @@ -39,7 +40,7 @@ func makeTestInfo(version string) beat.Info { } func TestLoadPipeline(t *testing.T) { - client := estest.GetTestingElasticsearch(t) + client := eslegtest.GetTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -77,7 +78,7 @@ func TestLoadPipeline(t *testing.T) { checkUploadedPipeline(t, client, "describe pipeline 2") } -func checkUploadedPipeline(t *testing.T, client *elasticsearch.Client, expectedDescription string) { +func checkUploadedPipeline(t *testing.T, client *esclientleg.Connection, expectedDescription string) { status, response, err := client.Request("GET", "/_ingest/pipeline/my-pipeline-id", "", nil, nil) assert.NoError(t, err) assert.Equal(t, 200, status) @@ -90,7 +91,7 @@ func checkUploadedPipeline(t *testing.T, client *elasticsearch.Client, expectedD } func TestSetupNginx(t *testing.T) { - client := estest.GetTestingElasticsearch(t) + client := eslegtest.GetTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -122,7 +123,7 @@ func TestSetupNginx(t *testing.T) { } func TestAvailableProcessors(t *testing.T) { - client := estest.GetTestingElasticsearch(t) + client := eslegtest.GetTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -147,18 +148,18 @@ func TestAvailableProcessors(t *testing.T) { assert.Contains(t, err.Error(), "ingest-hello") } -func hasIngest(client *elasticsearch.Client) bool { +func hasIngest(client *esclientleg.Connection) bool { v := client.GetVersion() return v.Major >= 5 } -func hasIngestPipelineProcessor(client *elasticsearch.Client) bool { +func hasIngestPipelineProcessor(client *esclientleg.Connection) bool { v := client.GetVersion() return v.Major > 6 || (v.Major == 6 && v.Minor >= 5) } func TestLoadMultiplePipelines(t *testing.T) { - client := estest.GetTestingElasticsearch(t) + client := eslegtest.GetTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -203,7 +204,7 @@ func TestLoadMultiplePipelines(t *testing.T) { } func TestLoadMultiplePipelinesWithRollback(t *testing.T) { - client := estest.GetTestingElasticsearch(t) + client := eslegtest.GetTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } diff --git a/libbeat/esclientleg/api_integration_test.go b/libbeat/esclientleg/api_integration_test.go index e480a39c99d..4cebd6f587b 100644 --- a/libbeat/esclientleg/api_integration_test.go +++ b/libbeat/esclientleg/api_integration_test.go @@ -29,15 +29,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/libbeat/outputs/transport" -) - -const ( - // ElasticsearchDefaultHost is the default host for elasticsearch. - ElasticsearchDefaultHost = "localhost" - // ElasticsearchDefaultPort is the default port for elasticsearch. - ElasticsearchDefaultPort = "9200" ->>>>>>> Moving API integration tests ) func TestIndex(t *testing.T) { @@ -173,70 +164,3 @@ func TestIngest(t *testing.T) { assert.Equal(t, "test", doc.Field) } - -// FIXME: Below code is repeated in outputs/elasticsearch/internal/testing.go - -// TestLogger is used to report fatal errors to the testing framework. -type TestLogger interface { - Fatal(args ...interface{}) -} - -// Connectable defines the minimum interface required to initialize a connected -// client. -type Connectable interface { - Connect() error -} - -// getTestingElasticsearch creates a test client. -func getTestingElasticsearch(t TestLogger) *Connection { - conn, err := NewConnection(ConnectionSettings{ - URL: getURL(), - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(0).Dial, - }, - Timeout: 0, - }, - }) - if err != nil { - t.Fatal(err) - panic(err) // panic in case TestLogger did not stop test - } - - conn.Encoder = NewJSONEncoder(nil, false) - - err = conn.Connect() - if err != nil { - t.Fatal(err) - panic(err) // panic in case TestLogger did not stop test - } - - return conn -} - -func getURL() string { - return fmt.Sprintf("http://%v:%v", getEsHost(), getEsPort()) -} - -func getEsHost() string { - return getEnv("ES_HOST", ElasticsearchDefaultHost) -} - -func getEsPort() string { - return getEnv("ES_PORT", ElasticsearchDefaultPort) -} - -func getUser() string { - return getEnv("ES_USER", "") -} - -func getPass() string { - return getEnv("ES_PASS", "") -} - -func getEnv(name, def string) string { - if v := os.Getenv(name); len(v) > 0 { - return v - } - return def -} diff --git a/libbeat/esclientleg/connection_integration_test.go b/libbeat/esclientleg/connection_integration_test.go new file mode 100644 index 00000000000..ef6ef894e53 --- /dev/null +++ b/libbeat/esclientleg/connection_integration_test.go @@ -0,0 +1,170 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package esclientleg + +import ( + "context" + "io/ioutil" + "math/rand" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/esclientleg/eslegtest" + "github.com/elastic/beats/libbeat/idxmgmt" + "github.com/elastic/beats/libbeat/outputs" + "github.com/elastic/beats/libbeat/outputs/elasticsearch/internal" +) + +func TestConnect(t *testing.T) { + conn := getTestingElasticsearch(t) + err := conn.Connect() + assert.NoError(t, err) +} + +func TestConnectWithProxy(t *testing.T) { + wrongPort, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + go func() { + c, err := wrongPort.Accept() + if err == nil { + // Provoke an early-EOF error on client + c.Close() + } + }() + defer wrongPort.Close() + + proxy := startTestProxy(t, eslegtest.GetURL()) + defer proxy.Close() + + // Use connectTestEs instead of getTestingElasticsearch to make use of makeES + _, client := connectTestEs(t, map[string]interface{}{ + "hosts": "http://" + wrongPort.Addr().String(), + "timeout": 5, // seconds + }) + assert.Error(t, client.Connect(), "it should fail without proxy") + + _, client = connectTestEs(t, map[string]interface{}{ + "hosts": "http://" + wrongPort.Addr().String(), + "proxy_url": proxy.URL, + "timeout": 5, // seconds + }) + assert.NoError(t, client.Connect()) +} + +func connectTestEs(t *testing.T, cfg interface{}) (outputs.Client, *Client) { + config, err := common.NewConfigFrom(map[string]interface{}{ + "hosts": eslegtest.GetEsHost(), + "username": eslegtest.GetUser(), + "password": eslegtest.GetPass(), + "template.enabled": false, + }) + if err != nil { + t.Fatal(err) + } + + tmp, err := common.NewConfigFrom(cfg) + if err != nil { + t.Fatal(err) + } + + err = config.Merge(tmp) + if err != nil { + t.Fatal(err) + } + + info := beat.Info{Beat: "libbeat"} + im, _ := idxmgmt.DefaultSupport(nil, info, nil) + output, err := makeES(im, info, outputs.NewNilObserver(), config) + if err != nil { + t.Fatal(err) + } + + type clientWrap interface { + outputs.NetworkClient + Client() outputs.NetworkClient + } + client := randomClient(output).(clientWrap).Client().(*Client) + + // Load version number + client.Connect() + + return client, client +} + +// getTestingElasticsearch creates a test client. +func getTestingElasticsearch(t internal.TestLogger) *Client { + conn, err := NewConnection(ConnectionSettings{ + URL: internal.GetURL(), + Username: internal.GetUser(), + Password: internal.GetUser(), + Timeout: 60 * time.Second, + CompressionLevel: 3, + }, nil) + eslegtest.InitConnection(t, conn, err) + return conn +} + +func randomClient(grp outputs.Group) outputs.NetworkClient { + L := len(grp.Clients) + if L == 0 { + panic("no elasticsearch client") + } + + client := grp.Clients[rand.Intn(L)] + return client.(outputs.NetworkClient) +} + +// startTestProxy starts a proxy that redirects all connections to the specified URL +func startTestProxy(t *testing.T, redirectURL string) *httptest.Server { + t.Helper() + + realURL, err := url.Parse(redirectURL) + require.NoError(t, err) + + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + req := r.Clone(context.Background()) + req.RequestURI = "" + req.URL.Scheme = realURL.Scheme + req.URL.Host = realURL.Host + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + for _, header := range []string{"Content-Encoding", "Content-Type"} { + w.Header().Set(header, resp.Header.Get(header)) + } + w.WriteHeader(resp.StatusCode) + w.Write(body) + })) + return proxy +} diff --git a/libbeat/esclientleg/eslegtest/util.go b/libbeat/esclientleg/eslegtest/util.go new file mode 100644 index 00000000000..3cc2d78b8e7 --- /dev/null +++ b/libbeat/esclientleg/eslegtest/util.go @@ -0,0 +1,119 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package eslegtest + +import ( + "fmt" + "net/http" + "os" + + "github.com/elastic/beats/libbeat/esclientleg" + + "github.com/elastic/beats/libbeat/outputs/transport" +) + +const ( + // ElasticsearchDefaultHost is the default host for elasticsearch. + ElasticsearchDefaultHost = "localhost" + // ElasticsearchDefaultPort is the default port for elasticsearch. + ElasticsearchDefaultPort = "9200" +) + +// TestLogger is used to report fatal errors to the testing framework. +type TestLogger interface { + Fatal(args ...interface{}) +} + +// Connectable defines the minimum interface required to initialize a connected +// client. +type Connectable interface { + Connect() error +} + +// InitConnection initializes a new connection if the no error value from creating the +// connection instance is reported. +// The test logger will be used if an error is found. +func InitConnection(t TestLogger, conn Connectable, err error) { + if err == nil { + err = conn.Connect() + } + + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } +} + +// GetTestingElasticsearch creates a test connection. +func GetTestingElasticsearch(t TestLogger) *esclientleg.Connection { + conn, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ + URL: GetURL(), + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(0).Dial, + }, + Timeout: 0, + }, + }) + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + conn.Encoder = esclientleg.NewJSONEncoder(nil, false) + + err = conn.Connect() + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + return conn +} + +// GetURL return the Elasticsearch testing URL. +func GetURL() string { + return fmt.Sprintf("http://%v:%v", GetEsHost(), getEsPort()) +} + +// GetEsHost returns the Elasticsearch testing host. +func GetEsHost() string { + return getEnv("ES_HOST", ElasticsearchDefaultHost) +} + +// getEsPort returns the Elasticsearch testing port. +func getEsPort() string { + return getEnv("ES_PORT", ElasticsearchDefaultPort) +} + +// GetUser returns the Elasticsearch testing user. +func GetUser() string { + return getEnv("ES_USER", "") +} + +// GetPass returns the Elasticsearch testing user's password. +func GetPass() string { + return getEnv("ES_PASS", "") +} + +func getEnv(name, def string) string { + if v := os.Getenv(name); len(v) > 0 { + return v + } + return def +} diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index 8daff6e9291..c9aea66cbd3 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -23,13 +23,14 @@ import ( "context" "io/ioutil" "math/rand" - "net" "net/http" "net/http/httptest" "net/url" "testing" "time" + "github.com/elastic/beats/libbeat/esclientleg/eslegtest" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -43,42 +44,6 @@ import ( "github.com/elastic/beats/v7/libbeat/outputs/outil" ) -func TestClientConnect(t *testing.T) { - client := getTestingElasticsearch(t) - err := client.Connect() - assert.NoError(t, err) -} - -func TestClientConnectWithProxy(t *testing.T) { - wrongPort, err := net.Listen("tcp", "localhost:0") - require.NoError(t, err) - go func() { - c, err := wrongPort.Accept() - if err == nil { - // Provoke an early-EOF error on client - c.Close() - } - }() - defer wrongPort.Close() - - proxy := startTestProxy(t, internal.GetURL()) - defer proxy.Close() - - // Use connectTestEs instead of getTestingElasticsearch to make use of makeES - _, client := connectTestEs(t, map[string]interface{}{ - "hosts": "http://" + wrongPort.Addr().String(), - "timeout": 5, // seconds - }) - assert.Error(t, client.Connect(), "it should fail without proxy") - - _, client = connectTestEs(t, map[string]interface{}{ - "hosts": "http://" + wrongPort.Addr().String(), - "proxy_url": proxy.URL, - "timeout": 5, // seconds - }) - assert.NoError(t, client.Connect()) -} - func TestClientPublishEvent(t *testing.T) { index := "beat-int-pub-single-event" output, client := connectTestEs(t, map[string]interface{}{ @@ -281,9 +246,9 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { func connectTestEs(t *testing.T, cfg interface{}) (outputs.Client, *Client) { config, err := common.NewConfigFrom(map[string]interface{}{ - "hosts": internal.GetEsHost(), - "username": internal.GetUser(), - "password": internal.GetPass(), + "hosts": eslegtest.GetEsHost(), + "username": eslegtest.GetUser(), + "password": eslegtest.GetPass(), "template.enabled": false, }) if err != nil { @@ -329,7 +294,7 @@ func getTestingElasticsearch(t internal.TestLogger) *Client { Timeout: 60 * time.Second, CompressionLevel: 3, }, nil) - internal.InitClient(t, client, err) + eslegtest.InitConnection(t, client, err) return client } diff --git a/libbeat/outputs/elasticsearch/estest/estest.go b/libbeat/outputs/elasticsearch/estest/estest.go deleted file mode 100644 index 3aafc7da0d7..00000000000 --- a/libbeat/outputs/elasticsearch/estest/estest.go +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package estest - -import ( - "time" - - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch/internal" - "github.com/elastic/beats/v7/libbeat/outputs/outil" -) - -// GetTestingElasticsearch creates a test client. -func GetTestingElasticsearch(t internal.TestLogger) *elasticsearch.Client { - client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ - URL: internal.GetURL(), - Index: outil.MakeSelector(), - Username: internal.GetUser(), - Password: internal.GetPass(), - Timeout: 60 * time.Second, - CompressionLevel: 3, - }, nil) - internal.InitClient(t, client, err) - return client -} diff --git a/libbeat/outputs/elasticsearch/internal/testing.go b/libbeat/outputs/elasticsearch/internal/testing.go index b0702ec210a..8bcbca6b411 100644 --- a/libbeat/outputs/elasticsearch/internal/testing.go +++ b/libbeat/outputs/elasticsearch/internal/testing.go @@ -16,68 +16,3 @@ // under the License. package internal - -import ( - "fmt" - "os" -) - -const ( - // ElasticsearchDefaultHost is the default host for elasticsearch. - ElasticsearchDefaultHost = "localhost" - // ElasticsearchDefaultPort is the default port for elasticsearch. - ElasticsearchDefaultPort = "9200" -) - -// TestLogger is used to report fatal errors to the testing framework. -type TestLogger interface { - Fatal(args ...interface{}) -} - -// Connectable defines the minimum interface required to initialize a connected -// client. -type Connectable interface { - Connect() error -} - -// InitClient initializes a new client if the no error value from creating the -// client instance is reported. -// The test logger will be used if an error is found. -func InitClient(t TestLogger, client Connectable, err error) { - if err == nil { - err = client.Connect() - } - - if err != nil { - t.Fatal(err) - panic(err) // panic in case TestLogger did not stop test - } -} - -// GetEsHost returns the Elasticsearch testing host. -func GetEsHost() string { - return getEnv("ES_HOST", ElasticsearchDefaultHost) -} - -// GetEsPort returns the Elasticsearch testing port. -func GetEsPort() string { - return getEnv("ES_PORT", ElasticsearchDefaultPort) -} - -// GetURL return the Elasticsearch testing URL. -func GetURL() string { - return fmt.Sprintf("http://%v:%v", GetEsHost(), GetEsPort()) -} - -// GetUser returns the Elasticsearch testing user. -func GetUser() string { return getEnv("ES_USER", "") } - -// GetPass returns the Elasticsearch testing user's password. -func GetPass() string { return getEnv("ES_PASS", "") } - -func getEnv(name, def string) string { - if v := os.Getenv(name); len(v) > 0 { - return v - } - return def -} diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 958b311057f..d5ee4e14267 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -34,8 +34,8 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esclientleg/eslegtest" "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch/estest" "github.com/elastic/beats/v7/libbeat/version" ) @@ -60,7 +60,7 @@ func newTestSetup(t *testing.T, cfg TemplateConfig) *testSetup { if cfg.Name == "" { cfg.Name = fmt.Sprintf("load-test-%+v", rand.Int()) } - client := estest.GetTestingElasticsearch(t) + client := eslegtest.GetTestingElasticsearch(t) if err := client.Connect(); err != nil { t.Fatal(err) } From fe0f221a0e6a8e6341c810e40ebdec1c42f9e752 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 25 Feb 2020 18:03:39 -0800 Subject: [PATCH 28/91] Replace uses of elasticsearch output client with esclientleg.NewConnection --- filebeat/fileset/pipelines_test.go | 18 +++++++++++----- .../ilm/client_handler_integration_test.go | 20 +++++++++++------- .../logstash/logstash_integration_test.go | 18 ++++++++++------ .../elastic_fetcher_integration_test.go | 21 ++++++++++++------- .../libbeat/licenser/elastic_fetcher_test.go | 17 +++++++++++---- 5 files changed, 65 insertions(+), 29 deletions(-) diff --git a/filebeat/fileset/pipelines_test.go b/filebeat/fileset/pipelines_test.go index d808639cda8..c81ae75eb97 100644 --- a/filebeat/fileset/pipelines_test.go +++ b/filebeat/fileset/pipelines_test.go @@ -23,11 +23,13 @@ import ( "net/http" "net/http/httptest" "testing" - - "github.com/stretchr/testify/assert" + "time" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" + "github.com/elastic/beats/v7/libbeat/esclientleg" + "github.com/elastic/beats/v7/libbeat/outputs/transport" + + "github.com/stretchr/testify/assert" ) func TestLoadPipelinesWithMultiPipelineFileset(t *testing.T) { @@ -87,9 +89,15 @@ func TestLoadPipelinesWithMultiPipelineFileset(t *testing.T) { })) defer testESServer.Close() - testESClient, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ + testESClient, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ URL: testESServer.URL, - }, nil) + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(90 * time.Second).Dial, + }, + Timeout: 90 * time.Second, + }, + }) assert.NoError(t, err) err = testESClient.Connect() diff --git a/libbeat/idxmgmt/ilm/client_handler_integration_test.go b/libbeat/idxmgmt/ilm/client_handler_integration_test.go index 2d9c2ca721e..76caa5c78e5 100644 --- a/libbeat/idxmgmt/ilm/client_handler_integration_test.go +++ b/libbeat/idxmgmt/ilm/client_handler_integration_test.go @@ -22,6 +22,7 @@ package ilm_test import ( "encoding/json" "fmt" + "net/http" "os" "testing" "time" @@ -31,9 +32,9 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/idxmgmt/ilm" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/outil" + "github.com/elastic/beats/v7/libbeat/outputs/transport" "github.com/elastic/beats/v7/libbeat/version" ) @@ -179,11 +180,16 @@ func newESClientHandler(t *testing.T) ilm.ClientHandler { } func newRawESClient(t *testing.T) ilm.ESClient { - client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ - URL: getURL(), - Index: outil.MakeSelector(), - Username: getUser(), - Password: getUser(), + client, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ + URL: getURL(), + Username: getUser(), + Password: getPass(), + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(60 * time.Second).Dial, + }, + Timeout: 0, + }, Timeout: 60 * time.Second, CompressionLevel: 3, }, nil) diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index c2170cb25d8..901319eb2e2 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -22,6 +22,7 @@ package logstash import ( "encoding/json" "fmt" + "net/http" "os" "sync" "testing" @@ -32,11 +33,12 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/fmtstr" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/outputs" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" "github.com/elastic/beats/v7/libbeat/outputs/outest" "github.com/elastic/beats/v7/libbeat/outputs/outil" + "github.com/elastic/beats/v7/libbeat/outputs/transport" ) const ( @@ -49,7 +51,7 @@ const ( ) type esConnection struct { - *elasticsearch.Client + *esclientleg.Connection t *testing.T index string } @@ -100,13 +102,17 @@ func esConnect(t *testing.T, index string) *esConnection { username := os.Getenv("ES_USER") password := os.Getenv("ES_PASS") - client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ + client, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ URL: host, - Index: indexSel, Username: username, Password: password, - Timeout: 60 * time.Second, - }, nil) + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(60 * time.Second).Dial, + }, + Timeout: 60 * time.Second, + }, + }) if err != nil { t.Fatal(err) } diff --git a/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go b/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go index 63b38fdb962..35995bf238f 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go +++ b/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go @@ -7,14 +7,17 @@ package licenser import ( + "net/http" "testing" "time" + "github.com/elastic/beats/libbeat/esclientleg" + "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/common/cli" "github.com/elastic/beats/v7/libbeat/common/cli" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/outil" + "github.com/elastic/beats/v7/libbeat/outputs/transport" ) const ( @@ -22,16 +25,20 @@ const ( elasticsearchPort = "9200" ) -func getTestClient() *elasticsearch.Client { +func getTestClient() *esclientleg.Connection { host := "http://" + cli.GetEnvOr("ES_HOST", elasticsearchHost) + ":" + cli.GetEnvOr("ES_POST", elasticsearchPort) - client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{ + client, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ URL: host, - Index: outil.MakeSelector(), Username: "myelastic", // NOTE: I will refactor this in a followup PR Password: "changeme", - Timeout: 60 * time.Second, CompressionLevel: 3, - }, nil) + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(60 * time.Second).Dial, + }, + Timeout: 60 * time.Second, + }, + }) if err != nil { panic(err) diff --git a/x-pack/libbeat/licenser/elastic_fetcher_test.go b/x-pack/libbeat/licenser/elastic_fetcher_test.go index 87b92d6bcca..d10f774d0ae 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher_test.go +++ b/x-pack/libbeat/licenser/elastic_fetcher_test.go @@ -14,18 +14,27 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/libbeat/esclientleg" + "github.com/elastic/beats/v7/libbeat/outputs/transport" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" + "github.com/stretchr/testify/assert" ) -func newServerClientPair(t *testing.T, handler http.HandlerFunc) (*httptest.Server, *elasticsearch.Client) { +func newServerClientPair(t *testing.T, handler http.HandlerFunc) (*httptest.Server, *esclientleg.Connection) { mux := http.NewServeMux() mux.Handle("/_license/", http.HandlerFunc(handler)) server := httptest.NewServer(mux) - client, err := elasticsearch.NewClient(elasticsearch.ClientSettings{URL: server.URL}, nil) + client, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ + URL: server.URL, + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(90 * time.Second).Dial, + }, + Timeout: 90 * time.Second, + }, + }) if err != nil { t.Fatalf("could not create the elasticsearch client, error: %s", err) } From eec5149ca5badac2d1861e52af4461fa1ca22df6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 25 Feb 2020 18:18:10 -0800 Subject: [PATCH 29/91] Replacing uses of ES output client struct with esclientleg.Connection --- filebeat/fileset/modules_integration_test.go | 3 +-- libbeat/template/load_integration_test.go | 4 ++-- x-pack/libbeat/licenser/elastic_fetcher.go | 8 ++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index b18dcd1d78b..f140aed2d81 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -24,11 +24,10 @@ import ( "path/filepath" "testing" - "github.com/elastic/beats/v7/libbeat/esclientleg" - "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/esclientleg/eslegtest" ) diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index d5ee4e14267..39de6d3e8e7 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -34,8 +34,8 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/esclientleg/eslegtest" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" "github.com/elastic/beats/v7/libbeat/version" ) @@ -290,7 +290,7 @@ func TestTemplateWithData(t *testing.T) { setup := newTestSetup(t, TemplateConfig{Enabled: true}) require.NoError(t, setup.loadFromFile([]string{"testdata", "fields.yml"})) require.True(t, setup.loader.templateExists(setup.config.Name)) - esClient := setup.client.(*elasticsearch.Client) + esClient := setup.client.(*esclientleg.Connection) for _, test := range dataTests { _, _, err := esClient.Index(setup.config.Name, "_doc", "", nil, test.data) if test.error { diff --git a/x-pack/libbeat/licenser/elastic_fetcher.go b/x-pack/libbeat/licenser/elastic_fetcher.go index 1edde4e7992..44103489e5b 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher.go +++ b/x-pack/libbeat/licenser/elastic_fetcher.go @@ -14,8 +14,8 @@ import ( "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" ) const xPackURL = "/_license" @@ -146,7 +146,7 @@ func (f *ElasticFetcher) parseJSON(b []byte) (*License, error) { // esClientMux is taking care of round robin request over an array of elasticsearch client, note that // calling request is not threadsafe. type esClientMux struct { - clients []elasticsearch.Client + clients []esclientleg.Connection idx int } @@ -177,12 +177,12 @@ func (mux *esClientMux) Request( // newESClientMux takes a list of clients and randomize where we start and the list of host we are // querying. -func newESClientMux(clients []elasticsearch.Client) *esClientMux { +func newESClientMux(clients []esclientleg.Connection) *esClientMux { // randomize where we start idx := rand.Intn(len(clients)) // randomize the list of round robin hosts. - tmp := make([]elasticsearch.Client, len(clients)) + tmp := make([]esclientleg.Connection, len(clients)) copy(tmp, clients) rand.Shuffle(len(tmp), func(i, j int) { tmp[i], tmp[j] = tmp[j], tmp[i] From 1c55cd29f04d5a7d212bea9714f11ecb81a2131b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 25 Feb 2020 18:23:35 -0800 Subject: [PATCH 30/91] Handle callbacks --- filebeat/beater/filebeat.go | 22 +++++++++---------- filebeat/fileset/factory.go | 3 ++- libbeat/cmd/instance/beat.go | 5 +++-- libbeat/outputs/elasticsearch/callbacks.go | 4 +++- libbeat/outputs/elasticsearch/client.go | 4 ++-- .../elasticsearch/elasticsearch_test.go | 14 +++++++----- x-pack/libbeat/licenser/es_callback.go | 4 +++- 7 files changed, 31 insertions(+), 25 deletions(-) diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index db7df730d7b..0eed451d8c1 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -22,29 +22,27 @@ import ( "fmt" "strings" - "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/pkg/errors" + fbautodiscover "github.com/elastic/beats/v7/filebeat/autodiscover" + "github.com/elastic/beats/v7/filebeat/channel" + cfg "github.com/elastic/beats/v7/filebeat/config" + "github.com/elastic/beats/v7/filebeat/fileset" + _ "github.com/elastic/beats/v7/filebeat/include" + "github.com/elastic/beats/v7/filebeat/input" + "github.com/elastic/beats/v7/filebeat/registrar" "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/common/reload" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - fbautodiscover "github.com/elastic/beats/v7/filebeat/autodiscover" - "github.com/elastic/beats/v7/filebeat/channel" - cfg "github.com/elastic/beats/v7/filebeat/config" - "github.com/elastic/beats/v7/filebeat/fileset" - "github.com/elastic/beats/v7/filebeat/input" - "github.com/elastic/beats/v7/filebeat/registrar" - - _ "github.com/elastic/beats/v7/filebeat/include" - // Add filebeat level processors _ "github.com/elastic/beats/v7/filebeat/processor/add_kubernetes_metadata" _ "github.com/elastic/beats/v7/libbeat/processors/decode_csv_fields" @@ -184,7 +182,7 @@ func (fb *Filebeat) loadModulesPipelines(b *beat.Beat) error { // register pipeline loading to happen every time a new ES connection is // established - callback := func(esClient *elasticsearch.Client) error { + callback := func(esClient *esclientleg.Connection) error { return fb.moduleRegistry.LoadPipelines(esClient, overwritePipelines) } _, err := elasticsearch.RegisterConnectCallback(callback) diff --git a/filebeat/fileset/factory.go b/filebeat/fileset/factory.go index c207a160819..6409808acca 100644 --- a/filebeat/fileset/factory.go +++ b/filebeat/fileset/factory.go @@ -18,6 +18,7 @@ package fileset import ( + "github.com/elastic/beats/libbeat/esclientleg" "github.com/gofrs/uuid" "github.com/elastic/beats/v7/libbeat/beat" @@ -134,7 +135,7 @@ func (p *inputsRunner) Start() { } // Register callback to try to load pipelines when connecting to ES. - callback := func(esClient *elasticsearch.Client) error { + callback := func(esClient *esclientleg.Connection) error { return p.moduleRegistry.LoadPipelines(esClient, p.overwritePipelines) } p.pipelineCallbackID, err = elasticsearch.RegisterConnectCallback(callback) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index d9d486c62a8..9f0ac89701a 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -33,6 +33,7 @@ import ( "strings" "time" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/kibana" "github.com/gofrs/uuid" @@ -799,7 +800,7 @@ func (b *Beat) registerESIndexManagement() error { } func (b *Beat) indexSetupCallback() elasticsearch.ConnectCallback { - return func(esClient *elasticsearch.Client) error { + return func(esClient *esclientleg.Connection) error { m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields)) return m.Setup(idxmgmt.LoadModeEnabled, idxmgmt.LoadModeEnabled) } @@ -845,7 +846,7 @@ func (b *Beat) clusterUUIDFetchingCallback() (elasticsearch.ConnectCallback, err elasticsearchRegistry := stateRegistry.NewRegistry("outputs.elasticsearch") clusterUUIDRegVar := monitoring.NewString(elasticsearchRegistry, "cluster_uuid") - callback := func(esClient *elasticsearch.Client) error { + callback := func(esClient *esclientleg.Connection) error { var response struct { ClusterUUID string `json:"cluster_uuid"` } diff --git a/libbeat/outputs/elasticsearch/callbacks.go b/libbeat/outputs/elasticsearch/callbacks.go index 76383e884f8..c05f1943fbe 100644 --- a/libbeat/outputs/elasticsearch/callbacks.go +++ b/libbeat/outputs/elasticsearch/callbacks.go @@ -20,11 +20,13 @@ package elasticsearch import ( "sync" + "github.com/elastic/beats/libbeat/esclientleg" + "github.com/gofrs/uuid" ) // ConnectCallback defines the type for the function to be called when the Elasticsearch client successfully connects to the cluster -type ConnectCallback func(client *Client) error +type ConnectCallback func(*esclientleg.Connection) error // Callbacks must not depend on the result of a previous one, // because the ordering is not fixed. diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index a7820fa2bc6..51ca82c3728 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -171,7 +171,7 @@ func NewClient( defer globalCallbackRegistry.mutex.Unlock() for _, callback := range globalCallbackRegistry.callbacks { - err := callback(client) + err := callback(conn) if err != nil { return err } @@ -182,7 +182,7 @@ func NewClient( defer onConnect.mutex.Unlock() for _, callback := range onConnect.callbacks { - err := callback(client) + err := callback(conn) if err != nil { return err } diff --git a/libbeat/outputs/elasticsearch/elasticsearch_test.go b/libbeat/outputs/elasticsearch/elasticsearch_test.go index 0fd3ae91db3..e8a0a6bcafd 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch_test.go +++ b/libbeat/outputs/elasticsearch/elasticsearch_test.go @@ -20,12 +20,14 @@ package elasticsearch import ( "fmt" "testing" + + "github.com/elastic/beats/libbeat/esclientleg" ) func TestConnectCallbacksManagement(t *testing.T) { - f0 := func(client *Client) error { fmt.Println("i am function #0"); return nil } - f1 := func(client *Client) error { fmt.Println("i am function #1"); return nil } - f2 := func(client *Client) error { fmt.Println("i am function #2"); return nil } + f0 := func(client *esclientleg.Connection) error { fmt.Println("i am function #0"); return nil } + f1 := func(client *esclientleg.Connection) error { fmt.Println("i am function #1"); return nil } + f2 := func(client *esclientleg.Connection) error { fmt.Println("i am function #2"); return nil } _, err := RegisterConnectCallback(f0) if err != nil { @@ -48,9 +50,9 @@ func TestConnectCallbacksManagement(t *testing.T) { } func TestGlobalConnectCallbacksManagement(t *testing.T) { - f0 := func(client *Client) error { fmt.Println("i am function #0"); return nil } - f1 := func(client *Client) error { fmt.Println("i am function #1"); return nil } - f2 := func(client *Client) error { fmt.Println("i am function #2"); return nil } + f0 := func(client *esclientleg.Connection) error { fmt.Println("i am function #0"); return nil } + f1 := func(client *esclientleg.Connection) error { fmt.Println("i am function #1"); return nil } + f2 := func(client *esclientleg.Connection) error { fmt.Println("i am function #2"); return nil } _, err := RegisterGlobalCallback(f0) if err != nil { diff --git a/x-pack/libbeat/licenser/es_callback.go b/x-pack/libbeat/licenser/es_callback.go index 5c15c3eaab4..59610db4826 100644 --- a/x-pack/libbeat/licenser/es_callback.go +++ b/x-pack/libbeat/licenser/es_callback.go @@ -8,6 +8,8 @@ import ( "fmt" "strings" + "github.com/elastic/beats/libbeat/esclientleg" + "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/logp" @@ -21,7 +23,7 @@ const licenseDebugK = "license" func Enforce(name string, checks ...CheckFunc) { name = strings.Title(name) - cb := func(client *elasticsearch.Client) error { + cb := func(client *esclientleg.Connection) error { // Logger created earlier than this place are at risk of discarding any log statement. log := logp.NewLogger(licenseDebugK) From b0d3767025efdc4f09fc3ce96134d2cd4e71e7e0 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 25 Feb 2020 18:32:16 -0800 Subject: [PATCH 31/91] Fixing formatting --- filebeat/fileset/factory.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/fileset/factory.go b/filebeat/fileset/factory.go index 6409808acca..3e377c7cbb7 100644 --- a/filebeat/fileset/factory.go +++ b/filebeat/fileset/factory.go @@ -18,12 +18,12 @@ package fileset import ( - "github.com/elastic/beats/libbeat/esclientleg" "github.com/gofrs/uuid" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" From 16069d7b07e8ed848210ac24a67452a6371a4f40 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 25 Feb 2020 20:04:04 -0800 Subject: [PATCH 32/91] Fixing import cycle --- filebeat/beater/filebeat.go | 4 ++-- filebeat/fileset/factory.go | 4 ++-- filebeat/fileset/modules_integration_test.go | 10 ++++----- filebeat/fileset/pipelines_test.go | 4 ++-- libbeat/cmd/instance/beat.go | 16 ++++++-------- .../{esclientleg => esleg/eslegclient}/api.go | 2 +- .../eslegclient}/api_integration_test.go | 2 +- .../eslegclient}/api_mock_test.go | 2 +- .../eslegclient}/api_test.go | 2 +- .../eslegclient}/bulkapi.go | 2 +- .../eslegclient}/bulkapi_integration_test.go | 2 +- .../eslegclient}/bulkapi_mock_test.go | 2 +- .../eslegclient}/bulkapi_test.go | 2 +- .../eslegclient}/connection.go | 2 +- .../connection_integration_test.go | 2 +- .../{esclientleg => esleg/eslegclient}/enc.go | 2 +- .../eslegclient}/enc_test.go | 2 +- .../eslegclient}/errors.go | 2 +- .../eslegclient}/json_read.go | 2 +- .../{esclientleg => esleg/eslegclient}/url.go | 2 +- .../eslegclient}/url_test.go | 2 +- .../{esclientleg => esleg}/eslegtest/util.go | 9 ++++---- .../ilm/client_handler_integration_test.go | 4 ++-- .../monitoring/report/elasticsearch/client.go | 13 +++-------- libbeat/outputs/elasticsearch/callbacks.go | 4 ++-- libbeat/outputs/elasticsearch/client.go | 22 +++++++++---------- libbeat/outputs/elasticsearch/client_test.go | 8 +++---- .../elasticsearch/elasticsearch_test.go | 14 ++++++------ .../logstash/logstash_integration_test.go | 6 ++--- libbeat/template/load_integration_test.go | 6 ++--- x-pack/libbeat/licenser/elastic_fetcher.go | 8 +++---- .../elastic_fetcher_integration_test.go | 6 ++--- .../libbeat/licenser/elastic_fetcher_test.go | 6 ++--- x-pack/libbeat/licenser/es_callback.go | 4 ++-- 34 files changed, 85 insertions(+), 95 deletions(-) rename libbeat/{esclientleg => esleg/eslegclient}/api.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/api_integration_test.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/api_mock_test.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/api_test.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/bulkapi.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/bulkapi_integration_test.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/bulkapi_mock_test.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/bulkapi_test.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/connection.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/connection_integration_test.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/enc.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/enc_test.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/errors.go (98%) rename libbeat/{esclientleg => esleg/eslegclient}/json_read.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/url.go (99%) rename libbeat/{esclientleg => esleg/eslegclient}/url_test.go (99%) rename libbeat/{esclientleg => esleg}/eslegtest/util.go (91%) diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 0eed451d8c1..67f163d30c4 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -37,7 +37,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/elastic/beats/v7/libbeat/esclientleg" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/monitoring" @@ -182,7 +182,7 @@ func (fb *Filebeat) loadModulesPipelines(b *beat.Beat) error { // register pipeline loading to happen every time a new ES connection is // established - callback := func(esClient *esclientleg.Connection) error { + callback := func(esClient *eslegclient.Connection) error { return fb.moduleRegistry.LoadPipelines(esClient, overwritePipelines) } _, err := elasticsearch.RegisterConnectCallback(callback) diff --git a/filebeat/fileset/factory.go b/filebeat/fileset/factory.go index 3e377c7cbb7..92e626795c9 100644 --- a/filebeat/fileset/factory.go +++ b/filebeat/fileset/factory.go @@ -23,7 +23,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/esclientleg" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" @@ -135,7 +135,7 @@ func (p *inputsRunner) Start() { } // Register callback to try to load pipelines when connecting to ES. - callback := func(esClient *esclientleg.Connection) error { + callback := func(esClient *eslegclient.Connection) error { return p.moduleRegistry.LoadPipelines(esClient, p.overwritePipelines) } p.pipelineCallbackID, err = elasticsearch.RegisterConnectCallback(callback) diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index f140aed2d81..6a8714323b6 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -27,8 +27,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/esclientleg" - "github.com/elastic/beats/v7/libbeat/esclientleg/eslegtest" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" + "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" ) func makeTestInfo(version string) beat.Info { @@ -77,7 +77,7 @@ func TestLoadPipeline(t *testing.T) { checkUploadedPipeline(t, client, "describe pipeline 2") } -func checkUploadedPipeline(t *testing.T, client *esclientleg.Connection, expectedDescription string) { +func checkUploadedPipeline(t *testing.T, client *eslegclient.Connection, expectedDescription string) { status, response, err := client.Request("GET", "/_ingest/pipeline/my-pipeline-id", "", nil, nil) assert.NoError(t, err) assert.Equal(t, 200, status) @@ -147,12 +147,12 @@ func TestAvailableProcessors(t *testing.T) { assert.Contains(t, err.Error(), "ingest-hello") } -func hasIngest(client *esclientleg.Connection) bool { +func hasIngest(client *esleg.Connection) bool { v := client.GetVersion() return v.Major >= 5 } -func hasIngestPipelineProcessor(client *esclientleg.Connection) bool { +func hasIngestPipelineProcessor(client *eslegclient.Connection) bool { v := client.GetVersion() return v.Major > 6 || (v.Major == 6 && v.Minor >= 5) } diff --git a/filebeat/fileset/pipelines_test.go b/filebeat/fileset/pipelines_test.go index c81ae75eb97..d7861957092 100644 --- a/filebeat/fileset/pipelines_test.go +++ b/filebeat/fileset/pipelines_test.go @@ -25,8 +25,8 @@ import ( "testing" "time" + "github.com/elastic/beats/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/outputs/transport" "github.com/stretchr/testify/assert" @@ -89,7 +89,7 @@ func TestLoadPipelinesWithMultiPipelineFileset(t *testing.T) { })) defer testESServer.Close() - testESClient, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ + testESClient, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: testESServer.URL, HTTP: &http.Client{ Transport: &http.Transport{ diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 9f0ac89701a..b8f29fe073d 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -33,17 +33,10 @@ import ( "strings" "time" - "github.com/elastic/beats/v7/libbeat/esclientleg" - "github.com/elastic/beats/v7/libbeat/kibana" - "github.com/gofrs/uuid" errw "github.com/pkg/errors" "go.uber.org/zap" - sysinfo "github.com/elastic/go-sysinfo" - "github.com/elastic/go-sysinfo/types" - ucfg "github.com/elastic/go-ucfg" - "github.com/elastic/beats/v7/libbeat/api" "github.com/elastic/beats/v7/libbeat/asset" "github.com/elastic/beats/v7/libbeat/beat" @@ -54,8 +47,10 @@ import ( "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/common/seccomp" "github.com/elastic/beats/v7/libbeat/dashboards" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/beats/v7/libbeat/kibana" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/logp/configure" "github.com/elastic/beats/v7/libbeat/management" @@ -71,6 +66,9 @@ import ( "github.com/elastic/beats/v7/libbeat/publisher/processing" svc "github.com/elastic/beats/v7/libbeat/service" "github.com/elastic/beats/v7/libbeat/version" + sysinfo "github.com/elastic/go-sysinfo" + "github.com/elastic/go-sysinfo/types" + ucfg "github.com/elastic/go-ucfg" ) // Beat provides the runnable and configurable instance of a beat. @@ -800,7 +798,7 @@ func (b *Beat) registerESIndexManagement() error { } func (b *Beat) indexSetupCallback() elasticsearch.ConnectCallback { - return func(esClient *esclientleg.Connection) error { + return func(esClient *eslegclient.Connection) error { m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields)) return m.Setup(idxmgmt.LoadModeEnabled, idxmgmt.LoadModeEnabled) } @@ -846,7 +844,7 @@ func (b *Beat) clusterUUIDFetchingCallback() (elasticsearch.ConnectCallback, err elasticsearchRegistry := stateRegistry.NewRegistry("outputs.elasticsearch") clusterUUIDRegVar := monitoring.NewString(elasticsearchRegistry, "cluster_uuid") - callback := func(esClient *esclientleg.Connection) error { + callback := func(esClient *eslegclient.Connection) error { var response struct { ClusterUUID string `json:"cluster_uuid"` } diff --git a/libbeat/esclientleg/api.go b/libbeat/esleg/eslegclient/api.go similarity index 99% rename from libbeat/esclientleg/api.go rename to libbeat/esleg/eslegclient/api.go index 15becfabe21..ba15d0ad26a 100644 --- a/libbeat/esclientleg/api.go +++ b/libbeat/esleg/eslegclient/api.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package esclientleg +package eslegclient import ( "encoding/json" diff --git a/libbeat/esclientleg/api_integration_test.go b/libbeat/esleg/eslegclient/api_integration_test.go similarity index 99% rename from libbeat/esclientleg/api_integration_test.go rename to libbeat/esleg/eslegclient/api_integration_test.go index 4cebd6f587b..986f012fa04 100644 --- a/libbeat/esclientleg/api_integration_test.go +++ b/libbeat/esleg/eslegclient/api_integration_test.go @@ -17,7 +17,7 @@ // +build integration -package esclientleg +package eslegclient import ( "encoding/json" diff --git a/libbeat/esclientleg/api_mock_test.go b/libbeat/esleg/eslegclient/api_mock_test.go similarity index 99% rename from libbeat/esclientleg/api_mock_test.go rename to libbeat/esleg/eslegclient/api_mock_test.go index ea44f249ead..cfb9b9b722b 100644 --- a/libbeat/esclientleg/api_mock_test.go +++ b/libbeat/esleg/eslegclient/api_mock_test.go @@ -17,7 +17,7 @@ // +build !integration -package esclientleg +package eslegclient import ( "encoding/json" diff --git a/libbeat/esclientleg/api_test.go b/libbeat/esleg/eslegclient/api_test.go similarity index 99% rename from libbeat/esclientleg/api_test.go rename to libbeat/esleg/eslegclient/api_test.go index 3a59b65da32..0d48ae14b6b 100644 --- a/libbeat/esclientleg/api_test.go +++ b/libbeat/esleg/eslegclient/api_test.go @@ -16,7 +16,7 @@ // under the License. // Need for unit and integration tests -package esclientleg +package eslegclient import ( "encoding/json" diff --git a/libbeat/esclientleg/bulkapi.go b/libbeat/esleg/eslegclient/bulkapi.go similarity index 99% rename from libbeat/esclientleg/bulkapi.go rename to libbeat/esleg/eslegclient/bulkapi.go index 75e0e8f2e74..f71ebffa5e7 100644 --- a/libbeat/esclientleg/bulkapi.go +++ b/libbeat/esleg/eslegclient/bulkapi.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package esclientleg +package eslegclient import ( "bytes" diff --git a/libbeat/esclientleg/bulkapi_integration_test.go b/libbeat/esleg/eslegclient/bulkapi_integration_test.go similarity index 99% rename from libbeat/esclientleg/bulkapi_integration_test.go rename to libbeat/esleg/eslegclient/bulkapi_integration_test.go index 7f6c16cc8b9..2ab696a0d13 100644 --- a/libbeat/esclientleg/bulkapi_integration_test.go +++ b/libbeat/esleg/eslegclient/bulkapi_integration_test.go @@ -17,7 +17,7 @@ // +build integration -package esclientleg +package eslegclient import ( "fmt" diff --git a/libbeat/esclientleg/bulkapi_mock_test.go b/libbeat/esleg/eslegclient/bulkapi_mock_test.go similarity index 99% rename from libbeat/esclientleg/bulkapi_mock_test.go rename to libbeat/esleg/eslegclient/bulkapi_mock_test.go index cad04de0df8..3f3646ba1f0 100644 --- a/libbeat/esclientleg/bulkapi_mock_test.go +++ b/libbeat/esleg/eslegclient/bulkapi_mock_test.go @@ -17,7 +17,7 @@ // +build !integration -package esclientleg +package eslegclient import ( "fmt" diff --git a/libbeat/esclientleg/bulkapi_test.go b/libbeat/esleg/eslegclient/bulkapi_test.go similarity index 99% rename from libbeat/esclientleg/bulkapi_test.go rename to libbeat/esleg/eslegclient/bulkapi_test.go index a08c6b99123..2c707943a09 100644 --- a/libbeat/esclientleg/bulkapi_test.go +++ b/libbeat/esleg/eslegclient/bulkapi_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package esclientleg +package eslegclient import ( "testing" diff --git a/libbeat/esclientleg/connection.go b/libbeat/esleg/eslegclient/connection.go similarity index 99% rename from libbeat/esclientleg/connection.go rename to libbeat/esleg/eslegclient/connection.go index e300b28a31f..2bca4a8396c 100644 --- a/libbeat/esclientleg/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package esclientleg +package eslegclient import ( "encoding/json" diff --git a/libbeat/esclientleg/connection_integration_test.go b/libbeat/esleg/eslegclient/connection_integration_test.go similarity index 99% rename from libbeat/esclientleg/connection_integration_test.go rename to libbeat/esleg/eslegclient/connection_integration_test.go index ef6ef894e53..40f076a02d3 100644 --- a/libbeat/esclientleg/connection_integration_test.go +++ b/libbeat/esleg/eslegclient/connection_integration_test.go @@ -17,7 +17,7 @@ // +build integration -package esclientleg +package eslegclient import ( "context" diff --git a/libbeat/esclientleg/enc.go b/libbeat/esleg/eslegclient/enc.go similarity index 99% rename from libbeat/esclientleg/enc.go rename to libbeat/esleg/eslegclient/enc.go index 79eab7af9fd..3116dc2f537 100644 --- a/libbeat/esclientleg/enc.go +++ b/libbeat/esleg/eslegclient/enc.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package esclientleg +package eslegclient import ( "bytes" diff --git a/libbeat/esclientleg/enc_test.go b/libbeat/esleg/eslegclient/enc_test.go similarity index 99% rename from libbeat/esclientleg/enc_test.go rename to libbeat/esleg/eslegclient/enc_test.go index c3997fd5b9b..32b2f35e1f3 100644 --- a/libbeat/esclientleg/enc_test.go +++ b/libbeat/esleg/eslegclient/enc_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package esclientleg +package eslegclient import ( "testing" diff --git a/libbeat/esclientleg/errors.go b/libbeat/esleg/eslegclient/errors.go similarity index 98% rename from libbeat/esclientleg/errors.go rename to libbeat/esleg/eslegclient/errors.go index f74599f8f01..c3636c71477 100644 --- a/libbeat/esclientleg/errors.go +++ b/libbeat/esleg/eslegclient/errors.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package esclientleg +package eslegclient import "errors" diff --git a/libbeat/esclientleg/json_read.go b/libbeat/esleg/eslegclient/json_read.go similarity index 99% rename from libbeat/esclientleg/json_read.go rename to libbeat/esleg/eslegclient/json_read.go index 94ca887b092..77b62197582 100644 --- a/libbeat/esclientleg/json_read.go +++ b/libbeat/esleg/eslegclient/json_read.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package esclientleg +package eslegclient import ( "errors" diff --git a/libbeat/esclientleg/url.go b/libbeat/esleg/eslegclient/url.go similarity index 99% rename from libbeat/esclientleg/url.go rename to libbeat/esleg/eslegclient/url.go index 95f0877b720..89b03c5bdb8 100644 --- a/libbeat/esclientleg/url.go +++ b/libbeat/esleg/eslegclient/url.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package esclientleg +package eslegclient import ( "fmt" diff --git a/libbeat/esclientleg/url_test.go b/libbeat/esleg/eslegclient/url_test.go similarity index 99% rename from libbeat/esclientleg/url_test.go rename to libbeat/esleg/eslegclient/url_test.go index 7893aac62f0..8e444a660e4 100644 --- a/libbeat/esclientleg/url_test.go +++ b/libbeat/esleg/eslegclient/url_test.go @@ -17,7 +17,7 @@ // +build !integration -package esclientleg +package eslegclient import ( "testing" diff --git a/libbeat/esclientleg/eslegtest/util.go b/libbeat/esleg/eslegtest/util.go similarity index 91% rename from libbeat/esclientleg/eslegtest/util.go rename to libbeat/esleg/eslegtest/util.go index 3cc2d78b8e7..636fef7f541 100644 --- a/libbeat/esclientleg/eslegtest/util.go +++ b/libbeat/esleg/eslegtest/util.go @@ -22,7 +22,8 @@ import ( "net/http" "os" - "github.com/elastic/beats/libbeat/esclientleg" + "github.com/elastic/beats/libbeat/esleg" + "github.com/elastic/beats/libbeat/esleg/eslegclient" "github.com/elastic/beats/libbeat/outputs/transport" ) @@ -60,8 +61,8 @@ func InitConnection(t TestLogger, conn Connectable, err error) { } // GetTestingElasticsearch creates a test connection. -func GetTestingElasticsearch(t TestLogger) *esclientleg.Connection { - conn, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ +func GetTestingElasticsearch(t TestLogger) *eslegclient.Connection { + conn, err := esleg.NewConnection(eslegclient.ConnectionSettings{ URL: GetURL(), HTTP: &http.Client{ Transport: &http.Transport{ @@ -75,7 +76,7 @@ func GetTestingElasticsearch(t TestLogger) *esclientleg.Connection { panic(err) // panic in case TestLogger did not stop test } - conn.Encoder = esclientleg.NewJSONEncoder(nil, false) + conn.Encoder = eslegclient.NewJSONEncoder(nil, false) err = conn.Connect() if err != nil { diff --git a/libbeat/idxmgmt/ilm/client_handler_integration_test.go b/libbeat/idxmgmt/ilm/client_handler_integration_test.go index 76caa5c78e5..b8e881aa2db 100644 --- a/libbeat/idxmgmt/ilm/client_handler_integration_test.go +++ b/libbeat/idxmgmt/ilm/client_handler_integration_test.go @@ -32,7 +32,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/esclientleg" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/idxmgmt/ilm" "github.com/elastic/beats/v7/libbeat/outputs/transport" "github.com/elastic/beats/v7/libbeat/version" @@ -180,7 +180,7 @@ func newESClientHandler(t *testing.T) ilm.ClientHandler { } func newRawESClient(t *testing.T) ilm.ESClient { - client, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ + client, err := esleg.NewConnection(eslegclient.ConnectionSettings{ URL: getURL(), Username: getUser(), Password: getPass(), diff --git a/libbeat/monitoring/report/elasticsearch/client.go b/libbeat/monitoring/report/elasticsearch/client.go index 77876816405..186eabca229 100644 --- a/libbeat/monitoring/report/elasticsearch/client.go +++ b/libbeat/monitoring/report/elasticsearch/client.go @@ -27,6 +27,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/esclientleg" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring/report" esout "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" @@ -97,11 +98,7 @@ func (c *publishClient) Connect() error { return errNoMonitoring } -<<<<<<< HEAD c.log.Debug("XPack monitoring is enabled") -======= - c.log.Debug(("XPack monitoring is enabled")) ->>>>>>> Reducing global logging usage return nil } @@ -119,11 +116,7 @@ func (c *publishClient) Publish(batch publisher.Batch) error { // Extract type t, err := event.Content.Meta.GetValue("type") if err != nil { -<<<<<<< HEAD c.log.Errorf("Type not available in monitoring reported. Please report this error: %+v", err) -======= - c.log.Errorf("Type not available in monitoring reported. Please report this error: %s", err) ->>>>>>> Reducing global logging usage continue } @@ -261,8 +254,8 @@ func getMonitoringIndexName() string { } func logBulkFailures(log *logp.Logger, result esclientleg.BulkResult, events []report.Event) { - reader := esclientleg.NewJSONReader(result) - err := esclientleg.BulkReadToItems(reader) + reader := eslegclient.NewJSONReader(result) + err := eslegclient.BulkReadToItems(reader) if err != nil { log.Errorf("failed to parse monitoring bulk items: %+v", err) return diff --git a/libbeat/outputs/elasticsearch/callbacks.go b/libbeat/outputs/elasticsearch/callbacks.go index c05f1943fbe..909ed10af6b 100644 --- a/libbeat/outputs/elasticsearch/callbacks.go +++ b/libbeat/outputs/elasticsearch/callbacks.go @@ -20,13 +20,13 @@ package elasticsearch import ( "sync" - "github.com/elastic/beats/libbeat/esclientleg" + "github.com/elastic/beats/libbeat/esleg/eslegclient" "github.com/gofrs/uuid" ) // ConnectCallback defines the type for the function to be called when the Elasticsearch client successfully connects to the cluster -type ConnectCallback func(*esclientleg.Connection) error +type ConnectCallback func(*eslegclient.Connection) error // Callbacks must not depend on the result of a previous one, // because the ordering is not fixed. diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 51ca82c3728..76c80d82b6f 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -31,7 +31,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/libbeat/esclientleg" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" "github.com/elastic/beats/v7/libbeat/outputs/outil" @@ -41,7 +41,7 @@ import ( // Client is an elasticsearch client. type Client struct { - esclientleg.Connection + eslegclient.Connection index outputs.IndexSelector pipeline *outil.Selector @@ -130,7 +130,7 @@ func NewClient( tlsDialer = transport.StatsDialer(tlsDialer, st) } - conn, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ + conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: s.URL, Username: s.Username, Password: s.Password, @@ -404,7 +404,7 @@ func (client *Client) publishEvents( if failed > 0 { if sendErr == nil { - sendErr = esclientleg.ErrTempBulkFailure + sendErr = eslegclient.ErrTempBulkFailure } return failedEvents, sendErr } @@ -416,7 +416,7 @@ func (client *Client) publishEvents( func bulkEncodePublishRequest( log *logp.Logger, version common.Version, - body esclientleg.BulkWriter, + body eslegclient.BulkWriter, index outputs.IndexSelector, pipeline *outil.Selector, eventType string, @@ -472,7 +472,7 @@ func createEventBulkMeta( } } - meta := esclientleg.BulkMeta{ + meta := eslegclient.BulkMeta{ Index: index, DocType: eventType, Pipeline: pipeline, @@ -480,9 +480,9 @@ func createEventBulkMeta( } if id != "" || version.Major > 7 || (version.Major == 7 && version.Minor >= 5) { - return esclientleg.BulkCreateAction{Create: meta}, nil + return eslegclient.BulkCreateAction{Create: meta}, nil } - return esclientleg.BulkIndexAction{Index: meta}, nil + return eslegclient.BulkIndexAction{Index: meta}, nil } func getPipeline(event *beat.Event, pipelineSel *outil.Selector) (string, error) { @@ -510,8 +510,8 @@ func bulkCollectPublishFails( result esclientleg.BulkResult, data []publisher.Event, ) ([]publisher.Event, bulkResultStats) { - reader := esclientleg.NewJSONReader(result) - if err := esclientleg.BulkReadToItems(reader); err != nil { + reader := eslegclient.NewJSONReader(result) + if err := eslegclient.BulkReadToItems(reader); err != nil { log.Errorf("failed to parse bulk response: %v", err.Error()) return nil, bulkResultStats{} } @@ -520,7 +520,7 @@ func bulkCollectPublishFails( failed := data[:0] stats := bulkResultStats{} for i := 0; i < count; i++ { - status, msg, err := esclientleg.BulkReadItemStatus(log, reader) + status, msg, err := eslegclient.BulkReadItemStatus(log, reader) if err != nil { log.Error(err) return nil, bulkResultStats{} diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index 8f0b3926486..a3e142f986b 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -27,7 +27,7 @@ import ( "testing" "time" - "github.com/elastic/beats/libbeat/esclientleg" + "github.com/elastic/beats/libbeat/esleg/eslegclient" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -291,11 +291,11 @@ func TestBulkEncodeEvents(t *testing.T) { // check meta-data for each event for i := 0; i < len(recorder.data); i += 2 { - var meta esclientleg.BulkMeta + var meta eslegclient.BulkMeta switch v := recorder.data[i].(type) { - case esclientleg.BulkCreateAction: + case eslegclient.BulkCreateAction: meta = v.Create - case esclientleg.BulkIndexAction: + case eslegclient.BulkIndexAction: meta = v.Index default: panic("unknown type") diff --git a/libbeat/outputs/elasticsearch/elasticsearch_test.go b/libbeat/outputs/elasticsearch/elasticsearch_test.go index e8a0a6bcafd..58d9df7cf93 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch_test.go +++ b/libbeat/outputs/elasticsearch/elasticsearch_test.go @@ -21,13 +21,13 @@ import ( "fmt" "testing" - "github.com/elastic/beats/libbeat/esclientleg" + "github.com/elastic/beats/libbeat/esleg/eslegclient" ) func TestConnectCallbacksManagement(t *testing.T) { - f0 := func(client *esclientleg.Connection) error { fmt.Println("i am function #0"); return nil } - f1 := func(client *esclientleg.Connection) error { fmt.Println("i am function #1"); return nil } - f2 := func(client *esclientleg.Connection) error { fmt.Println("i am function #2"); return nil } + f0 := func(client *eslegclient.Connection) error { fmt.Println("i am function #0"); return nil } + f1 := func(client *eslegclient.Connection) error { fmt.Println("i am function #1"); return nil } + f2 := func(client *eslegclient.Connection) error { fmt.Println("i am function #2"); return nil } _, err := RegisterConnectCallback(f0) if err != nil { @@ -50,9 +50,9 @@ func TestConnectCallbacksManagement(t *testing.T) { } func TestGlobalConnectCallbacksManagement(t *testing.T) { - f0 := func(client *esclientleg.Connection) error { fmt.Println("i am function #0"); return nil } - f1 := func(client *esclientleg.Connection) error { fmt.Println("i am function #1"); return nil } - f2 := func(client *esclientleg.Connection) error { fmt.Println("i am function #2"); return nil } + f0 := func(client *eslegclient.Connection) error { fmt.Println("i am function #0"); return nil } + f1 := func(client *eslegclient.Connection) error { fmt.Println("i am function #1"); return nil } + f2 := func(client *eslegclient.Connection) error { fmt.Println("i am function #2"); return nil } _, err := RegisterGlobalCallback(f0) if err != nil { diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 901319eb2e2..563844259b2 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -30,10 +30,10 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/fmtstr" - "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/outputs" "github.com/elastic/beats/v7/libbeat/outputs/outest" @@ -51,7 +51,7 @@ const ( ) type esConnection struct { - *esclientleg.Connection + *eslegclient.Connection t *testing.T index string } @@ -102,7 +102,7 @@ func esConnect(t *testing.T, index string) *esConnection { username := os.Getenv("ES_USER") password := os.Getenv("ES_PASS") - client, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ + client, err := esleg.NewConnection(eslegclient.ConnectionSettings{ URL: host, Username: username, Password: password, diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 39de6d3e8e7..de1938eb63d 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -32,10 +32,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/esclientleg" - "github.com/elastic/beats/v7/libbeat/esclientleg/eslegtest" + "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" "github.com/elastic/beats/v7/libbeat/version" ) @@ -290,7 +290,7 @@ func TestTemplateWithData(t *testing.T) { setup := newTestSetup(t, TemplateConfig{Enabled: true}) require.NoError(t, setup.loadFromFile([]string{"testdata", "fields.yml"})) require.True(t, setup.loader.templateExists(setup.config.Name)) - esClient := setup.client.(*esclientleg.Connection) + esClient := setup.client.(*eslegclient.Connection) for _, test := range dataTests { _, _, err := esClient.Index(setup.config.Name, "_doc", "", nil, test.data) if test.error { diff --git a/x-pack/libbeat/licenser/elastic_fetcher.go b/x-pack/libbeat/licenser/elastic_fetcher.go index 44103489e5b..6ffa5f6fa37 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher.go +++ b/x-pack/libbeat/licenser/elastic_fetcher.go @@ -14,7 +14,7 @@ import ( "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/esclientleg" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -146,7 +146,7 @@ func (f *ElasticFetcher) parseJSON(b []byte) (*License, error) { // esClientMux is taking care of round robin request over an array of elasticsearch client, note that // calling request is not threadsafe. type esClientMux struct { - clients []esclientleg.Connection + clients []eslegclient.Connection idx int } @@ -177,12 +177,12 @@ func (mux *esClientMux) Request( // newESClientMux takes a list of clients and randomize where we start and the list of host we are // querying. -func newESClientMux(clients []esclientleg.Connection) *esClientMux { +func newESClientMux(clients []eslegclient.Connection) *esClientMux { // randomize where we start idx := rand.Intn(len(clients)) // randomize the list of round robin hosts. - tmp := make([]esclientleg.Connection, len(clients)) + tmp := make([]eslegclient.Connection, len(clients)) copy(tmp, clients) rand.Shuffle(len(tmp), func(i, j int) { tmp[i], tmp[j] = tmp[j], tmp[i] diff --git a/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go b/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go index 35995bf238f..c2c953a05ee 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go +++ b/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go @@ -11,8 +11,6 @@ import ( "testing" "time" - "github.com/elastic/beats/libbeat/esclientleg" - "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/common/cli" @@ -25,9 +23,9 @@ const ( elasticsearchPort = "9200" ) -func getTestClient() *esclientleg.Connection { +func getTestClient() *esleg.Connection { host := "http://" + cli.GetEnvOr("ES_HOST", elasticsearchHost) + ":" + cli.GetEnvOr("ES_POST", elasticsearchPort) - client, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ + client, err := esleg.NewConnection(esleg.ConnectionSettings{ URL: host, Username: "myelastic", // NOTE: I will refactor this in a followup PR Password: "changeme", diff --git a/x-pack/libbeat/licenser/elastic_fetcher_test.go b/x-pack/libbeat/licenser/elastic_fetcher_test.go index d10f774d0ae..165c2a94417 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher_test.go +++ b/x-pack/libbeat/licenser/elastic_fetcher_test.go @@ -14,19 +14,19 @@ import ( "testing" "time" - "github.com/elastic/beats/v7/libbeat/esclientleg" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/outputs/transport" "github.com/stretchr/testify/assert" ) -func newServerClientPair(t *testing.T, handler http.HandlerFunc) (*httptest.Server, *esclientleg.Connection) { +func newServerClientPair(t *testing.T, handler http.HandlerFunc) (*httptest.Server, *eslegclient.Connection) { mux := http.NewServeMux() mux.Handle("/_license/", http.HandlerFunc(handler)) server := httptest.NewServer(mux) - client, err := esclientleg.NewConnection(esclientleg.ConnectionSettings{ + client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: server.URL, HTTP: &http.Client{ Transport: &http.Transport{ diff --git a/x-pack/libbeat/licenser/es_callback.go b/x-pack/libbeat/licenser/es_callback.go index 59610db4826..9bb58775bab 100644 --- a/x-pack/libbeat/licenser/es_callback.go +++ b/x-pack/libbeat/licenser/es_callback.go @@ -8,7 +8,7 @@ import ( "fmt" "strings" - "github.com/elastic/beats/libbeat/esclientleg" + "github.com/elastic/beats/libbeat/esleg/eslegclient" "github.com/pkg/errors" @@ -23,7 +23,7 @@ const licenseDebugK = "license" func Enforce(name string, checks ...CheckFunc) { name = strings.Title(name) - cb := func(client *esclientleg.Connection) error { + cb := func(client *eslegclient.Connection) error { // Logger created earlier than this place are at risk of discarding any log statement. log := logp.NewLogger(licenseDebugK) From 32c338df99452137875c1d874cd74db18b575570 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 25 Feb 2020 20:09:32 -0800 Subject: [PATCH 33/91] Fixing import and package name --- libbeat/esleg/eslegtest/util.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libbeat/esleg/eslegtest/util.go b/libbeat/esleg/eslegtest/util.go index 636fef7f541..4aa4ee99fe1 100644 --- a/libbeat/esleg/eslegtest/util.go +++ b/libbeat/esleg/eslegtest/util.go @@ -22,7 +22,6 @@ import ( "net/http" "os" - "github.com/elastic/beats/libbeat/esleg" "github.com/elastic/beats/libbeat/esleg/eslegclient" "github.com/elastic/beats/libbeat/outputs/transport" @@ -62,7 +61,7 @@ func InitConnection(t TestLogger, conn Connectable, err error) { // GetTestingElasticsearch creates a test connection. func GetTestingElasticsearch(t TestLogger) *eslegclient.Connection { - conn, err := esleg.NewConnection(eslegclient.ConnectionSettings{ + conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: GetURL(), HTTP: &http.Client{ Transport: &http.Transport{ From f83d0c1f87f88b7bba15a50adb32b92a82eabf56 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 25 Feb 2020 20:48:10 -0800 Subject: [PATCH 34/91] Fixing imports --- libbeat/esleg/eslegclient/connection_integration_test.go | 2 +- libbeat/outputs/elasticsearch/client_integration_test.go | 3 +-- libbeat/template/load_integration_test.go | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection_integration_test.go b/libbeat/esleg/eslegclient/connection_integration_test.go index 40f076a02d3..9b9f687a0b6 100644 --- a/libbeat/esleg/eslegclient/connection_integration_test.go +++ b/libbeat/esleg/eslegclient/connection_integration_test.go @@ -35,7 +35,7 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/esclientleg/eslegtest" + "github.com/elastic/beats/libbeat/esleg/eslegtest" "github.com/elastic/beats/libbeat/idxmgmt" "github.com/elastic/beats/libbeat/outputs" "github.com/elastic/beats/libbeat/outputs/elasticsearch/internal" diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index c9aea66cbd3..eb6d84c1caf 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -29,13 +29,12 @@ import ( "testing" "time" - "github.com/elastic/beats/libbeat/esclientleg/eslegtest" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index de1938eb63d..61e5fe0ac1a 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -32,9 +32,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" "github.com/elastic/beats/v7/libbeat/version" ) From b3ded80d3bf05d4cebed8780057267dc5b964d05 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 25 Feb 2020 20:52:00 -0800 Subject: [PATCH 35/91] More fixes --- libbeat/idxmgmt/ilm/client_handler_integration_test.go | 7 +++---- libbeat/outputs/logstash/logstash_integration_test.go | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/libbeat/idxmgmt/ilm/client_handler_integration_test.go b/libbeat/idxmgmt/ilm/client_handler_integration_test.go index b8e881aa2db..794f72d3917 100644 --- a/libbeat/idxmgmt/ilm/client_handler_integration_test.go +++ b/libbeat/idxmgmt/ilm/client_handler_integration_test.go @@ -180,7 +180,7 @@ func newESClientHandler(t *testing.T) ilm.ClientHandler { } func newRawESClient(t *testing.T) ilm.ESClient { - client, err := esleg.NewConnection(eslegclient.ConnectionSettings{ + client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: getURL(), Username: getUser(), Password: getPass(), @@ -188,11 +188,10 @@ func newRawESClient(t *testing.T) ilm.ESClient { Transport: &http.Transport{ Dial: transport.NetDialer(60 * time.Second).Dial, }, - Timeout: 0, + Timeout: 60 * time.Second, }, - Timeout: 60 * time.Second, CompressionLevel: 3, - }, nil) + }) if err != nil { t.Fatal(err) } diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 563844259b2..f2f5ed0d927 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -102,7 +102,7 @@ func esConnect(t *testing.T, index string) *esConnection { username := os.Getenv("ES_USER") password := os.Getenv("ES_PASS") - client, err := esleg.NewConnection(eslegclient.ConnectionSettings{ + client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: host, Username: username, Password: password, @@ -132,7 +132,6 @@ func esConnect(t *testing.T, index string) *esConnection { es := &esConnection{} es.t = t - es.Client = client es.index = index return es } From f4746aaa74fb86890f89bad7cc62059daf6b0e16 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 02:54:39 -0800 Subject: [PATCH 36/91] Breaking import cycle --- filebeat/fileset/modules_integration_test.go | 39 +++++++++++++++++--- libbeat/esleg/eslegtest/util.go | 32 ---------------- libbeat/template/load_integration_test.go | 30 ++++++++++++++- 3 files changed, 62 insertions(+), 39 deletions(-) diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index 6a8714323b6..ee9a6973102 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -21,6 +21,7 @@ package fileset import ( "encoding/json" + "net/http" "path/filepath" "testing" @@ -28,7 +29,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" - "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" + "github.com/elastic/beats/v7/libbeat/outputs/transport" ) func makeTestInfo(version string) beat.Info { @@ -39,7 +40,7 @@ func makeTestInfo(version string) beat.Info { } func TestLoadPipeline(t *testing.T) { - client := eslegtest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -90,7 +91,7 @@ func checkUploadedPipeline(t *testing.T, client *eslegclient.Connection, expecte } func TestSetupNginx(t *testing.T) { - client := eslegtest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -122,7 +123,7 @@ func TestSetupNginx(t *testing.T) { } func TestAvailableProcessors(t *testing.T) { - client := eslegtest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -158,7 +159,7 @@ func hasIngestPipelineProcessor(client *eslegclient.Connection) bool { } func TestLoadMultiplePipelines(t *testing.T) { - client := eslegtest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -203,7 +204,7 @@ func TestLoadMultiplePipelines(t *testing.T) { } func TestLoadMultiplePipelinesWithRollback(t *testing.T) { - client := eslegtest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if !hasIngest(client) { t.Skip("Skip tests because ingest is missing in this elasticsearch version: ", client.GetVersion()) } @@ -245,3 +246,29 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { status, _, _ = client.Request("GET", "/_ingest/pipeline/filebeat-6.6.0-foo-multibad-plain_logs_bad", "", nil, nil) assert.Equal(t, 404, status) } + +func getTestingElasticsearch(t TestLogger) *eslegclient.Connection { + conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ + URL: GetURL(), + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(0).Dial, + }, + Timeout: 0, + }, + }) + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + conn.Encoder = eslegclient.NewJSONEncoder(nil, false) + + err = conn.Connect() + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + return conn +} diff --git a/libbeat/esleg/eslegtest/util.go b/libbeat/esleg/eslegtest/util.go index 4aa4ee99fe1..8da334dc3a4 100644 --- a/libbeat/esleg/eslegtest/util.go +++ b/libbeat/esleg/eslegtest/util.go @@ -19,12 +19,7 @@ package eslegtest import ( "fmt" - "net/http" "os" - - "github.com/elastic/beats/libbeat/esleg/eslegclient" - - "github.com/elastic/beats/libbeat/outputs/transport" ) const ( @@ -59,33 +54,6 @@ func InitConnection(t TestLogger, conn Connectable, err error) { } } -// GetTestingElasticsearch creates a test connection. -func GetTestingElasticsearch(t TestLogger) *eslegclient.Connection { - conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ - URL: GetURL(), - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(0).Dial, - }, - Timeout: 0, - }, - }) - if err != nil { - t.Fatal(err) - panic(err) // panic in case TestLogger did not stop test - } - - conn.Encoder = eslegclient.NewJSONEncoder(nil, false) - - err = conn.Connect() - if err != nil { - t.Fatal(err) - panic(err) // panic in case TestLogger did not stop test - } - - return conn -} - // GetURL return the Elasticsearch testing URL. func GetURL() string { return fmt.Sprintf("http://%v:%v", GetEsHost(), getEsPort()) diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 61e5fe0ac1a..9fb84b63a59 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -24,6 +24,7 @@ import ( "fmt" "io/ioutil" "math/rand" + "net/http" "path/filepath" "strconv" "testing" @@ -32,6 +33,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/outputs/transport" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" @@ -60,7 +62,7 @@ func newTestSetup(t *testing.T, cfg TemplateConfig) *testSetup { if cfg.Name == "" { cfg.Name = fmt.Sprintf("load-test-%+v", rand.Int()) } - client := eslegtest.GetTestingElasticsearch(t) + client := getTestingElasticsearch(t) if err := client.Connect(); err != nil { t.Fatal(err) } @@ -350,3 +352,29 @@ func path(t *testing.T, fileElems []string) string { require.NoError(t, err) return fieldsPath } + +func getTestingElasticsearch(t TestLogger) *eslegclient.Connection { + conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ + URL: eslegtest.GetURL(), + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(0).Dial, + }, + Timeout: 0, + }, + }) + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + conn.Encoder = eslegclient.NewJSONEncoder(nil, false) + + err = conn.Connect() + if err != nil { + t.Fatal(err) + panic(err) // panic in case TestLogger did not stop test + } + + return conn +} From 7b27826a621b1c7fac752003ddb867d818fb7495 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 02:57:56 -0800 Subject: [PATCH 37/91] Removing unused function --- .../elasticsearch/client_integration_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index eb6d84c1caf..80f15e2afa8 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -38,9 +38,7 @@ import ( "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch/internal" "github.com/elastic/beats/v7/libbeat/outputs/outest" - "github.com/elastic/beats/v7/libbeat/outputs/outil" ) func TestClientPublishEvent(t *testing.T) { @@ -283,20 +281,6 @@ func connectTestEs(t *testing.T, cfg interface{}) (outputs.Client, *Client) { return client, client } -// getTestingElasticsearch creates a test client. -func getTestingElasticsearch(t internal.TestLogger) *Client { - client, err := NewClient(ClientSettings{ - URL: internal.GetURL(), - Index: outil.MakeSelector(), - Username: internal.GetUser(), - Password: internal.GetPass(), - Timeout: 60 * time.Second, - CompressionLevel: 3, - }, nil) - eslegtest.InitConnection(t, client, err) - return client -} - func randomClient(grp outputs.Group) outputs.NetworkClient { L := len(grp.Clients) if L == 0 { From cc4bf4836f05b76e2fc13ca58465b13e6436d702 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 03:09:18 -0800 Subject: [PATCH 38/91] Adding back missing statement --- libbeat/outputs/logstash/logstash_integration_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index f2f5ed0d927..80ded46044b 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -132,6 +132,7 @@ func esConnect(t *testing.T, index string) *esConnection { es := &esConnection{} es.t = t + es.Connection = client es.index = index return es } From 1af576b1495e075ae0fc98fe84158fb341ddbc4e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 04:16:42 -0800 Subject: [PATCH 39/91] Fixing param --- filebeat/fileset/modules_integration_test.go | 4 +++- libbeat/template/load_integration_test.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index ee9a6973102..8fbcd5ebd43 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -25,6 +25,8 @@ import ( "path/filepath" "testing" + "github.com/elastic/beats/libbeat/esleg/eslegtest" + "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/beat" @@ -247,7 +249,7 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { assert.Equal(t, 404, status) } -func getTestingElasticsearch(t TestLogger) *eslegclient.Connection { +func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: GetURL(), HTTP: &http.Client{ diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 9fb84b63a59..91895c641e0 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -353,7 +353,7 @@ func path(t *testing.T, fileElems []string) string { return fieldsPath } -func getTestingElasticsearch(t TestLogger) *eslegclient.Connection { +func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: eslegtest.GetURL(), HTTP: &http.Client{ From f55621cf3c1d7d76efd3bd89941502189a201445 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 05:29:02 -0800 Subject: [PATCH 40/91] Fixing package name --- libbeat/esleg/eslegclient/connection_integration_test.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection_integration_test.go b/libbeat/esleg/eslegclient/connection_integration_test.go index 9b9f687a0b6..f7d6d1da273 100644 --- a/libbeat/esleg/eslegclient/connection_integration_test.go +++ b/libbeat/esleg/eslegclient/connection_integration_test.go @@ -38,7 +38,6 @@ import ( "github.com/elastic/beats/libbeat/esleg/eslegtest" "github.com/elastic/beats/libbeat/idxmgmt" "github.com/elastic/beats/libbeat/outputs" - "github.com/elastic/beats/libbeat/outputs/elasticsearch/internal" ) func TestConnect(t *testing.T) { @@ -118,11 +117,11 @@ func connectTestEs(t *testing.T, cfg interface{}) (outputs.Client, *Client) { } // getTestingElasticsearch creates a test client. -func getTestingElasticsearch(t internal.TestLogger) *Client { +func getTestingElasticsearch(t eslegtest.TestLogger) *Client { conn, err := NewConnection(ConnectionSettings{ - URL: internal.GetURL(), - Username: internal.GetUser(), - Password: internal.GetUser(), + URL: eslegtest.GetURL(), + Username: eslegtest.GetUser(), + Password: eslegtest.GetPass(), Timeout: 60 * time.Second, CompressionLevel: 3, }, nil) From 7b37e1f93747130489f3bbd2f133c61557ba21de Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 05:32:23 -0800 Subject: [PATCH 41/91] Include ES output plugin so it's registered --- libbeat/outputs/logstash/logstash_integration_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 80ded46044b..8f2dce4e5b0 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -36,6 +36,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common/fmtstr" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/outputs" + _ "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" "github.com/elastic/beats/v7/libbeat/outputs/outest" "github.com/elastic/beats/v7/libbeat/outputs/outil" "github.com/elastic/beats/v7/libbeat/outputs/transport" From c37ee06871214733bf3af18375b02686cb4f7a5b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 07:21:56 -0800 Subject: [PATCH 42/91] Proxy handling --- libbeat/esleg/eslegclient/connection.go | 19 +++- .../connection_integration_test.go | 88 +++++++++++-------- libbeat/outputs/elasticsearch/client.go | 6 +- 3 files changed, 70 insertions(+), 43 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 2bca4a8396c..1ff45a5130a 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -47,8 +47,9 @@ type Connection struct { // ConnectionSettings are the settings needed for a Connection type ConnectionSettings struct { - URL string - ProxyURL *url.URL + URL string + Proxy *url.URL + ProxyDisable bool Username string Password string @@ -84,6 +85,20 @@ func NewConnection(settings ConnectionSettings) (*Connection, error) { return nil, err } + var proxy func(*http.Request) (*url.URL, error) + if !settings.ProxyDisable { + proxy = http.ProxyFromEnvironment + if settings.Proxy != nil { + proxy = http.ProxyURL(settings.Proxy) + } + } + + if settings.HTTP != nil { + if t, ok := settings.HTTP.Transport.(*http.Transport); ok { + t.Proxy = proxy + } + } + return &Connection{ ConnectionSettings: settings, Encoder: encoder, diff --git a/libbeat/esleg/eslegclient/connection_integration_test.go b/libbeat/esleg/eslegclient/connection_integration_test.go index f7d6d1da273..498547030e4 100644 --- a/libbeat/esleg/eslegclient/connection_integration_test.go +++ b/libbeat/esleg/eslegclient/connection_integration_test.go @@ -33,11 +33,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/esleg/eslegtest" - "github.com/elastic/beats/libbeat/idxmgmt" "github.com/elastic/beats/libbeat/outputs" + "github.com/elastic/beats/libbeat/outputs/transport" ) func TestConnect(t *testing.T) { @@ -62,69 +61,82 @@ func TestConnectWithProxy(t *testing.T) { defer proxy.Close() // Use connectTestEs instead of getTestingElasticsearch to make use of makeES - _, client := connectTestEs(t, map[string]interface{}{ + client, err := connectTestEs(t, map[string]interface{}{ "hosts": "http://" + wrongPort.Addr().String(), "timeout": 5, // seconds }) + require.NoError(t, err) assert.Error(t, client.Connect(), "it should fail without proxy") - _, client = connectTestEs(t, map[string]interface{}{ + client, err = connectTestEs(t, map[string]interface{}{ "hosts": "http://" + wrongPort.Addr().String(), "proxy_url": proxy.URL, "timeout": 5, // seconds }) + require.NoError(t, err) assert.NoError(t, client.Connect()) } -func connectTestEs(t *testing.T, cfg interface{}) (outputs.Client, *Client) { +func connectTestEs(t *testing.T, cfg interface{}) (*Connection, error) { config, err := common.NewConfigFrom(map[string]interface{}{ - "hosts": eslegtest.GetEsHost(), - "username": eslegtest.GetUser(), - "password": eslegtest.GetPass(), - "template.enabled": false, + "username": eslegtest.GetUser(), + "password": eslegtest.GetPass(), }) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) tmp, err := common.NewConfigFrom(cfg) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) err = config.Merge(tmp) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) - info := beat.Info{Beat: "libbeat"} - im, _ := idxmgmt.DefaultSupport(nil, info, nil) - output, err := makeES(im, info, outputs.NewNilObserver(), config) - if err != nil { - t.Fatal(err) - } + hosts, err := config.String("hosts", -1) + require.NoError(t, err) - type clientWrap interface { - outputs.NetworkClient - Client() outputs.NetworkClient + username, err := config.String("username", -1) + require.NoError(t, err) + + password, err := config.String("password", -1) + require.NoError(t, err) + + proxy, err := config.String("proxy", -1) + require.NoError(t, err) + + timeout, err := config.Int("timeout", -1) + require.NoError(t, err) + + s := ConnectionSettings{ + URL: hosts, + Username: username, + Password: password, + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(timeout * time.Second).Dial, + }, + Timeout: timeout * time.Second, + }, CompressionLevel: 3, } - client := randomClient(output).(clientWrap).Client().(*Client) - // Load version number - client.Connect() + if proxy != "" { + s.Proxy = proxy + } - return client, client + return NewConnection(s) } // getTestingElasticsearch creates a test client. -func getTestingElasticsearch(t eslegtest.TestLogger) *Client { +func getTestingElasticsearch(t eslegtest.TestLogger) *Connection { conn, err := NewConnection(ConnectionSettings{ - URL: eslegtest.GetURL(), - Username: eslegtest.GetUser(), - Password: eslegtest.GetPass(), - Timeout: 60 * time.Second, - CompressionLevel: 3, - }, nil) + URL: eslegtest.GetURL(), + Username: eslegtest.GetUser(), + Password: eslegtest.GetPass(), + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: transport.NetDialer(60 * time.Second).Dial, + }, + Timeout: 60 * time.Second, + }, CompressionLevel: 3, + }) eslegtest.InitConnection(t, conn, err) return conn } diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 76c80d82b6f..bc8a81e72d0 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -146,7 +146,7 @@ func NewClient( }, Timeout: s.Timeout, }, - ProxyURL: s.Proxy, + Proxy: s.Proxy, Parameters: s.Parameters, CompressionLevel: s.CompressionLevel, EscapeHTML: s.EscapeHTML, @@ -299,11 +299,11 @@ func (client *Client) Clone() *Client { URL: client.URL, Index: client.index, Pipeline: client.pipeline, - Proxy: client.ProxyURL, + Proxy: client.Proxy, // Without the following nil check on proxyURL, a nil Proxy field will try // reloading proxy settings from the environment instead of leaving them // empty. - ProxyDisable: client.ProxyURL == nil, + ProxyDisable: client.Proxy == nil, TLS: client.TLSConfig, Username: client.Username, Password: client.Password, From 6b32d9b034524149fb7f12ffdc2e97f4e4356b6f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 10:12:23 -0800 Subject: [PATCH 43/91] Let Connection handle ProxyDisable setting --- libbeat/outputs/elasticsearch/client.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index bc8a81e72d0..13e6bf95e6a 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -87,14 +87,6 @@ func NewClient( s ClientSettings, onConnect *callbacksRegistry, ) (*Client, error) { - var proxy func(*http.Request) (*url.URL, error) - if !s.ProxyDisable { - proxy = http.ProxyFromEnvironment - if s.Proxy != nil { - proxy = http.ProxyURL(s.Proxy) - } - } - pipeline := s.Pipeline if pipeline != nil && pipeline.IsEmpty() { pipeline = nil @@ -142,11 +134,11 @@ func NewClient( Dial: dialer.Dial, DialTLS: tlsDialer.Dial, TLSClientConfig: s.TLS.ToConfig(), - Proxy: proxy, }, Timeout: s.Timeout, }, Proxy: s.Proxy, + ProxyDisable: s.ProxyDisable, Parameters: s.Parameters, CompressionLevel: s.CompressionLevel, EscapeHTML: s.EscapeHTML, From 2f39d6f771972a6310cbb646f9bbf0603480180e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 11:04:56 -0800 Subject: [PATCH 44/91] Only parse proxy field from config if set --- libbeat/esleg/eslegclient/connection_integration_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection_integration_test.go b/libbeat/esleg/eslegclient/connection_integration_test.go index 498547030e4..cae382c7142 100644 --- a/libbeat/esleg/eslegclient/connection_integration_test.go +++ b/libbeat/esleg/eslegclient/connection_integration_test.go @@ -99,12 +99,15 @@ func connectTestEs(t *testing.T, cfg interface{}) (*Connection, error) { password, err := config.String("password", -1) require.NoError(t, err) - proxy, err := config.String("proxy", -1) - require.NoError(t, err) - timeout, err := config.Int("timeout", -1) require.NoError(t, err) + var proxy string + if config.HasField("proxy") { + proxy, err = config.String("proxy", -1) + require.NoError(t, err) + } + s := ConnectionSettings{ URL: hosts, Username: username, From 6acbb8d79735327f453cde78edc5787344b8e767 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 11:05:23 -0800 Subject: [PATCH 45/91] Cast timeout ints --- libbeat/esleg/eslegclient/connection_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection_integration_test.go b/libbeat/esleg/eslegclient/connection_integration_test.go index cae382c7142..d7cfc23a02f 100644 --- a/libbeat/esleg/eslegclient/connection_integration_test.go +++ b/libbeat/esleg/eslegclient/connection_integration_test.go @@ -114,9 +114,9 @@ func connectTestEs(t *testing.T, cfg interface{}) (*Connection, error) { Password: password, HTTP: &http.Client{ Transport: &http.Transport{ - Dial: transport.NetDialer(timeout * time.Second).Dial, + Dial: transport.NetDialer(time.Duration(timeout) * time.Second).Dial, }, - Timeout: timeout * time.Second, + Timeout: time.Duration(timeout) * time.Second, }, CompressionLevel: 3, } From 32836d54404479d5132ef389d92b194d68e7dcb3 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 11:05:32 -0800 Subject: [PATCH 46/91] Parse proxy URL --- libbeat/esleg/eslegclient/connection_integration_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libbeat/esleg/eslegclient/connection_integration_test.go b/libbeat/esleg/eslegclient/connection_integration_test.go index d7cfc23a02f..8d880d6b0ae 100644 --- a/libbeat/esleg/eslegclient/connection_integration_test.go +++ b/libbeat/esleg/eslegclient/connection_integration_test.go @@ -121,7 +121,9 @@ func connectTestEs(t *testing.T, cfg interface{}) (*Connection, error) { } if proxy != "" { - s.Proxy = proxy + p, err := url.Parse(proxy) + require.NoError(t, err) + s.Proxy = p } return NewConnection(s) From b6edfc48065540fe948dabc19a26e03d1af93761 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 26 Feb 2020 12:17:39 -0800 Subject: [PATCH 47/91] Fixing proxy integration test --- libbeat/esleg/eslegclient/connection_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection_integration_test.go b/libbeat/esleg/eslegclient/connection_integration_test.go index 8d880d6b0ae..2267bc6be57 100644 --- a/libbeat/esleg/eslegclient/connection_integration_test.go +++ b/libbeat/esleg/eslegclient/connection_integration_test.go @@ -103,8 +103,8 @@ func connectTestEs(t *testing.T, cfg interface{}) (*Connection, error) { require.NoError(t, err) var proxy string - if config.HasField("proxy") { - proxy, err = config.String("proxy", -1) + if config.HasField("proxy_url") { + proxy, err = config.String("proxy_url", -1) require.NoError(t, err) } From 1e3a37ab410838a7eae6f5fb8c652862c12c467f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 27 Feb 2020 14:01:18 -0800 Subject: [PATCH 48/91] Fixing ILM test --- libbeat/esleg/eslegclient/connection.go | 34 ++++++++++++++++--------- libbeat/outputs/elasticsearch/client.go | 24 ++++++++--------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 1ff45a5130a..9de0928d9ee 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -111,21 +111,12 @@ func NewConnection(settings ConnectionSettings) (*Connection, error) { // the configured host, updates the known Elasticsearch version and calls // globally configured handlers. func (conn *Connection) Connect() error { - versionString, err := conn.Ping() - if err != nil { + if err := conn.getVersion(); err != nil { return err } - if version, err := common.NewVersion(versionString); err != nil { - conn.log.Errorf("Invalid version from Elasticsearch: %v", versionString) - conn.version = common.Version{} - } else { - conn.version = *version - } - if conn.OnConnectCallback != nil { - err = conn.OnConnectCallback() - if err != nil { + if err := conn.OnConnectCallback(); err != nil { return fmt.Errorf("Connection marked as failed because the onConnect callback failed: %v", err) } } @@ -215,11 +206,30 @@ func (conn *Connection) execRequest( } // GetVersion returns the elasticsearch version the client is connected to. -// The version is read and updated on 'Connect'. func (conn *Connection) GetVersion() common.Version { + if !conn.version.IsValid() { + conn.getVersion() + } + return conn.version } +func (conn *Connection) getVersion() error { + versionString, err := conn.Ping() + if err != nil { + return err + } + + if version, err := common.NewVersion(versionString); err != nil { + conn.log.Errorf("Invalid version from Elasticsearch: %v", versionString) + conn.version = common.Version{} + } else { + conn.version = *version + } + + return nil +} + // LoadJSON creates a PUT request based on a JSON document. func (conn *Connection) LoadJSON(path string, json map[string]interface{}) ([]byte, error) { status, body, err := conn.Request("PUT", path, "", nil, json) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 13e6bf95e6a..69853655500 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -147,18 +147,7 @@ func NewClient( return nil, err } - client := &Client{ - Connection: *conn, - index: s.Index, - pipeline: pipeline, - timeout: s.Timeout, - - observer: s.Observer, - - log: logp.NewLogger("elasticsearch"), - } - - client.Connection.OnConnectCallback = func() error { + conn.OnConnectCallback = func() error { globalCallbackRegistry.mutex.Lock() defer globalCallbackRegistry.mutex.Unlock() @@ -183,6 +172,17 @@ func NewClient( return nil } + client := &Client{ + Connection: *conn, + index: s.Index, + pipeline: pipeline, + timeout: s.Timeout, + + observer: s.Observer, + + log: logp.NewLogger("elasticsearch"), + } + return client, nil } From bc1d1a7e8890772ff6f1c58374a00ee34c5d5496 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 27 Feb 2020 15:55:49 -0800 Subject: [PATCH 49/91] Updating expected request count in test --- libbeat/outputs/elasticsearch/client_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index a3e142f986b..c1fc3d812bb 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -207,8 +207,14 @@ func TestClientWithHeaders(t *testing.T) { // Request.Host field and removed from the Header map. assert.Equal(t, "myhost.local", r.Host) - bulkResponse := `{"items":[{"index":{}},{"index":{}},{"index":{}}]}` - fmt.Fprintln(w, bulkResponse) + var response string + if r.URL.Path == "/" { + response = `{ "version": { "number": "7.6.0" } }` + } else { + response = `{"items":[{"index":{}},{"index":{}},{"index":{}}]}` + + } + fmt.Fprintln(w, response) requestCount++ })) defer ts.Close() @@ -237,7 +243,7 @@ func TestClientWithHeaders(t *testing.T) { batch := outest.NewBatch(event, event, event) err = client.Publish(batch) assert.NoError(t, err) - assert.Equal(t, 2, requestCount) + assert.Equal(t, 3, requestCount) } type testBulkRecorder struct { From 144e7c40057d8b85e0b993eaa86a4e8b02b46778 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 27 Feb 2020 17:57:01 -0800 Subject: [PATCH 50/91] Fixing package names --- filebeat/fileset/modules_integration_test.go | 4 ++-- x-pack/libbeat/licenser/elastic_fetcher_integration_test.go | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index 8fbcd5ebd43..8e8423ca729 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -150,7 +150,7 @@ func TestAvailableProcessors(t *testing.T) { assert.Contains(t, err.Error(), "ingest-hello") } -func hasIngest(client *esleg.Connection) bool { +func hasIngest(client *eslegclient.Connection) bool { v := client.GetVersion() return v.Major >= 5 } @@ -251,7 +251,7 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ - URL: GetURL(), + URL: eslegtest.GetURL(), HTTP: &http.Client{ Transport: &http.Transport{ Dial: transport.NetDialer(0).Dial, diff --git a/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go b/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go index c2c953a05ee..42e9e89c44b 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go +++ b/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go @@ -11,6 +11,8 @@ import ( "testing" "time" + "github.com/elastic/beats/libbeat/esleg/eslegclient" + "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/common/cli" @@ -23,9 +25,9 @@ const ( elasticsearchPort = "9200" ) -func getTestClient() *esleg.Connection { +func getTestClient() *eslegclient.Connection { host := "http://" + cli.GetEnvOr("ES_HOST", elasticsearchHost) + ":" + cli.GetEnvOr("ES_POST", elasticsearchPort) - client, err := esleg.NewConnection(esleg.ConnectionSettings{ + client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ URL: host, Username: "myelastic", // NOTE: I will refactor this in a followup PR Password: "changeme", From 76f2ddbdc544647f0bd9fd601b4af48788ab4c83 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 06:13:33 -0800 Subject: [PATCH 51/91] Lots more refactoring!!! --- filebeat/beater/filebeat.go | 4 +- libbeat/cmd/instance/beat.go | 2 +- libbeat/common/transport/transport.go | 2 +- libbeat/esleg/eslegclient/config.go | 78 +++++++ libbeat/esleg/eslegclient/connection.go | 170 ++++++++++++--- .../report/elasticsearch/elasticsearch.go | 27 +-- libbeat/outputs/elasticsearch/client.go | 197 +++--------------- .../outputs/elasticsearch/elasticsearch.go | 34 +-- 8 files changed, 291 insertions(+), 223 deletions(-) create mode 100644 libbeat/esleg/eslegclient/config.go diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 67f163d30c4..e368cd89615 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -148,7 +148,7 @@ func (fb *Filebeat) setupPipelineLoaderCallback(b *beat.Beat) error { overwritePipelines := true b.OverwritePipelinesCallback = func(esConfig *common.Config) error { - esClient, err := elasticsearch.NewConnectedClient(esConfig) + esClient, err := eslegclient.NewConnectedClient(esConfig) if err != nil { return err } @@ -356,7 +356,7 @@ func (fb *Filebeat) Stop() { // Create a new pipeline loader (es client) factory func newPipelineLoaderFactory(esConfig *common.Config) fileset.PipelineLoaderFactory { pipelineLoaderFactory := func() (fileset.PipelineLoader, error) { - esClient, err := elasticsearch.NewConnectedClient(esConfig) + esClient, err := eslegclient.NewConnectedClient(esConfig) if err != nil { return nil, errors.Wrap(err, "Error creating Elasticsearch client") } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index b8f29fe073d..a903fcdc523 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -495,7 +495,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er if outCfg.Name() != "elasticsearch" { return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled") } - esClient, err := elasticsearch.NewConnectedClient(outCfg.Config()) + esClient, err := eslegclient.NewConnectedClient(outCfg.Config()) if err != nil { return err } diff --git a/libbeat/common/transport/transport.go b/libbeat/common/transport/transport.go index 35b397c5876..8c88d857a90 100644 --- a/libbeat/common/transport/transport.go +++ b/libbeat/common/transport/transport.go @@ -21,7 +21,7 @@ import ( "errors" "net" - "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/libbeat/logp" ) type Dialer interface { diff --git a/libbeat/esleg/eslegclient/config.go b/libbeat/esleg/eslegclient/config.go new file mode 100644 index 00000000000..46a17f3ca3a --- /dev/null +++ b/libbeat/esleg/eslegclient/config.go @@ -0,0 +1,78 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package eslegclient + +import ( + "fmt" + "time" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" +) + +type config struct { + Hosts []string `config:"hosts" validate:"required"` + Protocol string `config:"protocol"` + Path string `config:"path"` + Params map[string]string `config:"parameters"` + Headers map[string]string `config:"headers"` + + TLS *tlscommon.Config `config:"ssl"` + + ProxyURL string `config:"proxy_url"` + ProxyDisable bool `config:"proxy_disable"` + + Username string `config:"username"` + Password string `config:"password"` + APIKey string `config:"api_key"` + + CompressionLevel int `config:"compression_level" validate:"min=0, max=9"` + EscapeHTML bool `config:"escape_html"` + Timeout time.Duration `config:"timeout"` +} + +var ( + defaultConfig = config{ + Protocol: "", + Path: "", + ProxyURL: "", + ProxyDisable: false, + Params: nil, + Username: "", + Password: "", + APIKey: "", + Timeout: 90 * time.Second, + CompressionLevel: 0, + EscapeHTML: false, + TLS: nil, + } +) + +func (c *config) Validate() error { + if c.ProxyURL != "" && !c.ProxyDisable { + if _, err := common.ParseURL(c.ProxyURL); err != nil { + return err + } + } + + if c.APIKey != "" && (c.Username != "" || c.Password != "") { + return fmt.Errorf("cannot set both api_key and username/password") + } + + return nil +} diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 9de0928d9ee..58eca2271c9 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -24,11 +24,12 @@ import ( "io/ioutil" "net/http" "net/url" + "time" - "github.com/elastic/beats/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/libbeat/logp" - - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/logp" ) // Connection manages the connection for a given client. @@ -40,6 +41,8 @@ type Connection struct { // buffered bulk requests BulkRequ *BulkRequest + HTTP *http.Client + version common.Version log *logp.Logger @@ -56,55 +59,172 @@ type ConnectionSettings struct { APIKey string Headers map[string]string - TLSConfig *tlscommon.TLSConfig - - HTTP *http.Client + TLS *tlscommon.TLSConfig OnConnectCallback func() error + Observer transport.IOStatser Parameters map[string]string CompressionLevel int EscapeHTML bool + Timeout time.Duration } -func NewConnection(settings ConnectionSettings) (*Connection, error) { +// NewConnection returns a new Elasticsearch client +func NewConnection(s ConnectionSettings) (*Connection, error) { + u, err := url.Parse(s.URL) + if err != nil { + return nil, fmt.Errorf("failed to parse elasticsearch URL: %v", err) + } + + if u.User != nil { + s.Username = u.User.Username() + s.Password, _ = u.User.Password() + u.User = nil + + // Re-write URL without credentials. + s.URL = u.String() + } + logp.Info("elasticsearch url: %s", s.URL) + + // TODO: add socks5 proxy support + var dialer, tlsDialer transport.Dialer + + dialer = transport.NetDialer(s.Timeout) + tlsDialer, err = transport.TLSDialer(dialer, s.TLS, s.Timeout) + if err != nil { + return nil, err + } + + if st := s.Observer; st != nil { + dialer = transport.StatsDialer(dialer, st) + tlsDialer = transport.StatsDialer(tlsDialer, st) + } + var encoder BodyEncoder - var err error - compression := settings.CompressionLevel + compression := s.CompressionLevel if compression == 0 { - encoder = NewJSONEncoder(nil, settings.EscapeHTML) + encoder = NewJSONEncoder(nil, s.EscapeHTML) } else { - encoder, err = NewGzipEncoder(compression, nil, settings.EscapeHTML) + encoder, err = NewGzipEncoder(compression, nil, s.EscapeHTML) if err != nil { return nil, err } } - bulkRequ, err := NewBulkRequest(settings.URL, "", "", settings.Parameters, nil) + bulkRequ, err := NewBulkRequest(s.URL, "", "", s.Parameters, nil) if err != nil { return nil, err } var proxy func(*http.Request) (*url.URL, error) - if !settings.ProxyDisable { + if !s.ProxyDisable { proxy = http.ProxyFromEnvironment - if settings.Proxy != nil { - proxy = http.ProxyURL(settings.Proxy) + if s.Proxy != nil { + proxy = http.ProxyURL(s.Proxy) + } + } + + return &Connection{ + ConnectionSettings: s, + HTTP: &http.Client{ + Transport: &http.Transport{ + Dial: dialer.Dial, + DialTLS: tlsDialer.Dial, + TLSClientConfig: s.TLS.ToConfig(), + Proxy: proxy, + }, + Timeout: s.Timeout, + }, + Encoder: encoder, + BulkRequ: bulkRequ, + log: logp.NewLogger("esclientleg"), + }, nil +} + +// NewClients returns a list of Elasticsearch clients based on the given +// configuration. It accepts the same configuration parameters as the Elasticsearch +// output, except for the output specific configuration options. If multiple hosts +// are defined in the configuration, a client is returned for each of them. +func NewClients(cfg *common.Config) ([]Connection, error) { + config := defaultConfig + if err := cfg.Unpack(&config); err != nil { + return nil, err + } + + tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS) + if err != nil { + return nil, err + } + + var proxyURL *url.URL + if !config.ProxyDisable { + proxyURL, err = common.ParseURL(config.ProxyURL) + if err != nil { + return nil, err } + if proxyURL != nil { + logp.Info("using proxy URL: %s", proxyURL) + } + } + + params := config.Params + if len(params) == 0 { + params = nil } - if settings.HTTP != nil { - if t, ok := settings.HTTP.Transport.(*http.Transport); ok { - t.Proxy = proxy + clients := []Connection{} + for _, host := range config.Hosts { + esURL, err := common.MakeURL(config.Protocol, config.Path, host, 9200) + if err != nil { + logp.Err("invalid host param set: %s, Error: %v", host, err) + return nil, err } + + client, err := NewConnection(ConnectionSettings{ + URL: esURL, + Proxy: proxyURL, + ProxyDisable: config.ProxyDisable, + TLS: tlsConfig, + Username: config.Username, + Password: config.Password, + APIKey: config.APIKey, + Parameters: params, + Headers: config.Headers, + Timeout: config.Timeout, + CompressionLevel: config.CompressionLevel, + }) + if err != nil { + return clients, err + } + clients = append(clients, *client) + } + if len(clients) == 0 { + return clients, fmt.Errorf("no hosts defined in the config") } + return clients, nil +} - return &Connection{ - ConnectionSettings: settings, - Encoder: encoder, - BulkRequ: bulkRequ, - log: logp.NewLogger("esclientleg"), - }, nil +func NewConnectedClient(cfg *common.Config) (*Connection, error) { + clients, err := NewClients(cfg) + if err != nil { + return nil, err + } + + errors := []string{} + + for _, client := range clients { + err = client.Connect() + if err != nil { + const errMsg = "error connecting to Elasticsearch at %v: %v" + client.log.Errorf(errMsg, client.URL, err) + err = fmt.Errorf(errMsg, client.URL, err) + errors = append(errors, err.Error()) + continue + } + return &client, nil + } + return nil, fmt.Errorf("couldn't connect to any of the configured Elasticsearch hosts. Errors: %v", errors) } // Connect connects the client. It runs a GET request against the root URL of diff --git a/libbeat/monitoring/report/elasticsearch/elasticsearch.go b/libbeat/monitoring/report/elasticsearch/elasticsearch.go index b530a8c081e..fef360079e7 100644 --- a/libbeat/monitoring/report/elasticsearch/elasticsearch.go +++ b/libbeat/monitoring/report/elasticsearch/elasticsearch.go @@ -29,6 +29,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/monitoring/report" @@ -337,18 +338,20 @@ func makeClient( } esClient, err := esout.NewClient(esout.ClientSettings{ - URL: url, - Proxy: proxyURL, - TLS: tlsConfig, - Username: config.Username, - Password: config.Password, - APIKey: config.APIKey, - Parameters: params, - Headers: config.Headers, - Index: outil.MakeSelector(outil.ConstSelectorExpr("_xpack")), - Pipeline: nil, - Timeout: config.Timeout, - CompressionLevel: config.CompressionLevel, + ConnectionSettings: eslegclient.ConnectionSettings{ + URL: url, + Proxy: proxyURL, + TLS: tlsConfig, + Username: config.Username, + Password: config.Password, + APIKey: config.APIKey, + Parameters: params, + Headers: config.Headers, + Timeout: config.Timeout, + CompressionLevel: config.CompressionLevel, + }, + Index: outil.MakeSelector(outil.ConstSelectorExpr("_xpack")), + Pipeline: nil, }, nil) if err != nil { return nil, err diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 69853655500..5562b34e0f1 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -30,7 +30,6 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport" - "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" @@ -54,20 +53,10 @@ type Client struct { // ClientSettings contains the settings for a client. type ClientSettings struct { - URL string - Proxy *url.URL - ProxyDisable bool - TLS *tlscommon.TLSConfig - Username, Password string - APIKey string - EscapeHTML bool - Parameters map[string]string - Headers map[string]string - Index outputs.IndexSelector - Pipeline *outil.Selector - Timeout time.Duration - CompressionLevel int - Observer outputs.Observer + eslegclient.ConnectionSettings + Index outputs.IndexSelector + Pipeline *outil.Selector + Observer outputs.Observer } type bulkResultStats struct { @@ -92,51 +81,13 @@ func NewClient( pipeline = nil } - u, err := url.Parse(s.URL) - if err != nil { - return nil, fmt.Errorf("failed to parse elasticsearch URL: %v", err) - } - if u.User != nil { - s.Username = u.User.Username() - s.Password, _ = u.User.Password() - u.User = nil - - // Re-write URL without credentials. - s.URL = u.String() - } - - log := logp.NewLogger(logSelector) - log.Infof("Elasticsearch url: %s", s.URL) - - // TODO: add socks5 proxy support - var dialer, tlsDialer transport.Dialer - - dialer = transport.NetDialer(s.Timeout) - tlsDialer, err = transport.TLSDialer(dialer, s.TLS, s.Timeout) - if err != nil { - return nil, err - } - - if st := s.Observer; st != nil { - dialer = transport.StatsDialer(dialer, st) - tlsDialer = transport.StatsDialer(tlsDialer, st) - } - conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ - URL: s.URL, - Username: s.Username, - Password: s.Password, - APIKey: base64.StdEncoding.EncodeToString([]byte(s.APIKey)), - Headers: s.Headers, - TLSConfig: s.TLS, - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: dialer.Dial, - DialTLS: tlsDialer.Dial, - TLSClientConfig: s.TLS.ToConfig(), - }, - Timeout: s.Timeout, - }, + URL: s.URL, + Username: s.Username, + Password: s.Password, + APIKey: base64.StdEncoding.EncodeToString([]byte(s.APIKey)), + Headers: s.Headers, + TLS: s.TLS, Proxy: s.Proxy, ProxyDisable: s.ProxyDisable, Parameters: s.Parameters, @@ -186,99 +137,6 @@ func NewClient( return client, nil } -// NewConnectedClient creates a new Elasticsearch client based on the given config. -// It uses the NewElasticsearchClients to create a list of clients then returns -// the first from the list that successfully connects. -func NewConnectedClient(cfg *common.Config) (*Client, error) { - clients, err := NewElasticsearchClients(cfg) - if err != nil { - return nil, err - } - - errors := []string{} - - for _, client := range clients { - err = client.Connect() - if err != nil { - logp.Err("Error connecting to Elasticsearch at %v: %v", client.Connection.URL, err) - err = fmt.Errorf("Error connection to Elasticsearch %v: %v", client.Connection.URL, err) - errors = append(errors, err.Error()) - continue - } - return &client, nil - } - return nil, fmt.Errorf("Couldn't connect to any of the configured Elasticsearch hosts. Errors: %v", errors) -} - -// NewElasticsearchClients returns a list of Elasticsearch clients based on the given -// configuration. It accepts the same configuration parameters as the output, -// except for the output specific configuration options (index, pipeline, -// template) .If multiple hosts are defined in the configuration, a client is returned -// for each of them. -func NewElasticsearchClients(cfg *common.Config) ([]Client, error) { - hosts, err := outputs.ReadHostList(cfg) - if err != nil { - return nil, err - } - - config := defaultConfig - if err = cfg.Unpack(&config); err != nil { - return nil, err - } - - tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS) - if err != nil { - return nil, err - } - - var proxyURL *url.URL - if !config.ProxyDisable { - proxyURL, err = common.ParseURL(config.ProxyURL) - if err != nil { - return nil, err - } - if proxyURL != nil { - logp.Info("Using proxy URL: %s", proxyURL) - } - } - - params := config.Params - if len(params) == 0 { - params = nil - } - - clients := []Client{} - for _, host := range hosts { - esURL, err := common.MakeURL(config.Protocol, config.Path, host, 9200) - if err != nil { - logp.Err("Invalid host param set: %s, Error: %v", host, err) - return nil, err - } - - client, err := NewClient(ClientSettings{ - URL: esURL, - Proxy: proxyURL, - ProxyDisable: config.ProxyDisable, - TLS: tlsConfig, - Username: config.Username, - Password: config.Password, - APIKey: config.APIKey, - Parameters: params, - Headers: config.Headers, - Timeout: config.Timeout, - CompressionLevel: config.CompressionLevel, - }, nil) - if err != nil { - return clients, err - } - clients = append(clients, *client) - } - if len(clients) == 0 { - return clients, fmt.Errorf("No hosts defined in the Elasticsearch output") - } - return clients, nil -} - // Clone clones a client. func (client *Client) Clone() *Client { // when cloning the connection callback and params are not copied. A @@ -288,22 +146,27 @@ func (client *Client) Clone() *Client { c, _ := NewClient( ClientSettings{ - URL: client.URL, + ConnectionSettings: eslegclient.ConnectionSettings{ + URL: client.URL, + Proxy: client.Proxy, + // Without the following nil check on proxyURL, a nil Proxy field will try + // reloading proxy settings from the environment instead of leaving them + // empty. + ProxyDisable: client.Proxy == nil, + TLS: client.TLS, + Username: client.Username, + Password: client.Password, + APIKey: client.APIKey, + Parameters: nil, // XXX: do not pass params? + Headers: client.Headers, + Timeout: client.HTTP.Timeout, + CompressionLevel: client.CompressionLevel, + OnConnectCallback: nil, + Observer: nil, + EscapeHTML: false, + }, Index: client.index, Pipeline: client.pipeline, - Proxy: client.Proxy, - // Without the following nil check on proxyURL, a nil Proxy field will try - // reloading proxy settings from the environment instead of leaving them - // empty. - ProxyDisable: client.Proxy == nil, - TLS: client.TLSConfig, - Username: client.Username, - Password: client.Password, - APIKey: client.APIKey, - Parameters: nil, // XXX: do not pass params? - Headers: client.Headers, - Timeout: client.HTTP.Timeout, - CompressionLevel: client.CompressionLevel, }, nil, // XXX: do not pass connection callback? ) @@ -567,7 +430,7 @@ func (client *Client) Test(d testing.Driver) { } else { d.Run("TLS", func(d testing.Driver) { netDialer := transport.NetDialer(client.timeout) - tlsDialer, err := transport.TestTLSDialer(d, netDialer, client.TLSConfig, client.timeout) + tlsDialer, err := transport.TestTLSDialer(d, netDialer, client.TLS, client.timeout) _, err = tlsDialer.Dial("tcp", address) d.Fatal("dial up", err) }) diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index 8e46b4f3c3f..b6c3bd797a9 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -23,6 +23,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" "github.com/elastic/beats/v7/libbeat/outputs/outil" @@ -91,21 +92,24 @@ func makeES( var client outputs.NetworkClient client, err = NewClient(ClientSettings{ - URL: esURL, - Index: index, - Pipeline: pipeline, - Proxy: proxyURL, - ProxyDisable: config.ProxyDisable, - TLS: tlsConfig, - Username: config.Username, - Password: config.Password, - APIKey: config.APIKey, - Parameters: params, - Headers: config.Headers, - Timeout: config.Timeout, - CompressionLevel: config.CompressionLevel, - Observer: observer, - EscapeHTML: config.EscapeHTML, + ConnectionSettings: eslegclient.ConnectionSettings{ + URL: esURL, + Proxy: proxyURL, + ProxyDisable: config.ProxyDisable, + TLS: tlsConfig, + Username: config.Username, + Password: config.Password, + APIKey: config.APIKey, + Parameters: params, + Headers: config.Headers, + Timeout: config.Timeout, + CompressionLevel: config.CompressionLevel, + Observer: observer, + EscapeHTML: config.EscapeHTML, + }, + Index: index, + Pipeline: pipeline, + Observer: observer, }, &connectCallbackRegistry) if err != nil { return outputs.Fail(err) From 7be3905d542368ca467c1f125c38360e52037445 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 06:21:37 -0800 Subject: [PATCH 52/91] Move timeout field --- libbeat/esleg/eslegclient/connection.go | 32 ++++++++++++++++++++++ libbeat/outputs/elasticsearch/client.go | 36 +------------------------ 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 58eca2271c9..6e52f5abe11 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -30,6 +30,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common/transport" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/testing" ) // Connection manages the connection for a given client. @@ -279,6 +280,37 @@ func (conn *Connection) Close() error { return nil } +func (conn *Connection) Test(d testing.Driver) { + d.Run("elasticsearch: "+conn.URL, func(d testing.Driver) { + u, err := url.Parse(conn.URL) + d.Fatal("parse url", err) + + address := u.Host + + d.Run("connection", func(d testing.Driver) { + netDialer := transport.TestNetDialer(d, conn.timeout) + _, err = netDialer.Dial("tcp", address) + d.Fatal("dial up", err) + }) + + if u.Scheme != "https" { + d.Warn("TLS", "secure connection disabled") + } else { + d.Run("TLS", func(d testing.Driver) { + netDialer := transport.NetDialer(conn.timeout) + tlsDialer, err := transport.TestTLSDialer(d, netDialer, conn.TLS, conn.timeout) + _, err = tlsDialer.Dial("tcp", address) + d.Fatal("dial up", err) + }) + } + + err = conn.Connect() + d.Fatal("talk to server", err) + version := conn.GetVersion() + d.Info("version", version.String()) + }) +} + // Request sends a request via the connection. func (conn *Connection) Request( method, path string, diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 5562b34e0f1..e01fce2d8d9 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "net/http" - "net/url" "time" "github.com/pkg/errors" @@ -35,7 +34,6 @@ import ( "github.com/elastic/beats/v7/libbeat/outputs" "github.com/elastic/beats/v7/libbeat/outputs/outil" "github.com/elastic/beats/v7/libbeat/publisher" - "github.com/elastic/beats/v7/libbeat/testing" ) // Client is an elasticsearch client. @@ -44,7 +42,6 @@ type Client struct { index outputs.IndexSelector pipeline *outil.Selector - timeout time.Duration observer outputs.Observer @@ -93,6 +90,7 @@ func NewClient( Parameters: s.Parameters, CompressionLevel: s.CompressionLevel, EscapeHTML: s.EscapeHTML, + Timeout: s.Timeout, }) if err != nil { return nil, err @@ -127,7 +125,6 @@ func NewClient( Connection: *conn, index: s.Index, pipeline: pipeline, - timeout: s.Timeout, observer: s.Observer, @@ -412,37 +409,6 @@ func bulkCollectPublishFails( return failed, stats } -func (client *Client) Test(d testing.Driver) { - d.Run("elasticsearch: "+client.URL, func(d testing.Driver) { - u, err := url.Parse(client.URL) - d.Fatal("parse url", err) - - address := u.Host - - d.Run("connection", func(d testing.Driver) { - netDialer := transport.TestNetDialer(d, client.timeout) - _, err = netDialer.Dial("tcp", address) - d.Fatal("dial up", err) - }) - - if u.Scheme != "https" { - d.Warn("TLS", "secure connection disabled") - } else { - d.Run("TLS", func(d testing.Driver) { - netDialer := transport.NetDialer(client.timeout) - tlsDialer, err := transport.TestTLSDialer(d, netDialer, client.TLS, client.timeout) - _, err = tlsDialer.Dial("tcp", address) - d.Fatal("dial up", err) - }) - } - - err = client.Connect() - d.Fatal("talk to server", err) - version := client.GetVersion() - d.Info("version", version.String()) - }) -} - func (client *Client) String() string { return "elasticsearch(" + client.Connection.URL + ")" } From 7f9b52d382c6c73c67dedb466112d3d12328b457 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 06:29:41 -0800 Subject: [PATCH 53/91] More fixes --- libbeat/esleg/eslegclient/api_test.go | 10 ++-------- libbeat/esleg/eslegclient/connection.go | 6 +++--- .../outputs/elasticsearch/client_proxy_test.go | 12 ++++++++---- libbeat/outputs/elasticsearch/client_test.go | 18 +++++++++++------- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/libbeat/esleg/eslegclient/api_test.go b/libbeat/esleg/eslegclient/api_test.go index 0d48ae14b6b..8000be0dfa7 100644 --- a/libbeat/esleg/eslegclient/api_test.go +++ b/libbeat/esleg/eslegclient/api_test.go @@ -20,7 +20,6 @@ package eslegclient import ( "encoding/json" - "net/http" "testing" "github.com/stretchr/testify/assert" @@ -174,13 +173,8 @@ func TestReadSearchResult_invalid(t *testing.T) { func newTestConnection(url string) *Connection { conn, _ := NewConnection(ConnectionSettings{ - URL: url, - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(0).Dial, - }, - Timeout: 0, - }, + URL: url, + Timeout: 0, }) conn.Encoder = NewJSONEncoder(nil, false) return conn diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 6e52f5abe11..d8bd39ed29d 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -288,7 +288,7 @@ func (conn *Connection) Test(d testing.Driver) { address := u.Host d.Run("connection", func(d testing.Driver) { - netDialer := transport.TestNetDialer(d, conn.timeout) + netDialer := transport.TestNetDialer(d, conn.Timeout) _, err = netDialer.Dial("tcp", address) d.Fatal("dial up", err) }) @@ -297,8 +297,8 @@ func (conn *Connection) Test(d testing.Driver) { d.Warn("TLS", "secure connection disabled") } else { d.Run("TLS", func(d testing.Driver) { - netDialer := transport.NetDialer(conn.timeout) - tlsDialer, err := transport.TestTLSDialer(d, netDialer, conn.TLS, conn.timeout) + netDialer := transport.NetDialer(conn.Timeout) + tlsDialer, err := transport.TestTLSDialer(d, netDialer, conn.TLS, conn.Timeout) _, err = tlsDialer.Dial("tcp", address) d.Fatal("dial up", err) }) diff --git a/libbeat/outputs/elasticsearch/client_proxy_test.go b/libbeat/outputs/elasticsearch/client_proxy_test.go index 4d57f87fb78..34aa986e711 100644 --- a/libbeat/outputs/elasticsearch/client_proxy_test.go +++ b/libbeat/outputs/elasticsearch/client_proxy_test.go @@ -30,6 +30,8 @@ import ( "os/exec" "testing" + "github.com/elastic/beats/libbeat/esleg/eslegclient" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -184,10 +186,12 @@ func doClientPing(t *testing.T) { // if TEST_PROXY_DISABLE is nonempty, set ClientSettings.ProxyDisable. proxyDisable := os.Getenv("TEST_PROXY_DISABLE") clientSettings := ClientSettings{ - URL: serverURL, - Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), - Headers: map[string]string{headerTestField: headerTestValue}, - ProxyDisable: proxyDisable != "", + ConnectionSettings: eslegclient.ConnectionSettings{ + URL: serverURL, + Headers: map[string]string{headerTestField: headerTestValue}, + ProxyDisable: proxyDisable != "", + }, + Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), } if proxy != "" { proxyURL, err := url.Parse(proxy) diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index c1fc3d812bb..ab5a92e2d5a 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -220,12 +220,14 @@ func TestClientWithHeaders(t *testing.T) { defer ts.Close() client, err := NewClient(ClientSettings{ - URL: ts.URL, - Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), - Headers: map[string]string{ - "host": "myhost.local", - "X-Test": "testing value", + ConnectionSettings: eslegclient.ConnectionSettings{ + URL: ts.URL, + Headers: map[string]string{ + "host": "myhost.local", + "X-Test": "testing value", + }, }, + Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), }, nil) assert.NoError(t, err) @@ -341,8 +343,10 @@ func TestClientWithAPIKey(t *testing.T) { defer ts.Close() client, err := NewClient(ClientSettings{ - URL: ts.URL, - APIKey: "hyokHG4BfWk5viKZ172X:o45JUkyuS--yiSAuuxl8Uw", + ConnectionSettings: eslegclient.ConnectionSettings{ + URL: ts.URL, + APIKey: "hyokHG4BfWk5viKZ172X:o45JUkyuS--yiSAuuxl8Uw", + }, }, nil) assert.NoError(t, err) From 9a0549e75ac5a843b33e8edd8250cd1f5adf1970 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 06:30:48 -0800 Subject: [PATCH 54/91] Adding missing files --- libbeat/common/transport/transptest/testing.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libbeat/common/transport/transptest/testing.go b/libbeat/common/transport/transptest/testing.go index 64763ba9b71..4d758617993 100644 --- a/libbeat/common/transport/transptest/testing.go +++ b/libbeat/common/transport/transptest/testing.go @@ -169,7 +169,11 @@ func connectTCP(timeout time.Duration) TransportFactory { Proxy: proxy, Timeout: timeout, } +<<<<<<< HEAD return transport.NewClient(cfg, "tcp", addr, 0) +======= + return transport.NewClient(&cfg, "tcp", addr, 0) +>>>>>>> Adding missing files } } @@ -187,7 +191,11 @@ func connectTLS(timeout time.Duration, certName string) TransportFactory { TLS: tlsConfig, Timeout: timeout, } +<<<<<<< HEAD return transport.NewClient(cfg, "tcp", addr, 0) +======= + return transport.NewClient(&cfg, "tcp", addr, 0) +>>>>>>> Adding missing files } } From f10972edf78af205c06db204db44c8e95c3a2f49 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 07:42:05 -0800 Subject: [PATCH 55/91] No need to pass HTTP any more --- filebeat/fileset/modules_integration_test.go | 11 ++--------- filebeat/fileset/pipelines_test.go | 12 +++--------- x-pack/libbeat/licenser/elastic_fetcher_test.go | 10 ++-------- 3 files changed, 7 insertions(+), 26 deletions(-) diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index 8e8423ca729..2078d0bde73 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -21,7 +21,6 @@ package fileset import ( "encoding/json" - "net/http" "path/filepath" "testing" @@ -31,7 +30,6 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" - "github.com/elastic/beats/v7/libbeat/outputs/transport" ) func makeTestInfo(version string) beat.Info { @@ -251,13 +249,8 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ - URL: eslegtest.GetURL(), - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(0).Dial, - }, - Timeout: 0, - }, + URL: eslegtest.GetURL(), + Timeout: 0, }) if err != nil { t.Fatal(err) diff --git a/filebeat/fileset/pipelines_test.go b/filebeat/fileset/pipelines_test.go index d7861957092..648e82a1c2e 100644 --- a/filebeat/fileset/pipelines_test.go +++ b/filebeat/fileset/pipelines_test.go @@ -25,9 +25,8 @@ import ( "testing" "time" - "github.com/elastic/beats/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/outputs/transport" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/stretchr/testify/assert" ) @@ -90,13 +89,8 @@ func TestLoadPipelinesWithMultiPipelineFileset(t *testing.T) { defer testESServer.Close() testESClient, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ - URL: testESServer.URL, - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(90 * time.Second).Dial, - }, - Timeout: 90 * time.Second, - }, + URL: testESServer.URL, + Timeout: 90 * time.Second, }) assert.NoError(t, err) diff --git a/x-pack/libbeat/licenser/elastic_fetcher_test.go b/x-pack/libbeat/licenser/elastic_fetcher_test.go index 165c2a94417..660df43fbba 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher_test.go +++ b/x-pack/libbeat/licenser/elastic_fetcher_test.go @@ -15,7 +15,6 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" - "github.com/elastic/beats/v7/libbeat/outputs/transport" "github.com/stretchr/testify/assert" ) @@ -27,13 +26,8 @@ func newServerClientPair(t *testing.T, handler http.HandlerFunc) (*httptest.Serv server := httptest.NewServer(mux) client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ - URL: server.URL, - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(90 * time.Second).Dial, - }, - Timeout: 90 * time.Second, - }, + URL: server.URL, + Timeout: 90 * time.Second, }) if err != nil { t.Fatalf("could not create the elasticsearch client, error: %s", err) From 95f75b0c07b74255b1f877ca66ee065e7c26ade0 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 08:53:17 -0800 Subject: [PATCH 56/91] Simplifying Bulk API usage --- libbeat/esleg/eslegclient/bulkapi.go | 15 +-------------- libbeat/monitoring/report/elasticsearch/client.go | 2 +- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/libbeat/esleg/eslegclient/bulkapi.go b/libbeat/esleg/eslegclient/bulkapi.go index f71ebffa5e7..5f3ba118e2d 100644 --- a/libbeat/esleg/eslegclient/bulkapi.go +++ b/libbeat/esleg/eslegclient/bulkapi.go @@ -73,19 +73,6 @@ type BulkResult json.RawMessage func (conn *Connection) Bulk( index, docType string, params map[string]string, body []interface{}, -) (BulkResult, error) { - return conn.BulkWith(index, docType, params, nil, body) -} - -// BulkWith creates a HTTP request containing a bunch of operations and send -// them to Elasticsearch. The request is retransmitted up to max_retries before -// returning an error. -func (conn *Connection) BulkWith( - index string, - docType string, - params map[string]string, - metaBuilder MetaBuilder, - body []interface{}, ) (BulkResult, error) { if len(body) == 0 { return nil, nil @@ -93,7 +80,7 @@ func (conn *Connection) BulkWith( enc := conn.Encoder enc.Reset() - if err := bulkEncode(conn.log, enc, metaBuilder, body); err != nil { + if err := bulkEncode(conn.log, enc, nil, body); err != nil { return nil, err } diff --git a/libbeat/monitoring/report/elasticsearch/client.go b/libbeat/monitoring/report/elasticsearch/client.go index 186eabca229..25d1f5a5963 100644 --- a/libbeat/monitoring/report/elasticsearch/client.go +++ b/libbeat/monitoring/report/elasticsearch/client.go @@ -238,7 +238,7 @@ func (c *publishClient) publishBulk(event publisher.Event, typ string) error { // Currently one request per event is sent. Reason is that each event can contain different // interval params and X-Pack requires to send the interval param. // FIXME: index name (first param below) - result, err := c.es.BulkWith(getMonitoringIndexName(), "", nil, nil, bulk[:]) + result, err := c.es.Bulk(getMonitoringIndexName(), "", nil, bulk[:]) if err != nil { return err } From 9915ab3837865c2c1eef70c5968c758256d3411c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 08:58:08 -0800 Subject: [PATCH 57/91] Removing unused code --- libbeat/esleg/eslegclient/bulkapi.go | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/libbeat/esleg/eslegclient/bulkapi.go b/libbeat/esleg/eslegclient/bulkapi.go index 5f3ba118e2d..aeec67dcffa 100644 --- a/libbeat/esleg/eslegclient/bulkapi.go +++ b/libbeat/esleg/eslegclient/bulkapi.go @@ -58,9 +58,6 @@ type BulkMeta struct { ID string `json:"_id,omitempty" struct:"_id,omitempty"` } -// MetaBuilder creates meta data for bulk requests -type MetaBuilder func(interface{}) interface{} - type BulkRequest struct { requ *http.Request } @@ -80,7 +77,7 @@ func (conn *Connection) Bulk( enc := conn.Encoder enc.Reset() - if err := bulkEncode(conn.log, enc, nil, body); err != nil { + if err := bulkEncode(conn.log, enc, body); err != nil { return nil, err } @@ -109,7 +106,7 @@ func (conn *Connection) SendMonitoringBulk( enc := conn.Encoder enc.Reset() - if err := bulkEncode(conn.log, enc, nil, body); err != nil { + if err := bulkEncode(conn.log, enc, body); err != nil { return nil, err } @@ -339,20 +336,11 @@ func itemStatusInner(reader *JSONReader, logger *logp.Logger) (int, []byte, erro return status, msg, nil } -func bulkEncode(log *logp.Logger, out BulkWriter, metaBuilder MetaBuilder, body []interface{}) error { - if metaBuilder == nil { - for _, obj := range body { - if err := out.AddRaw(obj); err != nil { - log.Debugf("Failed to encode message: %+v", err) - return err - } - } - } else { - for _, obj := range body { - meta := metaBuilder(obj) - if err := out.Add(meta, obj); err != nil { - log.Debugf("Failed to encode event (dropping event): %+v", err) - } +func bulkEncode(log *logp.Logger, out BulkWriter, body []interface{}) error { + for _, obj := range body { + if err := out.AddRaw(obj); err != nil { + log.Debugf("Failed to encode message: %s", err) + return err } } return nil From 9114f968428b4a2a7f8a1982fbfbf322c172ed60 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 09:48:40 -0800 Subject: [PATCH 58/91] Remove bulk state from Connection --- libbeat/esleg/eslegclient/bulkapi.go | 39 +++++++++-------- .../esleg/eslegclient/bulkapi_mock_test.go | 6 +-- libbeat/esleg/eslegclient/connection.go | 19 ++------- .../monitoring/report/elasticsearch/client.go | 2 +- libbeat/outputs/elasticsearch/client.go | 42 +++++++------------ libbeat/outputs/elasticsearch/client_test.go | 42 +++++++------------ 6 files changed, 56 insertions(+), 94 deletions(-) diff --git a/libbeat/esleg/eslegclient/bulkapi.go b/libbeat/esleg/eslegclient/bulkapi.go index aeec67dcffa..31def378834 100644 --- a/libbeat/esleg/eslegclient/bulkapi.go +++ b/libbeat/esleg/eslegclient/bulkapi.go @@ -58,7 +58,7 @@ type BulkMeta struct { ID string `json:"_id,omitempty" struct:"_id,omitempty"` } -type BulkRequest struct { +type bulkRequest struct { requ *http.Request } @@ -70,27 +70,23 @@ type BulkResult json.RawMessage func (conn *Connection) Bulk( index, docType string, params map[string]string, body []interface{}, -) (BulkResult, error) { +) (int, BulkResult, error) { if len(body) == 0 { - return nil, nil + return 0, nil, nil } enc := conn.Encoder enc.Reset() if err := bulkEncode(conn.log, enc, body); err != nil { - return nil, err + return 0, nil, err } - requ, err := NewBulkRequest(conn.URL, index, docType, params, enc) + requ, err := newBulkRequest(conn.URL, index, docType, params, enc) if err != nil { - return nil, err + return 0, nil, err } - _, result, err := conn.SendBulkRequest(requ) - if err != nil { - return nil, err - } - return result, nil + return conn.sendBulkRequest(requ) } // SendMonitoringBulk creates a HTTP request to the X-Pack Monitoring API containing a bunch of @@ -121,19 +117,19 @@ func (conn *Connection) SendMonitoringBulk( return nil, err } - _, result, err := conn.SendBulkRequest(requ) + _, result, err := conn.sendBulkRequest(requ) if err != nil { return nil, err } return result, nil } -func NewBulkRequest( +func newBulkRequest( urlStr string, index, docType string, params map[string]string, body BodyEncoder, -) (*BulkRequest, error) { +) (*bulkRequest, error) { path, err := makePath(index, docType, "_bulk") if err != nil { return nil, err @@ -147,7 +143,7 @@ func newMonitoringBulkRequest( urlStr string, params map[string]string, body BodyEncoder, -) (*BulkRequest, error) { +) (*bulkRequest, error) { var path string var err error if esVersion.Major < 7 { @@ -168,7 +164,7 @@ func newBulkRequestWithPath( path string, params map[string]string, body BodyEncoder, -) (*BulkRequest, error) { +) (*bulkRequest, error) { url := addToURL(urlStr, path, "", params) var reader io.Reader @@ -185,12 +181,15 @@ func newBulkRequestWithPath( body.AddHeader(&requ.Header) } - return &BulkRequest{ + r := bulkRequest{ requ: requ, - }, nil + } + r.reset(body) + + return &r, nil } -func (r *BulkRequest) Reset(body BodyEncoder) { +func (r *bulkRequest) reset(body BodyEncoder) { bdy := body.Reader() rc, ok := bdy.(io.ReadCloser) @@ -213,7 +212,7 @@ func (r *BulkRequest) Reset(body BodyEncoder) { body.AddHeader(&r.requ.Header) } -func (conn *Connection) SendBulkRequest(requ *BulkRequest) (int, BulkResult, error) { +func (conn *Connection) sendBulkRequest(requ *bulkRequest) (int, BulkResult, error) { status, resp, err := conn.execHTTPRequest(requ.requ) return status, BulkResult(resp), err } diff --git a/libbeat/esleg/eslegclient/bulkapi_mock_test.go b/libbeat/esleg/eslegclient/bulkapi_mock_test.go index 3f3646ba1f0..1fbd53d9425 100644 --- a/libbeat/esleg/eslegclient/bulkapi_mock_test.go +++ b/libbeat/esleg/eslegclient/bulkapi_mock_test.go @@ -60,7 +60,7 @@ func TestOneHostSuccessResp_Bulk(t *testing.T) { params := map[string]string{ "refresh": "true", } - _, err := client.Bulk(index, "type1", params, body) + _, _, err := client.Bulk(index, "type1", params, body) if err != nil { t.Errorf("Bulk() returns error: %s", err) } @@ -96,7 +96,7 @@ func TestOneHost500Resp_Bulk(t *testing.T) { params := map[string]string{ "refresh": "true", } - _, err := client.Bulk(index, "type1", params, body) + _, _, err := client.Bulk(index, "type1", params, body) if err == nil { t.Errorf("Bulk() should return error.") } @@ -136,7 +136,7 @@ func TestOneHost503Resp_Bulk(t *testing.T) { params := map[string]string{ "refresh": "true", } - _, err := client.Bulk(index, "type1", params, body) + _, _, err := client.Bulk(index, "type1", params, body) if err == nil { t.Errorf("Bulk() should return error.") } diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index d8bd39ed29d..db84fe2a086 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -38,15 +38,10 @@ type Connection struct { ConnectionSettings Encoder BodyEncoder - - // buffered bulk requests - BulkRequ *BulkRequest - - HTTP *http.Client + HTTP *http.Client version common.Version - - log *logp.Logger + log *logp.Logger } // ConnectionSettings are the settings needed for a Connection @@ -113,11 +108,6 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { } } - bulkRequ, err := NewBulkRequest(s.URL, "", "", s.Parameters, nil) - if err != nil { - return nil, err - } - var proxy func(*http.Request) (*url.URL, error) if !s.ProxyDisable { proxy = http.ProxyFromEnvironment @@ -137,9 +127,8 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { }, Timeout: s.Timeout, }, - Encoder: encoder, - BulkRequ: bulkRequ, - log: logp.NewLogger("esclientleg"), + Encoder: encoder, + log: logp.NewLogger("esclientleg"), }, nil } diff --git a/libbeat/monitoring/report/elasticsearch/client.go b/libbeat/monitoring/report/elasticsearch/client.go index 25d1f5a5963..1f0cc5c29c9 100644 --- a/libbeat/monitoring/report/elasticsearch/client.go +++ b/libbeat/monitoring/report/elasticsearch/client.go @@ -238,7 +238,7 @@ func (c *publishClient) publishBulk(event publisher.Event, typ string) error { // Currently one request per event is sent. Reason is that each event can contain different // interval params and X-Pack requires to send the interval param. // FIXME: index name (first param below) - result, err := c.es.Bulk(getMonitoringIndexName(), "", nil, bulk[:]) + _, result, err := c.es.Bulk(getMonitoringIndexName(), "", nil, bulk[:]) if err != nil { return err } diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index e01fce2d8d9..9f9d78d5d86 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -28,7 +28,6 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/transport" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs" @@ -198,19 +197,10 @@ func (client *Client) publishEvents( return nil, nil } - body := client.Encoder - body.Reset() - // encode events into bulk request buffer, dropping failed elements from // events slice - - eventType := "" - if client.GetVersion().Major < 7 { - eventType = defaultEventType - } - origCount := len(data) - data = bulkEncodePublishRequest(client.GetVersion(), body, client.index, client.pipeline, eventType, data, client.log) + data, bulkItems := bulkEncodePublishRequest(client.GetVersion(), client.index, client.pipeline, data, client.log) newCount := len(data) if st != nil && origCount > newCount { st.Dropped(origCount - newCount) @@ -219,9 +209,7 @@ func (client *Client) publishEvents( return nil, nil } - requ := client.BulkRequ - requ.Reset(body) - status, result, sendErr := client.SendBulkRequest(requ) + status, result, sendErr := client.Bulk("", "", nil, bulkItems) if sendErr != nil { client.log.Errorf("Failed to perform any bulk index operations: %s", sendErr) return data, sendErr @@ -263,33 +251,29 @@ func (client *Client) publishEvents( return nil, nil } -// fillBulkRequest encodes all bulk requests and returns slice of events -// successfully added to bulk request. +// bulkEncodePublishRequest encodes all bulk requests and returns slice of events +// successfully added to the list of bulk items and the list of bulk items. func bulkEncodePublishRequest( log *logp.Logger, version common.Version, - body eslegclient.BulkWriter, index outputs.IndexSelector, pipeline *outil.Selector, - eventType string, data []publisher.Event, -) []publisher.Event { +) ([]publisher.Event, []interface{}) { + okEvents := data[:0] + bulkItems := []interface{}{} for i := range data { event := &data[i].Content - meta, err := createEventBulkMeta(log, version, index, pipeline, eventType, event) + meta, err := createEventBulkMeta(log, version, index, pipeline, event) if err != nil { log.Errorf("Failed to encode event meta data: %+v", err) continue } - if err := body.Add(meta, event); err != nil { - log.Errorf("Failed to encode event: %+v", err) - log.Debugf("Failed event: %v", event) - continue - } + bulkItems = append(bulkItems, meta, event) okEvents = append(okEvents, data[i]) } - return okEvents + return okEvents, bulkItems } func createEventBulkMeta( @@ -297,10 +281,14 @@ func createEventBulkMeta( version common.Version, indexSel outputs.IndexSelector, pipelineSel *outil.Selector, - eventType string, event *beat.Event, logger *logp.Logger, ) (interface{}, error) { + eventType := "" + if version.Major < 7 { + eventType = defaultEventType + } + pipeline, err := getPipeline(event, pipelineSel) if err != nil { err := fmt.Errorf("failed to select pipeline: %v", err) diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index ab5a92e2d5a..e002369aa61 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -248,22 +248,25 @@ func TestClientWithHeaders(t *testing.T) { assert.Equal(t, 3, requestCount) } -type testBulkRecorder struct { - data []interface{} - inAction bool -} - func TestBulkEncodeEvents(t *testing.T) { cases := map[string]struct { + version string docType string config common.MapStr events []common.MapStr }{ - "Beats 7.x event": { + "6.x": { + version: "6.8.0", docType: "doc", config: common.MapStr{}, events: []common.MapStr{{"message": "test"}}, }, + "latest": { + version: version.GetDefaultVersion(), + docType: "", + config: common.MapStr{}, + events: []common.MapStr{{"message": "test"}}, + }, } for name, test := range cases { @@ -272,7 +275,7 @@ func TestBulkEncodeEvents(t *testing.T) { cfg := common.MustNewConfigFrom(test.config) info := beat.Info{ IndexPrefix: "test", - Version: version.GetDefaultVersion(), + Version: test.version, } im, err := idxmgmt.DefaultSupport(nil, info, common.NewConfig()) @@ -291,16 +294,14 @@ func TestBulkEncodeEvents(t *testing.T) { } } - recorder := &testBulkRecorder{} - - encoded := bulkEncodePublishRequest(logp.L(), common.Version{Major: 7, Minor: 5}, recorder, index, pipeline, test.docType, events) + encoded, bulkItems := bulkEncodePublishRequest(logp.L(), *common.MustNewVersion(test.version), index, pipeline, events) assert.Equal(t, len(events), len(encoded), "all events should have been encoded") - assert.False(t, recorder.inAction, "incomplete bulk") + assert.Equal(t, 2*len(events), len(bulkItems), "incomplete bulk") // check meta-data for each event - for i := 0; i < len(recorder.data); i += 2 { + for i := 0; i < len(bulkItems); i += 2 { var meta eslegclient.BulkMeta - switch v := recorder.data[i].(type) { + switch v := bulkItems[i].(type) { case eslegclient.BulkCreateAction: meta = v.Create case eslegclient.BulkIndexAction: @@ -318,21 +319,6 @@ func TestBulkEncodeEvents(t *testing.T) { } } -func (r *testBulkRecorder) Add(meta, obj interface{}) error { - if r.inAction { - panic("can not add a new action if other action is active") - } - - r.data = append(r.data, meta, obj) - return nil -} - -func (r *testBulkRecorder) AddRaw(raw interface{}) error { - r.data = append(r.data) - r.inAction = !r.inAction - return nil -} - func TestClientWithAPIKey(t *testing.T) { var headers http.Header From c3ee32086ed6ee1d925c7721d2fc749e1df63373 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 11:17:27 -0800 Subject: [PATCH 59/91] Removing empty file --- .../outputs/elasticsearch/internal/testing.go | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 libbeat/outputs/elasticsearch/internal/testing.go diff --git a/libbeat/outputs/elasticsearch/internal/testing.go b/libbeat/outputs/elasticsearch/internal/testing.go deleted file mode 100644 index 8bcbca6b411..00000000000 --- a/libbeat/outputs/elasticsearch/internal/testing.go +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package internal From 8fbcbce419dfa039aebc0532577473f5579209ea Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 11:26:52 -0800 Subject: [PATCH 60/91] Moving Bulk API response streaming parsing code back into ES output package --- libbeat/esleg/eslegclient/bulkapi.go | 129 +--- .../monitoring/report/elasticsearch/client.go | 18 +- libbeat/outputs/elasticsearch/bulk.go | 155 +++++ .../elasticsearch/bulk_test.go} | 8 +- libbeat/outputs/elasticsearch/client.go | 6 +- libbeat/outputs/elasticsearch/json_read.go | 587 ++++++++++++++++++ 6 files changed, 755 insertions(+), 148 deletions(-) create mode 100644 libbeat/outputs/elasticsearch/bulk.go rename libbeat/{esleg/eslegclient/bulkapi_test.go => outputs/elasticsearch/bulk_test.go} (95%) create mode 100644 libbeat/outputs/elasticsearch/json_read.go diff --git a/libbeat/esleg/eslegclient/bulkapi.go b/libbeat/esleg/eslegclient/bulkapi.go index 31def378834..86b518eeea1 100644 --- a/libbeat/esleg/eslegclient/bulkapi.go +++ b/libbeat/esleg/eslegclient/bulkapi.go @@ -31,16 +31,7 @@ import ( ) var ( - errExpectedItemsArray = errors.New("expected items array") - errExpectedItemObject = errors.New("expected item response object") - errExpectedStatusCode = errors.New("expected item status code") - errUnexpectedEmptyObject = errors.New("empty object") - errExpectedObjectEnd = errors.New("expected end of object") - ErrTempBulkFailure = errors.New("temporary bulk send failure") - - nameItems = []byte("items") - nameStatus = []byte("status") - nameError = []byte("error") + ErrTempBulkFailure = errors.New("temporary bulk send failure") ) type BulkIndexAction struct { @@ -217,124 +208,6 @@ func (conn *Connection) sendBulkRequest(requ *bulkRequest) (int, BulkResult, err return status, BulkResult(resp), err } -// BulkReadToItems reads the bulk response up to (but not including) items -func BulkReadToItems(reader *JSONReader) error { - if err := reader.ExpectDict(); err != nil { - return errExpectedObject - } - - // find 'items' field in response - for { - kind, name, err := reader.nextFieldName() - if err != nil { - return err - } - - if kind == dictEnd { - return errExpectedItemsArray - } - - // found items array -> continue - if bytes.Equal(name, nameItems) { - break - } - - reader.ignoreNext() - } - - // check items field is an array - if err := reader.ExpectArray(); err != nil { - return errExpectedItemsArray - } - - return nil -} - -// BulkReadItemStatus reads the status and error fields from the bulk item -func BulkReadItemStatus(reader *JSONReader, logger *logp.Logger) (int, []byte, error) { - // skip outer dictionary - if err := reader.ExpectDict(); err != nil { - return 0, nil, errExpectedItemObject - } - - // find first field in outer dictionary (e.g. 'create') - kind, _, err := reader.nextFieldName() - if err != nil { - log.Errorf("Failed to parse bulk response item: %s", err) - return 0, nil, err - } - if kind == dictEnd { - err = errUnexpectedEmptyObject - log.Errorf("Failed to parse bulk response item: %s", err) - return 0, nil, err - } - - // parse actual item response code and error message - status, msg, err := itemStatusInner(reader, logger) - if err != nil { - log.Errorf("Failed to parse bulk response item: %s", err) - return 0, nil, err - } - - // close dictionary. Expect outer dictionary to have only one element - kind, _, err = reader.step() - if err != nil { - log.Errorf("Failed to parse bulk response item: %s", err) - return 0, nil, err - } - if kind != dictEnd { - err = errExpectedObjectEnd - log.Errorf("Failed to parse bulk response item: %s", err) - return 0, nil, err - } - - return status, msg, nil -} - -func itemStatusInner(reader *JSONReader, logger *logp.Logger) (int, []byte, error) { - if err := reader.ExpectDict(); err != nil { - return 0, nil, errExpectedItemObject - } - - status := -1 - var msg []byte - for { - kind, name, err := reader.nextFieldName() - if err != nil { - logger.Errorf("Failed to parse bulk response item: %s", err) - } - if kind == dictEnd { - break - } - - switch { - case bytes.Equal(name, nameStatus): // name == "status" - status, err = reader.nextInt() - if err != nil { - logger.Errorf("Failed to parse bulk response item: %s", err) - return 0, nil, err - } - - case bytes.Equal(name, nameError): // name == "error" - msg, err = reader.ignoreNext() // collect raw string for "error" field - if err != nil { - return 0, nil, err - } - - default: // ignore unknown fields - _, err = reader.ignoreNext() - if err != nil { - return 0, nil, err - } - } - } - - if status < 0 { - return 0, nil, errExpectedStatusCode - } - return status, msg, nil -} - func bulkEncode(log *logp.Logger, out BulkWriter, body []interface{}) error { for _, obj := range body { if err := out.AddRaw(obj); err != nil { diff --git a/libbeat/monitoring/report/elasticsearch/client.go b/libbeat/monitoring/report/elasticsearch/client.go index 1f0cc5c29c9..643d34de8a6 100644 --- a/libbeat/monitoring/report/elasticsearch/client.go +++ b/libbeat/monitoring/report/elasticsearch/client.go @@ -26,11 +26,10 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/esclientleg" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring/report" - esout "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" + "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" "github.com/elastic/beats/v7/libbeat/publisher" "github.com/elastic/beats/v7/libbeat/testing" ) @@ -38,8 +37,7 @@ import ( var createDocPrivAvailableESVersion = common.MustNewVersion("7.5.0") type publishClient struct { - log *logp.Logger - es *esout.Client + es *elasticsearch.Client params map[string]string format report.Format @@ -47,12 +45,11 @@ type publishClient struct { } func newPublishClient( - es *esout.Client, + es *elasticsearch.Client, params map[string]string, format report.Format, ) (*publishClient, error) { p := &publishClient{ - log: logp.NewLogger(selector), es: es, params: params, format: format, @@ -253,16 +250,17 @@ func getMonitoringIndexName() string { return fmt.Sprintf(".monitoring-beats-%v-%s", version, date) } -func logBulkFailures(log *logp.Logger, result esclientleg.BulkResult, events []report.Event) { - reader := eslegclient.NewJSONReader(result) - err := eslegclient.BulkReadToItems(reader) +// TODO: reimplement without using streaming JSON reader, since monitoring bulk results only contain single items +func logBulkFailures(log *logp.Logger, result eslegclient.BulkResult, events []report.Event) { + reader := elasticsearch.NewJSONReader(result) + err := elasticsearch.BulkReadToItems(reader) if err != nil { log.Errorf("failed to parse monitoring bulk items: %+v", err) return } for i := range events { - status, msg, err := esclientleg.BulkReadItemStatus(log, reader) + status, msg, err := elasticsearch.BulkReadItemStatus(log, reader) if err != nil { log.Errorf("failed to parse monitoring bulk item status: %+v", err) return diff --git a/libbeat/outputs/elasticsearch/bulk.go b/libbeat/outputs/elasticsearch/bulk.go new file mode 100644 index 00000000000..067ea830050 --- /dev/null +++ b/libbeat/outputs/elasticsearch/bulk.go @@ -0,0 +1,155 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package elasticsearch + +import ( + "bytes" + "errors" + + "github.com/elastic/beats/libbeat/logp" +) + +var ( + errExpectedItemsArray = errors.New("expected items array") + errExpectedItemObject = errors.New("expected item response object") + errExpectedStatusCode = errors.New("expected item status code") + errUnexpectedEmptyObject = errors.New("empty object") + errExpectedObjectEnd = errors.New("expected end of object") + + nameItems = []byte("items") + nameStatus = []byte("status") + nameError = []byte("error") +) + +// BulkReadToItems reads the bulk response up to (but not including) items +func BulkReadToItems(reader *JSONReader) error { + if err := reader.ExpectDict(); err != nil { + return errExpectedObject + } + + // find 'items' field in response + for { + kind, name, err := reader.nextFieldName() + if err != nil { + return err + } + + if kind == dictEnd { + return errExpectedItemsArray + } + + // found items array -> continue + if bytes.Equal(name, nameItems) { + break + } + + reader.ignoreNext() + } + + // check items field is an array + if err := reader.ExpectArray(); err != nil { + return errExpectedItemsArray + } + + return nil +} + +// BulkReadItemStatus reads the status and error fields from the bulk item +func BulkReadItemStatus(reader *JSONReader, logger *logp.Logger) (int, []byte, error) { + // skip outer dictionary + if err := reader.ExpectDict(); err != nil { + return 0, nil, errExpectedItemObject + } + + // find first field in outer dictionary (e.g. 'create') + kind, _, err := reader.nextFieldName() + if err != nil { + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + if kind == dictEnd { + err = errUnexpectedEmptyObject + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + // parse actual item response code and error message + status, msg, err := itemStatusInner(reader, logger) + if err != nil { + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + // close dictionary. Expect outer dictionary to have only one element + kind, _, err = reader.step() + if err != nil { + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + if kind != dictEnd { + err = errExpectedObjectEnd + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + return status, msg, nil +} + +func itemStatusInner(reader *JSONReader, logger *logp.Logger) (int, []byte, error) { + if err := reader.ExpectDict(); err != nil { + return 0, nil, errExpectedItemObject + } + + status := -1 + var msg []byte + for { + kind, name, err := reader.nextFieldName() + if err != nil { + logger.Errorf("Failed to parse bulk response item: %s", err) + } + if kind == dictEnd { + break + } + + switch { + case bytes.Equal(name, nameStatus): // name == "status" + status, err = reader.nextInt() + if err != nil { + logger.Errorf("Failed to parse bulk response item: %s", err) + return 0, nil, err + } + + case bytes.Equal(name, nameError): // name == "error" + msg, err = reader.ignoreNext() // collect raw string for "error" field + if err != nil { + return 0, nil, err + } + + default: // ignore unknown fields + _, err = reader.ignoreNext() + if err != nil { + return 0, nil, err + } + } + } + + if status < 0 { + return 0, nil, errExpectedStatusCode + } + return status, msg, nil +} diff --git a/libbeat/esleg/eslegclient/bulkapi_test.go b/libbeat/outputs/elasticsearch/bulk_test.go similarity index 95% rename from libbeat/esleg/eslegclient/bulkapi_test.go rename to libbeat/outputs/elasticsearch/bulk_test.go index 2c707943a09..c975270f128 100644 --- a/libbeat/esleg/eslegclient/bulkapi_test.go +++ b/libbeat/outputs/elasticsearch/bulk_test.go @@ -15,13 +15,11 @@ // specific language governing permissions and limitations // under the License. -package eslegclient +package elasticsearch import ( "testing" - "github.com/elastic/beats/libbeat/logp" - "github.com/stretchr/testify/assert" ) @@ -127,7 +125,3 @@ func readStatusItem(in []byte) (int, string, error) { code, msg, err := BulkReadItemStatus(reader, testLogger()) return code, string(msg), err } - -func testLogger() *logp.Logger { - return logp.NewLogger("test_esclientleg") -} diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 9f9d78d5d86..a44a1668b44 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -350,8 +350,8 @@ func bulkCollectPublishFails( result esclientleg.BulkResult, data []publisher.Event, ) ([]publisher.Event, bulkResultStats) { - reader := eslegclient.NewJSONReader(result) - if err := eslegclient.BulkReadToItems(reader); err != nil { + reader := NewJSONReader(result) + if err := BulkReadToItems(reader); err != nil { log.Errorf("failed to parse bulk response: %v", err.Error()) return nil, bulkResultStats{} } @@ -360,7 +360,7 @@ func bulkCollectPublishFails( failed := data[:0] stats := bulkResultStats{} for i := 0; i < count; i++ { - status, msg, err := eslegclient.BulkReadItemStatus(log, reader) + status, msg, err := BulkReadItemStatus(log, reader) if err != nil { log.Error(err) return nil, bulkResultStats{} diff --git a/libbeat/outputs/elasticsearch/json_read.go b/libbeat/outputs/elasticsearch/json_read.go new file mode 100644 index 00000000000..896ec89f2fa --- /dev/null +++ b/libbeat/outputs/elasticsearch/json_read.go @@ -0,0 +1,587 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package elasticsearch + +import ( + "errors" + + "github.com/elastic/beats/libbeat/common/streambuf" +) + +// SAX like json parser. But instead of relying on callbacks, state machine +// returns raw item plus entity. On top of state machine additional helper methods +// like ExpectDict, ExpectArray, nextFieldName and nextInt are available for +// low-level parsing/stepping through a json document. +// +// Due to parser simply stepping through the input buffer, almost no additional +// allocations are required. +type JSONReader struct { + streambuf.Buffer + + // parser state machine + states []state // state stack for nested arrays/objects + currentState state + + // preallocate stack memory for up to 32 nested arrays/objects + statesBuf [32]state +} + +var ( + errFailing = errors.New("JSON parser failed") + errUnknownChar = errors.New("unknown character") + errQuoteMissing = errors.New("missing closing quote") + errExpectColon = errors.New("expected ':' after map key") + errUnexpectedDictClose = errors.New("unexpected '}'") + errUnexpectedArrClose = errors.New("unexpected ']'") + errExpectedDigit = errors.New("expected a digit") + errExpectedObject = errors.New("expected JSON object") + errExpectedArray = errors.New("expected JSON array") + errExpectedFieldName = errors.New("expected JSON object field name") + errExpectedInteger = errors.New("expected integer value") + errExpectedNull = errors.New("expected null value") + errExpectedFalse = errors.New("expected false value") + errExpectedTrue = errors.New("expected true value") + errExpectedArrayField = errors.New("expected ']' or ','") +) + +var ( + nullSymbol = []byte("null") + trueSymbol = []byte("true") + falseSymbol = []byte("false") +) + +type entity uint8 + +const ( + failEntity entity = iota + trueValue + falseValue + nullValue + dictStart + dictEnd + arrStart + arrEnd + stringEntity + mapKeyEntity + intEntity + doubleEntity +) + +type state uint8 + +const ( + failedState state = iota + startState + arrState + arrStateNext + dictState + dictFieldState + dictFieldStateEnd +) + +var entityNames = map[entity]string{ + failEntity: "failEntity", + trueValue: "trueValue", + falseValue: "falseValue", + nullValue: "nullValue", + dictStart: "dictStart", + dictEnd: "dictEnd", + arrStart: "arrStart", + arrEnd: "arrEnd", + stringEntity: "stringEntity", + mapKeyEntity: "mapKeyEntity", + intEntity: "intEntity", + doubleEntity: "doubleEntity", +} + +var stateNames = map[state]string{ + failedState: "failed", + startState: "start", + arrState: "array", + arrStateNext: "arrayNext", + dictState: "dict", + dictFieldState: "dictValue", + dictFieldStateEnd: "dictNext", +} + +func (e entity) String() string { + if name, ok := entityNames[e]; ok { + return name + } + return "unknown" +} + +func (s state) String() string { + if name, ok := stateNames[s]; ok { + return name + } + return "unknown" +} + +// NewJSONReader returns a new JSONReader initialized with in +func NewJSONReader(in []byte) *JSONReader { + r := &JSONReader{} + r.init(in) + return r +} + +func (r *JSONReader) init(in []byte) { + r.Buffer.Init(in, true) + r.currentState = startState + r.states = r.statesBuf[:0] +} + +var whitespace = []byte(" \t\r\n") + +func (r *JSONReader) skipWS() { + r.IgnoreSymbols(whitespace) +} + +func (r *JSONReader) pushState(next state) { + if r.currentState != failedState { + r.states = append(r.states, r.currentState) + } + r.currentState = next +} + +func (r *JSONReader) popState() { + if len(r.states) == 0 { + r.currentState = failedState + } else { + last := len(r.states) - 1 + r.currentState = r.states[last] + r.states = r.states[:last] + } +} + +// ExpectDict checks if the next entity is a json object +func (r *JSONReader) ExpectDict() error { + e, _, err := r.step() + + if err != nil { + return err + } + + if e != dictStart { + return r.SetError(errExpectedObject) + } + + return nil +} + +// ExpectArray checks if the next entity is a json array +func (r *JSONReader) ExpectArray() error { + e, _, err := r.step() + if err != nil { + return err + } + + if e != arrStart { + return r.SetError(errExpectedArray) + } + + return nil +} + +func (r *JSONReader) nextFieldName() (entity, []byte, error) { + e, raw, err := r.step() + if err != nil { + return e, raw, err + } + + if e != mapKeyEntity && e != dictEnd { + return e, nil, r.SetError(errExpectedFieldName) + } + + return e, raw, err +} + +func (r *JSONReader) nextInt() (int, error) { + e, raw, err := r.step() + if err != nil { + return 0, err + } + + if e != intEntity { + return 0, errExpectedInteger + } + + tmp := streambuf.NewFixed(raw) + i, err := tmp.IntASCII(false) + return int(i), err +} + +// ignore type of next element and return raw content. +func (r *JSONReader) ignoreNext() (raw []byte, err error) { + r.skipWS() + + snapshot := r.Snapshot() + before := r.Len() + + e, _, err := r.step() + if err != nil { + return nil, err + } + + switch e { + case arrStart: + err = ignoreKind(r, arrEnd) + case dictStart: + err = ignoreKind(r, dictEnd) + default: + } + if err != nil { + return nil, err + } + + after := r.Len() + r.Restore(snapshot) + + bytes, _ := r.Collect(before - after) + return bytes, nil +} + +func ignoreKind(r *JSONReader, kind entity) error { + for { + e, _, err := r.step() + if err != nil { + return err + } + + switch e { + case kind: + return nil + case arrStart: + if err := ignoreKind(r, arrEnd); err != nil { + return err + } + case dictStart: + if err := ignoreKind(r, dictEnd); err != nil { + return err + } + } + } +} + +// step continues the JSON parser state machine until next entity has been parsed. +func (r *JSONReader) step() (entity, []byte, error) { + r.skipWS() + switch r.currentState { + case failedState: + return r.stepFailing() + case startState: + return r.stepStart() + case arrState: + return r.stepArray() + case arrStateNext: + return r.stepArrayNext() + case dictState: + return r.stepDict() + case dictFieldState: + return r.stepDictValue() + case dictFieldStateEnd: + return r.stepDictValueEnd() + default: + return r.failWith(errFailing) + } +} + +func (r *JSONReader) stepFailing() (entity, []byte, error) { + return failEntity, nil, r.Err() +} + +func (r *JSONReader) stepStart() (entity, []byte, error) { + c, err := r.PeekByte() + if err != nil { + return r.failWith(err) + } + + return r.tryStepPrimitive(c) +} + +func (r *JSONReader) stepArray() (entity, []byte, error) { + return r.doStepArray(true) +} + +func (r *JSONReader) stepArrayNext() (entity, []byte, error) { + c, err := r.PeekByte() + if err != nil { + return r.failWith(errFailing) + } + + switch c { + case ']': + return r.endArray() + case ',': + r.Advance(1) + r.skipWS() + r.currentState = arrState + return r.doStepArray(false) + default: + return r.failWith(errExpectedArrayField) + } +} + +func (r *JSONReader) doStepArray(allowArrayEnd bool) (entity, []byte, error) { + c, err := r.PeekByte() + if err != nil { + return r.failWith(err) + } + + if c == ']' { + if !allowArrayEnd { + return r.failWith(errUnexpectedArrClose) + } + return r.endArray() + } + + r.currentState = arrStateNext + return r.tryStepPrimitive(c) +} + +func (r *JSONReader) stepDict() (entity, []byte, error) { + return r.doStepDict(true) +} + +func (r *JSONReader) doStepDict(allowEnd bool) (entity, []byte, error) { + c, err := r.PeekByte() + if err != nil { + return r.failWith(err) + } + + switch c { + case '}': + if !allowEnd { + return r.failWith(errUnexpectedDictClose) + } + return r.endDict() + case '"': + r.currentState = dictFieldState + return r.stepMapKey() + default: + return r.failWith(errExpectedFieldName) + } +} + +func (r *JSONReader) stepDictValue() (entity, []byte, error) { + c, err := r.PeekByte() + if err != nil { + return r.failWith(err) + } + + r.currentState = dictFieldStateEnd + return r.tryStepPrimitive(c) +} + +func (r *JSONReader) stepDictValueEnd() (entity, []byte, error) { + c, err := r.PeekByte() + if err != nil { + return r.failWith(err) + } + + switch c { + case '}': + return r.endDict() + case ',': + r.Advance(1) + r.skipWS() + r.currentState = dictState + return r.doStepDict(false) + default: + return r.failWith(errUnknownChar) + } +} + +func (r *JSONReader) tryStepPrimitive(c byte) (entity, []byte, error) { + switch c { + case '{': // start dictionary + return r.startDict() + case '[': // start array + return r.startArray() + case 'n': // null + return r.stepNull() + case 'f': // false + return r.stepFalse() + case 't': // true + return r.stepTrue() + case '"': + return r.stepString() + default: + // parse number? + if c == '-' || c == '+' || c == '.' || ('0' <= c && c <= '9') { + return r.stepNumber() + } + + err := r.Err() + if err == nil { + err = r.SetError(errUnknownChar) + } + return r.failWith(err) + } +} + +func (r *JSONReader) stepNull() (entity, []byte, error) { + return stepSymbol(r, nullValue, nullSymbol, errExpectedNull) +} + +func (r *JSONReader) stepTrue() (entity, []byte, error) { + return stepSymbol(r, trueValue, trueSymbol, errExpectedTrue) +} + +func (r *JSONReader) stepFalse() (entity, []byte, error) { + return stepSymbol(r, falseValue, falseSymbol, errExpectedFalse) +} + +func stepSymbol(r *JSONReader, e entity, symb []byte, fail error) (entity, []byte, error) { + ok, err := r.MatchASCII(symb) + if err != nil { + return failEntity, nil, err + } + if !ok { + return failEntity, nil, fail + } + + r.Advance(len(symb)) + return e, nil, nil +} + +func (r *JSONReader) stepMapKey() (entity, []byte, error) { + e, key, err := r.stepString() + if err != nil { + return e, key, err + } + + r.skipWS() + c, err := r.ReadByte() + if err != nil { + return failEntity, nil, err + } + + if c != ':' { + return r.failWith(r.SetError(errExpectColon)) + } + + if err := r.Err(); err != nil { + return r.failWith(err) + } + return mapKeyEntity, key, nil +} + +func (r *JSONReader) stepString() (entity, []byte, error) { + start := 1 + for { + idxQuote := r.IndexByteFrom(start, '"') + if idxQuote == -1 { + return failEntity, nil, r.SetError(errQuoteMissing) + } + + if b, _ := r.PeekByteFrom(idxQuote - 1); b == '\\' { // escaped quote? + start = idxQuote + 1 + continue + } + + // found string end + str, err := r.Collect(idxQuote + 1) + str = str[1 : len(str)-1] + return stringEntity, str, err + } +} + +func (r *JSONReader) startDict() (entity, []byte, error) { + r.Advance(1) + r.pushState(dictState) + return dictStart, nil, nil +} + +func (r *JSONReader) endDict() (entity, []byte, error) { + r.Advance(1) + r.popState() + return dictEnd, nil, nil +} + +func (r *JSONReader) startArray() (entity, []byte, error) { + r.Advance(1) + r.pushState(arrState) + return arrStart, nil, nil +} + +func (r *JSONReader) endArray() (entity, []byte, error) { + r.Advance(1) + r.popState() + return arrEnd, nil, nil +} + +func (r *JSONReader) failWith(err error) (entity, []byte, error) { + r.currentState = failedState + return failEntity, nil, r.SetError(err) +} + +func (r *JSONReader) stepNumber() (entity, []byte, error) { + snapshot := r.Snapshot() + lenBefore := r.Len() + isDouble := false + + if err := r.Err(); err != nil { + return failEntity, nil, err + } + + // parse '+', '-' or '.' + if b, _ := r.PeekByte(); b == '-' || b == '+' { + r.Advance(1) + } + if b, _ := r.PeekByte(); b == '.' { + r.Advance(1) + isDouble = true + } + + // parse digits + buf, _ := r.CollectWhile(isDigit) + if len(buf) == 0 { + return failEntity, nil, r.SetError(errExpectedDigit) + } + + if !isDouble { + // parse optional '.' + if b, _ := r.PeekByte(); b == '.' { + r.Advance(1) + isDouble = true + + // parse optional digits + r.CollectWhile(isDigit) + } + } + + lenAfter := r.Len() + r.Restore(snapshot) + total := lenBefore - lenAfter - 1 + if total == 0 { + return failEntity, nil, r.SetError(errExpectedDigit) + } + + raw, _ := r.Collect(total) + state := intEntity + if isDouble { + state = doubleEntity + } + + return state, raw, nil +} + +func isDigit(c byte) bool { + return '0' <= c && c <= '9' +} From c0c42e0962adca3e1bc2fdcc9fbde83aa81792a1 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 11:50:04 -0800 Subject: [PATCH 61/91] Don't make monitoring bulk parsing code use streaming parser --- .../monitoring/report/elasticsearch/client.go | 42 ++++++++++++------- libbeat/outputs/elasticsearch/bulk.go | 8 ++-- libbeat/outputs/elasticsearch/bulk_test.go | 6 +-- libbeat/outputs/elasticsearch/client.go | 4 +- 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/libbeat/monitoring/report/elasticsearch/client.go b/libbeat/monitoring/report/elasticsearch/client.go index 643d34de8a6..b561ff1a623 100644 --- a/libbeat/monitoring/report/elasticsearch/client.go +++ b/libbeat/monitoring/report/elasticsearch/client.go @@ -250,26 +250,38 @@ func getMonitoringIndexName() string { return fmt.Sprintf(".monitoring-beats-%v-%s", version, date) } -// TODO: reimplement without using streaming JSON reader, since monitoring bulk results only contain single items func logBulkFailures(log *logp.Logger, result eslegclient.BulkResult, events []report.Event) { - reader := elasticsearch.NewJSONReader(result) - err := elasticsearch.BulkReadToItems(reader) - if err != nil { - log.Errorf("failed to parse monitoring bulk items: %+v", err) + var response struct { + Items []map[string]map[string]interface{} `json:"items"` + } + + if err := json.Unmarshal(result, &response); err != nil { + log.Errorf("failed to parse monitoring bulk items: %v", err) return } for i := range events { - status, msg, err := elasticsearch.BulkReadItemStatus(log, reader) - if err != nil { - log.Errorf("failed to parse monitoring bulk item status: %+v", err) - return - } - switch { - case status < 300, status == http.StatusConflict: - continue - default: - log.Warnf("monitoring bulk item insert failed (i=%v, status=%v): %s", i, status, msg) + for _, innerItem := range response.Items[i] { + var status int + if s, exists := innerItem["status"]; exists { + if v, ok := s.(int); ok { + status = v + } + } + + var errorMsg string + if e, exists := innerItem["error"]; exists { + if v, ok := e.(string); ok { + errorMsg = v + } + } + + switch { + case status < 300, status == http.StatusConflict: + continue + default: + log.Warnf("monitoring bulk item insert failed (i=%v, status=%v): %s", i, status, errorMsg) + } } } } diff --git a/libbeat/outputs/elasticsearch/bulk.go b/libbeat/outputs/elasticsearch/bulk.go index 067ea830050..6cf604adc38 100644 --- a/libbeat/outputs/elasticsearch/bulk.go +++ b/libbeat/outputs/elasticsearch/bulk.go @@ -36,8 +36,8 @@ var ( nameError = []byte("error") ) -// BulkReadToItems reads the bulk response up to (but not including) items -func BulkReadToItems(reader *JSONReader) error { +// bulkReadToItems reads the bulk response up to (but not including) items +func bulkReadToItems(reader *JSONReader) error { if err := reader.ExpectDict(); err != nil { return errExpectedObject } @@ -69,8 +69,8 @@ func BulkReadToItems(reader *JSONReader) error { return nil } -// BulkReadItemStatus reads the status and error fields from the bulk item -func BulkReadItemStatus(reader *JSONReader, logger *logp.Logger) (int, []byte, error) { +// bulkReadItemStatus reads the status and error fields from the bulk item +func bulkReadItemStatus(reader *JSONReader, logger *logp.Logger) (int, []byte, error) { // skip outer dictionary if err := reader.ExpectDict(); err != nil { return 0, nil, errExpectedItemObject diff --git a/libbeat/outputs/elasticsearch/bulk_test.go b/libbeat/outputs/elasticsearch/bulk_test.go index c975270f128..ccad926b0b5 100644 --- a/libbeat/outputs/elasticsearch/bulk_test.go +++ b/libbeat/outputs/elasticsearch/bulk_test.go @@ -34,7 +34,7 @@ func TestBulkReadToItems(t *testing.T) { reader := NewJSONReader(response) - err := BulkReadToItems(reader) + err := bulkReadToItems(reader) assert.NoError(t, err) for status := 200; status <= 400; status += 100 { @@ -70,7 +70,7 @@ func TestBulkReadItemStatus(t *testing.T) { response := []byte(`{"create": {"status": 200}}`) reader := NewJSONReader(response) - code, _, err := BulkReadItemStatus(reader, testLogger()) + code, _, err := bulkReadItemStatus(reader, testLogger()) assert.NoError(t, err) assert.Equal(t, 200, code) } @@ -122,6 +122,6 @@ func TestES2StyleExtendedErrorStatus(t *testing.T) { func readStatusItem(in []byte) (int, string, error) { reader := NewJSONReader(in) - code, msg, err := BulkReadItemStatus(reader, testLogger()) + code, msg, err := bulkReadItemStatus(reader, testLogger()) return code, string(msg), err } diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index a44a1668b44..e22191f6392 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -351,7 +351,7 @@ func bulkCollectPublishFails( data []publisher.Event, ) ([]publisher.Event, bulkResultStats) { reader := NewJSONReader(result) - if err := BulkReadToItems(reader); err != nil { + if err := bulkReadToItems(reader); err != nil { log.Errorf("failed to parse bulk response: %v", err.Error()) return nil, bulkResultStats{} } @@ -360,7 +360,7 @@ func bulkCollectPublishFails( failed := data[:0] stats := bulkResultStats{} for i := 0; i < count; i++ { - status, msg, err := BulkReadItemStatus(log, reader) + status, msg, err := bulkReadItemStatus(log, reader) if err != nil { log.Error(err) return nil, bulkResultStats{} From fd4ccbfa669357ef1fc77387e6359235f4e8c76c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 12:51:14 -0800 Subject: [PATCH 62/91] Replacing old HTTP struct passing --- libbeat/template/load_integration_test.go | 11 ++--------- .../licenser/elastic_fetcher_integration_test.go | 13 ++----------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 91895c641e0..1a53cc75073 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -24,7 +24,6 @@ import ( "fmt" "io/ioutil" "math/rand" - "net/http" "path/filepath" "strconv" "testing" @@ -33,7 +32,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/outputs/transport" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" @@ -355,13 +353,8 @@ func path(t *testing.T, fileElems []string) string { func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ - URL: eslegtest.GetURL(), - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(0).Dial, - }, - Timeout: 0, - }, + URL: eslegtest.GetURL(), + Timeout: 0, }) if err != nil { t.Fatal(err) diff --git a/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go b/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go index 42e9e89c44b..8cf182ad717 100644 --- a/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go +++ b/x-pack/libbeat/licenser/elastic_fetcher_integration_test.go @@ -7,17 +7,13 @@ package licenser import ( - "net/http" "testing" "time" - "github.com/elastic/beats/libbeat/esleg/eslegclient" - "github.com/stretchr/testify/assert" - "github.com/elastic/beats/libbeat/common/cli" "github.com/elastic/beats/v7/libbeat/common/cli" - "github.com/elastic/beats/v7/libbeat/outputs/transport" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" ) const ( @@ -32,12 +28,7 @@ func getTestClient() *eslegclient.Connection { Username: "myelastic", // NOTE: I will refactor this in a followup PR Password: "changeme", CompressionLevel: 3, - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(60 * time.Second).Dial, - }, - Timeout: 60 * time.Second, - }, + Timeout: 60 * time.Second, }) if err != nil { From 0b870bdaa58233aad117919198f1b6090e86515f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 14:46:53 -0800 Subject: [PATCH 63/91] Removing HTTP use --- libbeat/outputs/logstash/logstash_integration_test.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 8f2dce4e5b0..6e7e3693dc3 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -22,7 +22,6 @@ package logstash import ( "encoding/json" "fmt" - "net/http" "os" "sync" "testing" @@ -30,16 +29,15 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/fmtstr" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/outputs" _ "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" "github.com/elastic/beats/v7/libbeat/outputs/outest" "github.com/elastic/beats/v7/libbeat/outputs/outil" - "github.com/elastic/beats/v7/libbeat/outputs/transport" ) const ( @@ -107,12 +105,7 @@ func esConnect(t *testing.T, index string) *esConnection { URL: host, Username: username, Password: password, - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(60 * time.Second).Dial, - }, - Timeout: 60 * time.Second, - }, + Timeout: 60 * time.Second, }) if err != nil { t.Fatal(err) From 8f2f34b5d76fd552ed99ceb62b354a20f295cc62 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 16:12:16 -0800 Subject: [PATCH 64/91] Adding build tag --- libbeat/outputs/elasticsearch/bulk_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libbeat/outputs/elasticsearch/bulk_test.go b/libbeat/outputs/elasticsearch/bulk_test.go index ccad926b0b5..f43c5eda5e3 100644 --- a/libbeat/outputs/elasticsearch/bulk_test.go +++ b/libbeat/outputs/elasticsearch/bulk_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build !integration + package elasticsearch import ( From 1cfd53bf5805a4ce23535d248d9d38d671415c78 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 28 Feb 2020 17:00:32 -0800 Subject: [PATCH 65/91] Fixing up tests --- .../eslegclient/bulkapi_integration_test.go | 6 ++-- .../connection_integration_test.go | 29 +++++++------------ .../ilm/client_handler_integration_test.go | 15 +++------- 3 files changed, 17 insertions(+), 33 deletions(-) diff --git a/libbeat/esleg/eslegclient/bulkapi_integration_test.go b/libbeat/esleg/eslegclient/bulkapi_integration_test.go index 2ab696a0d13..49bb7497d56 100644 --- a/libbeat/esleg/eslegclient/bulkapi_integration_test.go +++ b/libbeat/esleg/eslegclient/bulkapi_integration_test.go @@ -53,7 +53,7 @@ func TestBulk(t *testing.T) { params := map[string]string{ "refresh": "true", } - _, err := client.Bulk(index, "", params, body) + _, _, err := client.Bulk(index, "", params, body) if err != nil { t.Fatalf("Bulk() returned error: %s", err) } @@ -86,7 +86,7 @@ func TestEmptyBulk(t *testing.T) { params := map[string]string{ "refresh": "true", } - resp, err := client.Bulk(index, "", params, body) + _, resp, err := client.Bulk(index, "", params, body) if err != nil { t.Fatalf("Bulk() returned error: %s", err) } @@ -150,7 +150,7 @@ func TestBulkMoreOperations(t *testing.T) { params := map[string]string{ "refresh": "true", } - resp, err := client.Bulk(index, "", params, body) + _, resp, err := client.Bulk(index, "", params, body) if err != nil { t.Fatalf("Bulk() returned error: %s [%s]", err, resp) } diff --git a/libbeat/esleg/eslegclient/connection_integration_test.go b/libbeat/esleg/eslegclient/connection_integration_test.go index 2267bc6be57..c8a75f86525 100644 --- a/libbeat/esleg/eslegclient/connection_integration_test.go +++ b/libbeat/esleg/eslegclient/connection_integration_test.go @@ -36,7 +36,6 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/esleg/eslegtest" "github.com/elastic/beats/libbeat/outputs" - "github.com/elastic/beats/libbeat/outputs/transport" ) func TestConnect(t *testing.T) { @@ -109,15 +108,11 @@ func connectTestEs(t *testing.T, cfg interface{}) (*Connection, error) { } s := ConnectionSettings{ - URL: hosts, - Username: username, - Password: password, - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(time.Duration(timeout) * time.Second).Dial, - }, - Timeout: time.Duration(timeout) * time.Second, - }, CompressionLevel: 3, + URL: hosts, + Username: username, + Password: password, + Timeout: time.Duration(timeout) * time.Second, + CompressionLevel: 3, } if proxy != "" { @@ -132,15 +127,11 @@ func connectTestEs(t *testing.T, cfg interface{}) (*Connection, error) { // getTestingElasticsearch creates a test client. func getTestingElasticsearch(t eslegtest.TestLogger) *Connection { conn, err := NewConnection(ConnectionSettings{ - URL: eslegtest.GetURL(), - Username: eslegtest.GetUser(), - Password: eslegtest.GetPass(), - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(60 * time.Second).Dial, - }, - Timeout: 60 * time.Second, - }, CompressionLevel: 3, + URL: eslegtest.GetURL(), + Username: eslegtest.GetUser(), + Password: eslegtest.GetPass(), + Timeout: 60 * time.Second, + CompressionLevel: 3, }) eslegtest.InitConnection(t, conn, err) return conn diff --git a/libbeat/idxmgmt/ilm/client_handler_integration_test.go b/libbeat/idxmgmt/ilm/client_handler_integration_test.go index 794f72d3917..936eb35dcd8 100644 --- a/libbeat/idxmgmt/ilm/client_handler_integration_test.go +++ b/libbeat/idxmgmt/ilm/client_handler_integration_test.go @@ -22,7 +22,6 @@ package ilm_test import ( "encoding/json" "fmt" - "net/http" "os" "testing" "time" @@ -34,7 +33,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/idxmgmt/ilm" - "github.com/elastic/beats/v7/libbeat/outputs/transport" "github.com/elastic/beats/v7/libbeat/version" ) @@ -181,15 +179,10 @@ func newESClientHandler(t *testing.T) ilm.ClientHandler { func newRawESClient(t *testing.T) ilm.ESClient { client, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ - URL: getURL(), - Username: getUser(), - Password: getPass(), - HTTP: &http.Client{ - Transport: &http.Transport{ - Dial: transport.NetDialer(60 * time.Second).Dial, - }, - Timeout: 60 * time.Second, - }, + URL: getURL(), + Username: getUser(), + Password: getPass(), + Timeout: 60 * time.Second, CompressionLevel: 3, }) if err != nil { From a0144ea225bd78b2b9cbb945f69a43b2682856a0 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 2 Mar 2020 16:52:58 -0800 Subject: [PATCH 66/91] Allow default scheme to be configurable --- libbeat/common/url.go | 32 +++++++++++++++++++++++++------- libbeat/common/url_test.go | 17 ++++++++++++++--- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/libbeat/common/url.go b/libbeat/common/url.go index d4e55154dfb..7e19f618b7e 100644 --- a/libbeat/common/url.go +++ b/libbeat/common/url.go @@ -84,18 +84,36 @@ func EncodeURLParams(url string, params url.Values) string { return strings.Join([]string{url, "?", params.Encode()}, "") } +type ParseHint func(raw string) string + // ParseURL tries to parse a URL and return the parsed result. -func ParseURL(raw string) (*url.URL, error) { +func ParseURL(raw string, hints ...ParseHint) (*url.URL, error) { if raw == "" { return nil, nil } - url, err := url.Parse(raw) - if err == nil && strings.HasPrefix(url.Scheme, "http") { - return url, err + u, err := url.Parse(raw) + if err == nil && strings.HasPrefix(u.Scheme, "http") { + return u, err + } + + // Parsing failed. Try applying hints and parsing again. + if len(hints) == 0 { + hints = append(hints, WithDefaultScheme("http")) + } + + for _, hint := range hints { + raw = hint(raw) } - // Proxy was bogus. Try prepending "http://" to it and - // see if that parses correctly. - return url.Parse("http://" + raw) + return u.Parse(raw) +} + +func WithDefaultScheme(scheme string) ParseHint { + return func(raw string) string { + if !strings.Contains(raw, "://") { + return scheme + "://" + raw + } + return raw + } } diff --git a/libbeat/common/url_test.go b/libbeat/common/url_test.go index a0becf080fe..aaded710f83 100644 --- a/libbeat/common/url_test.go +++ b/libbeat/common/url_test.go @@ -119,26 +119,37 @@ func TestURLParamsEncode(t *testing.T) { func TestParseURL(t *testing.T) { tests := map[string]struct { input string + hints []ParseHint expected string errorAssertFunc require.ErrorAssertionFunc }{ "http": { "http://host:1234/path", + nil, "http://host:1234/path", require.NoError, }, "https": { "https://host:1234/path", + nil, "https://host:1234/path", require.NoError, }, "no_scheme": { "host:1234/path", + nil, "http://host:1234/path", require.NoError, }, + "default_scheme_https": { + "host:1234/path", + []ParseHint{WithDefaultScheme("https")}, + "https://host:1234/path", + require.NoError, + }, "invalid": { "foobar:port", + nil, "", require.Error, }, @@ -146,12 +157,12 @@ func TestParseURL(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - url, err := ParseURL(test.input) + u, err := ParseURL(test.input, test.hints...) test.errorAssertFunc(t, err) if test.expected != "" { - require.Equal(t, test.expected, url.String()) + require.Equal(t, test.expected, u.String()) } else { - require.Nil(t, url) + require.Nil(t, u) } }) } From 1139429caddb2952e4688ed9cbd327a791c84d33 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 3 Mar 2020 10:05:59 -0800 Subject: [PATCH 67/91] Adding versions to import paths --- filebeat/fileset/modules_integration_test.go | 3 +-- heartbeat/main.go | 1 - libbeat/common/transport/transport.go | 2 +- libbeat/common/transport/wrap.go | 4 +++- libbeat/esleg/eslegclient/connection_integration_test.go | 6 +++--- libbeat/outputs/elasticsearch/bulk.go | 2 +- libbeat/outputs/elasticsearch/callbacks.go | 2 +- libbeat/outputs/elasticsearch/client_proxy_test.go | 3 +-- libbeat/outputs/elasticsearch/client_test.go | 3 +-- libbeat/outputs/elasticsearch/elasticsearch_test.go | 2 +- libbeat/outputs/elasticsearch/json_read.go | 2 +- x-pack/libbeat/licenser/es_callback.go | 3 +-- 12 files changed, 15 insertions(+), 18 deletions(-) diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index 2078d0bde73..8c5bc91bf70 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -24,12 +24,11 @@ import ( "path/filepath" "testing" - "github.com/elastic/beats/libbeat/esleg/eslegtest" - "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" + "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" ) func makeTestInfo(version string) beat.Info { diff --git a/heartbeat/main.go b/heartbeat/main.go index 6218bf00432..887b4b1991f 100644 --- a/heartbeat/main.go +++ b/heartbeat/main.go @@ -21,7 +21,6 @@ import ( "os" "github.com/elastic/beats/v7/heartbeat/cmd" - _ "github.com/elastic/beats/v7/heartbeat/include" ) diff --git a/libbeat/common/transport/transport.go b/libbeat/common/transport/transport.go index 8c88d857a90..35b397c5876 100644 --- a/libbeat/common/transport/transport.go +++ b/libbeat/common/transport/transport.go @@ -21,7 +21,7 @@ import ( "errors" "net" - "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/logp" ) type Dialer interface { diff --git a/libbeat/common/transport/wrap.go b/libbeat/common/transport/wrap.go index 7bde4735354..7192899652b 100644 --- a/libbeat/common/transport/wrap.go +++ b/libbeat/common/transport/wrap.go @@ -17,7 +17,9 @@ package transport -import "net" +import ( + "net" +) func ConnWrapper(d Dialer, w func(net.Conn) net.Conn) Dialer { return DialerFunc(func(network, addr string) (net.Conn, error) { diff --git a/libbeat/esleg/eslegclient/connection_integration_test.go b/libbeat/esleg/eslegclient/connection_integration_test.go index c8a75f86525..225edd6f36c 100644 --- a/libbeat/esleg/eslegclient/connection_integration_test.go +++ b/libbeat/esleg/eslegclient/connection_integration_test.go @@ -33,9 +33,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/esleg/eslegtest" - "github.com/elastic/beats/libbeat/outputs" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esleg/eslegtest" + "github.com/elastic/beats/v7/libbeat/outputs" ) func TestConnect(t *testing.T) { diff --git a/libbeat/outputs/elasticsearch/bulk.go b/libbeat/outputs/elasticsearch/bulk.go index 6cf604adc38..05382ffe5e8 100644 --- a/libbeat/outputs/elasticsearch/bulk.go +++ b/libbeat/outputs/elasticsearch/bulk.go @@ -21,7 +21,7 @@ import ( "bytes" "errors" - "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/logp" ) var ( diff --git a/libbeat/outputs/elasticsearch/callbacks.go b/libbeat/outputs/elasticsearch/callbacks.go index 909ed10af6b..dcbbd971adb 100644 --- a/libbeat/outputs/elasticsearch/callbacks.go +++ b/libbeat/outputs/elasticsearch/callbacks.go @@ -20,7 +20,7 @@ package elasticsearch import ( "sync" - "github.com/elastic/beats/libbeat/esleg/eslegclient" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/gofrs/uuid" ) diff --git a/libbeat/outputs/elasticsearch/client_proxy_test.go b/libbeat/outputs/elasticsearch/client_proxy_test.go index 34aa986e711..b431fcc4668 100644 --- a/libbeat/outputs/elasticsearch/client_proxy_test.go +++ b/libbeat/outputs/elasticsearch/client_proxy_test.go @@ -30,12 +30,11 @@ import ( "os/exec" "testing" - "github.com/elastic/beats/libbeat/esleg/eslegclient" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/outputs/outil" ) diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index e002369aa61..764ad9a00d4 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -27,13 +27,12 @@ import ( "testing" "time" - "github.com/elastic/beats/libbeat/esleg/eslegclient" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs/outest" diff --git a/libbeat/outputs/elasticsearch/elasticsearch_test.go b/libbeat/outputs/elasticsearch/elasticsearch_test.go index 58d9df7cf93..60268b59602 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch_test.go +++ b/libbeat/outputs/elasticsearch/elasticsearch_test.go @@ -21,7 +21,7 @@ import ( "fmt" "testing" - "github.com/elastic/beats/libbeat/esleg/eslegclient" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" ) func TestConnectCallbacksManagement(t *testing.T) { diff --git a/libbeat/outputs/elasticsearch/json_read.go b/libbeat/outputs/elasticsearch/json_read.go index 896ec89f2fa..8df87e5cb43 100644 --- a/libbeat/outputs/elasticsearch/json_read.go +++ b/libbeat/outputs/elasticsearch/json_read.go @@ -20,7 +20,7 @@ package elasticsearch import ( "errors" - "github.com/elastic/beats/libbeat/common/streambuf" + "github.com/elastic/beats/v7/libbeat/common/streambuf" ) // SAX like json parser. But instead of relying on callbacks, state machine diff --git a/x-pack/libbeat/licenser/es_callback.go b/x-pack/libbeat/licenser/es_callback.go index 9bb58775bab..0ca99db8f5f 100644 --- a/x-pack/libbeat/licenser/es_callback.go +++ b/x-pack/libbeat/licenser/es_callback.go @@ -8,10 +8,9 @@ import ( "fmt" "strings" - "github.com/elastic/beats/libbeat/esleg/eslegclient" - "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" ) From 3da06a3e25e7b1b2bb9dc4cf1e4adaae7cc8fd39 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 3 Mar 2020 17:49:15 -0800 Subject: [PATCH 68/91] Remove redundant check --- libbeat/common/url.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/common/url.go b/libbeat/common/url.go index 7e19f618b7e..74778cc992f 100644 --- a/libbeat/common/url.go +++ b/libbeat/common/url.go @@ -93,7 +93,7 @@ func ParseURL(raw string, hints ...ParseHint) (*url.URL, error) { } u, err := url.Parse(raw) - if err == nil && strings.HasPrefix(u.Scheme, "http") { + if err == nil { return u, err } From 82c8c532f5292ecfa2e9d50d2e42e004969387d1 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 4 Mar 2020 10:34:56 -0800 Subject: [PATCH 69/91] Undoing unnecessary heartbeat import change --- heartbeat/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/heartbeat/main.go b/heartbeat/main.go index 887b4b1991f..6218bf00432 100644 --- a/heartbeat/main.go +++ b/heartbeat/main.go @@ -21,6 +21,7 @@ import ( "os" "github.com/elastic/beats/v7/heartbeat/cmd" + _ "github.com/elastic/beats/v7/heartbeat/include" ) From f8998795401d6beaee13dad049217f1a98e6c4d8 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 4 Mar 2020 10:44:39 -0800 Subject: [PATCH 70/91] Forgot to resolve conflicts --- libbeat/common/transport/transptest/testing.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/libbeat/common/transport/transptest/testing.go b/libbeat/common/transport/transptest/testing.go index 4d758617993..64763ba9b71 100644 --- a/libbeat/common/transport/transptest/testing.go +++ b/libbeat/common/transport/transptest/testing.go @@ -169,11 +169,7 @@ func connectTCP(timeout time.Duration) TransportFactory { Proxy: proxy, Timeout: timeout, } -<<<<<<< HEAD return transport.NewClient(cfg, "tcp", addr, 0) -======= - return transport.NewClient(&cfg, "tcp", addr, 0) ->>>>>>> Adding missing files } } @@ -191,11 +187,7 @@ func connectTLS(timeout time.Duration, certName string) TransportFactory { TLS: tlsConfig, Timeout: timeout, } -<<<<<<< HEAD return transport.NewClient(cfg, "tcp", addr, 0) -======= - return transport.NewClient(&cfg, "tcp", addr, 0) ->>>>>>> Adding missing files } } From 4d2a8679501a09e1ca840453575a35e03a23a786 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 4 Mar 2020 10:53:14 -0800 Subject: [PATCH 71/91] Fixing imports --- libbeat/esleg/eslegclient/api_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/esleg/eslegclient/api_test.go b/libbeat/esleg/eslegclient/api_test.go index 8000be0dfa7..b74239369eb 100644 --- a/libbeat/esleg/eslegclient/api_test.go +++ b/libbeat/esleg/eslegclient/api_test.go @@ -25,7 +25,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/libbeat/outputs/transport" ) func GetValidQueryResult() QueryResult { From 9443e04d7177eb8849acfdf4c4ba06d978a3dd95 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 4 Mar 2020 11:37:02 -0800 Subject: [PATCH 72/91] Running go mod tidy --- go.mod | 1 - go.sum | 3 --- 2 files changed, 4 deletions(-) diff --git a/go.mod b/go.mod index 5332f700520..d49fb8a0ebe 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,6 @@ require ( github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 - github.com/elastic/beats v7.6.0+incompatible github.com/elastic/ecs v1.4.0 github.com/elastic/go-libaudit v0.4.0 github.com/elastic/go-licenser v0.2.1 diff --git a/go.sum b/go.sum index 8bf26d36cfc..d95286d6c56 100644 --- a/go.sum +++ b/go.sum @@ -209,9 +209,6 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= -github.com/elastic/beats v7.6.0+incompatible h1:vUcDbZ/qsvox6pC/t/tSdSItlt1/pKbsKaHEVS4GWss= -github.com/elastic/beats v7.6.0+incompatible/go.mod h1:7cX7zGsOwJ01FLkZs9Tg5nBdnQi6XB3hYAyWekpKgeY= -github.com/elastic/beats v7.6.1+incompatible h1:4iPVpFr8BSJW2fUVi+/tYXQ4v/IYVx0k2PPLTg8cfks= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/ecs v1.4.0 h1:BGIUwWJhThRO2IQxzm7ekV9TMUGwZoYyevT5/1xmMf0= From fdd2131c8b9cadc4f858b18fcde64c67b3379e89 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 4 Mar 2020 13:03:42 -0800 Subject: [PATCH 73/91] Revert "Remove redundant check" This reverts commit c5fde6ff3be765a89c0bc20f9cae8f697d08d47e. --- libbeat/common/url.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/common/url.go b/libbeat/common/url.go index 74778cc992f..7e19f618b7e 100644 --- a/libbeat/common/url.go +++ b/libbeat/common/url.go @@ -93,7 +93,7 @@ func ParseURL(raw string, hints ...ParseHint) (*url.URL, error) { } u, err := url.Parse(raw) - if err == nil { + if err == nil && strings.HasPrefix(u.Scheme, "http") { return u, err } From f5c7ac6495feb63ae9f18e7b4db6f5c2ed65a23a Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 09:35:06 -0800 Subject: [PATCH 74/91] Fixing args order --- libbeat/outputs/elasticsearch/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index e22191f6392..a25886c9f90 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -200,7 +200,7 @@ func (client *Client) publishEvents( // encode events into bulk request buffer, dropping failed elements from // events slice origCount := len(data) - data, bulkItems := bulkEncodePublishRequest(client.GetVersion(), client.index, client.pipeline, data, client.log) + data, bulkItems := bulkEncodePublishRequest(client.log, client.GetVersion(), client.index, client.pipeline, data) newCount := len(data) if st != nil && origCount > newCount { st.Dropped(origCount - newCount) From b0e3dfb1045bc299a5513180a2c7eb371fd46769 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 09:37:20 -0800 Subject: [PATCH 75/91] Removing extraneous parameter --- libbeat/outputs/elasticsearch/client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index a25886c9f90..1b10e32c936 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -282,7 +282,6 @@ func createEventBulkMeta( indexSel outputs.IndexSelector, pipelineSel *outil.Selector, event *beat.Event, - logger *logp.Logger, ) (interface{}, error) { eventType := "" if version.Major < 7 { From 29b817c616cb24fcaf97f324acb621bf8cea54bb Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 09:58:56 -0800 Subject: [PATCH 76/91] Removing wrong errors package import --- libbeat/outputs/elasticsearch/client.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 1b10e32c936..e4fe94dcaf7 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -24,8 +24,6 @@ import ( "net/http" "time" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" From c77351c8d8bff7282da39d6a5fc774a87f02a852 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 10:00:56 -0800 Subject: [PATCH 77/91] Fixing order of arguments --- libbeat/outputs/elasticsearch/bulk.go | 2 +- libbeat/outputs/elasticsearch/client.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/outputs/elasticsearch/bulk.go b/libbeat/outputs/elasticsearch/bulk.go index 05382ffe5e8..e774a92916c 100644 --- a/libbeat/outputs/elasticsearch/bulk.go +++ b/libbeat/outputs/elasticsearch/bulk.go @@ -70,7 +70,7 @@ func bulkReadToItems(reader *JSONReader) error { } // bulkReadItemStatus reads the status and error fields from the bulk item -func bulkReadItemStatus(reader *JSONReader, logger *logp.Logger) (int, []byte, error) { +func bulkReadItemStatus(logger *logp.Logger, reader *JSONReader) (int, []byte, error) { // skip outer dictionary if err := reader.ExpectDict(); err != nil { return 0, nil, errExpectedItemObject diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index e4fe94dcaf7..5c5f721b7d8 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -224,7 +224,7 @@ func (client *Client) publishEvents( failedEvents = data stats.fails = len(failedEvents) } else { - failedEvents, stats = bulkCollectPublishFails(result, data, client.log) + failedEvents, stats = bulkCollectPublishFails(client.log, result, data) } failed := len(failedEvents) From 9d3869d2affeaf72cbafe5b8b272884c7e59b550 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 10:01:08 -0800 Subject: [PATCH 78/91] Fixing package name --- libbeat/outputs/elasticsearch/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 5c5f721b7d8..badedf162a1 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -344,7 +344,7 @@ func getPipeline(event *beat.Event, pipelineSel *outil.Selector) (string, error) // the event will be dropped. func bulkCollectPublishFails( log *logp.Logger, - result esclientleg.BulkResult, + result eslegclient.BulkResult, data []publisher.Event, ) ([]publisher.Event, bulkResultStats) { reader := NewJSONReader(result) From 0c2a8b1eb77a27090e159ac77d1663ca7098be6d Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 10:08:26 -0800 Subject: [PATCH 79/91] Instantiating logger for tests --- libbeat/esleg/eslegclient/api_test.go | 2 +- libbeat/outputs/elasticsearch/bulk_test.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libbeat/esleg/eslegclient/api_test.go b/libbeat/esleg/eslegclient/api_test.go index b74239369eb..9055eb1f942 100644 --- a/libbeat/esleg/eslegclient/api_test.go +++ b/libbeat/esleg/eslegclient/api_test.go @@ -182,7 +182,7 @@ func newTestConnection(url string) *Connection { func (r QueryResult) String() string { out, err := json.Marshal(r) if err != nil { - logp.NewLogger(logSelector).Warnf("failed to marshal QueryResult (%+v): %#v", err, r) + logp.L().Warnf("failed to marshal QueryResult (%+v): %#v", err, r) return "ERROR" } return string(out) diff --git a/libbeat/outputs/elasticsearch/bulk_test.go b/libbeat/outputs/elasticsearch/bulk_test.go index f43c5eda5e3..9bd1abdbccf 100644 --- a/libbeat/outputs/elasticsearch/bulk_test.go +++ b/libbeat/outputs/elasticsearch/bulk_test.go @@ -22,6 +22,8 @@ package elasticsearch import ( "testing" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/stretchr/testify/assert" ) @@ -72,7 +74,7 @@ func TestBulkReadItemStatus(t *testing.T) { response := []byte(`{"create": {"status": 200}}`) reader := NewJSONReader(response) - code, _, err := bulkReadItemStatus(reader, testLogger()) + code, _, err := bulkReadItemStatus(logp.L(), reader) assert.NoError(t, err) assert.Equal(t, 200, code) } @@ -124,6 +126,6 @@ func TestES2StyleExtendedErrorStatus(t *testing.T) { func readStatusItem(in []byte) (int, string, error) { reader := NewJSONReader(in) - code, msg, err := bulkReadItemStatus(reader, testLogger()) + code, msg, err := bulkReadItemStatus(logp.L(), reader) return code, string(msg), err } From f74cc1ac305899a5289779a74444fa8d27b27590 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 10:47:57 -0800 Subject: [PATCH 80/91] Making streaming JSON parser private to ES output package --- libbeat/esleg/eslegclient/json_read.go | 587 --------------------- libbeat/outputs/elasticsearch/bulk.go | 6 +- libbeat/outputs/elasticsearch/bulk_test.go | 6 +- libbeat/outputs/elasticsearch/client.go | 2 +- libbeat/outputs/elasticsearch/json_read.go | 73 ++- 5 files changed, 43 insertions(+), 631 deletions(-) delete mode 100644 libbeat/esleg/eslegclient/json_read.go diff --git a/libbeat/esleg/eslegclient/json_read.go b/libbeat/esleg/eslegclient/json_read.go deleted file mode 100644 index 77b62197582..00000000000 --- a/libbeat/esleg/eslegclient/json_read.go +++ /dev/null @@ -1,587 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package eslegclient - -import ( - "errors" - - "github.com/elastic/beats/v7/libbeat/common/streambuf" -) - -// SAX like json parser. But instead of relying on callbacks, state machine -// returns raw item plus entity. On top of state machine additional helper methods -// like ExpectDict, ExpectArray, nextFieldName and nextInt are available for -// low-level parsing/stepping through a json document. -// -// Due to parser simply stepping through the input buffer, almost no additional -// allocations are required. -type JSONReader struct { - streambuf.Buffer - - // parser state machine - states []state // state stack for nested arrays/objects - currentState state - - // preallocate stack memory for up to 32 nested arrays/objects - statesBuf [32]state -} - -var ( - errFailing = errors.New("JSON parser failed") - errUnknownChar = errors.New("unknown character") - errQuoteMissing = errors.New("missing closing quote") - errExpectColon = errors.New("expected ':' after map key") - errUnexpectedDictClose = errors.New("unexpected '}'") - errUnexpectedArrClose = errors.New("unexpected ']'") - errExpectedDigit = errors.New("expected a digit") - errExpectedObject = errors.New("expected JSON object") - errExpectedArray = errors.New("expected JSON array") - errExpectedFieldName = errors.New("expected JSON object field name") - errExpectedInteger = errors.New("expected integer value") - errExpectedNull = errors.New("expected null value") - errExpectedFalse = errors.New("expected false value") - errExpectedTrue = errors.New("expected true value") - errExpectedArrayField = errors.New("expected ']' or ','") -) - -var ( - nullSymbol = []byte("null") - trueSymbol = []byte("true") - falseSymbol = []byte("false") -) - -type entity uint8 - -const ( - failEntity entity = iota - trueValue - falseValue - nullValue - dictStart - dictEnd - arrStart - arrEnd - stringEntity - mapKeyEntity - intEntity - doubleEntity -) - -type state uint8 - -const ( - failedState state = iota - startState - arrState - arrStateNext - dictState - dictFieldState - dictFieldStateEnd -) - -var entityNames = map[entity]string{ - failEntity: "failEntity", - trueValue: "trueValue", - falseValue: "falseValue", - nullValue: "nullValue", - dictStart: "dictStart", - dictEnd: "dictEnd", - arrStart: "arrStart", - arrEnd: "arrEnd", - stringEntity: "stringEntity", - mapKeyEntity: "mapKeyEntity", - intEntity: "intEntity", - doubleEntity: "doubleEntity", -} - -var stateNames = map[state]string{ - failedState: "failed", - startState: "start", - arrState: "array", - arrStateNext: "arrayNext", - dictState: "dict", - dictFieldState: "dictValue", - dictFieldStateEnd: "dictNext", -} - -func (e entity) String() string { - if name, ok := entityNames[e]; ok { - return name - } - return "unknown" -} - -func (s state) String() string { - if name, ok := stateNames[s]; ok { - return name - } - return "unknown" -} - -// NewJSONReader returns a new JSONReader initialized with in -func NewJSONReader(in []byte) *JSONReader { - r := &JSONReader{} - r.init(in) - return r -} - -func (r *JSONReader) init(in []byte) { - r.Buffer.Init(in, true) - r.currentState = startState - r.states = r.statesBuf[:0] -} - -var whitespace = []byte(" \t\r\n") - -func (r *JSONReader) skipWS() { - r.IgnoreSymbols(whitespace) -} - -func (r *JSONReader) pushState(next state) { - if r.currentState != failedState { - r.states = append(r.states, r.currentState) - } - r.currentState = next -} - -func (r *JSONReader) popState() { - if len(r.states) == 0 { - r.currentState = failedState - } else { - last := len(r.states) - 1 - r.currentState = r.states[last] - r.states = r.states[:last] - } -} - -// ExpectDict checks if the next entity is a json object -func (r *JSONReader) ExpectDict() error { - e, _, err := r.step() - - if err != nil { - return err - } - - if e != dictStart { - return r.SetError(errExpectedObject) - } - - return nil -} - -// ExpectArray checks if the next entity is a json array -func (r *JSONReader) ExpectArray() error { - e, _, err := r.step() - if err != nil { - return err - } - - if e != arrStart { - return r.SetError(errExpectedArray) - } - - return nil -} - -func (r *JSONReader) nextFieldName() (entity, []byte, error) { - e, raw, err := r.step() - if err != nil { - return e, raw, err - } - - if e != mapKeyEntity && e != dictEnd { - return e, nil, r.SetError(errExpectedFieldName) - } - - return e, raw, err -} - -func (r *JSONReader) nextInt() (int, error) { - e, raw, err := r.step() - if err != nil { - return 0, err - } - - if e != intEntity { - return 0, errExpectedInteger - } - - tmp := streambuf.NewFixed(raw) - i, err := tmp.IntASCII(false) - return int(i), err -} - -// ignore type of next element and return raw content. -func (r *JSONReader) ignoreNext() (raw []byte, err error) { - r.skipWS() - - snapshot := r.Snapshot() - before := r.Len() - - e, _, err := r.step() - if err != nil { - return nil, err - } - - switch e { - case arrStart: - err = ignoreKind(r, arrEnd) - case dictStart: - err = ignoreKind(r, dictEnd) - default: - } - if err != nil { - return nil, err - } - - after := r.Len() - r.Restore(snapshot) - - bytes, _ := r.Collect(before - after) - return bytes, nil -} - -func ignoreKind(r *JSONReader, kind entity) error { - for { - e, _, err := r.step() - if err != nil { - return err - } - - switch e { - case kind: - return nil - case arrStart: - if err := ignoreKind(r, arrEnd); err != nil { - return err - } - case dictStart: - if err := ignoreKind(r, dictEnd); err != nil { - return err - } - } - } -} - -// step continues the JSON parser state machine until next entity has been parsed. -func (r *JSONReader) step() (entity, []byte, error) { - r.skipWS() - switch r.currentState { - case failedState: - return r.stepFailing() - case startState: - return r.stepStart() - case arrState: - return r.stepArray() - case arrStateNext: - return r.stepArrayNext() - case dictState: - return r.stepDict() - case dictFieldState: - return r.stepDictValue() - case dictFieldStateEnd: - return r.stepDictValueEnd() - default: - return r.failWith(errFailing) - } -} - -func (r *JSONReader) stepFailing() (entity, []byte, error) { - return failEntity, nil, r.Err() -} - -func (r *JSONReader) stepStart() (entity, []byte, error) { - c, err := r.PeekByte() - if err != nil { - return r.failWith(err) - } - - return r.tryStepPrimitive(c) -} - -func (r *JSONReader) stepArray() (entity, []byte, error) { - return r.doStepArray(true) -} - -func (r *JSONReader) stepArrayNext() (entity, []byte, error) { - c, err := r.PeekByte() - if err != nil { - return r.failWith(errFailing) - } - - switch c { - case ']': - return r.endArray() - case ',': - r.Advance(1) - r.skipWS() - r.currentState = arrState - return r.doStepArray(false) - default: - return r.failWith(errExpectedArrayField) - } -} - -func (r *JSONReader) doStepArray(allowArrayEnd bool) (entity, []byte, error) { - c, err := r.PeekByte() - if err != nil { - return r.failWith(err) - } - - if c == ']' { - if !allowArrayEnd { - return r.failWith(errUnexpectedArrClose) - } - return r.endArray() - } - - r.currentState = arrStateNext - return r.tryStepPrimitive(c) -} - -func (r *JSONReader) stepDict() (entity, []byte, error) { - return r.doStepDict(true) -} - -func (r *JSONReader) doStepDict(allowEnd bool) (entity, []byte, error) { - c, err := r.PeekByte() - if err != nil { - return r.failWith(err) - } - - switch c { - case '}': - if !allowEnd { - return r.failWith(errUnexpectedDictClose) - } - return r.endDict() - case '"': - r.currentState = dictFieldState - return r.stepMapKey() - default: - return r.failWith(errExpectedFieldName) - } -} - -func (r *JSONReader) stepDictValue() (entity, []byte, error) { - c, err := r.PeekByte() - if err != nil { - return r.failWith(err) - } - - r.currentState = dictFieldStateEnd - return r.tryStepPrimitive(c) -} - -func (r *JSONReader) stepDictValueEnd() (entity, []byte, error) { - c, err := r.PeekByte() - if err != nil { - return r.failWith(err) - } - - switch c { - case '}': - return r.endDict() - case ',': - r.Advance(1) - r.skipWS() - r.currentState = dictState - return r.doStepDict(false) - default: - return r.failWith(errUnknownChar) - } -} - -func (r *JSONReader) tryStepPrimitive(c byte) (entity, []byte, error) { - switch c { - case '{': // start dictionary - return r.startDict() - case '[': // start array - return r.startArray() - case 'n': // null - return r.stepNull() - case 'f': // false - return r.stepFalse() - case 't': // true - return r.stepTrue() - case '"': - return r.stepString() - default: - // parse number? - if c == '-' || c == '+' || c == '.' || ('0' <= c && c <= '9') { - return r.stepNumber() - } - - err := r.Err() - if err == nil { - err = r.SetError(errUnknownChar) - } - return r.failWith(err) - } -} - -func (r *JSONReader) stepNull() (entity, []byte, error) { - return stepSymbol(r, nullValue, nullSymbol, errExpectedNull) -} - -func (r *JSONReader) stepTrue() (entity, []byte, error) { - return stepSymbol(r, trueValue, trueSymbol, errExpectedTrue) -} - -func (r *JSONReader) stepFalse() (entity, []byte, error) { - return stepSymbol(r, falseValue, falseSymbol, errExpectedFalse) -} - -func stepSymbol(r *JSONReader, e entity, symb []byte, fail error) (entity, []byte, error) { - ok, err := r.MatchASCII(symb) - if err != nil { - return failEntity, nil, err - } - if !ok { - return failEntity, nil, fail - } - - r.Advance(len(symb)) - return e, nil, nil -} - -func (r *JSONReader) stepMapKey() (entity, []byte, error) { - e, key, err := r.stepString() - if err != nil { - return e, key, err - } - - r.skipWS() - c, err := r.ReadByte() - if err != nil { - return failEntity, nil, err - } - - if c != ':' { - return r.failWith(r.SetError(errExpectColon)) - } - - if err := r.Err(); err != nil { - return r.failWith(err) - } - return mapKeyEntity, key, nil -} - -func (r *JSONReader) stepString() (entity, []byte, error) { - start := 1 - for { - idxQuote := r.IndexByteFrom(start, '"') - if idxQuote == -1 { - return failEntity, nil, r.SetError(errQuoteMissing) - } - - if b, _ := r.PeekByteFrom(idxQuote - 1); b == '\\' { // escaped quote? - start = idxQuote + 1 - continue - } - - // found string end - str, err := r.Collect(idxQuote + 1) - str = str[1 : len(str)-1] - return stringEntity, str, err - } -} - -func (r *JSONReader) startDict() (entity, []byte, error) { - r.Advance(1) - r.pushState(dictState) - return dictStart, nil, nil -} - -func (r *JSONReader) endDict() (entity, []byte, error) { - r.Advance(1) - r.popState() - return dictEnd, nil, nil -} - -func (r *JSONReader) startArray() (entity, []byte, error) { - r.Advance(1) - r.pushState(arrState) - return arrStart, nil, nil -} - -func (r *JSONReader) endArray() (entity, []byte, error) { - r.Advance(1) - r.popState() - return arrEnd, nil, nil -} - -func (r *JSONReader) failWith(err error) (entity, []byte, error) { - r.currentState = failedState - return failEntity, nil, r.SetError(err) -} - -func (r *JSONReader) stepNumber() (entity, []byte, error) { - snapshot := r.Snapshot() - lenBefore := r.Len() - isDouble := false - - if err := r.Err(); err != nil { - return failEntity, nil, err - } - - // parse '+', '-' or '.' - if b, _ := r.PeekByte(); b == '-' || b == '+' { - r.Advance(1) - } - if b, _ := r.PeekByte(); b == '.' { - r.Advance(1) - isDouble = true - } - - // parse digits - buf, _ := r.CollectWhile(isDigit) - if len(buf) == 0 { - return failEntity, nil, r.SetError(errExpectedDigit) - } - - if !isDouble { - // parse optional '.' - if b, _ := r.PeekByte(); b == '.' { - r.Advance(1) - isDouble = true - - // parse optional digits - r.CollectWhile(isDigit) - } - } - - lenAfter := r.Len() - r.Restore(snapshot) - total := lenBefore - lenAfter - 1 - if total == 0 { - return failEntity, nil, r.SetError(errExpectedDigit) - } - - raw, _ := r.Collect(total) - state := intEntity - if isDouble { - state = doubleEntity - } - - return state, raw, nil -} - -func isDigit(c byte) bool { - return '0' <= c && c <= '9' -} diff --git a/libbeat/outputs/elasticsearch/bulk.go b/libbeat/outputs/elasticsearch/bulk.go index e774a92916c..c771184afb8 100644 --- a/libbeat/outputs/elasticsearch/bulk.go +++ b/libbeat/outputs/elasticsearch/bulk.go @@ -37,7 +37,7 @@ var ( ) // bulkReadToItems reads the bulk response up to (but not including) items -func bulkReadToItems(reader *JSONReader) error { +func bulkReadToItems(reader *jsonReader) error { if err := reader.ExpectDict(); err != nil { return errExpectedObject } @@ -70,7 +70,7 @@ func bulkReadToItems(reader *JSONReader) error { } // bulkReadItemStatus reads the status and error fields from the bulk item -func bulkReadItemStatus(logger *logp.Logger, reader *JSONReader) (int, []byte, error) { +func bulkReadItemStatus(logger *logp.Logger, reader *jsonReader) (int, []byte, error) { // skip outer dictionary if err := reader.ExpectDict(); err != nil { return 0, nil, errExpectedItemObject @@ -110,7 +110,7 @@ func bulkReadItemStatus(logger *logp.Logger, reader *JSONReader) (int, []byte, e return status, msg, nil } -func itemStatusInner(reader *JSONReader, logger *logp.Logger) (int, []byte, error) { +func itemStatusInner(reader *jsonReader, logger *logp.Logger) (int, []byte, error) { if err := reader.ExpectDict(); err != nil { return 0, nil, errExpectedItemObject } diff --git a/libbeat/outputs/elasticsearch/bulk_test.go b/libbeat/outputs/elasticsearch/bulk_test.go index 9bd1abdbccf..49e84128a47 100644 --- a/libbeat/outputs/elasticsearch/bulk_test.go +++ b/libbeat/outputs/elasticsearch/bulk_test.go @@ -36,7 +36,7 @@ func TestBulkReadToItems(t *testing.T) { {"create": {"status": 400}} ]}`) - reader := NewJSONReader(response) + reader := newJSONReader(response) err := bulkReadToItems(reader) assert.NoError(t, err) @@ -73,7 +73,7 @@ func TestBulkReadToItems(t *testing.T) { func TestBulkReadItemStatus(t *testing.T) { response := []byte(`{"create": {"status": 200}}`) - reader := NewJSONReader(response) + reader := newJSONReader(response) code, _, err := bulkReadItemStatus(logp.L(), reader) assert.NoError(t, err) assert.Equal(t, 200, code) @@ -125,7 +125,7 @@ func TestES2StyleExtendedErrorStatus(t *testing.T) { } func readStatusItem(in []byte) (int, string, error) { - reader := NewJSONReader(in) + reader := newJSONReader(in) code, msg, err := bulkReadItemStatus(logp.L(), reader) return code, string(msg), err } diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index badedf162a1..8afab2ccdc6 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -347,7 +347,7 @@ func bulkCollectPublishFails( result eslegclient.BulkResult, data []publisher.Event, ) ([]publisher.Event, bulkResultStats) { - reader := NewJSONReader(result) + reader := newJSONReader(result) if err := bulkReadToItems(reader); err != nil { log.Errorf("failed to parse bulk response: %v", err.Error()) return nil, bulkResultStats{} diff --git a/libbeat/outputs/elasticsearch/json_read.go b/libbeat/outputs/elasticsearch/json_read.go index 8df87e5cb43..eb7d3c696ee 100644 --- a/libbeat/outputs/elasticsearch/json_read.go +++ b/libbeat/outputs/elasticsearch/json_read.go @@ -30,7 +30,7 @@ import ( // // Due to parser simply stepping through the input buffer, almost no additional // allocations are required. -type JSONReader struct { +type jsonReader struct { streambuf.Buffer // parser state machine @@ -133,14 +133,13 @@ func (s state) String() string { return "unknown" } -// NewJSONReader returns a new JSONReader initialized with in -func NewJSONReader(in []byte) *JSONReader { - r := &JSONReader{} +func newJSONReader(in []byte) *jsonReader { + r := &jsonReader{} r.init(in) return r } -func (r *JSONReader) init(in []byte) { +func (r *jsonReader) init(in []byte) { r.Buffer.Init(in, true) r.currentState = startState r.states = r.statesBuf[:0] @@ -148,18 +147,18 @@ func (r *JSONReader) init(in []byte) { var whitespace = []byte(" \t\r\n") -func (r *JSONReader) skipWS() { +func (r *jsonReader) skipWS() { r.IgnoreSymbols(whitespace) } -func (r *JSONReader) pushState(next state) { +func (r *jsonReader) pushState(next state) { if r.currentState != failedState { r.states = append(r.states, r.currentState) } r.currentState = next } -func (r *JSONReader) popState() { +func (r *jsonReader) popState() { if len(r.states) == 0 { r.currentState = failedState } else { @@ -170,7 +169,7 @@ func (r *JSONReader) popState() { } // ExpectDict checks if the next entity is a json object -func (r *JSONReader) ExpectDict() error { +func (r *jsonReader) ExpectDict() error { e, _, err := r.step() if err != nil { @@ -185,7 +184,7 @@ func (r *JSONReader) ExpectDict() error { } // ExpectArray checks if the next entity is a json array -func (r *JSONReader) ExpectArray() error { +func (r *jsonReader) ExpectArray() error { e, _, err := r.step() if err != nil { return err @@ -198,7 +197,7 @@ func (r *JSONReader) ExpectArray() error { return nil } -func (r *JSONReader) nextFieldName() (entity, []byte, error) { +func (r *jsonReader) nextFieldName() (entity, []byte, error) { e, raw, err := r.step() if err != nil { return e, raw, err @@ -211,7 +210,7 @@ func (r *JSONReader) nextFieldName() (entity, []byte, error) { return e, raw, err } -func (r *JSONReader) nextInt() (int, error) { +func (r *jsonReader) nextInt() (int, error) { e, raw, err := r.step() if err != nil { return 0, err @@ -227,7 +226,7 @@ func (r *JSONReader) nextInt() (int, error) { } // ignore type of next element and return raw content. -func (r *JSONReader) ignoreNext() (raw []byte, err error) { +func (r *jsonReader) ignoreNext() (raw []byte, err error) { r.skipWS() snapshot := r.Snapshot() @@ -256,7 +255,7 @@ func (r *JSONReader) ignoreNext() (raw []byte, err error) { return bytes, nil } -func ignoreKind(r *JSONReader, kind entity) error { +func ignoreKind(r *jsonReader, kind entity) error { for { e, _, err := r.step() if err != nil { @@ -279,7 +278,7 @@ func ignoreKind(r *JSONReader, kind entity) error { } // step continues the JSON parser state machine until next entity has been parsed. -func (r *JSONReader) step() (entity, []byte, error) { +func (r *jsonReader) step() (entity, []byte, error) { r.skipWS() switch r.currentState { case failedState: @@ -301,11 +300,11 @@ func (r *JSONReader) step() (entity, []byte, error) { } } -func (r *JSONReader) stepFailing() (entity, []byte, error) { +func (r *jsonReader) stepFailing() (entity, []byte, error) { return failEntity, nil, r.Err() } -func (r *JSONReader) stepStart() (entity, []byte, error) { +func (r *jsonReader) stepStart() (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(err) @@ -314,11 +313,11 @@ func (r *JSONReader) stepStart() (entity, []byte, error) { return r.tryStepPrimitive(c) } -func (r *JSONReader) stepArray() (entity, []byte, error) { +func (r *jsonReader) stepArray() (entity, []byte, error) { return r.doStepArray(true) } -func (r *JSONReader) stepArrayNext() (entity, []byte, error) { +func (r *jsonReader) stepArrayNext() (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(errFailing) @@ -337,7 +336,7 @@ func (r *JSONReader) stepArrayNext() (entity, []byte, error) { } } -func (r *JSONReader) doStepArray(allowArrayEnd bool) (entity, []byte, error) { +func (r *jsonReader) doStepArray(allowArrayEnd bool) (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(err) @@ -354,11 +353,11 @@ func (r *JSONReader) doStepArray(allowArrayEnd bool) (entity, []byte, error) { return r.tryStepPrimitive(c) } -func (r *JSONReader) stepDict() (entity, []byte, error) { +func (r *jsonReader) stepDict() (entity, []byte, error) { return r.doStepDict(true) } -func (r *JSONReader) doStepDict(allowEnd bool) (entity, []byte, error) { +func (r *jsonReader) doStepDict(allowEnd bool) (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(err) @@ -378,7 +377,7 @@ func (r *JSONReader) doStepDict(allowEnd bool) (entity, []byte, error) { } } -func (r *JSONReader) stepDictValue() (entity, []byte, error) { +func (r *jsonReader) stepDictValue() (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(err) @@ -388,7 +387,7 @@ func (r *JSONReader) stepDictValue() (entity, []byte, error) { return r.tryStepPrimitive(c) } -func (r *JSONReader) stepDictValueEnd() (entity, []byte, error) { +func (r *jsonReader) stepDictValueEnd() (entity, []byte, error) { c, err := r.PeekByte() if err != nil { return r.failWith(err) @@ -407,7 +406,7 @@ func (r *JSONReader) stepDictValueEnd() (entity, []byte, error) { } } -func (r *JSONReader) tryStepPrimitive(c byte) (entity, []byte, error) { +func (r *jsonReader) tryStepPrimitive(c byte) (entity, []byte, error) { switch c { case '{': // start dictionary return r.startDict() @@ -435,19 +434,19 @@ func (r *JSONReader) tryStepPrimitive(c byte) (entity, []byte, error) { } } -func (r *JSONReader) stepNull() (entity, []byte, error) { +func (r *jsonReader) stepNull() (entity, []byte, error) { return stepSymbol(r, nullValue, nullSymbol, errExpectedNull) } -func (r *JSONReader) stepTrue() (entity, []byte, error) { +func (r *jsonReader) stepTrue() (entity, []byte, error) { return stepSymbol(r, trueValue, trueSymbol, errExpectedTrue) } -func (r *JSONReader) stepFalse() (entity, []byte, error) { +func (r *jsonReader) stepFalse() (entity, []byte, error) { return stepSymbol(r, falseValue, falseSymbol, errExpectedFalse) } -func stepSymbol(r *JSONReader, e entity, symb []byte, fail error) (entity, []byte, error) { +func stepSymbol(r *jsonReader, e entity, symb []byte, fail error) (entity, []byte, error) { ok, err := r.MatchASCII(symb) if err != nil { return failEntity, nil, err @@ -460,7 +459,7 @@ func stepSymbol(r *JSONReader, e entity, symb []byte, fail error) (entity, []byt return e, nil, nil } -func (r *JSONReader) stepMapKey() (entity, []byte, error) { +func (r *jsonReader) stepMapKey() (entity, []byte, error) { e, key, err := r.stepString() if err != nil { return e, key, err @@ -482,7 +481,7 @@ func (r *JSONReader) stepMapKey() (entity, []byte, error) { return mapKeyEntity, key, nil } -func (r *JSONReader) stepString() (entity, []byte, error) { +func (r *jsonReader) stepString() (entity, []byte, error) { start := 1 for { idxQuote := r.IndexByteFrom(start, '"') @@ -502,36 +501,36 @@ func (r *JSONReader) stepString() (entity, []byte, error) { } } -func (r *JSONReader) startDict() (entity, []byte, error) { +func (r *jsonReader) startDict() (entity, []byte, error) { r.Advance(1) r.pushState(dictState) return dictStart, nil, nil } -func (r *JSONReader) endDict() (entity, []byte, error) { +func (r *jsonReader) endDict() (entity, []byte, error) { r.Advance(1) r.popState() return dictEnd, nil, nil } -func (r *JSONReader) startArray() (entity, []byte, error) { +func (r *jsonReader) startArray() (entity, []byte, error) { r.Advance(1) r.pushState(arrState) return arrStart, nil, nil } -func (r *JSONReader) endArray() (entity, []byte, error) { +func (r *jsonReader) endArray() (entity, []byte, error) { r.Advance(1) r.popState() return arrEnd, nil, nil } -func (r *JSONReader) failWith(err error) (entity, []byte, error) { +func (r *jsonReader) failWith(err error) (entity, []byte, error) { r.currentState = failedState return failEntity, nil, r.SetError(err) } -func (r *JSONReader) stepNumber() (entity, []byte, error) { +func (r *jsonReader) stepNumber() (entity, []byte, error) { snapshot := r.Snapshot() lenBefore := r.Len() isDouble := false From 023409245b2497ea4af5c6e838f5da38bec00941 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 10:55:11 -0800 Subject: [PATCH 81/91] Detect and try to fix scheme before parsing URL --- libbeat/common/url.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/libbeat/common/url.go b/libbeat/common/url.go index 7e19f618b7e..809655bf207 100644 --- a/libbeat/common/url.go +++ b/libbeat/common/url.go @@ -92,21 +92,17 @@ func ParseURL(raw string, hints ...ParseHint) (*url.URL, error) { return nil, nil } - u, err := url.Parse(raw) - if err == nil && strings.HasPrefix(u.Scheme, "http") { - return u, err - } - - // Parsing failed. Try applying hints and parsing again. if len(hints) == 0 { hints = append(hints, WithDefaultScheme("http")) } - for _, hint := range hints { - raw = hint(raw) + if parts := strings.SplitN(raw, "://", 2); len(parts) != 2 { + for _, hint := range hints { + raw = hint(raw) + } } - return u.Parse(raw) + return url.Parse(raw) } func WithDefaultScheme(scheme string) ParseHint { From 32f0411f5d567a16fc2b210538a8e7db153ec925 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 11:29:56 -0800 Subject: [PATCH 82/91] Making Connection private to ES output Client --- .../monitoring/report/elasticsearch/client.go | 15 +++--- libbeat/outputs/elasticsearch/client.go | 46 ++++++++++++------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/libbeat/monitoring/report/elasticsearch/client.go b/libbeat/monitoring/report/elasticsearch/client.go index b561ff1a623..55f4673eb5d 100644 --- a/libbeat/monitoring/report/elasticsearch/client.go +++ b/libbeat/monitoring/report/elasticsearch/client.go @@ -70,7 +70,8 @@ func (c *publishClient) Connect() error { params := map[string]string{ "filter_path": "features.monitoring.enabled", } - status, body, err := c.es.Request("GET", "/_xpack", "", params, nil) + conn := c.es.Connection() + status, body, err := conn.Request("GET", "/_xpack", "", params, nil) if err != nil { return fmt.Errorf("X-Pack capabilities query failed with: %v", err) } @@ -160,7 +161,8 @@ func (c *publishClient) Publish(batch publisher.Batch) error { } func (c *publishClient) Test(d testing.Driver) { - c.es.Test(d) + conn := c.es.Connection() + conn.Test(d) } func (c *publishClient) String() string { @@ -183,7 +185,8 @@ func (c *publishClient) publishXPackBulk(params map[string]string, event publish // Currently one request per event is sent. Reason is that each event can contain different // interval params and X-Pack requires to send the interval param. - _, err := c.es.SendMonitoringBulk(params, bulk[:]) + conn := c.es.Connection() + _, err := conn.SendMonitoringBulk(params, bulk[:]) return err } @@ -193,7 +196,8 @@ func (c *publishClient) publishBulk(event publisher.Event, typ string) error { "_routing": nil, } - esVersion := c.es.GetVersion() + conn := c.es.Connection() + esVersion := conn.GetVersion() if esVersion.Major < 7 { meta["_type"] = "doc" } @@ -234,8 +238,7 @@ func (c *publishClient) publishBulk(event publisher.Event, typ string) error { // Currently one request per event is sent. Reason is that each event can contain different // interval params and X-Pack requires to send the interval param. - // FIXME: index name (first param below) - _, result, err := c.es.Bulk(getMonitoringIndexName(), "", nil, bulk[:]) + _, result, err := conn.Bulk(getMonitoringIndexName(), "", nil, bulk[:]) if err != nil { return err } diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 8afab2ccdc6..4205f794b68 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -35,7 +35,7 @@ import ( // Client is an elasticsearch client. type Client struct { - eslegclient.Connection + conn eslegclient.Connection index outputs.IndexSelector pipeline *outil.Selector @@ -119,9 +119,9 @@ func NewClient( } client := &Client{ - Connection: *conn, - index: s.Index, - pipeline: pipeline, + conn: *conn, + index: s.Index, + pipeline: pipeline, observer: s.Observer, @@ -141,20 +141,20 @@ func (client *Client) Clone() *Client { c, _ := NewClient( ClientSettings{ ConnectionSettings: eslegclient.ConnectionSettings{ - URL: client.URL, - Proxy: client.Proxy, + URL: client.conn.URL, + Proxy: client.conn.Proxy, // Without the following nil check on proxyURL, a nil Proxy field will try // reloading proxy settings from the environment instead of leaving them // empty. - ProxyDisable: client.Proxy == nil, - TLS: client.TLS, - Username: client.Username, - Password: client.Password, - APIKey: client.APIKey, + ProxyDisable: client.conn.Proxy == nil, + TLS: client.conn.TLS, + Username: client.conn.Username, + Password: client.conn.Password, + APIKey: client.conn.APIKey, Parameters: nil, // XXX: do not pass params? - Headers: client.Headers, - Timeout: client.HTTP.Timeout, - CompressionLevel: client.CompressionLevel, + Headers: client.conn.Headers, + Timeout: client.conn.HTTP.Timeout, + CompressionLevel: client.conn.CompressionLevel, OnConnectCallback: nil, Observer: nil, EscapeHTML: false, @@ -198,7 +198,7 @@ func (client *Client) publishEvents( // encode events into bulk request buffer, dropping failed elements from // events slice origCount := len(data) - data, bulkItems := bulkEncodePublishRequest(client.log, client.GetVersion(), client.index, client.pipeline, data) + data, bulkItems := bulkEncodePublishRequest(client.log, client.conn.GetVersion(), client.index, client.pipeline, data) newCount := len(data) if st != nil && origCount > newCount { st.Dropped(origCount - newCount) @@ -207,7 +207,7 @@ func (client *Client) publishEvents( return nil, nil } - status, result, sendErr := client.Bulk("", "", nil, bulkItems) + status, result, sendErr := client.conn.Bulk("", "", nil, bulkItems) if sendErr != nil { client.log.Errorf("Failed to perform any bulk index operations: %s", sendErr) return data, sendErr @@ -394,6 +394,18 @@ func bulkCollectPublishFails( return failed, stats } +func (client *Client) Connect() error { + return client.conn.Connect() +} + +func (client *Client) Close() error { + return client.conn.Close() +} + +func (client *Client) Connection() eslegclient.Connection { + return client.conn +} + func (client *Client) String() string { - return "elasticsearch(" + client.Connection.URL + ")" + return "elasticsearch(" + client.conn.URL + ")" } From be64ba44603a60c292f384912b652c92cd13c0ec Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 11:48:37 -0800 Subject: [PATCH 83/91] Update test --- libbeat/outputs/elasticsearch/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index 764ad9a00d4..fdb66d8c43d 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -335,6 +335,6 @@ func TestClientWithAPIKey(t *testing.T) { }, nil) assert.NoError(t, err) - client.Ping() + client.Connect() assert.Equal(t, "ApiKey aHlva0hHNEJmV2s1dmlLWjE3Mlg6bzQ1SlVreXVTLS15aVNBdXV4bDhVdw==", headers.Get("Authorization")) } From 26c28f293987749540cc8500e4394c32dea177b9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 11:56:11 -0800 Subject: [PATCH 84/91] Replace client.Ping() calls with client.Connect() calls in test code --- libbeat/outputs/elasticsearch/client_proxy_test.go | 2 +- libbeat/outputs/elasticsearch/client_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client_proxy_test.go b/libbeat/outputs/elasticsearch/client_proxy_test.go index b431fcc4668..b6751860e0a 100644 --- a/libbeat/outputs/elasticsearch/client_proxy_test.go +++ b/libbeat/outputs/elasticsearch/client_proxy_test.go @@ -203,7 +203,7 @@ func doClientPing(t *testing.T) { // This ping won't succeed; we aren't testing end-to-end communication // (which would require a lot more setup work), we just want to make sure // the client is pointed at the right server or proxy. - client.Ping() + client.Connect() } // serverState contains the state of the http listeners for proxy tests, diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index fdb66d8c43d..afda2797549 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -231,7 +231,7 @@ func TestClientWithHeaders(t *testing.T) { assert.NoError(t, err) // simple ping - client.Ping() + client.Connect() assert.Equal(t, 1, requestCount) // bulk request From addcfd45ddfb548afcfb4013e6d4cd079f9f1bec Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 11:57:47 -0800 Subject: [PATCH 85/91] Updating tests --- libbeat/outputs/elasticsearch/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index afda2797549..d69849dabab 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -244,7 +244,7 @@ func TestClientWithHeaders(t *testing.T) { batch := outest.NewBatch(event, event, event) err = client.Publish(batch) assert.NoError(t, err) - assert.Equal(t, 3, requestCount) + assert.Equal(t, 2, requestCount) } func TestBulkEncodeEvents(t *testing.T) { From 6788cfd81b5b5c63784de48c1a70dd5cb345fe26 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 12:16:40 -0800 Subject: [PATCH 86/91] Removing usage of ES output from monitoring code! --- .../monitoring/report/elasticsearch/client.go | 21 +++++-------- .../report/elasticsearch/elasticsearch.go | 30 ++++++++----------- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/libbeat/monitoring/report/elasticsearch/client.go b/libbeat/monitoring/report/elasticsearch/client.go index 55f4673eb5d..9e8469ab547 100644 --- a/libbeat/monitoring/report/elasticsearch/client.go +++ b/libbeat/monitoring/report/elasticsearch/client.go @@ -29,7 +29,6 @@ import ( "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring/report" - "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" "github.com/elastic/beats/v7/libbeat/publisher" "github.com/elastic/beats/v7/libbeat/testing" ) @@ -37,7 +36,7 @@ import ( var createDocPrivAvailableESVersion = common.MustNewVersion("7.5.0") type publishClient struct { - es *elasticsearch.Client + es *eslegclient.Connection params map[string]string format report.Format @@ -45,7 +44,7 @@ type publishClient struct { } func newPublishClient( - es *elasticsearch.Client, + es *eslegclient.Connection, params map[string]string, format report.Format, ) (*publishClient, error) { @@ -70,8 +69,7 @@ func (c *publishClient) Connect() error { params := map[string]string{ "filter_path": "features.monitoring.enabled", } - conn := c.es.Connection() - status, body, err := conn.Request("GET", "/_xpack", "", params, nil) + status, body, err := c.es.Request("GET", "/_xpack", "", params, nil) if err != nil { return fmt.Errorf("X-Pack capabilities query failed with: %v", err) } @@ -161,12 +159,11 @@ func (c *publishClient) Publish(batch publisher.Batch) error { } func (c *publishClient) Test(d testing.Driver) { - conn := c.es.Connection() - conn.Test(d) + c.es.Test(d) } func (c *publishClient) String() string { - return "publish(" + c.es.String() + ")" + return "monitoring(" + c.es.URL + ")" } func (c *publishClient) publishXPackBulk(params map[string]string, event publisher.Event, typ string) error { @@ -185,8 +182,7 @@ func (c *publishClient) publishXPackBulk(params map[string]string, event publish // Currently one request per event is sent. Reason is that each event can contain different // interval params and X-Pack requires to send the interval param. - conn := c.es.Connection() - _, err := conn.SendMonitoringBulk(params, bulk[:]) + _, err := c.es.SendMonitoringBulk(params, bulk[:]) return err } @@ -196,8 +192,7 @@ func (c *publishClient) publishBulk(event publisher.Event, typ string) error { "_routing": nil, } - conn := c.es.Connection() - esVersion := conn.GetVersion() + esVersion := c.es.GetVersion() if esVersion.Major < 7 { meta["_type"] = "doc" } @@ -238,7 +233,7 @@ func (c *publishClient) publishBulk(event publisher.Event, typ string) error { // Currently one request per event is sent. Reason is that each event can contain different // interval params and X-Pack requires to send the interval param. - _, result, err := conn.Bulk(getMonitoringIndexName(), "", nil, bulk[:]) + _, result, err := c.es.Bulk(getMonitoringIndexName(), "", nil, bulk[:]) if err != nil { return err } diff --git a/libbeat/monitoring/report/elasticsearch/elasticsearch.go b/libbeat/monitoring/report/elasticsearch/elasticsearch.go index fef360079e7..4bab9f3117d 100644 --- a/libbeat/monitoring/report/elasticsearch/elasticsearch.go +++ b/libbeat/monitoring/report/elasticsearch/elasticsearch.go @@ -34,8 +34,6 @@ import ( "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/monitoring/report" "github.com/elastic/beats/v7/libbeat/outputs" - esout "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" - "github.com/elastic/beats/v7/libbeat/outputs/outil" "github.com/elastic/beats/v7/libbeat/publisher/pipeline" "github.com/elastic/beats/v7/libbeat/publisher/processing" "github.com/elastic/beats/v7/libbeat/publisher/queue" @@ -337,22 +335,18 @@ func makeClient( return nil, err } - esClient, err := esout.NewClient(esout.ClientSettings{ - ConnectionSettings: eslegclient.ConnectionSettings{ - URL: url, - Proxy: proxyURL, - TLS: tlsConfig, - Username: config.Username, - Password: config.Password, - APIKey: config.APIKey, - Parameters: params, - Headers: config.Headers, - Timeout: config.Timeout, - CompressionLevel: config.CompressionLevel, - }, - Index: outil.MakeSelector(outil.ConstSelectorExpr("_xpack")), - Pipeline: nil, - }, nil) + esClient, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ + URL: url, + Proxy: proxyURL, + TLS: tlsConfig, + Username: config.Username, + Password: config.Password, + APIKey: config.APIKey, + Parameters: params, + Headers: config.Headers, + Timeout: config.Timeout, + CompressionLevel: config.CompressionLevel, + }) if err != nil { return nil, err } From 9de9b87e2ca5593e68ee86547d7f9c7f122ac10c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 12:21:29 -0800 Subject: [PATCH 87/91] Using strings.Index instead of strings.SplitN --- libbeat/common/url.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/common/url.go b/libbeat/common/url.go index 809655bf207..949c4631edf 100644 --- a/libbeat/common/url.go +++ b/libbeat/common/url.go @@ -96,7 +96,7 @@ func ParseURL(raw string, hints ...ParseHint) (*url.URL, error) { hints = append(hints, WithDefaultScheme("http")) } - if parts := strings.SplitN(raw, "://", 2); len(parts) != 2 { + if strings.Index(raw, "://") == -1 { for _, hint := range hints { raw = hint(raw) } From 63664e5f4d86da786de074f5793b8127e2c6121b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 12:34:29 -0800 Subject: [PATCH 88/91] Return default config via function call --- libbeat/esleg/eslegclient/config.go | 6 +++--- libbeat/esleg/eslegclient/connection.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libbeat/esleg/eslegclient/config.go b/libbeat/esleg/eslegclient/config.go index 46a17f3ca3a..5c171a4eb2b 100644 --- a/libbeat/esleg/eslegclient/config.go +++ b/libbeat/esleg/eslegclient/config.go @@ -46,8 +46,8 @@ type config struct { Timeout time.Duration `config:"timeout"` } -var ( - defaultConfig = config{ +func defaultConfig() config { + return config{ Protocol: "", Path: "", ProxyURL: "", @@ -61,7 +61,7 @@ var ( EscapeHTML: false, TLS: nil, } -) +} func (c *config) Validate() error { if c.ProxyURL != "" && !c.ProxyDisable { diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index db84fe2a086..b591307c444 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -137,7 +137,7 @@ func NewConnection(s ConnectionSettings) (*Connection, error) { // output, except for the output specific configuration options. If multiple hosts // are defined in the configuration, a client is returned for each of them. func NewClients(cfg *common.Config) ([]Connection, error) { - config := defaultConfig + config := defaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, err } From bfa05b7f8b12e7994622734927cacb0102dd2c25 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 12:43:01 -0800 Subject: [PATCH 89/91] Removing "escape hatch" method to expose underlying connection from ES output client --- libbeat/outputs/elasticsearch/client.go | 4 ---- libbeat/outputs/elasticsearch/client_integration_test.go | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 4205f794b68..045dfc77446 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -402,10 +402,6 @@ func (client *Client) Close() error { return client.conn.Close() } -func (client *Client) Connection() eslegclient.Connection { - return client.conn -} - func (client *Client) String() string { return "elasticsearch(" + client.conn.URL + ")" } diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index 80f15e2afa8..68155f6146b 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -91,7 +91,7 @@ func TestClientPublishEventWithPipeline(t *testing.T) { client.Delete(index, "", "", nil) // Check version - if client.Connection.GetVersion().Major < 5 { + if client.conn.GetVersion().Major < 5 { t.Skip("Skipping tests as pipeline not available in <5.x releases") } @@ -172,7 +172,7 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { }) client.Delete(index, "", "", nil) - if client.Connection.GetVersion().Major < 5 { + if client.conn.GetVersion().Major < 5 { t.Skip("Skipping tests as pipeline not available in <5.x releases") } From a61779d611fb90ee3bb06d46221eca91cc4baf07 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 13:43:36 -0800 Subject: [PATCH 90/91] Using client connection in tests --- .../elasticsearch/client_integration_test.go | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index 68155f6146b..1e01b757da0 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -48,7 +48,7 @@ func TestClientPublishEvent(t *testing.T) { }) // drop old index preparing test - client.Delete(index, "", "", nil) + client.conn.Delete(index, "", "", nil) batch := outest.NewBatch(beat.Event{ Timestamp: time.Now(), @@ -63,12 +63,12 @@ func TestClientPublishEvent(t *testing.T) { t.Fatal(err) } - _, _, err = client.Refresh(index) + _, _, err = client.conn.Refresh(index) if err != nil { t.Fatal(err) } - _, resp, err := client.CountSearchURI(index, "", nil) + _, resp, err := client.conn.CountSearchURI(index, "", nil) if err != nil { t.Fatal(err) } @@ -88,7 +88,7 @@ func TestClientPublishEventWithPipeline(t *testing.T) { "index": index, "pipeline": "%{[pipeline]}", }) - client.Delete(index, "", "", nil) + client.conn.Delete(index, "", "", nil) // Check version if client.conn.GetVersion().Major < 5 { @@ -103,7 +103,7 @@ func TestClientPublishEventWithPipeline(t *testing.T) { } getCount := func(query string) int { - _, resp, err := client.CountSearchURI(index, "", map[string]string{ + _, resp, err := client.conn.CountSearchURI(index, "", map[string]string{ "q": query, }) if err != nil { @@ -124,8 +124,8 @@ func TestClientPublishEventWithPipeline(t *testing.T) { }, } - client.DeletePipeline(pipeline, nil) - _, resp, err := client.CreatePipeline(pipeline, nil, pipelineBody) + client.conn.DeletePipeline(pipeline, nil) + _, resp, err := client.conn.CreatePipeline(pipeline, nil, pipelineBody) if err != nil { t.Fatal(err) } @@ -149,7 +149,7 @@ func TestClientPublishEventWithPipeline(t *testing.T) { "testfield": 0, }}) - _, _, err = client.Refresh(index) + _, _, err = client.conn.Refresh(index) if err != nil { t.Fatal(err) } @@ -170,7 +170,7 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { "index": index, "pipeline": "%{[pipeline]}", }) - client.Delete(index, "", "", nil) + client.conn.Delete(index, "", "", nil) if client.conn.GetVersion().Major < 5 { t.Skip("Skipping tests as pipeline not available in <5.x releases") @@ -184,7 +184,7 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { } getCount := func(query string) int { - _, resp, err := client.CountSearchURI(index, "", map[string]string{ + _, resp, err := client.conn.CountSearchURI(index, "", map[string]string{ "q": query, }) if err != nil { @@ -205,8 +205,8 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { }, } - client.DeletePipeline(pipeline, nil) - _, resp, err := client.CreatePipeline(pipeline, nil, pipelineBody) + client.conn.DeletePipeline(pipeline, nil) + _, resp, err := client.conn.CreatePipeline(pipeline, nil, pipelineBody) if err != nil { t.Fatal(err) } @@ -232,7 +232,7 @@ func TestClientBulkPublishEventsWithPipeline(t *testing.T) { }}, ) - _, _, err = client.Refresh(index) + _, _, err = client.conn.Refresh(index) if err != nil { t.Fatal(err) } From a416cfdc92b3549c14c641cde4e2c65b2b9e4a60 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Mar 2020 16:17:24 -0800 Subject: [PATCH 91/91] Re-implement Test() method for ES output client --- libbeat/outputs/elasticsearch/client.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 045dfc77446..2969c0f057b 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -24,6 +24,8 @@ import ( "net/http" "time" + "github.com/elastic/beats/v7/libbeat/testing" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" @@ -405,3 +407,7 @@ func (client *Client) Close() error { func (client *Client) String() string { return "elasticsearch(" + client.conn.URL + ")" } + +func (client *Client) Test(d testing.Driver) { + client.conn.Test(d) +}